From 08098235119b3927a17493db99d2c57456685586 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Wed, 26 Aug 2026 22:49:41 +0200 Subject: [PATCH] feat(openbao): run three raft replicas so a node drain is not an outage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenBao ran a single replica. Every node drain took the whole secret store down with it: today the k8s-1 Talos upgrade sealed it, the ClusterSecretStore went InvalidProviderConfig, and it stayed that way for two hours because nothing was watching. Three, not two. Raft needs a quorum and the quorum of two is two, so a two-replica cluster tolerates no failures — it would stop the moment one pod is sealed or rescheduled, which is worse than the single replica it replaced. Three tolerates one. What this buys: a drain seals the replica on that node, and raft elects a new leader from the remaining standbys within seconds. The service does not go down. What it does not buy, stated plainly because it is easy to assume otherwise: a sealed server can neither serve requests nor act as a standby, so that replica stays out of the cluster until someone unseals it. Scaling removes the outage, not the manual unseal. For rolling node maintenance the consequence matters — a Talos upgrade drains all three nodes in turn, so unseal each replica as it comes back, or the third drain takes the last unsealed one and quorum is lost anyway. retry_join is what makes a scaled replica join the existing cluster rather than come up isolated and uninitialized; without it each new pod would need a manual `bao operator raft join`. All three peers are listed on every node — the stanza is a list of possible leaders, a node ignores its own entry, and listing all three keeps the config identical across replicas regardless of which pod currently leads. Addressed through the headless service so each entry resolves to one specific pod. The snapshot job moves from the plain service to openbao-active. With one replica the plain service could only ever hit the one pod; with three it selects any of them, and a raft snapshot has to be taken against the leader. No affinity was added: the chart already applies a required podAntiAffinity on kubernetes.io/hostname, so the replicas spread across the three nodes on their own — verified against the running StatefulSet. Co-Authored-By: Claude Opus 5 (1M context) --- .../infrastructure/openbao/helmrelease.yaml | 48 +++++++++++++++++-- .../openbao/snapshot-cronjob.yaml | 8 +++- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/kubernetes/infrastructure/openbao/helmrelease.yaml b/kubernetes/infrastructure/openbao/helmrelease.yaml index 2e7d1d72..e7ecadb7 100644 --- a/kubernetes/infrastructure/openbao/helmrelease.yaml +++ b/kubernetes/infrastructure/openbao/helmrelease.yaml @@ -42,12 +42,28 @@ spec: # the agent-injector mutating webhook is unused surface. enabled: false server: - # Single-node raft (not standalone/file) so we get `bao operator raft - # snapshot` as a consistent backup primitive. TLS terminates at the - # Traefik ingress; in-cluster traffic is plain HTTP. + # Raft (not standalone/file) for two reasons: `bao operator raft + # snapshot` as a consistent backup primitive, and real HA across three + # nodes. TLS terminates at the Traefik ingress; in-cluster traffic is + # plain HTTP. ha: enabled: true - replicas: 1 + # Three, not two. Raft needs a quorum, and the quorum of two is two — + # a two-replica cluster tolerates no failures at all and would stop + # the moment one pod is sealed or rescheduled, which is worse than + # the single replica it replaced. Three tolerates one. + # + # What this buys, and what it does not: a node drain seals the replica + # on it, and raft elects a new leader from the remaining standbys + # within seconds, so the service does not go down. It does NOT remove + # the manual unseal — a sealed server can neither serve requests nor + # act as standby, so that replica stays out until someone unseals it. + # + # The consequence for rolling node maintenance is worth stating: a + # Talos upgrade drains all three nodes in turn, sealing each replica + # as it goes. Unseal each one as it comes back, or the third drain + # takes the last unsealed replica with it and quorum is lost anyway. + replicas: 3 raft: enabled: true setNodeId: true @@ -65,8 +81,32 @@ spec: cluster_address = "[::]:8201" } + # retry_join is what makes a replica join the existing raft + # cluster instead of coming up as an isolated, uninitialized + # node. Without it, scaling the StatefulSet produces pods that + # never become peers and would each need a manual + # `bao operator raft join`. + # + # All three peers are listed on every node, including the node + # itself. That is deliberate: the stanza is a list of possible + # leaders, a node ignores its own entry, and listing all three + # means the set is identical on every replica and stays correct + # regardless of which pod currently leads. + # + # Addressed through the headless service, so each entry resolves + # to one specific pod rather than load-balancing across them. storage "raft" { path = "/openbao/data" + + retry_join { + leader_api_addr = "http://openbao-0.openbao-internal:8200" + } + retry_join { + leader_api_addr = "http://openbao-1.openbao-internal:8200" + } + retry_join { + leader_api_addr = "http://openbao-2.openbao-internal:8200" + } } service_registration "kubernetes" {} diff --git a/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml b/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml index 6bc96259..d134fa4a 100644 --- a/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml +++ b/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml @@ -53,7 +53,13 @@ spec: command: ["/bin/sh", "-ec"] args: - | - export BAO_ADDR=http://openbao.openbao.svc.cluster.local:8200 + # openbao-active, not openbao: with more than one replica + # the plain service selects any pod, and a raft snapshot has + # to be taken against the leader. Standbys forward ordinary + # requests, but this is not one to leave to chance — the + # active service selects the leader by label, so the + # snapshot is always taken where it is consistent. + export BAO_ADDR=http://openbao-active.openbao.svc.cluster.local:8200 BAO_TOKEN=$(bao write -field=token auth/kubernetes/login \ role=openbao-snapshot \ jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)") -- 2.54.0