diff --git a/kubernetes/infrastructure/openbao/helmrelease.yaml b/kubernetes/infrastructure/openbao/helmrelease.yaml index 2e7d1d72..e7ecadb7 100644 --- a/kubernetes/infrastructure/openbao/helmrelease.yaml +++ b/kubernetes/infrastructure/openbao/helmrelease.yaml @@ -42,12 +42,28 @@ spec: # the agent-injector mutating webhook is unused surface. enabled: false server: - # Single-node raft (not standalone/file) so we get `bao operator raft - # snapshot` as a consistent backup primitive. TLS terminates at the - # Traefik ingress; in-cluster traffic is plain HTTP. + # Raft (not standalone/file) for two reasons: `bao operator raft + # snapshot` as a consistent backup primitive, and real HA across three + # nodes. TLS terminates at the Traefik ingress; in-cluster traffic is + # plain HTTP. ha: enabled: true - replicas: 1 + # Three, not two. Raft needs a quorum, and the quorum of two is two — + # a two-replica cluster tolerates no failures at all and would stop + # the moment one pod is sealed or rescheduled, which is worse than + # the single replica it replaced. Three tolerates one. + # + # What this buys, and what it does not: a node drain seals the replica + # on it, and raft elects a new leader from the remaining standbys + # within seconds, so the service does not go down. It does NOT remove + # the manual unseal — a sealed server can neither serve requests nor + # act as standby, so that replica stays out until someone unseals it. + # + # The consequence for rolling node maintenance is worth stating: a + # Talos upgrade drains all three nodes in turn, sealing each replica + # as it goes. Unseal each one as it comes back, or the third drain + # takes the last unsealed replica with it and quorum is lost anyway. + replicas: 3 raft: enabled: true setNodeId: true @@ -65,8 +81,32 @@ spec: cluster_address = "[::]:8201" } + # retry_join is what makes a replica join the existing raft + # cluster instead of coming up as an isolated, uninitialized + # node. Without it, scaling the StatefulSet produces pods that + # never become peers and would each need a manual + # `bao operator raft join`. + # + # All three peers are listed on every node, including the node + # itself. That is deliberate: the stanza is a list of possible + # leaders, a node ignores its own entry, and listing all three + # means the set is identical on every replica and stays correct + # regardless of which pod currently leads. + # + # Addressed through the headless service, so each entry resolves + # to one specific pod rather than load-balancing across them. storage "raft" { path = "/openbao/data" + + retry_join { + leader_api_addr = "http://openbao-0.openbao-internal:8200" + } + retry_join { + leader_api_addr = "http://openbao-1.openbao-internal:8200" + } + retry_join { + leader_api_addr = "http://openbao-2.openbao-internal:8200" + } } service_registration "kubernetes" {} diff --git a/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml b/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml index 6bc96259..d134fa4a 100644 --- a/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml +++ b/kubernetes/infrastructure/openbao/snapshot-cronjob.yaml @@ -53,7 +53,13 @@ spec: command: ["/bin/sh", "-ec"] args: - | - export BAO_ADDR=http://openbao.openbao.svc.cluster.local:8200 + # openbao-active, not openbao: with more than one replica + # the plain service selects any pod, and a raft snapshot has + # to be taken against the leader. Standbys forward ordinary + # requests, but this is not one to leave to chance — the + # active service selects the leader by label, so the + # snapshot is always taken where it is consistent. + export BAO_ADDR=http://openbao-active.openbao.svc.cluster.local:8200 BAO_TOKEN=$(bao write -field=token auth/kubernetes/login \ role=openbao-snapshot \ jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)")