fix: [alert] OpenBaoQuorumLost (openbao) (#920)

Reviewed-on: k.renner/homelab#919

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -57,10 +57,14 @@ spec:
           # the ClusterSecretStore goes InvalidProviderConfig and no
           # ExternalSecret can refresh.
           #
-          # No window: below quorum the secret store is down and every
-          # ExternalSecret stalls with it, so waiting only delays the page.
+          # Short window: readiness is only a proxy for quorum. A one-second
+          # node NotReady flap makes the node lifecycle controller mark that
+          # node's pods not-ready for ~30s while raft keeps its leader and
+          # serves (issue #920, during a rolling reboot with one replica
+          # already evicted). A really sealed or crashed majority persists,
+          # and existing Secrets keep their last values, so 2m costs little.
           expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
-          for: 0m
+          for: 2m
           labels:
             severity: critical
           annotations:

kubernetes/talos/patches/cluster.yaml

@@ -58,6 +58,12 @@ machine:
       imageGCHighThresholdPercent: 70
       imageGCLowThresholdPercent: 60
 cluster:
+  # etcd advertises on the management VLAN until the node↔node move (#576
+  # step 3). Pinned so the node0 addresses cannot be picked on a reboot: Talos
+  # never updates an existing member's peer URLs.
+  etcd:
+    advertisedSubnets:
+      - 10.2.40.0/24
   # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its
   # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443
   # there. Setting it again in this patch restated the same value in a second

kubernetes/talos/patches/k8s-1.yaml

@@ -17,6 +17,8 @@ machine:
     # pod0 is a closed L2 fabric between the three nodes: statically
     # addressed, no gateway, nothing to learn from an RA.
     net.ipv6.conf.pod0.accept_ra: "0"
+    # node0 is closed the same way.
+    net.ipv6.conf.node0.accept_ra: "0"
     # ingress0 keeps RAs, at the default accept_ra of 2: an IPv6 default
     # router is a link-local address learned only from the RA, and with
     # net.ipv6.conf.all.forwarding=1 — which Cilium sets — Linux ignores RAs
@@ -177,6 +179,19 @@ addresses:
   - address: 10.2.48.11/24
   - address: fd02:48::11/64
 ---
+# Node network, VLAN 46: traffic among the three nodes themselves (#576) — the
+# node IP, etcd, apiserver↔kubelet. Closed like pod0: statically addressed, no
+# gateway, never routed via the UDM. Unused until the node IP moves here.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::11/64
+  - address: 10.2.46.11/24
+---
 # Ingress network, VLAN 47. World-to-pod: the BGP session, the next hop for
 # every advertised LoadBalancer VIP, and the masquerade source for pod egress.
 # Publicly routed, which is what separates it from pod0.

kubernetes/talos/patches/k8s-2.yaml

@@ -16,6 +16,7 @@ machine:
     # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
     # and only the default route is held back — see k8s-1.yaml.
     net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.node0.accept_ra: "0"
     net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
@@ -122,6 +123,17 @@ addresses:
   - address: 10.2.48.12/24
   - address: fd02:48::12/64
 ---
+# Node network, VLAN 46 — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::12/64
+  - address: 10.2.46.12/24
+---
 # Ingress network, VLAN 47. World-to-pod, publicly routed, statically
 # addressed — see k8s-1.yaml for why it is not leased.
 apiVersion: v1alpha1

kubernetes/talos/patches/k8s-3.yaml

@@ -12,6 +12,7 @@ machine:
     # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
     # and only the default route is held back — see k8s-1.yaml.
     net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.node0.accept_ra: "0"
     net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
@@ -109,6 +110,17 @@ addresses:
   - address: 10.2.48.13/24
   - address: fd02:48::13/64
 ---
+# Node network, VLAN 46 — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::13/64
+  - address: 10.2.46.13/24
+---
 # Ingress network, VLAN 47. World-to-pod, publicly routed, statically
 # addressed — see k8s-1.yaml for why it is not leased.
 apiVersion: v1alpha1