fix: [alert] OpenBaoQuorumLost (openbao) (#920)

Reviewed-on: k.renner/homelab#919

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -57,10 +57,14 @@ spec:
           # the ClusterSecretStore goes InvalidProviderConfig and no
           # ExternalSecret can refresh.
           #
-          # No window: below quorum the secret store is down and every
-          # ExternalSecret stalls with it, so waiting only delays the page.
+          # Short window: readiness is only a proxy for quorum. A one-second
+          # node NotReady flap makes the node lifecycle controller mark that
+          # node's pods not-ready for ~30s while raft keeps its leader and
+          # serves (issue #920, during a rolling reboot with one replica
+          # already evicted). A really sealed or crashed majority persists,
+          # and existing Secrets keep their last values, so 2m costs little.
           expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
-          for: 0m
+          for: 2m
           labels:
             severity: critical
           annotations:

kubernetes/infrastructure/bird/bird.conf

@@ -23,7 +23,7 @@ protocol device { scan time 10; }
 # down. The usual advice is not to export these to the kernel because it
 # maintains them itself — true of main, not of a table the kernel never fills.
 protocol direct attached {
-  interface "mgmt0", "ingress0", "pod0";
+  interface "mgmt0", "ingress0", "pod0", "node0";
   ipv4 { table master4; };
   ipv6 { table master6; };
 }

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -118,14 +118,21 @@ spec:
       # traffic out to the VIP and back (which fails with L2 announcements).
       lbExternalClusterIP: true
     cni:
-      # Multus (kubernetes/infrastructure/multus) runs alongside Cilium and
-      # writes 00-multus.conf to /etc/cni/net.d so kubelet picks multus
-      # first; multus then delegates to Cilium's 05-cilium.conflist via
-      # clusterNetwork. The default `exclusive: true` makes cilium-cni's
-      # install step rename anything alphabetically before its own config
-      # to *.cilium_bak on every reconcile, which kept nuking multus.
-      # Setting it false lets multus's config coexist.
+      # Multus is the only writer of /etc/cni/net.d (#832): Cilium's conflist
+      # goes to /etc/cni/net.d/multus/, which the container runtime does not
+      # read, and Multus delegates to it via clusterNetwork. Without this a
+      # pod created after a reboot, before Multus rewrote 00-multus.conf, came
+      # up with Cilium alone. See kubernetes/infrastructure/multus.
+      confPath: /etc/cni/net.d/multus
+      # The default `exclusive: true` renames other configs in the conf dir to
+      # *.cilium_bak on every reconcile, which kept nuking multus.
       exclusive: false
+    # One agent at a time: the chart default of 2 restarts two of three
+    # nodes' datapath at once.
+    updateStrategy:
+      type: RollingUpdate
+      rollingUpdate:
+        maxUnavailable: 1
     # Talos-specific settings
     securityContext:
       capabilities:

kubernetes/infrastructure/multus/kustomization.yaml

@@ -4,7 +4,6 @@ resources:
   - cni-plugins-installer.yaml
   - cni-dhcp-daemon.yaml
   - multus-daemonset.yaml
-  - multus-subdir.yaml
   # Whereabouts — cluster-coordinated IPAM, for attachments on segments with
   # no DHCP server. CRDs first: the DaemonSet's control loop reads them at
   # startup. See the header in whereabouts-daemonset.yaml for why neither

kubernetes/infrastructure/multus/multus-daemonset.yaml

@@ -111,10 +111,18 @@ metadata:
   name: multus
   namespace: kube-system
 ---
+# Multus is the only writer of /etc/cni/net.d (#832). Cilium writes its conflist
+# to /etc/cni/net.d/multus/, a subdirectory the container runtime does not read
+# (libcni skips directories); Multus picks it up there via multusAutoconfigDir
+# and writes 00-multus.conf. Until that file exists the runtime has no network
+# config at all, so no pod can be created with Cilium alone after a reboot.
+#
+# The "-subdir" names date from the node-by-node rollout; renaming would roll
+# Multus again for nothing.
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: multus-daemon-config
+  name: multus-daemon-config-subdir
   namespace: kube-system
   labels:
     tier: node
@@ -127,7 +135,7 @@ data:
         "logLevel": "verbose",
         "logToStderr": true,
         "cniConfigDir": "/host/etc/cni/net.d",
-        "multusAutoconfigDir": "/host/etc/cni/net.d",
+        "multusAutoconfigDir": "/host/etc/cni/net.d/multus",
         "multusConfigFile": "auto",
         "socketDir": "/host/run/multus/"
     }
@@ -135,16 +143,16 @@ data:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kube-multus-ds
+  name: kube-multus-ds-subdir
   namespace: kube-system
   labels:
     tier: node
     app: multus
-    name: multus
+    name: multus-subdir
 spec:
   selector:
     matchLabels:
-      name: multus
+      name: multus-subdir
   updateStrategy:
     type: RollingUpdate
   template:
@@ -152,17 +160,8 @@ spec:
       labels:
         tier: node
         app: multus
-        name: multus
+        name: multus-subdir
     spec:
-      # Nodes converted under #832 run kube-multus-ds-subdir (multus-subdir.yaml).
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: cni.renner.uno/multus-sole-writer
-                    operator: NotIn
-                    values: ["true"]
       # Upstream ships this DaemonSet with no priority class, so it runs at 0
       # and can preempt nothing. Any event that makes the pod need a fresh
       # scheduling decision — a rollout, a node restart, an eviction — then has
@@ -233,6 +232,10 @@ spec:
           volumeMounts:
             - name: cni
               mountPath: /host/etc/cni/net.d
+            # Exists before Multus starts, so it waits for Cilium's conflist
+            # instead of exiting on a missing directory.
+            - name: cni-cilium-conf
+              mountPath: /host/etc/cni/net.d/multus
             # multus-daemon expects that cnibin path must be identical between pod and container host.
             # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
             # not to any other directory, like '/opt/bin' or '/usr/bin'.
@@ -288,6 +291,10 @@ spec:
         - name: cni
           hostPath:
             path: /etc/cni/net.d
+        - name: cni-cilium-conf
+          hostPath:
+            path: /etc/cni/net.d/multus
+            type: DirectoryOrCreate
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
@@ -296,7 +303,7 @@ spec:
             path: /
         - name: multus-daemon-config
           configMap:
-            name: multus-daemon-config
+            name: multus-daemon-config-subdir
             items:
               - key: daemon-config.json
                 path: daemon-config.json

kubernetes/infrastructure/multus/multus-subdir.yaml

@@ -1,165 +0,0 @@
-# Transitional (#832): Multus for nodes labelled
-# cni.renner.uno/multus-sole-writer=true. Identical to kube-multus-ds in
-# multus-daemonset.yaml except that it finds the Cilium conflist in
-# /etc/cni/net.d/multus/ (see post-crds/cilium-nodeconfig.yaml), so 00-multus.conf
-# is the only config the container runtime can load. Folded back into
-# kube-multus-ds once every node is converted.
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-daemon-config-subdir
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  daemon-config.json: |
-    {
-        "chrootDir": "/hostroot",
-        "cniVersion": "0.3.1",
-        "logLevel": "verbose",
-        "logToStderr": true,
-        "cniConfigDir": "/host/etc/cni/net.d",
-        "multusAutoconfigDir": "/host/etc/cni/net.d/multus",
-        "multusConfigFile": "auto",
-        "socketDir": "/host/run/multus/"
-    }
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-subdir
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus-subdir
-spec:
-  selector:
-    matchLabels:
-      name: multus-subdir
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus-subdir
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: cni.renner.uno/multus-sole-writer
-                    operator: In
-                    values: ["true"]
-      # Rationale for the class, resources and mounts: kube-multus-ds.
-      priorityClassName: system-node-critical
-      hostNetwork: true
-      hostPID: true
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-        - operator: Exists
-          effect: NoExecute
-      serviceAccountName: multus
-      containers:
-        - name: kube-multus
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.4-thick
-          command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "128Mi"
-            limits:
-              memory: "256Mi"
-          securityContext:
-            privileged: true
-          terminationMessagePolicy: FallbackToLogsOnError
-          volumeMounts:
-            - name: cni
-              mountPath: /host/etc/cni/net.d
-            - name: cnibin
-              mountPath: /opt/cni/bin
-            - name: host-run
-              mountPath: /host/run
-            - name: host-var-lib-cni-multus
-              mountPath: /var/lib/cni/multus
-            - name: host-var-lib-kubelet
-              mountPath: /var/lib/kubelet
-              mountPropagation: HostToContainer
-            - name: host-run-k8s-cni-cncf-io
-              mountPath: /run/k8s.cni.cncf.io
-            - name: host-run-netns
-              mountPath: /run/netns
-              mountPropagation: HostToContainer
-            - name: multus-daemon-config
-              mountPath: /etc/cni/net.d/multus.d
-              readOnly: true
-            - name: hostroot
-              mountPath: /hostroot
-              mountPropagation: HostToContainer
-            - mountPath: /etc/cni/multus/net.d
-              name: multus-conf-dir
-          env:
-            - name: MULTUS_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-      initContainers:
-        - name: install-multus-binary
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.4-thick
-          command:
-            - "/usr/src/multus-cni/bin/install_multus"
-            - "-d"
-            - "/host/opt/cni/bin"
-            - "-t"
-            - "thick"
-          resources:
-            requests:
-              cpu: "10m"
-              memory: "15Mi"
-          securityContext:
-            privileged: true
-          terminationMessagePolicy: FallbackToLogsOnError
-          volumeMounts:
-            - name: cnibin
-              mountPath: /host/opt/cni/bin
-              mountPropagation: Bidirectional
-      terminationGracePeriodSeconds: 10
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: hostroot
-          hostPath:
-            path: /
-        - name: multus-daemon-config
-          configMap:
-            name: multus-daemon-config-subdir
-            items:
-              - key: daemon-config.json
-                path: daemon-config.json
-        - name: host-run
-          hostPath:
-            path: /run
-        - name: host-var-lib-cni-multus
-          hostPath:
-            path: /var/lib/cni/multus
-        - name: host-var-lib-kubelet
-          hostPath:
-            path: /var/lib/kubelet
-        - name: host-run-k8s-cni-cncf-io
-          hostPath:
-            path: /run/k8s.cni.cncf.io
-        - name: host-run-netns
-          hostPath:
-            path: /run/netns/
-        - name: multus-conf-dir
-          hostPath:
-            path: /etc/cni/multus/net.d

kubernetes/infrastructure/post-crds/cilium-nodeconfig.yaml

@@ -1,32 +0,0 @@
-# Staged rollout of "Multus is the only writer of /etc/cni/net.d" (#832), one
-# node at a time. Each override applies only to nodes carrying its label (set
-# in kubernetes/talos/patches/k8s-N.yaml) and takes effect when that node's
-# cilium pod is re-created.
-#
-# Step 1 label: the agent's preStop removes its conflist from /etc/cni/net.d.
-apiVersion: cilium.io/v2
-kind: CiliumNodeConfig
-metadata:
-  name: cni-uninstall
-  namespace: kube-system
-spec:
-  nodeSelector:
-    matchLabels:
-      cni.renner.uno/cilium-uninstall: "true"
-  defaults:
-    cni-uninstall: "true"
----
-# Step 2 label: the agent writes its conflist to a subdirectory the container
-# runtime does not read (libcni ConfFiles skips directories). Multus on the
-# same node (kube-multus-ds-subdir) picks it up from there.
-apiVersion: cilium.io/v2
-kind: CiliumNodeConfig
-metadata:
-  name: multus-sole-writer
-  namespace: kube-system
-spec:
-  nodeSelector:
-    matchLabels:
-      cni.renner.uno/multus-sole-writer: "true"
-  defaults:
-    write-cni-conf-when-ready: /host/etc/cni/net.d/multus/05-cilium.conflist

kubernetes/infrastructure/post-crds/kustomization.yaml

@@ -4,7 +4,6 @@ resources:
   - clusterissuer.yaml
   - cloudflare-externalsecret.yaml
   - cilium-bgp.yaml
-  - cilium-nodeconfig.yaml
   - cnpg-barman-certs.yaml
   - velero-schedules.yaml
   # Only safe here because OpenBao is already initialized + k8s-auth is

kubernetes/talos/patches/cluster.yaml

@@ -58,6 +58,12 @@ machine:
       imageGCHighThresholdPercent: 70
       imageGCLowThresholdPercent: 60
 cluster:
+  # etcd advertises on the management VLAN until the node↔node move (#576
+  # step 3). Pinned so the node0 addresses cannot be picked on a reboot: Talos
+  # never updates an existing member's peer URLs.
+  etcd:
+    advertisedSubnets:
+      - 10.2.40.0/24
   # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its
   # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443
   # there. Setting it again in this patch restated the same value in a second

kubernetes/talos/patches/k8s-1.yaml

@@ -17,6 +17,8 @@ machine:
     # pod0 is a closed L2 fabric between the three nodes: statically
     # addressed, no gateway, nothing to learn from an RA.
     net.ipv6.conf.pod0.accept_ra: "0"
+    # node0 is closed the same way.
+    net.ipv6.conf.node0.accept_ra: "0"
     # ingress0 keeps RAs, at the default accept_ra of 2: an IPv6 default
     # router is a link-local address learned only from the RA, and with
     # net.ipv6.conf.all.forwarding=1 — which Cilium sets — Linux ignores RAs
@@ -177,6 +179,19 @@ addresses:
   - address: 10.2.48.11/24
   - address: fd02:48::11/64
 ---
+# Node network, VLAN 46: traffic among the three nodes themselves (#576) — the
+# node IP, etcd, apiserver↔kubelet. Closed like pod0: statically addressed, no
+# gateway, never routed via the UDM. Unused until the node IP moves here.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::11/64
+  - address: 10.2.46.11/24
+---
 # Ingress network, VLAN 47. World-to-pod: the BGP session, the next hop for
 # every advertised LoadBalancer VIP, and the masquerade source for pod egress.
 # Publicly routed, which is what separates it from pod0.

kubernetes/talos/patches/k8s-2.yaml

@@ -16,6 +16,7 @@ machine:
     # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
     # and only the default route is held back — see k8s-1.yaml.
     net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.node0.accept_ra: "0"
     net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
@@ -33,9 +34,6 @@ machine:
     topology.kubernetes.io/zone: saturn
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
-    # Staged #832 rollout, see infrastructure/post-crds/cilium-nodeconfig.yaml.
-    cni.renner.uno/cilium-uninstall: "true"
-    cni.renner.uno/multus-sole-writer: "true"
   # pod0 addresses for the bird peer list — see k8s-1.yaml.
   nodeAnnotations:
     net.renner.uno/pod0-v6: fd02:48::12
@@ -125,6 +123,17 @@ addresses:
   - address: 10.2.48.12/24
   - address: fd02:48::12/64
 ---
+# Node network, VLAN 46 — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::12/64
+  - address: 10.2.46.12/24
+---
 # Ingress network, VLAN 47. World-to-pod, publicly routed, statically
 # addressed — see k8s-1.yaml for why it is not leased.
 apiVersion: v1alpha1

kubernetes/talos/patches/k8s-3.yaml

@@ -12,6 +12,7 @@ machine:
     # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
     # and only the default route is held back — see k8s-1.yaml.
     net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.node0.accept_ra: "0"
     net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
@@ -29,9 +30,6 @@ machine:
     topology.kubernetes.io/zone: neptune
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
-    # Staged #832 rollout, see infrastructure/post-crds/cilium-nodeconfig.yaml.
-    cni.renner.uno/cilium-uninstall: "true"
-    cni.renner.uno/multus-sole-writer: "true"
   # pod0 addresses for the bird peer list — see k8s-1.yaml.
   nodeAnnotations:
     net.renner.uno/pod0-v6: fd02:48::13
@@ -112,6 +110,17 @@ addresses:
   - address: 10.2.48.13/24
   - address: fd02:48::13/64
 ---
+# Node network, VLAN 46 — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::13/64
+  - address: 10.2.46.13/24
+---
 # Ingress network, VLAN 47. World-to-pod, publicly routed, statically
 # addressed — see k8s-1.yaml for why it is not leased.
 apiVersion: v1alpha1