fix: [alert] OpenBaoReplicaNotReady (openbao) (#808 )

2026-09-07 07:48:24 +00:00
57 changed files with 2225 additions and 2342 deletions
@@ -42,7 +42,7 @@ when:

 steps:
  - name: fetch-alerts
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
    environment:
      SOPS_AGE_KEY:
        from_secret: sops_age_key
@@ -89,7 +89,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-vmtest
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/vmtest
    depends_on: []
    when:
@@ -349,56 +349,3 @@ steps:
          --dockerfile Containerfile \
          --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
          --destination gitea.renner.uno/gitea_admin/heating-can:latest
-
-  # netbird-proxy-services: the controller that reconciles NetBird reverse-proxy
-  # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
-  # Runtime artifact like heating-can; same registry so the audit tracks it.
-  - name: build-netbird-proxy-services
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
-    directory: ci/images/netbird-proxy-services
-    depends_on: []
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/netbird-proxy-services/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
-          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
-
-  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
-  # artifact like heating-can; same registry so the audit tracks it.
-  - name: build-bird
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
-    directory: ci/images/bird
-    depends_on: []
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/bird/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/bird:$TAG \
-          --destination gitea.renner.uno/gitea_admin/bird:latest
@@ -60,7 +60,7 @@ steps:
  # run that never touched them. Tracked in #461.
  - name: kustomize-build
    depends_on: [] # see note above `steps:`
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
    commands:
      - |
        set -eu
@@ -158,7 +158,7 @@ steps:
  # whenever it trails the deployed server. If you bump the server, bump here.
  - name: woodpecker-lint
    depends_on: [] # see note above `steps:`
-    image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
+    image: woodpeckerci/woodpecker-cli:v3.17.0-alpine
    commands:
      - |
        set -eu
@@ -246,7 +246,7 @@ steps:
  # this step fails permanently on its own documentation.
  - name: no-digest-pins
    depends_on: [] # see note above `steps:`
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
    commands:
      - |
        set -eu
@@ -4,7 +4,7 @@ when:

 steps:
  - name: e2e
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
    environment:
      SOPS_AGE_KEY:
        from_secret: sops_age_key
@@ -31,7 +31,7 @@ steps:
      - sh scripts/ci/test-model-selection.sh

  - name: e2e-test
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
    environment:
      SOPS_AGE_KEY:
        from_secret: sops_age_key
@@ -49,7 +49,7 @@ steps:
    # (the issue-pick logic needs jq, which only the ci-claude image has);
    # minting a 1h read-only token per run is cheap. Degrades to
    # snapshot-only if the ci-diagnostics ServiceAccount isn't reconciled.
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
    environment:
      SOPS_AGE_KEY:
        from_secret: sops_age_key
@@ -32,13 +32,8 @@ netbird_watchdog_fail_threshold: 3

 # Minimum seconds between watchdog-issued restarts, so a genuinely broken
 # backend can't put the proxy into a restart loop. Shared by both the URL-probe
-# watchdog and the flood watchdog below. One minute: the route-manager wedge
-# recurs on every routing-peer loss, and rolling node reboots lose one peer
-# every few minutes; a suppressed restart there leaves the wedge standing
-# until the URL probe's three rounds fire (3.5 min of 000 on every domain,
-# measured 2026-09-07). A minute still caps a loop at one restart per
-# probe interval.
-netbird_watchdog_cooldown_seconds: 60
+# watchdog and the flood watchdog below.
+netbird_watchdog_cooldown_seconds: 600

 # Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain
 # substring in the proxy's docker logs by the flood watchdog. On this build the
@@ -1,22 +0,0 @@
-FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
-
-# bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
-# step — the image is the binary plus its runtime.
-#
-# 3.22 rather than the 3.21 the other images here pin: bird2 first appears in
-# 3.22/community.
-#
-# Config is read from /etc/bird/bird.conf, not the package default
-# /etc/bird.conf, so the mount cannot shadow what the package ships.
-#
-# kubectl renders the init container's per-node config: go-template-file for
-# the templating, a field selector for excluding the node itself, in-cluster
-# config for the auth. From the distro, so there is no version literal to drift
-# and no download to trust. It is only ever used to read core/v1 Nodes.
-RUN apk add --no-cache bird2 kubectl
-
-# Fails the build rather than the first pod if a binary or its path is wrong.
-RUN /usr/sbin/bird --version \
- && kubectl version --client > /dev/null
-
-ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]
@@ -1,9 +1,9 @@
 FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d

-ARG TALOS_VERSION=v1.13.9
+ARG TALOS_VERSION=v1.12.6
 ARG VELERO_VERSION=v1.16.0
-ARG KUBECTL_VERSION=v1.36.3
-ARG FLUX_VERSION=2.9.5
+ARG KUBECTL_VERSION=v1.36.2
+ARG FLUX_VERSION=2.9.2

 RUN apk add --no-cache \
      aws-cli \
@@ -18,7 +18,7 @@ RUN apk add --no-cache \

 RUN curl -sfL -o /usr/local/bin/talosctl \
      "https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
- && echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e  /usr/local/bin/talosctl" | sha256sum -c - \
+ && echo "4c41a3b10b075292e64b182283d558e9d8e935f43239de1ed4e4b14359593efb  /usr/local/bin/talosctl" | sha256sum -c - \
 && chmod +x /usr/local/bin/talosctl

 RUN curl -sfL -o /usr/local/bin/kubectl \
@@ -31,7 +31,7 @@ RUN curl -sfL -o /usr/local/bin/kubectl \

 RUN curl -sfL -o /tmp/flux.tar.gz \
      "https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \
- && echo "b853df82adfd7736f580692f9f734473d571606307139f8fd20c2a80dd1ff473  /tmp/flux.tar.gz" | sha256sum -c - \
+ && echo "4092b367fd060097976fb7261601f79420ecf7266328417dfd8dfee27de0e6a3  /tmp/flux.tar.gz" | sha256sum -c - \
 && tar -xzf /tmp/flux.tar.gz -C /usr/local/bin flux \
 && chmod +x /usr/local/bin/flux \
 && rm -f /tmp/flux.tar.gz
@@ -1,12 +0,0 @@
-# netbird-proxy-services: reconciles NetBird reverse-proxy services (public
-# domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
-# public label. Standard library only, so the image is the interpreter plus one
-# file; runs unprivileged as its own user.
-FROM python:3.13-slim
-
-RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
-COPY controller.py /opt/controller.py
-
-ENV PYTHONUNBUFFERED=1
-USER controller
-ENTRYPOINT ["python3", "/opt/controller.py"]
@@ -1,275 +0,0 @@
-#!/usr/bin/env python3
-"""netbird-proxy-services: reconcile NetBird reverse-proxy services from Ingresses.
-
-Source of truth: every Ingress carrying the public label (dns.renner.uno/public).
-For each host on such an Ingress the NetBird management gets one reverse-proxy
-service in TLS-passthrough mode on :443, targeting Traefik's ClusterIP through a
-subnet resource that covers the cluster's service CIDR. SNI and Host reach
-Traefik untouched, the proxy terminates nothing, and no domain resource is
-involved -- the routing peers stay ordinary IP routers. The target sends a
-PROXY protocol v2 header so Traefik (and CrowdSec behind its access log) sees
-the WAN client address instead of the routing peer's pod IP.
-
-Ownership is derived from NetBird itself, not from local state: a service is
-managed iff its mode is "tls" and its target is this controller's subnet
-resource. Services that do not match (the hand-made HTTP/domain ones) are never
-touched; a desired host held by such a service is reported and skipped until
-that service is removed by hand.
-
-Credentials: the NetBird PAT is read from OpenBao on every pass through the
-Kubernetes auth role, exactly like the setup-key rotator; it never lands in a
-Kubernetes Secret or in a log line.
-"""
-
-import json
-import logging
-import os
-import sys
-import time
-import urllib.error
-import urllib.parse
-import urllib.request
-
-LOG = logging.getLogger("netbird-proxy-services")
-
-NB = os.environ.get("NB_MANAGEMENT_URL", "https://netbird.renner.uno").rstrip("/")
-BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
-BAO_ROLE = os.environ.get("BAO_ROLE", "netbird-proxy-services")
-BAO_PAT_PATH = os.environ.get("BAO_PAT_PATH", "homelab/data/infrastructure/secret-rotators-bootstrap/netbird")
-BAO_PAT_KEY = os.environ.get("BAO_PAT_KEY", "pat")
-SERVICE_CIDR = os.environ.get("SERVICE_CIDR", "10.96.0.0/12")
-NETWORK_NAME = os.environ.get("NETWORK_NAME", "k8s public ingress")
-ROUTER_GROUP = os.environ.get("ROUTER_GROUP_NAME", "k8s-ingress-router")
-SOURCE_GROUP = os.environ.get("SOURCE_GROUP_NAME", "All")
-TRAEFIK_NS, TRAEFIK_SVC = os.environ.get("TRAEFIK_SERVICE", "ingress/traefik").split("/", 1)
-PUBLIC_LABEL = os.environ.get("PUBLIC_LABEL", "dns.renner.uno/public")
-LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "443"))
-INTERVAL = int(os.environ.get("INTERVAL_SECONDS", "60"))
-HTTP_TIMEOUT = 20
-
-K8S_API = "https://kubernetes.default.svc"
-K8S_TOKEN_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-K8S_CA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-
-
-class ApiError(Exception):
-    pass
-
-
-def http(method, url, body=None, headers=None, cafile=None):
-    data = None if body is None else json.dumps(body).encode()
-    req = urllib.request.Request(url, data=data, method=method)
-    req.add_header("Accept", "application/json")
-    if data is not None:
-        req.add_header("Content-Type", "application/json")
-    for k, v in (headers or {}).items():
-        req.add_header(k, v)
-    ctx = None
-    if cafile:
-        import ssl
-        ctx = ssl.create_default_context(cafile=cafile)
-    try:
-        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT, context=ctx) as resp:
-            raw = resp.read()
-            return json.loads(raw) if raw else None
-    except urllib.error.HTTPError as e:
-        detail = e.read().decode(errors="replace")[:300]
-        raise ApiError(f"{method} {url} -> {e.code}: {detail}") from None
-
-
-# --- Kubernetes -------------------------------------------------------------
-
-def k8s(path):
-    with open(K8S_TOKEN_FILE) as f:
-        token = f.read().strip()
-    return http("GET", K8S_API + path, headers={"Authorization": f"Bearer {token}"}, cafile=K8S_CA_FILE)
-
-
-def desired_hosts():
-    sel = urllib.parse.quote(f"{PUBLIC_LABEL}=true")
-    ingresses = k8s(f"/apis/networking.k8s.io/v1/ingresses?labelSelector={sel}")
-    hosts = {}
-    for ing in ingresses.get("items", []):
-        ref = f'{ing["metadata"]["namespace"]}/{ing["metadata"]["name"]}'
-        for rule in ing.get("spec", {}).get("rules", []):
-            host = rule.get("host")
-            if not host or host.startswith("*"):
-                continue
-            hosts.setdefault(host, ref)
-    return hosts
-
-
-def traefik_cluster_ip():
-    svc = k8s(f"/api/v1/namespaces/{TRAEFIK_NS}/services/{TRAEFIK_SVC}")
-    for ip in svc["spec"].get("clusterIPs", []):
-        if ":" not in ip:
-            return ip
-    raise ApiError(f"{TRAEFIK_NS}/{TRAEFIK_SVC} has no IPv4 ClusterIP")
-
-
-# --- OpenBao ----------------------------------------------------------------
-
-def netbird_pat():
-    with open(K8S_TOKEN_FILE) as f:
-        jwt = f.read().strip()
-    login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
-    bao_token = login["auth"]["client_token"]
-    kv = http("GET", f"{BAO_ADDR}/v1/{BAO_PAT_PATH}", headers={"X-Vault-Token": bao_token})
-    pat = kv["data"]["data"].get(BAO_PAT_KEY)
-    if not pat:
-        raise ApiError(f"OpenBao {BAO_PAT_PATH} has no key {BAO_PAT_KEY}")
-    return pat
-
-
-# --- NetBird management -----------------------------------------------------
-
-class NetBird:
-    def __init__(self, pat):
-        self.h = {"Authorization": f"Token {pat}"}
-
-    def get(self, path):
-        return http("GET", f"{NB}/api/{path}", headers=self.h)
-
-    def list(self, path):
-        # An empty collection comes back as an empty body / null, not [].
-        return self.get(path) or []
-
-    def post(self, path, body):
-        return http("POST", f"{NB}/api/{path}", body, headers=self.h)
-
-    def put(self, path, body):
-        return http("PUT", f"{NB}/api/{path}", body, headers=self.h)
-
-    def delete(self, path):
-        return http("DELETE", f"{NB}/api/{path}", headers=self.h)
-
-
-def group_id(nb, name):
-    for g in nb.list("groups"):
-        if g["name"] == name:
-            return g["id"]
-    raise ApiError(f"NetBird group {name!r} not found")
-
-
-def ensure_network(nb):
-    for n in nb.list("networks"):
-        if n["name"] == NETWORK_NAME:
-            return n["id"]
-    n = nb.post("networks", {"name": NETWORK_NAME,
-                             "description": "managed by netbird-proxy-services; public ingress via TLS passthrough"})
-    LOG.info("created network %s (%s)", NETWORK_NAME, n["id"])
-    return n["id"]
-
-
-def ensure_resource(nb, net):
-    for r in nb.list(f"networks/{net}/resources"):
-        if r.get("address") == SERVICE_CIDR:
-            return r["id"]
-    r = nb.post(f"networks/{net}/resources", {"name": "cluster service CIDR", "address": SERVICE_CIDR,
-                                              "description": "managed by netbird-proxy-services",
-                                              "enabled": True, "groups": []})
-    LOG.info("created subnet resource %s (%s)", SERVICE_CIDR, r["id"])
-    return r["id"]
-
-
-def ensure_router(nb, net, router_group):
-    for r in nb.list(f"networks/{net}/routers"):
-        if router_group in (r.get("peer_groups") or []):
-            return r["id"]
-    r = nb.post(f"networks/{net}/routers", {"peer_groups": [router_group], "metric": 9999,
-                                            "masquerade": True, "enabled": True})
-    LOG.info("created router on group %s (%s)", ROUTER_GROUP, r["id"])
-    return r["id"]
-
-
-def ensure_policy(nb, resource, source_group):
-    name = f"{NETWORK_NAME} access"
-    for p in nb.list("policies"):
-        if p["name"] == name:
-            return p["id"]
-    p = nb.post("policies", {"name": name, "enabled": True, "rules": [{
-        "name": name, "enabled": True, "action": "accept", "protocol": "tcp",
-        "ports": [str(LISTEN_PORT)], "bidirectional": False, "sources": [source_group],
-        "destinationResource": {"id": resource, "type": "subnet"}}]})
-    LOG.info("created policy %s (%s)", name, p["id"])
-    return p["id"]
-
-
-def service_body(host, resource, target_ip):
-    # proxy_protocol: PROXY v2 header carries the WAN client address to
-    # Traefik (proxyProtocol.trustedIPs on websecure); TCP/TLS modes only.
-    return {"name": host, "domain": host, "mode": "tls", "listen_port": LISTEN_PORT, "enabled": True,
-            "targets": [{"target_type": "subnet", "target_id": resource, "protocol": "tcp",
-                         "host": target_ip, "port": 443, "enabled": True,
-                         "options": {"proxy_protocol": True}}]}
-
-
-def is_managed(svc, resource):
-    return svc.get("mode") == "tls" and any(t.get("target_id") == resource for t in svc.get("targets") or [])
-
-
-def target_matches(svc, target_ip):
-    if svc.get("listen_port") != LISTEN_PORT or not svc.get("enabled"):
-        return False
-    ts = svc.get("targets") or []
-    return len(ts) == 1 and ts[0].get("host") == target_ip and ts[0].get("port") == 443 \
-        and ts[0].get("protocol") == "tcp" and ts[0].get("enabled") \
-        and (ts[0].get("options") or {}).get("proxy_protocol") is True
-
-
-def reconcile():
-    hosts = desired_hosts()
-    target_ip = traefik_cluster_ip()
-    nb = NetBird(netbird_pat())
-
-    router_group = group_id(nb, ROUTER_GROUP)
-    source_group = group_id(nb, SOURCE_GROUP)
-    net = ensure_network(nb)
-    resource = ensure_resource(nb, net)
-    ensure_router(nb, net, router_group)
-    ensure_policy(nb, resource, source_group)
-
-    by_domain = {s["domain"]: s for s in nb.list("reverse-proxies/services")}
-    created = updated = deleted = held = 0
-
-    for host, ref in sorted(hosts.items()):
-        svc = by_domain.get(host)
-        if svc is None:
-            nb.post("reverse-proxies/services", service_body(host, resource, target_ip))
-            LOG.info("created %s (from %s) -> %s:443", host, ref, target_ip)
-            created += 1
-        elif not is_managed(svc, resource):
-            LOG.warning("%s is held by unmanaged service %s (mode %s); remove it to migrate",
-                        host, svc["id"], svc.get("mode"))
-            held += 1
-        elif not target_matches(svc, target_ip):
-            nb.put(f"reverse-proxies/services/{svc['id']}", service_body(host, resource, target_ip))
-            LOG.info("updated %s -> %s:443", host, target_ip)
-            updated += 1
-
-    for domain, svc in by_domain.items():
-        if is_managed(svc, resource) and domain not in hosts:
-            nb.delete(f"reverse-proxies/services/{svc['id']}")
-            LOG.info("deleted %s (no public Ingress carries it any more)", domain)
-            deleted += 1
-
-    LOG.info("reconciled: %d hosts desired, %d created, %d updated, %d deleted, %d held by unmanaged services",
-             len(hosts), created, updated, deleted, held)
-
-
-def main():
-    logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), stream=sys.stdout,
-                        format="%(asctime)s %(levelname)s %(message)s")
-    if os.environ.get("ONCE") == "1":
-        reconcile()
-        return
-    while True:
-        try:
-            reconcile()
-        except Exception as e:  # keep running; the next pass retries
-            LOG.error("reconcile failed: %s", e)
-        time.sleep(INTERVAL)
-
-
-if __name__ == "__main__":
-    main()
@@ -50,18 +50,15 @@ pass insert homelab/openbao/root-token
 kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste key
 ```

-**Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
-the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
-from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
-itself. The shamir key above became the **recovery key** in the migration; it
-is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
-runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
-one is neither a voter nor a standby, so a replica that stays sealed after a
-restart — wrong or missing seal-key Secret — costs redundancy, two cost
-quorum. The trade-off taken with the static seal: the key that unlocks the
-store sits in the same etcd as the Secrets ESO already syncs there in
-plaintext. Symptom of sealed: ESO stores go NotReady, synced Secrets keep
-their last value (they are real k8s Secrets — pods keep running).
+**Manual unseal is required after every pod restart** (node drain, upgrade,
+crash), **per replica** — the server runs three raft peers (see
+`infrastructure/openbao/helmrelease.yaml`), and a sealed one is neither a
+voter nor a standby, so it stays out of the cluster until unsealed:
+`kubectl -n openbao exec -it openbao-<N> -- bao operator unseal`, same key for
+all three. Three peers mean one sealed replica is survivable; two are not.
+Accepted for now; auto-unseal needs a second trust root (transit/KMS) and is
+explicitly out of scope. Symptom of sealed: ESO stores go NotReady, synced
+Secrets keep their last value (they are real k8s Secrets — pods keep running).

 ## Phase 2 — configure + ClusterSecretStore (follow-up PR)

@@ -237,19 +234,16 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
 - **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
  dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
  `openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
-  A raft snapshot restore needs the matching seal key — which lives in
+  A raft snapshot restore needs the matching unseal key — which lives in
  `pass`, outside the cluster (GPG key backed up externally 2026-07-14).
  A failed snapshot Job is the sealed-after-restart alarm. Still owed:
  one tested restore before phase 4.
- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
-  openbao (empty) → init → restore raft snapshot → replicas unseal themselves
-  → ClusterSecretStore Ready → ESO materializes Secrets → apps start
-  (`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
-  hand). Until then, secret-consuming apps simply stay Pending on missing
-  Secrets — same as any missing dependency.
- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
-  offline-capable). With those, everything is recoverable from a raft
-  snapshot alone.
+- **Cluster rebuild order:** Flux brings up openbao (sealed, empty) → restore
+  raft snapshot → unseal (key from `pass`) → ClusterSecretStore Ready → ESO
+  materializes Secrets → apps start. Until unseal, secret-consuming apps
+  simply stay Pending on missing Secrets — same as any missing dependency.
+- **Break-glass:** `pass` holds unseal key + root token (GPG, offline-capable).
+  With those two, everything is recoverable from a raft snapshot alone.
 - **Ansible Vault password:** `ansible/vault-password.sh` reads
  `ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
  2026-07-14. It gates decryption of several Ansible-Vault files (PBS
@@ -1,117 +0,0 @@
-# Traffic classes on the k8s nodes
-
-Issue #576. Three of the four classes are separated; management is not.
-
-## Where each class runs
-
-| class | link | mechanism |
-|---|---|---|
-| Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) |
-| world→pod | `ingress0`, VLAN 47 | BGP session sourced there, so the advertised next hop is that address |
-| pod→world | `ingress0`, VLAN 47 | routing rule on the pod source prefix → table 47 |
-| pod↔pod | `pod0`, VLAN 48 | routing rule on pod source **and** destination → table 48 |
-| management | `mgmt0`, VLAN 40 | not started |
-
-Hard constraint, set by the owner: inter-node and pod traffic never traverses
-the UDM. Everything above stays on the 10G switch.
-
-## Why policy routing rather than moving the node IP
-
-Cilium reaches a peer's pods at that peer's Kubernetes InternalIP, in native
-and tunnel mode alike. `--ipv4-node` is overwritten from the Node object, and
-`nodePort.directRoutingDevice` selects the BPF NodePort device, not a next hop.
-Both were established by test (#837, #838), not by reading.
-
-So the InternalIP decides where pod traffic goes — unless the lookup happens in
-a different table. The rules do that, and nothing about the node's identity
-moves. The routing decision runs before Cilium's masquerade hook, so a rule
-matching the pod source sees the pod address, and the masquerade then rewrites
-it to the egress link's.
-
-## Rule order
-
-Lower priority runs first.
-
-| prio | match | table |
-|---|---|---|
-| 980 | src pod, dst pod | 48 |
-| 981 | src pod, dst pod (v4) | 48 |
-| 990 | dst `fd00::/8` | main |
-| 991 | dst `10.0.0.0/8` | main |
-| 1000 | src pod | 47 |
-| 1001 | src pod (v4) | 47 |
-
-980/981 match source **and** destination. Source alone would also catch the
-node's own traffic to a pod, which would then leave with a `pod0` source — an
-address `CiliumNode.spec.addresses` does not list, so Cilium classifies it as
-`world` rather than `host`.
-
-990/991 are Cilium's `ipv4/ipv6NativeRoutingCIDR`, and they are what keeps
-cluster-internal traffic out of table 47. Without them a pod's packet to a
-service matches the source rule, finds only a default route, and leaves for the
-UDM — measured, in-cluster DNS stopped resolving.
-
-Traffic to a pod on the same node falls through on its own: table 48 holds only
-the peers' CIDRs, so the lookup misses and 990/991 send it to main.
-
-## Where each table comes from
-
- **47** — two default routes declared per node in `talos/patches/k8s-*.yaml`.
-  The IPv6 gateway is the UDM's configured address on the segment, not the
-  RA-learned link-local; `accept_ra_defrtr` stays `0` on `ingress0` so the RA
-  default does not land in main.
- **48** — filled by the `bird` DaemonSet: an iBGP mesh over `pod0`, one route
-  per peer pod CIDR with the peer's `pod0` address as next hop.
-
-## The bird mesh
-
-`kubernetes/infrastructure/bird/`. Each node advertises only its own pod CIDR
-and installs what it learns into table 48. Cilium's routes in main are
-untouched, which is why the two coexist.
-
-No address is written into this repo twice. Each node publishes its `pod0`
-addresses as `nodeAnnotations`, beside the `VLANConfig` that assigns them; the
-pod CIDRs come from the Node objects. An init container renders both from the
-API with `kubectl -o go-template-file`, excludes the node itself with a field
-selector, and ends with `bird -p` so BIRD's own parser gates startup.
-
-The node's own CIDR is **originated**, not read back from the kernel. Reading
-it back made BIRD depend on the shape Cilium installs, and the IPv4 route is
-rejected outright: its next hop sits inside the prefix it describes, which KRT
-refuses with `strange next-hop`. The IPv6 route is a device route and passed,
-so the failure was silent on one family only.
-
-## Failure behaviour
-
-Benign at every layer. An empty or absent table 48 makes 980/981 fall through
-to main, and pod traffic returns to VLAN 40 — unseparated, not broken. The same
-holds if the DaemonSet cannot start, which is the case during a cold rebuild
-until Gitea serves its image.
-
-## Management, the remaining class
-
-Not started. The levers differ per service and none of them is the node IP:
-
-| service | can leave the node IP? | lever |
-|---|---|---|
-| etcd | yes | `cluster.etcd.advertisedSubnets`; needs a reboot per node |
-| kube-apiserver endpoint | yes | it is a VIP — `Layer2VIPConfig.link` |
-| Talos API | n/a | `apid` answers on every node address; a firewall question |
-| kubelet | no | it *is* the node IP |
-
-Two approaches were rejected by the owner and are not to be proposed again:
-moving the node IP and etcd to `pod0` (#838), and relabelling VLAN 40 as the
-cluster VLAN while moving etcd and the API VIP to a new management VLAN.
-Replacing the CNI (#839, Antrea) was shelved.
-
-Note that the reasoning which produced that list also concluded pod↔pod was
-blocked, which policy routing disproved. Treat it as a record of decisions, not
-as an exhaustive analysis.
-
-## Sources
-
-Cilium v1.20.1 `pkg/datapath/linux/node.go`, `pkg/node/types/node.go`,
-`pkg/node/sync/local_node_sync.go`; BIRD 2.17 user manual (kernel protocol,
-`learn all`, `strict bind`, `direct`); Talos v1.13 `RoutingRuleConfig` and
-`machine.nodeAnnotations`. PRs #831, #833, #835, #836, #837, #842, #852, #854,
-#855; closed unmerged: #830, #838, #839.
@@ -0,0 +1,194 @@
+# Proposal — separate the four traffic classes onto their own VLANs
+
+Status: draft. The question this started with is **answered** — from the Cilium
+source and flag reference, not from a test — and the answer changes the shape
+of the proposal rather than confirming it.
+
+Supersedes the framing in issue #576, which described this as "give node
+management its own NIC, and Ceph public its own VLAN". That was two of four
+classes and one of them turned out already solved.
+
+## The four classes
+
+| class | what it is | where it goes today |
+|---|---|---|
+| **Ceph replication** | OSD-to-OSD heartbeat, replication, recovery | VLAN 49, its own X710 port — **separated** |
+| **cluster-internal** | pod-to-pod east-west, including Ceph client I/O | pod network, VXLAN over `mgmt0` |
+| **cluster-external** | ingress to LoadBalancer VIPs, egress to LAN and internet | `mgmt0` |
+| **management** | Talos API, etcd, kube-apiserver, kubelet | `mgmt0` |
+
+Three of four share one interface. That is not a design decision — until the
+bare-metal conversion there was one NIC to work with.
+
+Ceph client I/O living on the pod network is correct rather than a leftover:
+that traffic *is* cluster-internal. Ceph is done.
+
+## Principle
+
+**Each class gets its own VLAN. Which physical interface a VLAN lands on is a
+separate, changeable decision.**
+
+Defining classes by VLAN means the separation survives recabling, port failures
+and capacity changes, and it does not require one port per class. Defining them
+by interface would freeze a capacity decision into the architecture.
+
+## The finding that shapes everything else
+
+The original open question was whether `nodePort.directRoutingDevice` moves the
+next-hop of Cilium's direct routes, or only selects an interface. It is
+answerable from the source, and the answer is: **only the interface.**
+
+`installDirectRoute` takes the next-hop as a parameter:
+
+```go
+func installDirectRoute(log *slog.Logger, prefix netip.Prefix, nodeIP net.IP,
+    skipUnreachable bool) (routeSpec *netlink.Route, err error)
+```
+
+and `nodeUpdate` supplies it as `newNode.GetNodeIP(false)`. `GetNodeIP` returns,
+in priority order, `NodeInternalIP` → `NodeExternalIP` → anything else. The
+same `newIP4` is what tunnel mode uses as the VXLAN endpoint.
+
+`NodeInternalIP` comes from the Kubernetes Node object. `--ipv4-node` looks like
+an override but is not one: `initFromConfig()` calls `SetNodeInternalIP` and
+`initFromK8s()` runs **after** it and overwrites the value. In a Kubernetes
+cluster there is no path by which Cilium's node IP differs from the kubelet's
+`InternalIP`.
+
+`--direct-routing-device` is documented as "Device name used to connect nodes in
+direct routing mode" — a device, not an address.
+
+**So cluster-internal traffic is welded to the node IP, and the node IP is the
+kubelet InternalIP.** Two consequences:
+
+- The class cannot be moved off whatever VLAN the node IP lives on.
+- **Routing mode is irrelevant to this problem.** Tunnel mode sources the VXLAN
+  from the node IP; native mode uses it as the direct-route next-hop. Native
+  routing buys nothing here, so the disruptive, non-rolling step this proposal
+  previously contained is simply gone.
+
+## Consequence: draw the line the other way
+
+If cluster-internal cannot leave the node IP, then separation has to come from
+moving *management services* off it — and each one has to be checked
+individually, because they do not share a mechanism.
+
+| management service | can it leave the node IP? | lever |
+|---|---|---|
+| etcd | **yes** | `cluster.etcd.advertisedSubnets` (+ `listenSubnets`, which defaults to it) |
+| kube-apiserver endpoint | **yes** | already a VIP — `Layer2VIPConfig` with a `link:` field |
+| Talos API | **n/a** | `apid` answers on every node address (measured); a client and firewall choice, not a bind setting |
+| kubelet | **no** | it *is* the node IP, by definition |
+
+So four VLANs are reachable, but the boundary does not fall where the class
+names suggest: **kubelet stays with cluster-internal.** That is the honest
+result, and it is worth stating plainly rather than filing kubelet under
+management and hoping.
+
+The remaining classes keep the levers established earlier:
+
+- **cluster-external ingress** — `CiliumBGPNodeConfigOverride.spec.bgpInstances[].peers[].localAddress`.
+  For eBGP the session source is the next-hop, so pinning the session pins
+  where return traffic arrives. The session to the UDM has been running for
+  months.
+- **cluster-external egress** — eBPF masquerading follows the output device, so
+  this is a default-route decision plus `enable-masquerade-to-route-source`
+  (today `false`).
+- **Ceph replication** — Multus macvlan on VLAN 49, done.
+
+## The general escape hatch
+
+Ceph is separated because its OSDs got a second interface and stopped using the
+pod network for replication. That is the *only* mechanism that fully detaches a
+workload from the node IP, and it generalises: anything that must not ride the
+node IP gets its own `NetworkAttachmentDefinition`.
+
+It is also expensive per workload — an attachment, an IPAM pool, a pod spec
+change. Worth it for Ceph replication. Not worth it for east-west traffic in
+general.
+
+## Recommended mapping
+
+Two ways to arrange this, and they differ only in which VLAN gets relabelled:
+
+**A — move the node IP to a new cluster VLAN.** VLAN 40 keeps etcd, the API VIP
+and Talos API. Cleanest match to the class names, but it changes every node's
+identity, which is the one address the whole cluster is administered through.
+
+**B (recommended) — leave the node IP on VLAN 40 and move management off it.**
+VLAN 40 becomes the cluster VLAN; a new VLAN carries etcd and the API VIP.
+Identical end state, and no node ever changes its InternalIP.
+
+Under B:
+
+| VLAN | carries |
+|---|---|
+| 49 | Ceph replication |
+| 40 (relabelled *cluster*) | pod east-west, kubelet |
+| new *management* | etcd, kube-apiserver VIP, Talos API by convention |
+| new *external* | LoadBalancer ingress, egress default route |
+
+The property the user asked for — management carries only management traffic —
+holds under B. The label moves; the property is what matters.
+
+## Migration shape
+
+Every step is rolling and reversible. The routing-mode change is gone.
+
+1. **New VLANs** on the switch, trunked to the X710. Nothing uses them.
+2. **Addresses** on the new VLANs via Talos, one node at a time, `--mode=try`.
+   Still unused; this only proves the L2 works.
+3. **etcd** to the management VLAN via `advertisedSubnets`. One node at a time,
+   waiting for a healthy quorum between each.
+4. **API VIP** to the management VLAN via `Layer2VIPConfig.link`. DNS for
+   `k8s.renner.uno` follows.
+5. **cluster-external**: BGP `localAddress`, default route, masquerade flag.
+   Reversible by reverting the override; failure is visible immediately in
+   `cilium-dbg shell -- bgp/peers`.
+6. Verify VLAN 40 by what is *absent* from it, not by what still works.
+
+Step 3 is the one that deserves care — etcd is the only step where a mistake
+costs quorum rather than connectivity.
+
+## What this does not buy
+
+Management currently shares a 10 Gb link and is not contended. Nothing observed
+today is slow because of this. The case for the split is that traffic classes
+with different failure modes and security postures should not be
+indistinguishable — not that something is broken.
+
+## Sources and measurements
+
+Source, Cilium v1.20.1 (the running version):
+
+- `pkg/datapath/linux/node.go` — `installDirectRoute(…, nodeIP net.IP, …)`,
+  fed from `newIP4 = newNode.GetNodeIP(false)`; `createDirectRouteSpec` rejects
+  a next-hop that is not directly reachable.
+- `pkg/node/types/node.go` — `GetNodeIP` prefers `NodeInternalIP`.
+- `pkg/node/sync/local_node_sync.go` — `initFromConfig()` applies
+  `--ipv4-node`, `initFromK8s()` runs afterwards and overwrites it from the
+  Kubernetes Node.
+- `cilium-agent` flag reference — `--direct-routing-device`: "Device name used
+  to connect nodes in direct routing mode".
+
+Talos:
+
+- Multihoming guide — `cluster.etcd.advertisedSubnets`; `listenSubnets`
+  defaults to it. kube-apiserver and Talos API listen addresses are not
+  covered there.
+- `kubernetes/talos/patches/k8s-{1,2,3}.yaml` — `Layer2VIPConfig` for
+  `10.2.40.60` with `link: mgmt0`.
+
+Measured on the cluster:
+
+- `apid` answered on a node address that is not the InternalIP, so the Talos
+  API is not bound to one address.
+- k8s-1 currently holds one routable v4 address plus the VIP on `mgmt0`; its
+  InternalIP had changed since the last session, which is why nothing here
+  names an address as fixed.
+- Cilium runs `routing-mode: tunnel` with `enable-masquerade-to-route-source`
+  unset and no direct-routing device configured.
+
+Not verified: whether the kube-apiserver's `--advertise-address` follows the
+node IP independently of the VIP. It does not affect the recommendation, since
+clients reach the API through the VIP either way.
@@ -432,8 +432,8 @@ is why it belongs here.
  among its devices but not `ens19`, because `ens19` is deliberately
  address-less. Cilium therefore owns `mgmt0` and sees VLAN 40 frames already
  untagged by the 8021q layer, while VLAN 30 frames only ever meet the filter
-  on the shared parent `uplink`, still tagged. Any future address-less VLAN
-  carrying pod traffic will need the same treatment.
+  on the shared parent `enp2s0f0np0`, still tagged. Any future address-less
+  VLAN carrying pod traffic will need the same treatment.

  The symptom does not look like a VLAN problem. A tc-layer drop reaches an
  AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports `no buffer space
@@ -67,12 +67,10 @@ These must exist outside the cluster or the rebuild cannot complete:
      `flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
      files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
      `MIGRATION-sops-openbao.md`.)
- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
-      store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
-      former shamir key, now the recovery key — and `homelab/openbao/root-token`),
-      and the GPG private key backed up off-machine. The seal key is the one
-      the snapshots are encrypted under: without it, no OpenBao snapshot can be
-      restored and no secret-consuming app can start.
+- [ ] **OpenBao unseal key + root token** — in the GPG `pass` store
+      (`homelab/openbao/unseal-key`, `homelab/openbao/root-token`), and the GPG
+      private key backed up off-machine. Without the unseal key, no OpenBao
+      snapshot can be restored and no secret-consuming app can start.
 - [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
      (`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
      `openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
@@ -296,65 +294,42 @@ output** — a green step over a still-forming Ceph is a known trap.
 > Steady-state (non-rebuild) bootstrap is just `kubernetes/bootstrap.sh flux`
 > with `GITEA_TOKEN` set; `FLUX_SOURCE` defaults to `gitea`.

-## Step 4 — openbao: restore the snapshot
+## Step 4 — openbao: restore the snapshot + unseal

-`bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
-`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
-the empty server, restores the newest raft snapshot straight out of the Velero
-kopia repository, and waits until every replica has unsealed itself from the
-static seal. What follows is the same procedure by hand, for when the script
-is not an option.
-
-OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
-`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
-snapshots are encrypted under that key, so it must be the same key the backed
-up cluster used — a different one restores nothing. Until the snapshot is
-restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
-Secrets, and every secret-consuming app stays Pending.
+After Step 3, OpenBao comes up **sealed and empty**. Until it is restored and
+unsealed, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
+Secrets, and every secret-consuming app stays Pending. This step is **manual**
+(the unseal key lives in `pass`, off-cluster by design).

 ```sh
-# 1. The seal key, before the pods start (a pod without it never starts).
-kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
-kubectl -n openbao create secret generic openbao-seal-key \
-  --from-literal=key="$(pass show homelab/openbao/seal-key)"
+# 1. Wait for the OpenBao pods to exist (all three sealed, 0/1 Ready is expected).
+kubectl -n openbao get pods

-# 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
-#    initialise it. With an auto-seal, init hands out recovery keys and the
-#    server unseals itself. These keys are throwaway: the restore replaces them.
-kubectl -n openbao exec -it openbao-0 -- bao operator init \
-  -recovery-shares=1 -recovery-threshold=1
-
-# 3. Restore the most recent raft snapshot. Source it from the snapshot PVC
+# 2. Restore the most recent raft snapshot. Source it from the snapshot PVC
 #    (infrastructure/openbao/snapshot-cronjob.yaml) or, in a total-loss rebuild,
-#    from the Velero `openbao-daily` offsite copy. Root token from the init
-#    output. No -force: OpenBao checks the snapshot was written under this
-#    server's seal key and refuses otherwise ("different autoseal key") — that
-#    refusal is the signal the Secret is wrong. -force would skip the check and
-#    leave a server that can never unseal.
+#    from the Velero `openbao-daily` offsite copy. Copy it into the pod, then:
 kubectl -n openbao cp <snapshot>.snap openbao-0:/tmp/restore.snap
-kubectl -n openbao exec -it openbao-0 -- \
-  env BAO_TOKEN=<init root token> bao operator raft snapshot restore /tmp/restore.snap
+kubectl -n openbao exec -it openbao-0 -- bao operator raft snapshot restore /tmp/restore.snap

-# 4. The server seals against the snapshot's keyring and unseals itself from
-#    the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
-#    to type. Afterwards the tokens and recovery key from pass apply again.
-kubectl -n openbao get pods -w
+# 3. Unseal with the key from pass (threshold 1) — EVERY replica, same key.
+#    A sealed server is neither a raft voter nor a standby, so unsealing only
+#    openbao-0 leaves the three-peer cluster one node short of quorum.
+kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste homelab/openbao/unseal-key
+kubectl -n openbao exec -it openbao-1 -- bao operator unseal   # same key
+kubectl -n openbao exec -it openbao-2 -- bao operator unseal   # same key
 ```

 Verify the store recovers before moving on:

 ```sh
-kubectl -n openbao exec openbao-0 -- bao status   # want: Seal Type static, Sealed false
-kubectl get clustersecretstore openbao            # want: Valid / Ready
-kubectl -n <ns> get externalsecret                # want: SecretSynced
+kubectl get clustersecretstore openbao        # want: Valid / Ready
+kubectl -n <ns> get externalsecret            # want: SecretSynced
 ```

-> A replica that stays **sealed** after a restart could not use the static
-> seal — the Secret is missing or does not match the key the data was
-> written under. Fix the Secret and restart the pod; `bao operator unseal`
-> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
-> unseal an auto-sealed server and is only for recovery-mode operations.
-> Details + rationale: `MIGRATION-sops-openbao.md`.
+> **Manual unseal is required after every OpenBao pod restart**, not just
+> during rebuilds, and for *each* replica that restarts — a rolling node
+> drain seals all three in turn. Details + rationale:
+> `MIGRATION-sops-openbao.md`.

 ## Step 5 — apps: let the rest reconcile

@@ -418,15 +393,14 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
 - **`bootstrap.sh talos` is dead** — it cannot generate a config (#581), and
  Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
  Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
-  Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
-  other secrets are read from `pass` or streamed in at run time and never live
-  in CI.
- **OpenBao unseals itself, but only under the seal key the snapshot was
-  written with.** Three raft replicas mean the service survives losing one; a
-  sealed server is neither a voter nor a standby. A pod that stays sealed after
-  a restart has the wrong `openbao-seal-key` Secret — Step 4 covers this; it is
-  repeated here because it is the step most likely to be half-done under
-  pressure.
+  It has also never had an `openbao` phase — Step 4 is manual because the
+  unseal key must not live in CI; see issue #187 on whether to wire a
+  semi-automated OpenBao restore.
+- **OpenBao seals on every restart and each replica must be unsealed
+  individually.** Three raft replicas mean the service survives losing one, but
+  a sealed server is neither a voter nor a standby — unsealing only one leaves
+  the cluster short of quorum. Step 4 covers this; it is repeated here because
+  it is the step most likely to be half-done under pressure.
 - Ceph's cluster network is a Multus attachment on VLAN 49 with whereabouts
  IPAM, not host networking. Nothing in the rebuild needs special handling for
  it — the nodes hold no address on that segment — but a rebuilt cluster will
@@ -21,9 +21,6 @@ metadata:
  namespace: audiobookshelf
  labels:
    app.kubernetes.io/name: audiobookshelf
-  annotations:
-    # envFrom does not restart the pod when the ConfigMap changes; Reloader does.
-    configmap.reloader.stakater.com/reload: audiobookshelf-config
 spec:
  replicas: 1
  strategy:
@@ -95,7 +95,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.2 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.9.1 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -32,7 +32,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.1.0 # {"$imagepolicy": "flux-system:immich-machine-learning"}
          ports:
            - name: http
              containerPort: 3003
@@ -35,9 +35,6 @@ metadata:
  namespace: immich
  labels:
    app.kubernetes.io/name: immich-server
-  annotations:
-    # envFrom does not restart the pod when the ConfigMap changes; Reloader does.
-    configmap.reloader.stakater.com/reload: immich-config
 spec:
  # Scaled to 0 alongside immich-machine-learning during single-node
  # maintenance; see the note there. The CNPG Postgres cluster stays running,
@@ -87,7 +84,7 @@ spec:
              mountPath: /ca
      containers:
        - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.1.0 # {"$imagepolicy": "flux-system:immich-server"}
          ports:
            - name: http
              containerPort: 3001
@@ -1,20 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: intel-gpu-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: intel-gpu-exporter
-    release: kube-prometheus-stack
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: intel-gpu-exporter
-  endpoints:
-    - port: http-metrics
-      path: /metrics
-      interval: 30s
-      relabelings:
-        # The series would otherwise carry only the exporter pod's IP.
-        - sourceLabels: [__meta_kubernetes_pod_node_name]
-          targetLabel: node
@@ -39,5 +39,4 @@ resources:
  - immich-servicemonitor.yaml
  - loki-servicemonitor.yaml
  - smartctl-exporter-servicemonitor.yaml
-  - intel-gpu-exporter-servicemonitor.yaml
  - probes-blackbox.yaml
@@ -34,9 +34,7 @@ spec:
          # query parameter, and an exact match would break silently if the
          # instance label ever encodes it differently.
          expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
-          # Two consecutive failed probes: enough to skip a single lost probe
-          # through Traefik, not enough to sit on a sealed store.
-          for: 1m
+          for: 5m
          labels:
            severity: critical
          annotations:
@@ -27,14 +27,24 @@ spec:
        # not-Ready. The converse does not hold — a crash or a stuck
        # scheduling would also fire this — but both want a human too, so the
        # imprecision costs nothing.
+        #
+        # WHY THE TWO THRESHOLD ALERTS AGGREGATE AWAY THE SCRAPE TARGET
+        #
+        # kube-state-metrics stamps instance/pod/endpoint onto every series it
+        # exports, so a bare selector yields a *different* series each time the
+        # kube-state-metrics pod is replaced — and Prometheus tracks the `for:`
+        # timer per output series, so a replacement resets a pending alert to
+        # inactive. That is not hypothetical: during the 2026-09-07 quorum
+        # outage kube-state-metrics itself moved twice (the openbao replicas
+        # were on the same three nodes that were being rebooted), and the two
+        # alerts below fired ~14 min and ~6.5 min later than their `for:`
+        # windows say they should have. Aggregating to the identity that
+        # actually matters — the StatefulSet — makes the series survive a
+        # kube-state-metrics restart and the timer run continuously.
        - alert: OpenBaoReplicaNotReady
-          # A replica unseals itself from the static seal, so a restart is
-          # Ready again in well under a minute; the duration only ignores that
-          # ordinary rescheduling. A replica still sealed after it never
-          # recovers on its own: the seal failed (openbao-seal-key Secret
-          # missing or wrong).
-          # max by: kube-state-metrics stamps instance/pod on the series, so a
-          # rescheduled exporter starts a new series and resets the timer.
+          # A replica that is sealed never recovers on its own, so the
+          # duration is only there to ignore ordinary rescheduling: a healthy
+          # pod moving between nodes is Ready again in well under a minute.
          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 3
          for: 10m
          labels:
@@ -42,14 +52,14 @@ spec:
          annotations:
            summary: "OpenBao has {{ $value }} of 3 replicas ready"
            description: >-
-              An OpenBao replica has been unavailable for 10 minutes. A restart
-              unseals itself from the static seal, so this is not the ordinary
-              drain case. The service is still up on the remaining replicas,
-              so nothing is broken yet, but redundancy is reduced and a second
-              failure would cost quorum. Check `kubectl -n openbao get pods`,
-              the pod log and `bao status` in the not-ready pod. If it is
-              sealed, the static seal failed: verify the openbao-seal-key
-              Secret matches `pass show homelab/openbao/seal-key`.
+              An OpenBao replica has been unavailable for 10 minutes. The usual
+              cause is a sealed replica after a node drain — every restart
+              seals it, and it stays out of the raft cluster until unsealed by
+              hand. The service is still up on the remaining replicas, so
+              nothing is broken yet, but redundancy is reduced and a second
+              drain would cost quorum. Check `kubectl -n openbao get pods` and
+              `bao status` in the not-ready pod; unseal with the key from
+              `pass` (homelab/openbao/unseal-key).

        - alert: OpenBaoQuorumLost
          # Raft quorum of a three-node cluster is two. At one ready replica
@@ -57,10 +67,11 @@ spec:
          # the ClusterSecretStore goes InvalidProviderConfig and no
          # ExternalSecret can refresh.
          #
-          # No window: below quorum the secret store is down and every
-          # ExternalSecret stalls with it, so waiting only delays the page.
+          # Shorter window than the warning above because this is an outage,
+          # not a degradation — but not zero, so a rolling restart that
+          # briefly dips does not page.
          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
-          for: 0m
+          for: 5m
          labels:
            severity: critical
          annotations:
@@ -70,11 +81,9 @@ spec:
              quorum and the secret store is unavailable. ExternalSecrets stop
              refreshing; existing Secrets keep their last values, so failures
              surface only when something rotates or a new secret is needed.
-              Replicas unseal themselves from the static seal; one that stays
-              sealed could not. Check its log and the openbao-seal-key Secret
-              against `pass show homelab/openbao/seal-key`, fix the Secret and
-              restart the pod. The recovery key cannot unseal an auto-sealed
-              server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
+              Unseal the sealed replicas (key in `pass`,
+              homelab/openbao/unseal-key) — each one individually, since a
+              sealed server cannot be a standby.

        - alert: OpenBaoMetricsMissing
          # Guards the two alerts above. Both are threshold comparisons, so if
@@ -1,368 +0,0 @@
-{
-  "title": "Intel iGPU",
-  "uid": "intel-gpu",
-  "tags": ["gpu", "intel", "i915"],
-  "schemaVersion": 39,
-  "editable": true,
-  "timezone": "",
-  "refresh": "1m",
-  "time": { "from": "now-6h", "to": "now" },
-  "templating": {
-    "list": [
-      {
-        "name": "node",
-        "label": "Node",
-        "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
-        "query": {
-          "query": "label_values(gpumon_power, node)",
-          "refId": "PrometheusVariableQueryEditor-VariableQuery"
-        },
-        "definition": "label_values(gpumon_power, node)",
-        "includeAll": true,
-        "multi": true,
-        "refresh": 2,
-        "sort": 1,
-        "current": { "selected": true, "text": ["All"], "value": ["$__all"] }
-      }
-    ]
-  },
-  "panels": [
-    {
-      "type": "stat",
-      "title": "Render/3D busy now",
-      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "decimals": 0,
-          "min": 0,
-          "max": 100,
-          "color": { "mode": "fixed", "fixedColor": "text" }
-        },
-        "overrides": []
-      },
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "center",
-        "orientation": "horizontal",
-        "textMode": "value_and_name",
-        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "stat",
-      "title": "GPU power now",
-      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "watt",
-          "decimals": 1,
-          "color": { "mode": "fixed", "fixedColor": "text" }
-        },
-        "overrides": []
-      },
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "center",
-        "orientation": "horizontal",
-        "textMode": "value_and_name",
-        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "stat",
-      "title": "GT frequency now",
-      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "suffix: MHz",
-          "decimals": 0,
-          "color": { "mode": "fixed", "fixedColor": "text" }
-        },
-        "overrides": []
-      },
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "center",
-        "orientation": "horizontal",
-        "textMode": "value_and_name",
-        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "Render/3D busy",
-      "description": "Compute and 3D. llama.cpp's Vulkan backend runs here; 0 % during an inference means the model is on the CPU.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "Video busy",
-      "description": "Fixed-function decode/encode (QuickSync).",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Video\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "VideoEnhance busy",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"VideoEnhance\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "Blitter busy",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Blitter\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "GT frequency (actual)",
-      "description": "0 while the GT sits in RC6.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "suffix: MHz",
-          "min": 0,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "GPU power",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "watt",
-          "min": 0,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "Package power",
-      "description": "Whole SoC (CPU + GPU + uncore), from the same PMU.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "watt",
-          "min": 0,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_power{type=\"pkg\", node=~\"$node\"}",
-          "legendFormat": "{{node}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "type": "timeseries",
-      "title": "Memory controller bandwidth",
-      "description": "Reads solid, writes dashed. The iGPU shares DDR5 with the CPU, so this is the whole package's traffic.",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "MiBs",
-          "min": 0,
-          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
-        },
-        "overrides": [
-          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] },
-          { "matcher": { "id": "byRegexp", "options": "/write/" }, "properties": [{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] }
-        ]
-      },
-      "options": {
-        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
-        "tooltip": { "mode": "multi", "sort": "desc" }
-      },
-      "targets": [
-        {
-          "expr": "gpumon_imc_read{node=~\"$node\"}",
-          "legendFormat": "{{node}} read",
-          "refId": "A"
-        },
-        {
-          "expr": "gpumon_imc_write{node=~\"$node\"}",
-          "legendFormat": "{{node}} write",
-          "refId": "B"
-        }
-      ]
-    }
-  ]
-}
@@ -77,9 +77,6 @@ configMapGenerator:
  - name: dashboard-infrastructure-overview
    files:
      - infrastructure-overview.json
-  - name: dashboard-intel-gpu
-    files:
-      - intel-gpu.json
  - name: dashboard-netbird-overview
    files:
      - netbird-overview.json
@@ -1,81 +0,0 @@
-# Intel iGPU utilisation, frequency and power for every node (#752).
-#
-# Not privileged: /dev/dri comes from the device plugin's monitoring resource
-# (every GPU on the node, no hostPath), and the i915 PMU that intel_gpu_top
-# reads needs CAP_PERFMON — the one capability granted. No hostPID, no
-# hostNetwork.
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: intel-gpu-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: intel-gpu-exporter
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: intel-gpu-exporter
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: intel-gpu-exporter
-    spec:
-      # Every node is a control plane; without this the exporter covers nothing.
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      securityContext:
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: intel-gpu-exporter
-          image: ghcr.io/clambin/intel-gpu-exporter:0.7.4@sha256:d78761fef018834c759db154fe9dcf3dd9db5974703fa50ca21f6abc35221be1
-          args:
-            # No -device: intel_gpu_top picks the only GPU the plugin injects.
-            - -interval=5s
-          ports:
-            - name: http-metrics
-              containerPort: 9100
-              protocol: TCP
-          securityContext:
-            # Root so the PERFMON capability is effective without file
-            # capabilities on the binary.
-            runAsUser: 0
-            runAsNonRoot: false
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-              add: ["PERFMON"]
-          resources:
-            requests:
-              cpu: 10m
-              memory: 32Mi
-              gpu.intel.com/monitoring: 1
-            limits:
-              memory: 128Mi
-              gpu.intel.com/monitoring: 1
-          livenessProbe:
-            httpGet:
-              path: /metrics
-              port: http-metrics
-            initialDelaySeconds: 15
-            periodSeconds: 30
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: intel-gpu-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: intel-gpu-exporter
-spec:
-  clusterIP: None
-  selector:
-    app.kubernetes.io/name: intel-gpu-exporter
-  ports:
-    - name: http-metrics
-      port: 9100
-      targetPort: http-metrics
-      protocol: TCP
@@ -16,7 +16,6 @@ resources:
  - prometheus-gate-ingress.yaml
  - ci-diagnostics-rbac.yaml
  - smartctl-exporter.yaml
-  - intel-gpu-exporter.yaml
  - dashboards/
 # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
 # separate Flux Kustomization that dependsOn `monitoring` — otherwise the
@@ -28,7 +28,7 @@ spec:
        # net1 instead. master: ens19 resolves on every k8s-{1,2,3}/edge
        # worker, though it is no longer the same kind of device everywhere:
        # on the bare-metal control-plane nodes it is a VLAN 30 sub-interface
-        # of uplink rather than a NIC, which is why the datapath needs
+        # of enp2s0f0np0 rather than a NIC, which is why the datapath needs
        # bpf.vlanBypass to carry it (issue #592).
        k8s.v1.cni.cncf.io/networks: music-assistant-iot
    spec:
@@ -55,7 +55,7 @@ spec:
        # stdlib only; Flux keeps the tag in lockstep via the shared image
        # policy) — no extra dependency.
        - name: reconcile-publish-ip
-          image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
          command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
          volumeMounts:
            - name: data
@@ -65,7 +65,7 @@ spec:
              readOnly: true
      containers:
        - name: music-assistant
-          image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
          env:
            - name: LOG_LEVEL
              value: info
@@ -22,7 +22,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: gotenberg
-          image: gotenberg/gotenberg:8.37.0 # {"$imagepolicy": "flux-system:gotenberg"}
+          image: gotenberg/gotenberg:8.36.0 # {"$imagepolicy": "flux-system:gotenberg"}
          args:
            - "gotenberg"
            - "--chromium-disable-javascript=true"
@@ -134,7 +134,7 @@ spec:
        # is a device-bricking class of risk. Bumps are a reviewed, manual
        # act — see kubernetes/infrastructure/image-automation/README.md.
        - name: terminus
-          image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
+          image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
          ports:
            - name: http
              containerPort: 2345
@@ -197,7 +197,7 @@ spec:
            capabilities:
              drop: ["ALL"]
        - name: sidekiq
-          image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
+          image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
          command: ["bundle", "exec", "sidekiq", "-r", "./config/sidekiq.rb"]
          envFrom:
            - configMapRef:
@@ -18,7 +18,7 @@ spec:
    spec:
      containers:
        - name: whisper
-          image: rhasspy/wyoming-whisper:3.8.1 # {"$imagepolicy": "flux-system:wyoming-whisper"}
+          image: rhasspy/wyoming-whisper:3.7.0 # {"$imagepolicy": "flux-system:wyoming-whisper"}
          args:
            - --model=base-int8
            - --language=de
@@ -34,7 +34,7 @@ spec:
          # Image automation, range 3.x (see image-automation/woodpecker.yaml).
          # MUST stay version-matched with woodpecker-server (gRPC parity) — both
          # track the same range/filter so they bump together on each release.
-          image: woodpeckerci/woodpecker-agent:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-agent"}
+          image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
          env:
            - name: WOODPECKER_SERVER
              value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
@@ -40,7 +40,7 @@ spec:
          # minor + patch auto-update within 3.x; only the 4.0 major is held for
          # a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
          # -bump hazard is gone). Stays in lockstep with the agent.
-          image: woodpeckerci/woodpecker-server:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-server"}
+          image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
          env:
            - name: WOODPECKER_HOST
              value: https://ci.renner.uno
@@ -179,18 +179,16 @@ phase_runs() {
 OPENBAO_NS="openbao"

 # Where the secrets come from. On a workstation, the operator's pass store. In
-# the in-cluster Job there is no pass and no GPG key — and there should not be:
-# the pass store opens everything, and it stays on the operator's machine. Set
-# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
-# operator streams in when the run asks for them:
+# the in-cluster Job there is no pass and no GPG key — and there should not be,
+# because the key that protects OpenBao has no business living in the cluster
+# OpenBao secures. Set OPENBAO_SECRETS_DIR and the run reads them from files
+# instead, which the operator streams in when the run asks for them:
 #
-#     pass show homelab/openbao/seal-key \
-#       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
+#     pass show homelab/openbao/unseal-key \
+#       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/unseal-key'
 #
 # They exist in the pod's filesystem for the length of the run and are never
-# written to a Secret in this cluster, so they do not land in its etcd or in
-# any backup of it. The seal key alone becomes a Secret in the cluster being
-# built, which is what lets that OpenBao unseal itself.
+# written to a Secret, so they do not land in etcd or in any backup of it.
 OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}"

 # Every secret the run needs, named once, in one place. The list is here and
@@ -245,9 +243,7 @@ OPENBAO_REQUIRED_SECRETS=(
    "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
    "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
    "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
-    # The static seal key. The snapshots are encrypted under it, so it has to
-    # be the one the backed-up cluster ran with — a fresh key restores nothing.
-    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
+    "unseal-key|OpenBao unseal key|pass show homelab/openbao/unseal-key"
    # Read by the node registry auth (see registry_auth_patch): the images an app
    # needs while the cluster is still being built are mirrored to a private
    # ghcr.io namespace, and a node has to be able to pull them before anything
@@ -281,7 +277,7 @@ OPENBAO_REQUIRED_SECRETS=(
 # That phase used to need nothing but the machines, and `bootstrap.sh talos` is
 # documented as the part that can run unattended. Node registry auth changed
 # that: the machine config now carries a pull credential, so the phase needs
-# exactly one secret. Demanding all six there would drag in the OpenBao seal
+# exactly one secret. Demanding all six there would drag in the OpenBao unseal
 # key, which deliberately never reaches CI — and demanding none, as before,
 # left the phase to discover the missing file halfway through `talosctl gen`.
 require_secrets() {
@@ -412,7 +408,7 @@ read_secret() {
 # the kernel link name enp2s0f1np1, and that name cannot be manufactured here
 # without assuming a Hetzner private network passes VLAN tags. See
 # talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase
-# needs the OpenBao seal key, which deliberately never reaches CI, and this
+# needs the OpenBao unseal key, which deliberately never reaches CI, and this
 # test is meant to run unattended.
 # Talos version and Image Factory schematic, read out of the installer image in
 # cluster.yaml. Both throwaway-cluster phases need them and neither may state
@@ -610,8 +606,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
    # Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which
    # would not match the amd64 schematic this cluster installs. That leaves the
    # cx*/cpx*2 families in the German locations.
-    # Two segments only, unlike the homelab's split in
-    # NETWORK-traffic-classes.md: here the node address carries management,
+    # Today's traffic-class split, not the target one in
+    # PROPOSAL-traffic-class-separation.md: the node address carries management,
    # cluster-internal and external alike, and Ceph replication has its own.
    #
    # The order is load-bearing: it decides which kernel interface each segment
@@ -1125,7 +1121,7 @@ EOF
    # nothing has demanded the secrets yet — the full require_secrets is gated
    # on the flux phase. Ask for the one this needs, and only that one: pulling
    # in the whole list would make an unattended talos run depend on the OpenBao
-    # seal key, which never reaches CI.
+    # unseal key, which never reaches CI.
    require_secrets ghcr-pull
    REGISTRY_PATCH=$(registry_auth_patch)
    [[ -s "$REGISTRY_PATCH" ]] \
@@ -1410,11 +1406,9 @@ fi
 # waits on infrastructure-crds. A rebuild could not reach a reconciled state
 # without someone at a keyboard.
 #
-# The seal key stays where it was — the operator's `pass` store — and is read
+# The unseal key stays where it was — the operator's `pass` store — and is read
 # here at run time. It is never written to the repo and never reaches CI, which
-# is the reason this was left manual in the first place. It reaches the cluster
-# as the openbao-seal-key Secret, created by the flux phase before Flux starts
-# OpenBao, and from then on every replica unseals itself.
+# is the reason this was left manual in the first place.
 #
 # Everything below is idempotent: it is safe to re-run against an OpenBao that
 # is already unsealed and configured, which is what makes it usable as part of
@@ -1504,63 +1498,61 @@ restore_openbao() {
    echo "  extracted $(stat -c%s "$work/restore.snap") bytes"

    # A snapshot can only be loaded into a running OpenBao, so the fresh one is
-    # initialised first. Under the static seal, init hands out a recovery key
-    # rather than an unseal key and the server unseals itself; the only thing
-    # used from its output is the root token, for the next few seconds, and
-    # then it stops existing: restoring replaces the keyring with the
-    # snapshot's, after which the root token and recovery key from pass apply.
-    #
-    # No -force. Without it OpenBao checks that the snapshot was written under
-    # the seal key this server runs with, and refuses a mismatch by name
-    # ("possibly the snapshot is using a different autoseal key"). That check
-    # is the one guard against restoring under the wrong seal-key Secret;
-    # -force would skip it and leave a server that can never unseal.
-    #
-    # The init output is not echoed on a parse failure: it carries the root
-    # token and the recovery key of a server that is about to be overwritten,
-    # but a log line is a log line.
-    echo -n "  initialising the empty server (throwaway recovery key)... "
-    local init_json root
-    init_json=$(bao_exec "$pod" bao operator init -recovery-shares=1 -recovery-threshold=1 -format=json 2>&1) \
+    # initialised and unsealed with a throwaway key first. That key is used for
+    # the next few seconds and then stops existing: restoring replaces the
+    # keyring with the snapshot's, after which the real key from pass applies.
+    # -force is required for exactly that reason — without it OpenBao refuses a
+    # snapshot whose keys do not match the cluster's own.
+    echo -n "  initialising the empty server with a throwaway key... "
+    local init_json throwaway
+    init_json=$(bao_exec "$pod" bao operator init -key-shares=1 -key-threshold=1 -format=json 2>&1) \
        || { echo; echo "$init_json" >&2; fail "could not initialise OpenBao"; }
-    root=$(jq -r '.root_token // empty' <<<"$init_json")
-    [[ -n "$root" ]] || fail "init output carried no root token"
+    throwaway=$(jq -r '.unseal_keys_b64[0]' <<<"$init_json")
+    [[ -n "$throwaway" && "$throwaway" != "null" ]] || fail "could not read the throwaway unseal key from init output"
    ok

-    # Sealed after init means the static seal is not working — the Secret is
-    # missing or malformed — and the restore below cannot run against it.
-    echo -n "  waiting for it to unseal itself"
-    for i in $(seq 1 30); do
-        if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
-            | jq -e '.sealed == false' >/dev/null; then
-            echo; break
-        fi
-        printf '.'
-        sleep 2
-        if [[ $i -eq 30 ]]; then
-            echo
-            cat "$OPENBAO_SECRETS_DIR/.baoerr" >&2
-            fail "$pod did not unseal itself after init — check the openbao-seal-key Secret and the pod log"
-        fi
-    done
+    echo -n "  unsealing with it... "
+    # The key goes as an argument, which OpenBao's own error message asks for:
+    #
+    #   you are executing outside of a terminal (tty) ... If this is not an
+    #   option, the unseal key can be provided as the first argument
+    #
+    # Two earlier shapes failed. `unseal -` is Vault syntax; OpenBao takes
+    # argv[1] literally, so the dash was sent as the key. Omitting the argument
+    # makes it prompt, and the prompt needs a real tty — a pipe gives "file
+    # descriptor 0 is not a terminal". There is no stdin path at all.
+    #
+    # This does put the key in the pod's process list for the length of the
+    # call, which the code here previously went out of its way to avoid. It is
+    # accepted rather than worked around: reading that list requires exec into
+    # the very pod that holds the key in memory, and the alternative — a
+    # port-forward plus curl against /v1/sys/unseal — puts a background process
+    # in the most critical path of a recovery. The openbao image has no curl,
+    # wget or nc, so it would have to run from here.
+    bao_exec "$pod" bao operator unseal "$throwaway" >/dev/null \
+        || fail "could not unseal with the throwaway key"
+    ok

    echo -n "  loading the snapshot... "
    kubectl -n "$OPENBAO_NS" cp "$work/restore.snap" "$pod:/tmp/restore.snap" >/dev/null \
        || fail "could not copy the snapshot into $pod"
+    local root
+    root=$(jq -r '.root_token' <<<"$init_json")
    local out
-    out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore /tmp/restore.snap 2>&1) \
+    out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore -force /tmp/restore.snap 2>&1) \
        || { echo; echo "$out" >&2; fail "snapshot restore rejected"; }
    ok

    rm -rf "$work"
-    # The server seals itself against the keyring that came with the snapshot
-    # and unseals again from the static seal — the same key the snapshot was
-    # written under. bring_up_openbao waits for that on every replica.
-    echo "  restored — the server re-seals and unseals itself from the seal key"
+    # The server is now sealed again, against the keyring that came with the
+    # snapshot. unseal_openbao carries on from here with the real key.
+    echo "  restored — the server now expects the original unseal key"
 }

-bring_up_openbao() {
-    step 0 "Bringing OpenBao up..."
+unseal_openbao() {
+    step 0 "Unsealing OpenBao..."
+    [[ -n "$OPENBAO_SECRETS_DIR" ]] || command -v pass >/dev/null \
+        || fail "pass not found — the unseal key lives in the operator's pass store, or set OPENBAO_SECRETS_DIR and stream it in"
    command -v jq >/dev/null   || fail "jq not found (needed to read OpenBao's seal status)"

    # OpenBao is deployed by the infrastructure Kustomization. What this step
@@ -1666,28 +1658,40 @@ bring_up_openbao() {
        restore_openbao "${pods[0]}"
    fi

-    # Wait for every replica the StatefulSet wants, not the ones that happen to
-    # exist right now. On a cold start only openbao-0 exists when this runs:
-    # podManagementPolicy is OrderedReady, so openbao-1 is created once
-    # openbao-0 passes its readiness probe and openbao-2 after that. Each
-    # unseals itself from the static seal; nothing here types a key.
+    local key
+    key=$(read_secret unseal-key) \
+        || fail "could not read the unseal key"
+
+    # Unseal every replica the StatefulSet wants, not the ones that happen to
+    # exist right now. On a cold start only openbao-0 exists when this runs, and
+    # unsealing just that one deadlocks the StatefulSet: podManagementPolicy is
+    # OrderedReady, a sealed pod never passes its readiness probe, so openbao-1
+    # is created only after openbao-0 is unsealed and openbao-2 is never created
+    # at all. The cluster comes up at 1 of 3.
    #
-    # Waiting for the whole chain rather than the first pod matters because the
-    # openbao Service sets publishNotReadyAddresses — the chart does this so
-    # replicas can reach each other to join raft before any of them is Ready —
-    # so a replica that is still starting is a live backend on the service IP.
-    # Clients balanced onto it get `dial tcp <svc>:8200: connect: no route to
-    # host`, and the failure surfaces nowhere near here: external-secrets'
-    # ClusterSecretStore flaps, every ExternalSecret reports "could not get
-    # secret data from provider", and infrastructure-crds — which waits on those
-    # — never goes Ready, taking all twenty app Kustomizations with it.
+    # That alone would be survivable. What makes it fatal is that the openbao
+    # Service sets publishNotReadyAddresses — the chart does this so replicas can
+    # reach each other to join raft before any of them is unsealed — so the
+    # sealed openbao-1 is a live backend on the service IP. Clients are balanced
+    # onto a node that cannot answer, and the failure surfaces nowhere near here:
+    # external-secrets' ClusterSecretStore flaps between "store validated" and
+    # `dial tcp <svc>:8200: connect: no route to host`, every ExternalSecret in
+    # the cluster reports "could not get secret data from provider", and
+    # infrastructure-crds — which waits on those — never goes Ready, taking all
+    # twenty app Kustomizations with it.
+    #
+    # So this converges instead of iterating once: re-list the pods each round,
+    # unseal whatever is sealed, and finish when spec.replicas of them are
+    # unsealed. A pod that is still starting cannot be unsealed yet and is not an
+    # error — it is retried next round, which is exactly how the OrderedReady
+    # chain is meant to be walked.
    local want
    want=$(kubectl -n "$OPENBAO_NS" get statefulset openbao -o jsonpath='{.spec.replicas}') \
        || fail "could not read the openbao StatefulSet's replica count"
    [[ "$want" =~ ^[0-9]+$ ]] || fail "openbao StatefulSet reported a non-numeric replica count: '$want'"
-    echo "  waiting for $want replica(s) to unseal themselves"
+    echo "  unsealing $want replica(s)"

-    local unsealed=0 round pod
+    local unsealed=0 round pod err
    local -A seen=()
    for round in $(seq 1 180); do
        mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
@@ -1696,21 +1700,29 @@ bring_up_openbao() {
        for pod in "${pods[@]}"; do
            if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
                | jq -e '.sealed == false' >/dev/null; then
-                [[ -n "${seen[$pod]:-}" ]] || { echo -e "  $pod: ${GREEN}unsealed${NC}"; seen[$pod]=1; }
+                [[ -n "${seen[$pod]:-}" ]] || { echo "  $pod: already unsealed"; seen[$pod]=1; }
                unsealed=$(( unsealed + 1 ))
+                continue
+            fi
+            # Argument, not stdin — see the restore path above for why: OpenBao
+            # has no stdin form, and its prompt requires a tty a Job does not
+            # have. The key is exposed in the pod's process list for the length
+            # of the call.
+            if err=$(bao_exec "$pod" bao operator unseal "$key" 2>&1); then
+                echo -e "  $pod: ${GREEN}unsealed${NC}"
+                seen[$pod]=1
+                unsealed=$(( unsealed + 1 ))
+            else
+                # Not fatal on its own: a container still in Init or still
+                # pulling answers this way, and the next round finds it ready.
+                # Only running out of rounds is a failure.
+                echo "  $pod: not yet (${err##*$'\n'})"
            fi
        done
        (( unsealed >= want )) && break
        sleep 10
        if [[ $round -eq 180 ]]; then
-            # A replica that is running and still sealed did not get a working
-            # seal. Say which, with its own answer, rather than "timed out".
-            for pod in "${pods[@]}"; do
-                [[ -n "${seen[$pod]:-}" ]] && continue
-                echo "  $pod:" >&2
-                bao_exec "$pod" bao status 2>&1 | sed 's/^/    /' >&2 || true
-            done
-            fail "only $unsealed of $want OpenBao replicas unsealed after 30 minutes — a sealed one means the static seal failed (openbao-seal-key Secret, pod log)"
+            fail "only $unsealed of $want OpenBao replicas came unsealed after 30 minutes"
        fi
    done
    ok
@@ -1735,7 +1747,7 @@ if phase_runs flux; then
    command -v flux >/dev/null    || fail "flux not found"
    # In the in-cluster Job neither the SOPS key nor the Gitea token is present,
    # and neither should be baked in. Both are streamed into the secrets
-    # directory when the run asks for them, the same way the seal key is.
+    # directory when the run asks for them, the same way the unseal key is.
    # Unconditional: the same collection step on a laptop and in the Job, so the
    # recovery path is the one that gets exercised.
    require_secrets
@@ -1758,37 +1770,6 @@ if phase_runs flux; then
        --dry-run=client -o yaml | kubectl apply -f -
    ok

-    # The static seal key, before Flux brings OpenBao up: a pod whose env
-    # references a Secret that does not exist is never started, and OpenBao
-    # reads the key at start. Left alone when the cluster already holds the
-    # same value. A DIFFERENT value is refused rather than applied: the data
-    # is encrypted under whichever key OpenBao started with, and swapping the
-    # Secret would leave every replica sealed for good at its next restart.
-    # That is a migration, not a bootstrap step.
-    #
-    # --from-literal, not --from-file: `pass show` ends its output with a
-    # newline, the substitution strips it, and OpenBao rejects a key that is
-    # not exactly 32 bytes of base64. The key sits in this process's argument
-    # list for the length of the call, as it did in the pod's before.
-    echo -n "Creating OpenBao seal key secret... "
-    seal_key=$(read_secret seal-key) || fail "could not read the seal key"
-    kubectl create namespace "$OPENBAO_NS" --dry-run=client -o yaml | kubectl apply -f -
-    if existing=$(kubectl -n "$OPENBAO_NS" get secret openbao-seal-key \
-            -o go-template='{{index .data "key" | base64decode}}' 2>"$OPENBAO_SECRETS_DIR/.err"); then
-        [[ "$existing" == "$seal_key" ]] \
-            || fail "openbao-seal-key in $OPENBAO_NS holds a different key than pass — not replacing it (OpenBao's data is written under the key it started with)"
-        echo "already present"
-    elif grep -q NotFound "$OPENBAO_SECRETS_DIR/.err"; then
-        kubectl -n "$OPENBAO_NS" create secret generic openbao-seal-key \
-            --from-literal=key="$seal_key" >/dev/null \
-            || fail "could not create the openbao-seal-key Secret"
-        ok
-    else
-        echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
-        fail "could not read the openbao-seal-key Secret"
-    fi
-    unset seal_key existing
-
    if [[ "$FLUX_SOURCE" == "github" ]]; then
        step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..."
        echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}"
@@ -1839,7 +1820,7 @@ if phase_runs flux; then
            --token-auth --timeout 15m 2>&1
    fi

-    bring_up_openbao
+    unseal_openbao

    step 0 "Waiting for infrastructure..."
    for i in $(seq 1 60); do
@@ -1,67 +0,0 @@
-# Identical on every node and free of addresses. defines.conf and peers.conf
-# are rendered per node by the init container; the first is included above the
-# templates that reference it, the second below them.
-include "/etc/bird/defines.conf";
-
-router id from "pod0";
-log stderr all;
-
-protocol device { scan time 10; }
-
-# This node's own pod CIDR, originated rather than read back out of the kernel.
-# Reading it back made BIRD's view depend on the shape of Cilium's route, and
-# the IPv4 one is rejected outright: its next hop sits inside the prefix it
-# describes, which KRT refuses with "strange next-hop". The prefix comes from
-# the Node object either way, so there is nothing to learn.
-#
-# blackhole because this route is only ever advertised. Nothing installs it:
-# the only kernel protocol exporting from these tables takes BGP routes.
-protocol static own4 {
-  ipv4 { table master4; };
-  route OWN_POD4 blackhole;
-}
-protocol static own6 {
-  ipv6 { table master6; };
-  route OWN_POD6 blackhole;
-}
-
-# Only BGP-learned routes reach table 48 — not the static above, and not
-# anything else that may end up in the master tables.
-filter to_t48 {
-  if source != RTS_BGP then reject;
-  accept;
-}
-
-protocol kernel k48v4 {
-  kernel table 48;
-  metric 0;
-  ipv4 { table master4; import none; export filter to_t48; };
-}
-protocol kernel k48v6 {
-  kernel table 48;
-  metric 0;
-  ipv6 { table master6; import none; export filter to_t48; };
-}
-
-# iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM
-# from ingress0 and is unrelated.
-#
-# direct is explicit because iBGP defaults to multihop, which makes the gateway
-# recursive — BIRD would try to resolve the peer in the pod RIB. strict bind
-# keeps the listener off mgmt0 and ingress0.
-template bgp pod4 {
-  local as 64514;
-  source address MY4;
-  direct;
-  strict bind;
-  ipv4 { next hop self; import all; export all; };
-}
-template bgp pod6 {
-  local as 64514;
-  source address MY6;
-  direct;
-  strict bind;
-  ipv6 { next hop self; import all; export all; };
-}
-
-include "/etc/bird/peers.conf";
@@ -1,115 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: bird
-  namespace: kube-system
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: bird
-rules:
-  # The peer list is read from the nodes' pod0 annotations, so no address is
-  # written into this repo twice.
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: bird
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: bird
-subjects:
-  - kind: ServiceAccount
-    name: bird
-    namespace: kube-system
---
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: bird
-  namespace: kube-system
-  labels:
-    tier: node
-    app: bird
-spec:
-  selector:
-    matchLabels:
-      app: bird
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: bird
-    spec:
-      # Carries pod east-west once the routing rules land. Without it table 48
-      # stays empty and the rules fall through to main, i.e. back to VLAN 40.
-      priorityClassName: system-node-critical
-      # Writes host routes and peers on host addresses.
-      hostNetwork: true
-      serviceAccountName: bird
-      # Every node here is a control plane node, so this is not optional.
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-      initContainers:
-        - name: render-config
-          # By digest, not by the dated tag: the tag is rebuilt within the day
-          # it names, and the cluster's image path keeps resolving it to the
-          # first build of that day — imagePullPolicy: Always does not help,
-          # because the tag itself resolves stale. Update this when the image
-          # is rebuilt.
-          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              set -eu
-              cp /config/bird.conf /etc/bird/bird.conf
-              kubectl get node "$NODE_NAME" \
-                -o go-template-file=/config/defines.tmpl > /etc/bird/defines.conf
-              kubectl get nodes --field-selector "metadata.name!=$NODE_NAME" \
-                -o go-template-file=/config/peers.tmpl > /etc/bird/peers.conf
-              # BIRD's own parser is the check: a node whose annotation is
-              # missing renders as <no value> and fails here rather than
-              # crashlooping the daemon or forming a mesh with a hole.
-              bird -p -c /etc/bird/bird.conf
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          volumeMounts:
-            - name: config
-              mountPath: /config
-            - name: etc-bird
-              mountPath: /etc/bird
-      containers:
-        - name: bird
-          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
-          resources:
-            requests:
-              cpu: "50m"
-              memory: "32Mi"
-            limits:
-              cpu: "200m"
-              memory: "128Mi"
-          securityContext:
-            capabilities:
-              drop: ["ALL"]
-              # NET_ADMIN writes the routes; NET_BIND_SERVICE binds 179.
-              add: ["NET_ADMIN", "NET_BIND_SERVICE"]
-          volumeMounts:
-            - name: etc-bird
-              mountPath: /etc/bird
-      volumes:
-        - name: config
-          configMap:
-            name: bird-config
-        - name: etc-bird
-          emptyDir: {}
@@ -1,7 +0,0 @@
-define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }};
-define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }};
-{{/* podCIDRs follows the cluster's podSubnets order, v4 first. A flip puts an
-     IPv6 prefix into an ipv4 protocol, which fails the parse in the init
-     container rather than passing silently. */}}
-define OWN_POD4 = {{ index .spec.podCIDRs 0 }};
-define OWN_POD6 = {{ index .spec.podCIDRs 1 }};
@@ -1,12 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - daemonset.yaml
-configMapGenerator:
-  # Hashed name, so editing any of these rolls the DaemonSet.
-  - name: bird-config
-    namespace: kube-system
-    files:
-      - bird.conf
-      - defines.tmpl
-      - peers.tmpl
@@ -1,7 +0,0 @@
-{{- range $i, $n := .items -}}
-{{- $v6 := index $n.metadata.annotations "net.renner.uno/pod0-v6" -}}
-{{- $v4 := index $n.metadata.annotations "net.renner.uno/pod0-v4" -}}
-# {{ $n.metadata.name }}
-protocol bgp peer{{ $i }}_v6 from pod6 { neighbor {{ $v6 }} as 64514; }
-protocol bgp peer{{ $i }}_v4 from pod4 { neighbor {{ $v4 }} as 64514; }
-{{ end -}}
@@ -180,17 +180,3 @@ spec:
    # the supernets here cover the LAN as well, which is the point.
    ipv4NativeRoutingCIDR: 10.0.0.0/8
    ipv6NativeRoutingCIDR: fd00::/8
-    # Pod east-west leaves the node on pod0 (VLAN 48) rather than the
-    # management VLAN. Native routing hands forwarding to the kernel;
-    # autoDirectNodeRoutes installs the remote PodCIDR routes, which is
-    # allowed because all three nodes share pod0's L2 segment.
-    #
-    # directRoutingDevice is explicit on purpose: with several addressed
-    # devices Cilium falls back to the one carrying the Kubernetes InternalIP,
-    # which is mgmt0.
-    #
-    # Encapsulation is gone, so the datapath MTU goes 1450 -> 1500.
-    routingMode: native
-    autoDirectNodeRoutes: true
-    nodePort:
-      directRoutingDevice: pod0
@@ -72,9 +72,6 @@ spec:
            # they contend for it. Sized for an inference server plus room for
            # a transcoder and a benchmark pod alongside it.
            - -shared-dev-num=4
-            # Advertises gpu.intel.com/monitoring: one resource that grants a
-            # pod every GPU on the node instead of a slot. The exporter needs it.
-            - -enable-monitoring
          securityContext:
            seLinuxOptions:
              type: "container_device_plugin_t"
@@ -7,7 +7,6 @@ resources:
  - cluster-settings/
  - cilium/
  - multus/
-  - bird/
  - cert-manager/
  - snapshot-controller/
  - metrics-server/
@@ -19,7 +18,6 @@ resources:
  - cloudnative-pg/
  - mariadb-operator/
  - netbird-ingress/
-  - netbird-proxy-services/
  - coredns-config/
  - k8s-gateway/
  - external-dns-unifi/
@@ -1,108 +0,0 @@
-# Reconciles NetBird reverse-proxy services from Ingresses that carry
-# dns.renner.uno/public: one TLS-passthrough service per host on :443,
-# targeting Traefik's ClusterIP through a subnet resource over the service
-# CIDR (ci/images/netbird-proxy-services/controller.py).
-#
-# openbao namespace, like the rotators: its default-deny already allows egress
-# to OpenBao, DNS, the apiserver and `world` (netbird.renner.uno), and the
-# NetBird PAT is read from OpenBao on every pass through the
-# netbird-proxy-services role — policies/netbird-proxy-services.hcl, applied
-# once by hand like the rotator roles. Until that role exists the pod logs an
-# auth error each minute and does nothing.
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: netbird-proxy-services
-  namespace: openbao
-  labels:
-    app.kubernetes.io/name: netbird-proxy-services
---
-# Read-only, cluster-wide: public Ingresses live in many namespaces, the
-# Traefik Service in ingress/.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: netbird-proxy-services-read
-rules:
-  - apiGroups: ["networking.k8s.io"]
-    resources: ["ingresses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["get"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: netbird-proxy-services-read
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: netbird-proxy-services-read
-subjects:
-  - kind: ServiceAccount
-    name: netbird-proxy-services
-    namespace: openbao
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netbird-proxy-services
-  namespace: openbao
-  labels:
-    app.kubernetes.io/name: netbird-proxy-services
-spec:
-  replicas: 1
-  # One writer against the management API at a time.
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: netbird-proxy-services
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: netbird-proxy-services
-    spec:
-      serviceAccountName: netbird-proxy-services
-      securityContext:
-        runAsNonRoot: true
-        # Numeric, or the kubelet refuses the image's named user with
-        # "cannot verify user is non-root". The script needs no home and no
-        # writable path, so nobody/nogroup is enough.
-        runAsUser: 65534
-        runAsGroup: 65534
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: controller
-          # Pinned by digest: the date tag is rebuilt on every push of the
-          # image directory (branch and master alike) and the registry keeps
-          # only the newest manifest per tag, so the digest must come from
-          # the MASTER build of the code change.
-          image: gitea.renner.uno/gitea_admin/netbird-proxy-services:2026-09-07@sha256:caffa308d802c6fcc6aea919ac94657c08b993651baa6b8d18de395bcc40523e
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-          env:
-            - name: NB_MANAGEMENT_URL
-              value: https://netbird.renner.uno
-            - name: BAO_ADDR
-              value: http://openbao.openbao.svc.cluster.local:8200
-            - name: BAO_ROLE
-              value: netbird-proxy-services
-            # Must match cluster.network.serviceSubnets in talos/patches/cluster.yaml.
-            - name: SERVICE_CIDR
-              value: "10.96.0.0/12"
-            - name: TRAEFIK_SERVICE
-              value: ingress/traefik
-            - name: INTERVAL_SECONDS
-              value: "60"
-          resources:
-            requests:
-              cpu: 10m
-              memory: 32Mi
-            limits:
-              memory: 64Mi
@@ -1,4 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - deployment.yaml
@@ -25,10 +25,10 @@ spec:
        name: openbao
        namespace: openbao
  install:
-    # A fresh OpenBao pod is 0/1 Ready until `bao operator init` (readiness
-    # reports sealed). helm --wait would time out and remediation would
-    # uninstall the release, so don't wait here; init is manual (see
-    # kubernetes/MIGRATION-sops-openbao.md). Unseal is automatic (static seal).
+    # A fresh OpenBao pod is 0/1 Ready until `bao operator init` + unseal
+    # (readiness probe reports sealed). helm --wait would time out and
+    # remediation would uninstall the release, so don't wait here; the
+    # init/unseal procedure is manual (see kubernetes/MIGRATION-sops-openbao.md).
    disableWait: true
    remediation:
      retries: 3
@@ -42,11 +42,6 @@ spec:
      # the agent-injector mutating webhook is unused surface.
      enabled: false
    server:
-      # The static seal key, from the openbao-seal-key Secret (bootstrap.sh).
-      extraSecretEnvironmentVars:
-        - envName: BAO_SEAL_KEY
-          secretName: openbao-seal-key
-          secretKey: key
      # Raft (not standalone/file) for two reasons: `bao operator raft
      # snapshot` as a consistent backup primitive, and real HA across three
      # nodes. TLS terminates at the Traefik ingress; in-cluster traffic is
@@ -58,8 +53,16 @@ spec:
        # the moment one pod is sealed or rescheduled, which is worse than
        # the single replica it replaced. Three tolerates one.
        #
-        # A restarted replica unseals itself from the static seal and rejoins
-        # raft, so a rolling reboot only ever costs one replica at a time.
+        # What this buys, and what it does not: a node drain seals the replica
+        # on it, and raft elects a new leader from the remaining standbys
+        # within seconds, so the service does not go down. It does NOT remove
+        # the manual unseal — a sealed server can neither serve requests nor
+        # act as standby, so that replica stays out until someone unseals it.
+        #
+        # The consequence for rolling node maintenance is worth stating: a
+        # Talos upgrade drains all three nodes in turn, sealing each replica
+        # as it goes. Unseal each one as it comes back, or the third drain
+        # takes the last unsealed replica with it and quorum is lost anyway.
        replicas: 3
        raft:
          enabled: true
@@ -108,14 +111,6 @@ spec:

            service_registration "kubernetes" {}

-            # Auto-unseal. The key is a 32-byte AES key in the openbao-seal-key
-            # Secret, created from pass at bootstrap before this release. The
-            # former shamir key is the recovery key (#805).
-            seal "static" {
-              current_key_id = "1"
-              current_key    = "env://BAO_SEAL_KEY"
-            }
-
            audit "file" "stdout" {
              description = "audit trail to pod stdout -> Loki"
              options {
@@ -1,23 +0,0 @@
-# OpenBao policy: netbird-proxy-services
-#
-# The reverse-proxy-services controller (Deployment netbird-proxy-services in
-# the openbao namespace) reads the NetBird PAT on every reconcile pass and
-# calls the management API with it. Read-only on that one path; it shares the
-# bootstrap credential with rotator-netbird but not that role's PATCH rights.
-#
-# NOT applied by Flux. Apply ONCE with an admin token:
-#
-#   bao policy write netbird-proxy-services kubernetes/infrastructure/openbao/policies/netbird-proxy-services.hcl
-#   bao write auth/kubernetes/role/netbird-proxy-services \
-#     bound_service_account_names=netbird-proxy-services \
-#     bound_service_account_namespaces=openbao \
-#     policies=netbird-proxy-services ttl=15m
-#
-# The PAT itself is the one rotator-netbird already uses
-# (homelab/infrastructure/secret-rotators-bootstrap/netbird, key `pat`).
-#
-# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
-
-path "homelab/data/infrastructure/secret-rotators-bootstrap/netbird" {
-  capabilities = ["read"]
-}
@@ -11,8 +11,8 @@
 # secret is stored anywhere; each run logs in fresh.
 #
 # A SEALED OpenBao fails the login step and the Job goes Failed — that is
-# deliberate signal, not noise: it means the static seal could not unseal a
-# restarted pod and backups have stopped.
+# deliberate signal, not noise: it means unseal was forgotten after a pod
+# restart and backups have stopped.
 #
 # Retention: last 7 snapshots on the PVC; velero adds 30 d offsite.
 apiVersion: batch/v1
@@ -25,13 +25,13 @@ spec:
      stop: "fd99::ff"
 ---
 # Cilium BGPv2 control-plane config peering each node with the UDM over the
-# ingress VLAN 47 — never over v30iot, which is the untrusted IoT segment.
+# management LAN — never over v30iot, which is the untrusted IoT segment.
 # The source address comes from the kernel routing decision, so no explicit
 # localAddress override is needed.
 #
 # Two parallel sessions, one per address family:
-#   - v4 session: peer 10.2.47.1, family IPv4 unicast only
-#   - v6 session: peer fd02:47::1, family IPv6 unicast only
+#   - v4 session: peer 10.2.40.1, family IPv4 unicast only
+#   - v6 session: peer fd40::1, family IPv6 unicast only
 # This avoids the "v6 NLRIs over v4 transport" path, which requires
 # bilateral extended-nexthop capability negotiation (RFC 5549/8950).
 # Cilium 1.19 BGPv2 advertises the capability only for the IPv4 Unicast
@@ -74,14 +74,14 @@ spec:
      peers:
        - name: udm-v6
          peerASN: 64512
-          # ULA — stable across ISP renumber events. UDM's br47 has
-          # fd02:47::1/64; nodes get fd02:47::<eui-64> via SLAAC on ingress0.
-          peerAddress: fd02:47::1
+          # ULA — stable across ISP renumber events. UDM's br40 has
+          # fd40::1/64; nodes get fd40::<eui-64> via SLAAC on eth0.
+          peerAddress: fd40::1
          peerConfigRef:
            name: bgp-peer-udm-v6
        - name: udm-v4
          peerASN: 64512
-          peerAddress: 10.2.47.1
+          peerAddress: 10.2.40.1
          peerConfigRef:
            name: bgp-peer-udm-v4
 ---
@@ -93,17 +93,13 @@ spec:
        # Real client-IP visibility. The netbird-ingress routing peer
        # re-originates every mesh + WAN-via-proxy connection from its own
        # pod IP (rootless userspace WireGuard), so Traefik's TCP peer for
-        # that traffic is always a pod IP — collapsing all external clients
-        # to one source. The NetBird reverse-proxy services are TLS
-        # passthrough and carry the WAN client address in a PROXY protocol
-        # v2 header (options.proxy_protocol, set by netbird-proxy-services);
-        # X-Forwarded-For stays trusted for HTTP-mode services. Both are
-        # accepted from the pod CIDRs only; connections without a header
-        # (direct LAN/mesh) keep their RemoteAddr.
-        proxyProtocol:
-          trustedIPs:
-            - "10.244.0.0/16"
-            - "fd10:244::/56"
+        # that traffic is always a pod IP (e.g. fd10:244::2de1) — collapsing
+        # all external clients to one source. The NetBird reverse-proxy
+        # (HTTP mode, on the VPS) sets X-Forwarded-For with the real WAN
+        # client IP; trust the pod CIDRs so Traefik reads it instead. Without
+        # this, a CrowdSec bouncer bans the shared pod IP and 403s everyone
+        # (the 2026-06-27 IPv6 outage). Harmless no-op when no XFF is present
+        # (direct LAN/mesh) — those keep their RemoteAddr.
        forwardedHeaders:
          trustedIPs:
            - "10.244.0.0/16"
@@ -12,28 +12,11 @@ machine:
    # Left enabled they autoconfigure a ULA and an IPv6 default route from its
    # RAs, which ties with mgmt0's and costs the cluster IPv6 egress.
    # Disabling v6 on a VLAN parent does not affect the VLAN child.
-    net.ipv6.conf.uplink.disable_ipv6: "1"
-    net.ipv6.conf.mgmt.disable_ipv6: "1"
-    # pod0 is a closed L2 fabric between the three nodes: statically
-    # addressed, no gateway, nothing to learn from an RA.
-    net.ipv6.conf.pod0.accept_ra: "0"
-    # ingress0 keeps RAs, at the default accept_ra of 2: an IPv6 default
-    # router is a link-local address learned only from the RA, and with
-    # net.ipv6.conf.all.forwarding=1 — which Cilium sets — Linux ignores RAs
-    # entirely below 2. That is how mgmt0 has its IPv6 default today.
-    #
-    # The prefix is announced correctly, so SLAAC addresses this link the same
-    # way mgmt0 gets its fd40:: address: derived from the MAC, stable across
-    # reboots, nothing to allocate. Only IPv4 is declared here, and only
-    # because a second DHCP client is what broke the node's DNS record.
-    #
-    # Just the default route is suppressed, because mgmt0 owns it until the
-    # split flips it and two defaults at one metric tie. Dropping this line is
-    # what hands the IPv6 default over.
-    net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
+    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
+    net.ipv6.conf.enp87s0.disable_ipv6: "1"
    # That network also carries the mgmt VLAN's ARP. Answer ARP only on the
    # link that owns the address, or the UDM learns the node IP against
-    # mgmt's MAC and delivers IPv4 to a NIC Cilium is not attached to (#793).
+    # enp87s0's MAC and delivers IPv4 to a NIC Cilium is not attached to (#793).
    net.ipv4.conf.all.arp_ignore: "1"
    net.ipv4.conf.default.arp_ignore: "1"
    net.ipv4.conf.all.arp_announce: "2"
@@ -52,22 +35,13 @@ machine:
    # this label instead of listing hostnames, so which boxes carry storage is
    # stated once, here, by the machine that knows.
    storage.renner.uno/ceph-osd: "true"
-  # Publishes this node's pod0 addresses, declared below in its VLANConfig, so
-  # the bird DaemonSet reads its peers from the node list rather than from a
-  # copy. Annotations, not labels: a label value cannot contain a colon.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::11
-    net.renner.uno/pod0-v4: 10.2.48.11
  udev:
-    # Renames every port this node uses to its role at boot, matched by
-    # burned-in MAC — kernel names derive from PCI topology and shift when
-    # hardware moves. Shared manifests then name roles and only this file
-    # knows the hardware. A rule applied to a RUNNING node cannot rename
-    # (Resource busy) and takes effect at the next reboot.
+    # Renames the SAN port (X710 port 1, matched by its burned-in MAC) to the
+    # role name at boot, before the network stack opens links — a rule applied
+    # to a RUNNING node cannot rename (Resource busy) and takes effect at the
+    # next reboot.
    rules:
-      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:70:27", NAME="uplink"
      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:70:28", NAME="ceph-san"
-      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:70:29", NAME="mgmt"
 cluster:
  allowSchedulingOnControlPlanes: true
 ---
@@ -109,17 +83,20 @@ provisioning:
  grow: true
 ---
 # X710 port 0 carries the switch trunk and takes no addressing itself — its
-# untagged VLAN is a network this node has no business being on. Management,
-# pod, ingress and v30iot all ride it as tagged VLANs.
+# untagged VLAN is a network this node has no business being on. Management
+# and v30iot ride it as tagged VLANs.
 #
-# Named by the udev rule above, not by LinkAliasConfig. Aliases are valid VLAN
-# parents, but a selector matching zero links is skipped silently and Talos
-# disables default DHCP machine-wide as soon as any new-style network document
-# exists — so a silent miss leaves the node with no address at all. udev
-# renames the link itself, so a missed match is visible as a missing link.
+# Links are named by predictable kernel name, which derives from PCI topology
+# and can still shift when hardware is added.
+#
+# LinkAliasConfig would pin these better, but aliasing the X710 ports by MAC
+# and using the aliases as VLAN parents left the node with no address on any
+# interface — the aliases were never created and every document referencing
+# them silently did nothing. Prove any alias-based revision with
+# `talosctl apply-config --mode=try` first; it rolls back on its own.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: uplink
+name: enp2s0f0np0
 up: true
 ---
 # Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
@@ -128,7 +105,7 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: uplink
+parent: enp2s0f0np0
 up: true
 ---
 # DHCP, and the lease is not reserved in this repo — reservations live on the
@@ -157,59 +134,9 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: uplink
+parent: enp2s0f0np0
 up: true
 ---
-# Pod network, VLAN 48. Node-to-node only: the VXLAN overlay rides it once the
-# kubelet node IP moves here, and nothing on it is routed off the switch.
-#
-# Statically addressed, no gateway, no DHCP. A lease would make the cluster's
-# data path depend on the UDM being up in order to form, on a segment whose
-# traffic never reaches the UDM. Declared per node, the way ceph-san's segment
-# carries a declared range rather than a lease.
-apiVersion: v1alpha1
-kind: VLANConfig
-name: pod0
-vlanID: 48
-parent: uplink
-up: true
-addresses:
-  - address: 10.2.48.11/24
-  - address: fd02:48::11/64
---
-# Ingress network, VLAN 47. World-to-pod: the BGP session, the next hop for
-# every advertised LoadBalancer VIP, and the masquerade source for pod egress.
-# Publicly routed, which is what separates it from pod0.
-#
-# Statically addressed, not leased. A second DHCP client on the node announced
-# the same hostname as mgmt0's and the UDM registered the node against this
-# segment instead, so mgmt0's `srv.renner.uno` record disappeared and the
-# lease's resolver displaced the management one. The UDM cannot register a
-# network's clients selectively, so the client goes rather than the record.
-# DHCP is off on VLAN 47 for the same reason.
-#
-# Both defaults live in table 47, not in main, so only the pod-source rules in
-# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
-# reply the node sends from its own addresses.
-#
-# The v6 gateway is the UDM's configured ULA on this segment, the same address
-# BGP peers with. The RA-learned link-local would be the alternative and is not
-# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
-# default would land in main, where it does not belong.
-apiVersion: v1alpha1
-kind: VLANConfig
-name: ingress0
-vlanID: 47
-parent: uplink
-up: true
-addresses:
-  - address: 10.2.47.11/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication and nothing else. The udev rule in the machine
 # section above renames the port to ceph-san at boot, so shared manifests name
@@ -10,13 +10,8 @@ machine:
    # Same for the physical ports on the untagged fallback network: left
    # enabled they take an IPv6 default route from its RAs and cost the
    # cluster IPv6 egress.
-    net.ipv6.conf.uplink.disable_ipv6: "1"
-    net.ipv6.conf.mgmt.disable_ipv6: "1"
-    # pod0 is a closed segment with no router, so it ignores RAs outright.
-    # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
-    # and only the default route is held back — see k8s-1.yaml.
-    net.ipv6.conf.pod0.accept_ra: "0"
-    net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
+    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
+    net.ipv6.conf.enp87s0.disable_ipv6: "1"
    # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
    net.ipv4.conf.all.arp_ignore: "1"
    net.ipv4.conf.default.arp_ignore: "1"
@@ -33,16 +28,10 @@ machine:
    topology.kubernetes.io/zone: saturn
    # Holds a Ceph OSD drive — see k8s-1.yaml.
    storage.renner.uno/ceph-osd: "true"
-  # pod0 addresses for the bird peer list — see k8s-1.yaml.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::12
-    net.renner.uno/pod0-v4: 10.2.48.12
  udev:
-    # Renames every port to its role at boot — see k8s-1.yaml.
+    # Renames the SAN port to the role name at boot — see k8s-1.yaml.
    rules:
-      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:76:6f:73", NAME="uplink"
      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:76:6f:74", NAME="ceph-san"
-      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:76:6f:75", NAME="mgmt"
 cluster:
  allowSchedulingOnControlPlanes: true
 ---
@@ -72,14 +61,17 @@ provisioning:
  grow: true
 ---
 # X710 port 0 carries the switch trunk and takes no addressing itself — the
-# port's untagged VLAN is not one this node belongs on. Management, pod,
-# ingress and v30iot all ride it as tagged VLANs.
+# port's untagged VLAN is not one this node belongs on. Management and v30iot
+# both ride it as tagged VLANs.
 #
-# Named by the udev rule above, not by LinkAliasConfig — see k8s-1.yaml and
-# RUNBOOK-baremetal-conversion.md.
+# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are
+# valid VLAN parents, but a selector matching zero links is skipped silently
+# and Talos disables default DHCP machine-wide as soon as any new-style network
+# document exists — so a silent miss leaves the node with no address on any
+# interface. See RUNBOOK-baremetal-conversion.md.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: uplink
+name: enp2s0f0np0
 up: true
 ---
 # Management, VLAN 40. Node IP, etcd, the API and Ceph's public network.
@@ -87,7 +79,7 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: uplink
+parent: enp2s0f0np0
 up: true
 ---
 apiVersion: v1alpha1
@@ -107,37 +99,9 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: uplink
+parent: enp2s0f0np0
 up: true
 ---
-# Pod network, VLAN 48. Node-to-node only, statically addressed, no gateway
-# and no DHCP — see k8s-1.yaml.
-apiVersion: v1alpha1
-kind: VLANConfig
-name: pod0
-vlanID: 48
-parent: uplink
-up: true
-addresses:
-  - address: 10.2.48.12/24
-  - address: fd02:48::12/64
---
-# Ingress network, VLAN 47. World-to-pod, publicly routed, statically
-# addressed — see k8s-1.yaml for why it is not leased.
-apiVersion: v1alpha1
-kind: VLANConfig
-name: ingress0
-vlanID: 47
-parent: uplink
-up: true
-addresses:
-  - address: 10.2.47.12/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.
 # Renamed to the role name by the udev rule above — see k8s-1.yaml.
@@ -6,13 +6,8 @@ machine:
    net.ipv6.conf.ens19.disable_ipv6: "1"
    # Same for the physical ports on the untagged fallback network — see
    # k8s-1.yaml.
-    net.ipv6.conf.uplink.disable_ipv6: "1"
-    net.ipv6.conf.mgmt.disable_ipv6: "1"
-    # pod0 is a closed segment with no router, so it ignores RAs outright.
-    # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
-    # and only the default route is held back — see k8s-1.yaml.
-    net.ipv6.conf.pod0.accept_ra: "0"
-    net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
+    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
+    net.ipv6.conf.enp87s0.disable_ipv6: "1"
    # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
    net.ipv4.conf.all.arp_ignore: "1"
    net.ipv4.conf.default.arp_ignore: "1"
@@ -29,16 +24,10 @@ machine:
    topology.kubernetes.io/zone: neptune
    # Holds a Ceph OSD drive — see k8s-1.yaml.
    storage.renner.uno/ceph-osd: "true"
-  # pod0 addresses for the bird peer list — see k8s-1.yaml.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::13
-    net.renner.uno/pod0-v4: 10.2.48.13
  udev:
-    # Renames every port to its role at boot — see k8s-1.yaml.
+    # Renames the SAN port to the role name at boot — see k8s-1.yaml.
    rules:
-      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:72:d3", NAME="uplink"
      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:72:d4", NAME="ceph-san"
-      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:72:d5", NAME="mgmt"
 cluster:
  allowSchedulingOnControlPlanes: true
 ---
@@ -69,14 +58,14 @@ provisioning:
 ---
 apiVersion: v1alpha1
 kind: LinkConfig
-name: uplink
+name: enp2s0f0np0
 up: true
 ---
 apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: uplink
+parent: enp2s0f0np0
 up: true
 ---
 apiVersion: v1alpha1
@@ -94,37 +83,9 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: uplink
+parent: enp2s0f0np0
 up: true
 ---
-# Pod network, VLAN 48. Node-to-node only, statically addressed, no gateway
-# and no DHCP — see k8s-1.yaml.
-apiVersion: v1alpha1
-kind: VLANConfig
-name: pod0
-vlanID: 48
-parent: uplink
-up: true
-addresses:
-  - address: 10.2.48.13/24
-  - address: fd02:48::13/64
---
-# Ingress network, VLAN 47. World-to-pod, publicly routed, statically
-# addressed — see k8s-1.yaml for why it is not leased.
-apiVersion: v1alpha1
-kind: VLANConfig
-name: ingress0
-vlanID: 47
-parent: uplink
-up: true
-addresses:
-  - address: 10.2.47.13/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
 # here on purpose; the link is handed whole to the OSD pod (host-device).
 apiVersion: v1alpha1
@@ -20,70 +20,3 @@ machine:
      validSubnets:
        - 10.2.40.0/24
        - fd40::/64
---
-# Pod egress leaves over ingress0. Matching on the pod source prefix is what
-# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
-# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
-# source onto VLAN 47.
-#
-# The routing decision runs before Cilium's masquerade hook, so the rule sees
-# the pod address and the masquerade then rewrites it to ingress0's.
-#
-# Table 47 holds nothing but the two default routes, so the destination rules
-# below have to run first: without them a pod's packet to a service or to a pod
-# on another node also matches the source rule, finds only the default, and
-# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
-#
-# They send everything inside the cluster and the local networks back to main.
-# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
-# is not foreign traffic.
-#
-# Priorities are below main (32766) and clear of the rule Cilium installs at 9.
-# The two families are separate rule tables, so the priorities need not match;
-# the document name is the priority and must be unique.
-# Pod to pod between nodes goes over pod0, via table 48, which the bird mesh
-# fills with one route per peer pod CIDR and the peer's pod0 address as the
-# next hop. Ahead of 990/991 so it wins for that traffic, and matching source
-# AND destination so nothing else lands there: the node's own traffic to a pod
-# would otherwise leave with a pod0 source, which CiliumNode does not list, so
-# Cilium would classify it as world rather than host.
-#
-# Traffic to a pod on this node falls through — table 48 holds only the peers'
-# CIDRs, so the lookup misses and 990/991 send it to main and cilium_host.
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "980"
-src: fd10:244::/48
-dst: fd10:244::/48
-table: "48"
---
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "981"
-src: 10.244.0.0/16
-dst: 10.244.0.0/16
-table: "48"
---
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "990"
-dst: fd00::/8
-table: main
---
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "991"
-dst: 10.0.0.0/8
-table: main
---
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "1000"
-src: fd10:244::/48
-table: "47"
---
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "1001"
-src: 10.244.0.0/16
-table: "47"
@@ -10,10 +10,10 @@
 # session. The nodes have ~94G each with about 70G unused.
 #
 # The whole script runs here, flux and OpenBao included, so that what a rebuild
-# rehearses is what a real recovery would execute. The OpenBao seal key is not
-# baked in and not stored as a Secret here: the operator streams the secrets
-# into an in-memory emptyDir at the moment the run asks for them, and they die
-# with the Job without ever reaching this cluster's etcd or a backup of it.
+# rehearses is what a real recovery would execute. The unseal key is not baked
+# in and not stored as a Secret: the operator streams all five secrets into an
+# in-memory emptyDir at the moment the run asks for them, and they die with the
+# Job without ever reaching etcd or a backup of it.
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -28,19 +28,10 @@ router bgp 64512
 ! advertise extended-nexthop for the IPv6 Unicast SAFI when v6 routes
 ! are carried over a v4 transport, so we run a native v6 session that
 ! carries v6 routes with v6 next-hops and skip the capability dance
- ! entirely. Dynamic neighbors — nodes are SLAAC-addressed, so the listen
- ! ranges cover the SLAAC suffix space rather than naming each peer.
- !
- ! Both segments are listed while the ingress split lands (#576). The
- ! cluster peers from mgmt (40) today and moves to ingress (47), and a
- ! listen range only accepts a session, it does not open one — so having
- ! both costs nothing and means the move never has a window where the UDM
- ! is not listening where the nodes are calling from. Drop the 40 ranges
- ! once every session is established on 47.
+ ! entirely. Dynamic neighbors — k8s nodes are SLAAC-addressed on mgmt
+ ! LAN; the listen ranges cover the SLAAC suffix space.
 bgp listen range 10.2.40.0/24 peer-group K8S
 bgp listen range fd40::/64 peer-group K8S
- bgp listen range 10.2.47.0/24 peer-group K8S
- bgp listen range fd02:47::/64 peer-group K8S
 !
 address-family ipv4 unicast
  neighbor K8S activate