feat(bird): add curl and jq for the peer-list init container

The per-node peer list is rendered from the Kubernetes API rather than written into the repo a second time, so the init container needs an HTTP client that can verify against the API server's CA bundle. busybox wget cannot, and the query is not going to run unverified. Using the same image for the init container and the daemon keeps one lifecycle instead of borrowing a CI image at runtime.

Refs #576

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.woodpecker/alert-monitor.yml

@@ -42,7 +42,7 @@ when:
 
 steps:
   - name: fetch-alerts
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
     environment:
       SOPS_AGE_KEY:
         from_secret: sops_age_key

.woodpecker/build-ci-images.yml

@@ -89,7 +89,7 @@ steps:
           --destination gitea.renner.uno/gitea_admin/ci-k8s:latest
 
   - name: build-vmtest
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/vmtest
     depends_on: []
     when:
@@ -349,3 +349,56 @@ steps:
           --dockerfile Containerfile \
           --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
           --destination gitea.renner.uno/gitea_admin/heating-can:latest
+
+  # netbird-proxy-services: the controller that reconciles NetBird reverse-proxy
+  # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
+  # Runtime artifact like heating-can; same registry so the audit tracks it.
+  - name: build-netbird-proxy-services
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images/netbird-proxy-services
+    depends_on: []
+    when:
+      - event: deployment
+      - event: push
+        path: ["ci/images/netbird-proxy-services/**", ".woodpecker/build-ci-images.yml"]
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
+          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
+
+  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
+  # artifact like heating-can; same registry so the audit tracks it.
+  - name: build-bird
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images/bird
+    depends_on: []
+    when:
+      - event: deployment
+      - event: push
+        path: ["ci/images/bird/**", ".woodpecker/build-ci-images.yml"]
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/bird:$TAG \
+          --destination gitea.renner.uno/gitea_admin/bird:latest

.woodpecker/ci-manifests-test.yml

@@ -60,7 +60,7 @@ steps:
   # run that never touched them. Tracked in #461.
   - name: kustomize-build
     depends_on: [] # see note above `steps:`
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
     commands:
       - |
         set -eu
@@ -158,7 +158,7 @@ steps:
   # whenever it trails the deployed server. If you bump the server, bump here.
   - name: woodpecker-lint
     depends_on: [] # see note above `steps:`
-    image: woodpeckerci/woodpecker-cli:v3.17.0-alpine
+    image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
     commands:
       - |
         set -eu
@@ -246,7 +246,7 @@ steps:
   # this step fails permanently on its own documentation.
   - name: no-digest-pins
     depends_on: [] # see note above `steps:`
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
     commands:
       - |
         set -eu

.woodpecker/e2e-test.yml

@@ -4,7 +4,7 @@ when:
 
 steps:
   - name: e2e
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
     environment:
       SOPS_AGE_KEY:
         from_secret: sops_age_key

.woodpecker/e2e.yml

@@ -31,7 +31,7 @@ steps:
       - sh scripts/ci/test-model-selection.sh
 
   - name: e2e-test
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
     environment:
       SOPS_AGE_KEY:
         from_secret: sops_age_key

.woodpecker/issue-fix.yml

@@ -49,7 +49,7 @@ steps:
     # (the issue-pick logic needs jq, which only the ci-claude image has);
     # minting a 1h read-only token per run is cheap. Degrades to
     # snapshot-only if the ci-diagnostics ServiceAccount isn't reconciled.
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
     environment:
       SOPS_AGE_KEY:
         from_secret: sops_age_key

ansible/roles/netbird-proxy-watchdog/defaults/main.yml

@@ -32,8 +32,13 @@ netbird_watchdog_fail_threshold: 3
 
 # Minimum seconds between watchdog-issued restarts, so a genuinely broken
 # backend can't put the proxy into a restart loop. Shared by both the URL-probe
-# watchdog and the flood watchdog below.
-netbird_watchdog_cooldown_seconds: 600
+# watchdog and the flood watchdog below. One minute: the route-manager wedge
+# recurs on every routing-peer loss, and rolling node reboots lose one peer
+# every few minutes; a suppressed restart there leaves the wedge standing
+# until the URL probe's three rounds fire (3.5 min of 000 on every domain,
+# measured 2026-09-07). A minute still caps a loop at one restart per
+# probe interval.
+netbird_watchdog_cooldown_seconds: 60
 
 # Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain
 # substring in the proxy's docker logs by the flood watchdog. On this build the

ci/images/bird/Containerfile

@@ -0,0 +1,22 @@
+FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
+
+# bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
+# step — the image is the binary plus its runtime.
+#
+# 3.22 rather than the 3.21 the other images here pin: bird2 first appears in
+# 3.22/community.
+#
+# Config is read from /etc/bird/bird.conf, not the package default
+# /etc/bird.conf, so the mount cannot shadow what the package ships.
+#
+# curl and jq are for the init container, which renders the per-node peer list
+# from the Kubernetes API. busybox wget cannot verify against the API server's
+# CA bundle, and the query must not run unverified.
+RUN apk add --no-cache bird2 curl jq
+
+# Fails the build rather than the first pod if a binary or its path is wrong.
+RUN /usr/sbin/bird --version \
+ && curl --version > /dev/null \
+ && jq --version > /dev/null
+
+ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]

ci/images/k8s/Containerfile

@@ -1,9 +1,9 @@
 FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
 
-ARG TALOS_VERSION=v1.12.6
+ARG TALOS_VERSION=v1.13.9
 ARG VELERO_VERSION=v1.16.0
-ARG KUBECTL_VERSION=v1.36.2
-ARG FLUX_VERSION=2.9.2
+ARG KUBECTL_VERSION=v1.36.3
+ARG FLUX_VERSION=2.9.5
 
 RUN apk add --no-cache \
       aws-cli \
@@ -18,7 +18,7 @@ RUN apk add --no-cache \
 
 RUN curl -sfL -o /usr/local/bin/talosctl \
       "https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
- && echo "4c41a3b10b075292e64b182283d558e9d8e935f43239de1ed4e4b14359593efb  /usr/local/bin/talosctl" | sha256sum -c - \
+ && echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e  /usr/local/bin/talosctl" | sha256sum -c - \
  && chmod +x /usr/local/bin/talosctl
 
 RUN curl -sfL -o /usr/local/bin/kubectl \
@@ -31,7 +31,7 @@ RUN curl -sfL -o /usr/local/bin/kubectl \
 
 RUN curl -sfL -o /tmp/flux.tar.gz \
       "https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \
- && echo "4092b367fd060097976fb7261601f79420ecf7266328417dfd8dfee27de0e6a3  /tmp/flux.tar.gz" | sha256sum -c - \
+ && echo "b853df82adfd7736f580692f9f734473d571606307139f8fd20c2a80dd1ff473  /tmp/flux.tar.gz" | sha256sum -c - \
  && tar -xzf /tmp/flux.tar.gz -C /usr/local/bin flux \
  && chmod +x /usr/local/bin/flux \
  && rm -f /tmp/flux.tar.gz

ci/images/netbird-proxy-services/Containerfile

@@ -0,0 +1,12 @@
+# netbird-proxy-services: reconciles NetBird reverse-proxy services (public
+# domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
+# public label. Standard library only, so the image is the interpreter plus one
+# file; runs unprivileged as its own user.
+FROM python:3.13-slim
+
+RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
+COPY controller.py /opt/controller.py
+
+ENV PYTHONUNBUFFERED=1
+USER controller
+ENTRYPOINT ["python3", "/opt/controller.py"]

ci/images/netbird-proxy-services/controller.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+"""netbird-proxy-services: reconcile NetBird reverse-proxy services from Ingresses.
+
+Source of truth: every Ingress carrying the public label (dns.renner.uno/public).
+For each host on such an Ingress the NetBird management gets one reverse-proxy
+service in TLS-passthrough mode on :443, targeting Traefik's ClusterIP through a
+subnet resource that covers the cluster's service CIDR. SNI and Host reach
+Traefik untouched, the proxy terminates nothing, and no domain resource is
+involved -- the routing peers stay ordinary IP routers. The target sends a
+PROXY protocol v2 header so Traefik (and CrowdSec behind its access log) sees
+the WAN client address instead of the routing peer's pod IP.
+
+Ownership is derived from NetBird itself, not from local state: a service is
+managed iff its mode is "tls" and its target is this controller's subnet
+resource. Services that do not match (the hand-made HTTP/domain ones) are never
+touched; a desired host held by such a service is reported and skipped until
+that service is removed by hand.
+
+Credentials: the NetBird PAT is read from OpenBao on every pass through the
+Kubernetes auth role, exactly like the setup-key rotator; it never lands in a
+Kubernetes Secret or in a log line.
+"""
+
+import json
+import logging
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+LOG = logging.getLogger("netbird-proxy-services")
+
+NB = os.environ.get("NB_MANAGEMENT_URL", "https://netbird.renner.uno").rstrip("/")
+BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
+BAO_ROLE = os.environ.get("BAO_ROLE", "netbird-proxy-services")
+BAO_PAT_PATH = os.environ.get("BAO_PAT_PATH", "homelab/data/infrastructure/secret-rotators-bootstrap/netbird")
+BAO_PAT_KEY = os.environ.get("BAO_PAT_KEY", "pat")
+SERVICE_CIDR = os.environ.get("SERVICE_CIDR", "10.96.0.0/12")
+NETWORK_NAME = os.environ.get("NETWORK_NAME", "k8s public ingress")
+ROUTER_GROUP = os.environ.get("ROUTER_GROUP_NAME", "k8s-ingress-router")
+SOURCE_GROUP = os.environ.get("SOURCE_GROUP_NAME", "All")
+TRAEFIK_NS, TRAEFIK_SVC = os.environ.get("TRAEFIK_SERVICE", "ingress/traefik").split("/", 1)
+PUBLIC_LABEL = os.environ.get("PUBLIC_LABEL", "dns.renner.uno/public")
+LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "443"))
+INTERVAL = int(os.environ.get("INTERVAL_SECONDS", "60"))
+HTTP_TIMEOUT = 20
+
+K8S_API = "https://kubernetes.default.svc"
+K8S_TOKEN_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+K8S_CA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+
+class ApiError(Exception):
+    pass
+
+
+def http(method, url, body=None, headers=None, cafile=None):
+    data = None if body is None else json.dumps(body).encode()
+    req = urllib.request.Request(url, data=data, method=method)
+    req.add_header("Accept", "application/json")
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
+    for k, v in (headers or {}).items():
+        req.add_header(k, v)
+    ctx = None
+    if cafile:
+        import ssl
+        ctx = ssl.create_default_context(cafile=cafile)
+    try:
+        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT, context=ctx) as resp:
+            raw = resp.read()
+            return json.loads(raw) if raw else None
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode(errors="replace")[:300]
+        raise ApiError(f"{method} {url} -> {e.code}: {detail}") from None
+
+
+# --- Kubernetes -------------------------------------------------------------
+
+def k8s(path):
+    with open(K8S_TOKEN_FILE) as f:
+        token = f.read().strip()
+    return http("GET", K8S_API + path, headers={"Authorization": f"Bearer {token}"}, cafile=K8S_CA_FILE)
+
+
+def desired_hosts():
+    sel = urllib.parse.quote(f"{PUBLIC_LABEL}=true")
+    ingresses = k8s(f"/apis/networking.k8s.io/v1/ingresses?labelSelector={sel}")
+    hosts = {}
+    for ing in ingresses.get("items", []):
+        ref = f'{ing["metadata"]["namespace"]}/{ing["metadata"]["name"]}'
+        for rule in ing.get("spec", {}).get("rules", []):
+            host = rule.get("host")
+            if not host or host.startswith("*"):
+                continue
+            hosts.setdefault(host, ref)
+    return hosts
+
+
+def traefik_cluster_ip():
+    svc = k8s(f"/api/v1/namespaces/{TRAEFIK_NS}/services/{TRAEFIK_SVC}")
+    for ip in svc["spec"].get("clusterIPs", []):
+        if ":" not in ip:
+            return ip
+    raise ApiError(f"{TRAEFIK_NS}/{TRAEFIK_SVC} has no IPv4 ClusterIP")
+
+
+# --- OpenBao ----------------------------------------------------------------
+
+def netbird_pat():
+    with open(K8S_TOKEN_FILE) as f:
+        jwt = f.read().strip()
+    login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
+    bao_token = login["auth"]["client_token"]
+    kv = http("GET", f"{BAO_ADDR}/v1/{BAO_PAT_PATH}", headers={"X-Vault-Token": bao_token})
+    pat = kv["data"]["data"].get(BAO_PAT_KEY)
+    if not pat:
+        raise ApiError(f"OpenBao {BAO_PAT_PATH} has no key {BAO_PAT_KEY}")
+    return pat
+
+
+# --- NetBird management -----------------------------------------------------
+
+class NetBird:
+    def __init__(self, pat):
+        self.h = {"Authorization": f"Token {pat}"}
+
+    def get(self, path):
+        return http("GET", f"{NB}/api/{path}", headers=self.h)
+
+    def list(self, path):
+        # An empty collection comes back as an empty body / null, not [].
+        return self.get(path) or []
+
+    def post(self, path, body):
+        return http("POST", f"{NB}/api/{path}", body, headers=self.h)
+
+    def put(self, path, body):
+        return http("PUT", f"{NB}/api/{path}", body, headers=self.h)
+
+    def delete(self, path):
+        return http("DELETE", f"{NB}/api/{path}", headers=self.h)
+
+
+def group_id(nb, name):
+    for g in nb.list("groups"):
+        if g["name"] == name:
+            return g["id"]
+    raise ApiError(f"NetBird group {name!r} not found")
+
+
+def ensure_network(nb):
+    for n in nb.list("networks"):
+        if n["name"] == NETWORK_NAME:
+            return n["id"]
+    n = nb.post("networks", {"name": NETWORK_NAME,
+                             "description": "managed by netbird-proxy-services; public ingress via TLS passthrough"})
+    LOG.info("created network %s (%s)", NETWORK_NAME, n["id"])
+    return n["id"]
+
+
+def ensure_resource(nb, net):
+    for r in nb.list(f"networks/{net}/resources"):
+        if r.get("address") == SERVICE_CIDR:
+            return r["id"]
+    r = nb.post(f"networks/{net}/resources", {"name": "cluster service CIDR", "address": SERVICE_CIDR,
+                                              "description": "managed by netbird-proxy-services",
+                                              "enabled": True, "groups": []})
+    LOG.info("created subnet resource %s (%s)", SERVICE_CIDR, r["id"])
+    return r["id"]
+
+
+def ensure_router(nb, net, router_group):
+    for r in nb.list(f"networks/{net}/routers"):
+        if router_group in (r.get("peer_groups") or []):
+            return r["id"]
+    r = nb.post(f"networks/{net}/routers", {"peer_groups": [router_group], "metric": 9999,
+                                            "masquerade": True, "enabled": True})
+    LOG.info("created router on group %s (%s)", ROUTER_GROUP, r["id"])
+    return r["id"]
+
+
+def ensure_policy(nb, resource, source_group):
+    name = f"{NETWORK_NAME} access"
+    for p in nb.list("policies"):
+        if p["name"] == name:
+            return p["id"]
+    p = nb.post("policies", {"name": name, "enabled": True, "rules": [{
+        "name": name, "enabled": True, "action": "accept", "protocol": "tcp",
+        "ports": [str(LISTEN_PORT)], "bidirectional": False, "sources": [source_group],
+        "destinationResource": {"id": resource, "type": "subnet"}}]})
+    LOG.info("created policy %s (%s)", name, p["id"])
+    return p["id"]
+
+
+def service_body(host, resource, target_ip):
+    # proxy_protocol: PROXY v2 header carries the WAN client address to
+    # Traefik (proxyProtocol.trustedIPs on websecure); TCP/TLS modes only.
+    return {"name": host, "domain": host, "mode": "tls", "listen_port": LISTEN_PORT, "enabled": True,
+            "targets": [{"target_type": "subnet", "target_id": resource, "protocol": "tcp",
+                         "host": target_ip, "port": 443, "enabled": True,
+                         "options": {"proxy_protocol": True}}]}
+
+
+def is_managed(svc, resource):
+    return svc.get("mode") == "tls" and any(t.get("target_id") == resource for t in svc.get("targets") or [])
+
+
+def target_matches(svc, target_ip):
+    if svc.get("listen_port") != LISTEN_PORT or not svc.get("enabled"):
+        return False
+    ts = svc.get("targets") or []
+    return len(ts) == 1 and ts[0].get("host") == target_ip and ts[0].get("port") == 443 \
+        and ts[0].get("protocol") == "tcp" and ts[0].get("enabled") \
+        and (ts[0].get("options") or {}).get("proxy_protocol") is True
+
+
+def reconcile():
+    hosts = desired_hosts()
+    target_ip = traefik_cluster_ip()
+    nb = NetBird(netbird_pat())
+
+    router_group = group_id(nb, ROUTER_GROUP)
+    source_group = group_id(nb, SOURCE_GROUP)
+    net = ensure_network(nb)
+    resource = ensure_resource(nb, net)
+    ensure_router(nb, net, router_group)
+    ensure_policy(nb, resource, source_group)
+
+    by_domain = {s["domain"]: s for s in nb.list("reverse-proxies/services")}
+    created = updated = deleted = held = 0
+
+    for host, ref in sorted(hosts.items()):
+        svc = by_domain.get(host)
+        if svc is None:
+            nb.post("reverse-proxies/services", service_body(host, resource, target_ip))
+            LOG.info("created %s (from %s) -> %s:443", host, ref, target_ip)
+            created += 1
+        elif not is_managed(svc, resource):
+            LOG.warning("%s is held by unmanaged service %s (mode %s); remove it to migrate",
+                        host, svc["id"], svc.get("mode"))
+            held += 1
+        elif not target_matches(svc, target_ip):
+            nb.put(f"reverse-proxies/services/{svc['id']}", service_body(host, resource, target_ip))
+            LOG.info("updated %s -> %s:443", host, target_ip)
+            updated += 1
+
+    for domain, svc in by_domain.items():
+        if is_managed(svc, resource) and domain not in hosts:
+            nb.delete(f"reverse-proxies/services/{svc['id']}")
+            LOG.info("deleted %s (no public Ingress carries it any more)", domain)
+            deleted += 1
+
+    LOG.info("reconciled: %d hosts desired, %d created, %d updated, %d deleted, %d held by unmanaged services",
+             len(hosts), created, updated, deleted, held)
+
+
+def main():
+    logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), stream=sys.stdout,
+                        format="%(asctime)s %(levelname)s %(message)s")
+    if os.environ.get("ONCE") == "1":
+        reconcile()
+        return
+    while True:
+        try:
+            reconcile()
+        except Exception as e:  # keep running; the next pass retries
+            LOG.error("reconcile failed: %s", e)
+        time.sleep(INTERVAL)
+
+
+if __name__ == "__main__":
+    main()

kubernetes/MIGRATION-sops-openbao.md

@@ -50,15 +50,18 @@ pass insert homelab/openbao/root-token
 kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste key
 ```
 
-**Manual unseal is required after every pod restart** (node drain, upgrade,
-crash), **per replica** — the server runs three raft peers (see
-`infrastructure/openbao/helmrelease.yaml`), and a sealed one is neither a
-voter nor a standby, so it stays out of the cluster until unsealed:
-`kubectl -n openbao exec -it openbao-<N> -- bao operator unseal`, same key for
-all three. Three peers mean one sealed replica is survivable; two are not.
-Accepted for now; auto-unseal needs a second trust root (transit/KMS) and is
-explicitly out of scope. Symptom of sealed: ESO stores go NotReady, synced
-Secrets keep their last value (they are real k8s Secrets — pods keep running).
+**Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
+the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
+from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
+itself. The shamir key above became the **recovery key** in the migration; it
+is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
+runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
+one is neither a voter nor a standby, so a replica that stays sealed after a
+restart — wrong or missing seal-key Secret — costs redundancy, two cost
+quorum. The trade-off taken with the static seal: the key that unlocks the
+store sits in the same etcd as the Secrets ESO already syncs there in
+plaintext. Symptom of sealed: ESO stores go NotReady, synced Secrets keep
+their last value (they are real k8s Secrets — pods keep running).
 
 ## Phase 2 — configure + ClusterSecretStore (follow-up PR)
 
@@ -234,16 +237,19 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
 - **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
   dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
   `openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
-  A raft snapshot restore needs the matching unseal key — which lives in
+  A raft snapshot restore needs the matching seal key — which lives in
   `pass`, outside the cluster (GPG key backed up externally 2026-07-14).
   A failed snapshot Job is the sealed-after-restart alarm. Still owed:
   one tested restore before phase 4.
-- **Cluster rebuild order:** Flux brings up openbao (sealed, empty) → restore
-  raft snapshot → unseal (key from `pass`) → ClusterSecretStore Ready → ESO
-  materializes Secrets → apps start. Until unseal, secret-consuming apps
-  simply stay Pending on missing Secrets — same as any missing dependency.
-- **Break-glass:** `pass` holds unseal key + root token (GPG, offline-capable).
-  With those two, everything is recoverable from a raft snapshot alone.
+- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
+  openbao (empty) → init → restore raft snapshot → replicas unseal themselves
+  → ClusterSecretStore Ready → ESO materializes Secrets → apps start
+  (`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
+  hand). Until then, secret-consuming apps simply stay Pending on missing
+  Secrets — same as any missing dependency.
+- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
+  offline-capable). With those, everything is recoverable from a raft
+  snapshot alone.
 - **Ansible Vault password:** `ansible/vault-password.sh` reads
   `ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
   2026-07-14. It gates decryption of several Ansible-Vault files (PBS

kubernetes/PROPOSAL-node-roll-downtime.md

@@ -1,304 +0,0 @@
-# Proposal — a node roll without the ingress hiccup
-
-Status: draft, grounded in one measured roll. On 2026-09-07 all three MS-01
-were drained, patched and rebooted back to back (06:52–07:08) for the SAN
-rollout. Everything below is read off that window: Traefik access logs and
-Cilium BGP logs in Loki, the drain logs, and the live cluster afterwards.
-
-The short version: there are two ingresses and they failed differently.
-**The LAN ingress (Traefik behind the BGP VIP) did not hiccup** — it
-answered every request; the 5xx it returned were for backends mid-move.
-**The public edge (proxies → NetBird mesh → Traefik) went dark for 40–80 s
-on two of the three rolls, and would have for 3½ minutes on the third had
-its routes not recovered by luck** — a known client bug the repo already
-documents and works around with a watchdog, whose 600 s restart cooldown is
-what turns a 3-second blip into minutes when rolls follow each other. So
-this is two projects: make evictions cheap, and make a peer loss on the
-mesh cost seconds, deterministically.
-
-## What was measured
-
-Traefik, 06:48–07:12, 24 minutes, 3 391 requests, of which 5xx:
-
-| router | status | count |
-|---|---|---|
-| gitea | 503 | 56 |
-| home-assistant | 503 | 36 |
-| shelly-manager | 502 | 25 |
-| immich | 503 | 19 |
-| openbao | 503 | 13 |
-| grafana | 503 | 10 |
-| music-assistant | 503 | 9 |
-| crowdsec-lapi | 503 | 7 |
-| wordpress | 503 | 6 |
-| woodpecker | 503 | 4 |
-
-The quiet 24 minutes before the roll had zero 5xx. Every entry above is a
-single-replica workload (or its single database) that was evicted and had to
-come up on another node — pod reschedule, RBD volume detach and re-attach,
-process start. That is 30–90 s of 503 per app per drain.
-
-Two things made it worse than one move per app:
-
-- **Double moves.** The drain of k8s-1 pushed pods onto k8s-2, which was
-  drained next: `ha-pg-1`, `ak-pg-1`, `im-pg-1`, `pprl-pg-2`, `tr-pg-1`,
-  `wp-pg-1`, `influxdb`, `protonmail-bridge`, `prometheus`, `wordpress-mariadb`
-  each appear in two drain logs. Half the downtime was paid twice.
-- **`--disable-eviction`.** Ten PodDisruptionBudgets allow zero disruptions
-  (every CNPG primary, gitea, influxdb, syncthing, protonmail-bridge,
-  woodpecker-server). A plain `kubectl drain` hangs on them, so the roll used
-  `--disable-eviction`, which deletes pods outright. For the two-instance CNPG
-  clusters (authentik, immich, paperless) that skipped the switchover CNPG
-  would have done: the operator's documented behaviour is "if a node is to be
-  drained and contains a cluster's primary instance, a switchover happens
-  ahead of the drain" — but only when the drain goes through the eviction API.
-  We turned a 2-second switchover into a 30–60 s primary restart.
-
-Beside the HTTP path, one reboot side effect took a dependency down for
-half an hour: openbao-0 came back **sealed**, and because the StatefulSet is
-`OrderedReady`, openbao-1/2 were never recreated after their eviction until
-openbao-0 was unsealed by hand. ESO could not refresh a secret for 35 minutes.
-
-### The public edge
-
-Public traffic reaches Traefik with client address `127.0.0.1` (the rootless
-netbird-ingress peer forwards from inside its pod through socket-LB), LAN
-traffic with `10.2.x`. That splits the access log by path. At that hour the
-only public traffic was the proxies' own watchdog — two hosts probed once a
-minute from each of two proxies, a flat 4 requests/min — which makes it a
-clean synthetic probe of the whole external chain:
-
-| minute (UTC) | LAN req | public req | what was happening |
-|---|---|---|---|
-| 04:55 | 128 | 4 | k8s-1 rebooting (04:55:10) |
-| 04:56 | 109 | 2 | k8s-1 back, its peer re-enrolled 04:56:24 |
-| 04:57 | 127 | **0** | — |
-| 04:58 | 126 | 4 | proxies restarted by their watchdog |
-| 05:00–05:03 | 186–233 | 4 | k8s-2 roll: **no dip** |
-| 05:06–05:07 | 97–135 | 2 | k8s-3 roll: half |
-
-The external chain was down for roughly two minutes at the k8s-1 roll, one
-at k8s-3, and not at all at k8s-2. The mechanism is in
-`ansible/roles/netbird-proxy-watchdog/defaults/main.yml`: the proxy's
-embedded NetBird client pins each domain route to one routing-peer
-*identity*; the netbird-ingress DaemonSet enrols with a setup key, so every
-pod start is a **new identity** (the k8s-1 pod registered fresh at
-04:56:24); and upstream's dns-interceptor never rebinds a route whose
-owning peer is gone ("already routed by peer … HA routing disabled"). The
-proxy then serves 502 for every domain until its container restarts —
-which the watchdog does after three failed one-minute probes, or at once
-when the flood watchdog sees the wedge in the proxy log. Which roll hurts
-depends on which of the three peers the proxies happened to pin. k8s-2's
-peer was pinned by neither.
-
-Note what did *not* help: the DaemonSet's preStop (`netbird down`) took the
-peer out of the mesh cleanly at 04:55:11, and the peer was back 73 s later.
-Neither matters to a client that will not rebind and will not recognise
-the returning pod as the same peer.
-
-## What was not the cause
-
-**BGP.** Per node the UDM sessions were down for 47–51 s (Peer Down at the
-Cilium shutdown, Peer Up ~50 s later). That gap did not carry traffic: every
-LoadBalancer here is `externalTrafficPolicy: Local`, so a VIP is only
-advertised from nodes with a local endpoint, and the drain had already
-evicted those endpoints — Cilium withdrew the routes while the session was
-still healthy (the "soft reset out" bursts during each drain). The 90 s hold
-time and 120 s graceful-restart window never came into play. Leave them.
-
-**Traefik.** Two replicas on k8s-1/k8s-2, one evicted per roll, the other
-carried the VIP alone. No PDB, no spread constraint, no preStop — which held
-by luck (the two pods happened to sit on different nodes), and is fixed below
-because luck is not a design.
-
-## Target state, in order of measured impact
-
-### 1. Drain in a way that lets operators do their job (procedure only)
-
-- **Cordon the next node before draining the current one.** Evicted pods
-  then land on the node that stays up, and nothing moves twice. Order:
-  cordon B → drain A → reboot A → uncordon A → cordon C → drain B → …
-- **Never `--disable-eviction`.** It exists in the roll only because ten
-  PDBs say "never". Stage 2 makes those PDBs honest, after which a plain
-  `kubectl drain --ignore-daemonsets --delete-emptydir-data` completes and
-  CNPG switches primaries over before its replicas are evicted.
-- **After every reboot, three gates before uncordon:** `00-multus.conf` on
-  the host and Multus + whereabouts Running (k8s-1's OSD came up
-  Cilium-only today because it beat the Multus DaemonSet by 30 s); OpenBao
-  unsealed (`bao status` on every pod that exists, then the ones the
-  StatefulSet creates after it); Ceph back at baseline.
-- This is `rollout-node.sh` from today's window, minus `--disable-eviction`,
-  plus the OpenBao gate, checked into `scripts/`. A reboot runbook that lives
-  in a scratchpad is a runbook nobody else can run.
-
-### 2. PodDisruptionBudgets that mean what they say
-
-A `minAvailable: 1` PDB on a single-replica workload does not protect
-anything; it only forces the operator to bypass all PDBs. Two changes:
-
-- **CNPG single-instance clusters** (grafana-db, home-assistant, trmnl,
-  woodpecker): `spec.enablePDB: false`. CNPG documents exactly this — "for
-  single-instance clusters the operator will prevent draining the node" and
-  recommends moving to direct control of disruption budgets. The instance
-  restarts on drain (it does today too, just through the back door), and the
-  multi-instance clusters keep their switchover-before-eviction behaviour.
-- **gitea, influxdb, syncthing, protonmail-bridge, woodpecker-server**: drop
-  the `minAvailable: 1` PDBs. They are single replicas with RWO volumes; the
-  PDB cannot buy availability, only stalls.
-
-### 3. The ingress itself: three replicas, one per node, drained gracefully
-
-Traefik goes to `replicas: 3` with a required `podAntiAffinity` on
-hostname, `PodDisruptionBudget maxUnavailable: 1`, and a preStop that lets
-in-flight requests finish before the endpoint is withdrawn:
-
-```yaml
-# chart keys as in traefik/values.yaml
-deployment:
-  replicas: 3
-  lifecycle:
-    preStop:
-      sleep:
-        seconds: 5
-affinity:
-  podAntiAffinity:
-    requiredDuringSchedulingIgnoredDuringExecution:
-      - topologyKey: kubernetes.io/hostname
-        labelSelector:
-          matchLabels:
-            app.kubernetes.io/name: traefik
-podDisruptionBudget:
-  enabled: true
-  maxUnavailable: 1
-ports:
-  websecure:
-    transport:
-      lifeCycle:
-        graceTimeOut: 15s
-```
-
-With `Local` traffic policy every node then advertises the VIP; a roll takes
-one of three ECMP paths away instead of one of two, and the UDM sees the
-withdrawal before the pod stops accepting. Also gives the UDM a route from
-k8s-3, which today advertises nothing for the ingress.
-
-### 4. Databases the apps actually wait on
-
-The 503 table is a priority list. Each of these is a single Postgres today:
-
-| app | 5xx | cluster | today |
-|---|---|---|---|
-| home-assistant | 36 | ha-pg | `instances: 1` |
-| grafana | 10 | gf-pg | `instances: 1` |
-| woodpecker | 4 | wp-pg | `instances: 1` |
-| trmnl | — | tr-pg | `instances: 1` |
-
-`instances: 2` for ha-pg and gf-pg turns a drain into a CNPG switchover
-(seconds). The cost is one more Postgres pod and a second RBD volume each.
-trmnl and woodpecker can stay at one: they were barely visible.
-
-gitea (56) is the largest number and has no cheap answer: one StatefulSet,
-one RWO volume, an application that does not run two instances against the
-same data directory without shared storage. It moves once per roll (with
-stage 1) and that is its floor unless its repositories move to CephFS.
-
-### 5. The public edge: what the proxies' own journals say
-
-This is the only *complete* outage in the window, so despite the numbering
-it ships right after stage 1. It was measured twice: from the proxies'
-watchdog journals for the morning roll, and in a controlled afternoon
-experiment (three routing-peer kills, probed from proxy-2 itself at 2 s —
-the LAN cannot probe this path, the UDM hairpin-DNATs the proxy addresses
-straight to the VIP). Proxy clocks run ~3 min behind the cluster.
-
-**What a peer loss does.** When the routing peer the proxy has pinned
-disappears, the proxy's route manager wedges (`couldn't fetch peer state`
-burst) and the *whole* process stops answering — every domain, and in the
-experiment also services that never used a domain route. The flood
-watchdog restarts the container within 2 s of the burst; the probe saw
-one failed round (≈3 s) at the first two kills. The third kill fell inside
-the flood watchdog's **600 s restart cooldown**: restart suppressed, wedge
-left standing, and the URL watchdog needed its three one-minute rounds —
-**3½ minutes of 000 on every public domain**. The morning roll had the
-same shape: k8s-1 (both proxies restarted, 40–80 s), k8s-2 (wedge at
-06:58, restart suppressed on both proxies, 278 s after the previous one —
-the routes happened to recover on their own), k8s-3 (restart, ~10 s).
-Three rolls in sixteen minutes guarantee the suppressed case.
-
-- **5a. Cooldown 600 s → 60 s** in
-  `ansible/roles/netbird-proxy-watchdog/defaults/main.yml`. The cooldown
-  exists so a genuinely broken backend cannot restart-loop the proxy; a
-  minute still does that (the URL watchdog needs three minutes to fire
-  anyway). With it, every peer loss — rolls back to back included — is a
-  ~3 s blip instead of a lottery between 3 s and 3½ min. Cheapest change
-  in this document, ships first.
-- **5b. Persist the peer identity.** Replace the DaemonSet with a
-  StatefulSet of three, required anti-affinity on hostname,
-  `PodDisruptionBudget maxUnavailable: 1`, and a small RBD volume per
-  replica holding NetBird's state directory. A drain then evicts the pod
-  through the PDB (the DaemonSet pod is not evicted at all today, it dies
-  with the reboot) and it returns within ~30 s **as the same peer**. Whether
-  the route manager then un-wedges without a restart is the hypothesis to
-  measure; the setup-key constraint (ephemeral peers reaped after >10 min
-  offline, NeedsLogin liveness restart) holds unchanged.
-- **5c. The structural exit: no domain routes at all.** The subnet-target
-  experiment settled what works: an HTTP-mode service with an IP target
-  forwards `Host: <ip>` and Traefik answers 404 (NetBird rejects a Host
-  override); an HTTP-mode service with a domain target needs mesh DNS on
-  the proxy, which answers SERVFAIL there; a **TLS-passthrough service
-  (`mode: tls`, `listen_port: 443`) with a subnet target works** — SNI and
-  Host reach Traefik untouched, it coexists with the HTTP listener on 443,
-  no `skip_tls_verify`. If every public service became such a passthrough
-  over a subnet resource covering the Cilium LB pool (`10.2.99.0/24`,
-  `fd99::/64`), there would be no domain resource left to wedge on, and
-  peer loss would be NetBird's documented HA failover (seconds). Costs: the
-  target's `host` is the VIP, so a reconciler (the external-dns-unifi
-  pattern, `PUT /api/reverse-proxies/services/{id}` from the Traefik
-  Service's `status.loadBalancer.ingress`) has to own it; the proxy's L7
-  features (auth, path rules) are lost — none are in use; and the pure
-  failover time is unmeasured, because today every kill wedged the domain
-  routes first. Decide after 5a and 5b have been measured on a real roll.
-- **Track the upstream bug.** The repo's note dates the dns-interceptor
-  behaviour to 2026-07; a search of NetBird's releases and issues turned up
-  no fix for the "already routed by peer / HA routing disabled" path.
-  Re-check on each NetBird bump.
-- **Side findings, each its own small change:** both proxy clocks are
-  ~3 min slow (the proxy logs SCT timestamps "in the future"; TLS
-  validity windows care) — NTP on proxy-1/proxy-2; a new service's
-  certificate reaches the IPv6-only member only after ~20 min of ACME
-  retries (Let's Encrypt validates over IPv4 and lands on the other member,
-  which has no token for it); proxy-1 and proxy-2 do not ship syslog to
-  Loki (edge-1 does), so every public-edge timeline is reconstructed over
-  SSH — point them at the alloy-syslog VIP.
-
-## Expected result
-
-| stage | removes |
-|---|---|
-| 1 | every double move (half of today's 5xx), the primary restarts, the 35-minute OpenBao/ESO outage, the Multus race |
-| 2 | the need for `--disable-eviction` at all |
-| 3 | the single-node ingress exposure; ECMP loses 1/3 instead of 1/2 |
-| 4 | home-assistant and grafana 503s entirely |
-| 5a | the 3½-minute public-edge blackout of a second peer loss within 10 min; every peer loss becomes a ~3 s restart |
-| 5b/5c | the restart itself, if the returning identity (5b) or the absence of domain routes (5c) proves to un-wedge — to be measured |
-
-What remains after all of it: one move per single-replica app per roll
-(gitea, immich, shelly-manager, music-assistant, wordpress), 30–60 s each,
-once, and a few seconds of public edge per roll. That is the honest floor
-for single-replica services on RWO storage and a mesh client that pins.
-
-## Transition
-
-1. Stage 5a is one Ansible default and a proxy deploy — ships first.
-2. Stage 1 is a script and a runbook change — no cluster change.
-3. Stage 5b's StatefulSet changes how the public edge enrols, so it gets
-   its own roll to measure, probed from a proxy.
-4. Stage 2 is one field on four CNPG clusters and five deleted PDBs; each
-   Kustomization applies without disruption.
-5. Stage 3 is a Traefik HelmRelease change; the rollout itself is a rolling
-   update under the new PDB.
-6. Stage 4 is `instances: 2` on two clusters; CNPG adds the replica online.
-7. Stage 5c only if 5a+5b leave a measurable public gap.
-
-Measure the next roll the same way: 5xx by router, the proxies' watchdog
-journals (or their syslog in Loki), BGP Down→Up gaps, pods per drain log.

kubernetes/RUNBOOK-baremetal-conversion.md

@@ -432,8 +432,8 @@ is why it belongs here.
   among its devices but not `ens19`, because `ens19` is deliberately
   address-less. Cilium therefore owns `mgmt0` and sees VLAN 40 frames already
   untagged by the 8021q layer, while VLAN 30 frames only ever meet the filter
-  on the shared parent `enp2s0f0np0`, still tagged. Any future address-less
-  VLAN carrying pod traffic will need the same treatment.
+  on the shared parent `uplink`, still tagged. Any future address-less VLAN
+  carrying pod traffic will need the same treatment.
 
   The symptom does not look like a VLAN problem. A tc-layer drop reaches an
   AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports `no buffer space

kubernetes/RUNBOOK-cold-rebuild.md

@@ -67,10 +67,12 @@ These must exist outside the cluster or the rebuild cannot complete:
       `flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
       files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
       `MIGRATION-sops-openbao.md`.)
-- [ ] **OpenBao unseal key + root token** — in the GPG `pass` store
-      (`homelab/openbao/unseal-key`, `homelab/openbao/root-token`), and the GPG
-      private key backed up off-machine. Without the unseal key, no OpenBao
-      snapshot can be restored and no secret-consuming app can start.
+- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
+      store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
+      former shamir key, now the recovery key — and `homelab/openbao/root-token`),
+      and the GPG private key backed up off-machine. The seal key is the one
+      the snapshots are encrypted under: without it, no OpenBao snapshot can be
+      restored and no secret-consuming app can start.
 - [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
       (`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
       `openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
@@ -294,42 +296,65 @@ output** — a green step over a still-forming Ceph is a known trap.
 > Steady-state (non-rebuild) bootstrap is just `kubernetes/bootstrap.sh flux`
 > with `GITEA_TOKEN` set; `FLUX_SOURCE` defaults to `gitea`.
 
-## Step 4 — openbao: restore the snapshot + unseal
+## Step 4 — openbao: restore the snapshot
 
-After Step 3, OpenBao comes up **sealed and empty**. Until it is restored and
-unsealed, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
-Secrets, and every secret-consuming app stays Pending. This step is **manual**
-(the unseal key lives in `pass`, off-cluster by design).
+`bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
+`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
+the empty server, restores the newest raft snapshot straight out of the Velero
+kopia repository, and waits until every replica has unsealed itself from the
+static seal. What follows is the same procedure by hand, for when the script
+is not an option.
+
+OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
+`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
+snapshots are encrypted under that key, so it must be the same key the backed
+up cluster used — a different one restores nothing. Until the snapshot is
+restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
+Secrets, and every secret-consuming app stays Pending.
 
 ```sh
-# 1. Wait for the OpenBao pods to exist (all three sealed, 0/1 Ready is expected).
-kubectl -n openbao get pods
+# 1. The seal key, before the pods start (a pod without it never starts).
+kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
+kubectl -n openbao create secret generic openbao-seal-key \
+  --from-literal=key="$(pass show homelab/openbao/seal-key)"
 
-# 2. Restore the most recent raft snapshot. Source it from the snapshot PVC
+# 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
+#    initialise it. With an auto-seal, init hands out recovery keys and the
+#    server unseals itself. These keys are throwaway: the restore replaces them.
+kubectl -n openbao exec -it openbao-0 -- bao operator init \
+  -recovery-shares=1 -recovery-threshold=1
+
+# 3. Restore the most recent raft snapshot. Source it from the snapshot PVC
 #    (infrastructure/openbao/snapshot-cronjob.yaml) or, in a total-loss rebuild,
-#    from the Velero `openbao-daily` offsite copy. Copy it into the pod, then:
+#    from the Velero `openbao-daily` offsite copy. Root token from the init
+#    output. No -force: OpenBao checks the snapshot was written under this
+#    server's seal key and refuses otherwise ("different autoseal key") — that
+#    refusal is the signal the Secret is wrong. -force would skip the check and
+#    leave a server that can never unseal.
 kubectl -n openbao cp <snapshot>.snap openbao-0:/tmp/restore.snap
-kubectl -n openbao exec -it openbao-0 -- bao operator raft snapshot restore /tmp/restore.snap
+kubectl -n openbao exec -it openbao-0 -- \
+  env BAO_TOKEN=<init root token> bao operator raft snapshot restore /tmp/restore.snap
 
-# 3. Unseal with the key from pass (threshold 1) — EVERY replica, same key.
-#    A sealed server is neither a raft voter nor a standby, so unsealing only
-#    openbao-0 leaves the three-peer cluster one node short of quorum.
-kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste homelab/openbao/unseal-key
-kubectl -n openbao exec -it openbao-1 -- bao operator unseal   # same key
-kubectl -n openbao exec -it openbao-2 -- bao operator unseal   # same key
+# 4. The server seals against the snapshot's keyring and unseals itself from
+#    the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
+#    to type. Afterwards the tokens and recovery key from pass apply again.
+kubectl -n openbao get pods -w
 ```
 
 Verify the store recovers before moving on:
 
 ```sh
-kubectl get clustersecretstore openbao        # want: Valid / Ready
-kubectl -n <ns> get externalsecret            # want: SecretSynced
+kubectl -n openbao exec openbao-0 -- bao status   # want: Seal Type static, Sealed false
+kubectl get clustersecretstore openbao            # want: Valid / Ready
+kubectl -n <ns> get externalsecret                # want: SecretSynced
 ```
 
-> **Manual unseal is required after every OpenBao pod restart**, not just
-> during rebuilds, and for *each* replica that restarts — a rolling node
-> drain seals all three in turn. Details + rationale:
-> `MIGRATION-sops-openbao.md`.
+> A replica that stays **sealed** after a restart could not use the static
+> seal — the Secret is missing or does not match the key the data was
+> written under. Fix the Secret and restart the pod; `bao operator unseal`
+> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
+> unseal an auto-sealed server and is only for recovery-mode operations.
+> Details + rationale: `MIGRATION-sops-openbao.md`.
 
 ## Step 5 — apps: let the rest reconcile
 
@@ -393,14 +418,15 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
 - **`bootstrap.sh talos` is dead** — it cannot generate a config (#581), and
   Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
   Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
-  It has also never had an `openbao` phase — Step 4 is manual because the
-  unseal key must not live in CI; see issue #187 on whether to wire a
-  semi-automated OpenBao restore.
-- **OpenBao seals on every restart and each replica must be unsealed
-  individually.** Three raft replicas mean the service survives losing one, but
-  a sealed server is neither a voter nor a standby — unsealing only one leaves
-  the cluster short of quorum. Step 4 covers this; it is repeated here because
-  it is the step most likely to be half-done under pressure.
+  Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
+  other secrets are read from `pass` or streamed in at run time and never live
+  in CI.
+- **OpenBao unseals itself, but only under the seal key the snapshot was
+  written with.** Three raft replicas mean the service survives losing one; a
+  sealed server is neither a voter nor a standby. A pod that stays sealed after
+  a restart has the wrong `openbao-seal-key` Secret — Step 4 covers this; it is
+  repeated here because it is the step most likely to be half-done under
+  pressure.
 - Ceph's cluster network is a Multus attachment on VLAN 49 with whereabouts
   IPAM, not host networking. Nothing in the rebuild needs special handling for
   it — the nodes hold no address on that segment — but a rebuilt cluster will

kubernetes/apps/audiobookshelf/deployment.yaml

@@ -21,6 +21,9 @@ metadata:
   namespace: audiobookshelf
   labels:
     app.kubernetes.io/name: audiobookshelf
+  annotations:
+    # envFrom does not restart the pod when the ConfigMap changes; Reloader does.
+    configmap.reloader.stakater.com/reload: audiobookshelf-config
 spec:
   replicas: 1
   strategy:

kubernetes/apps/home-assistant/deployment.yaml

@@ -95,7 +95,7 @@ spec:
               memory: 32Mi
       containers:
         - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.1 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.9.2 # {"$imagepolicy": "flux-system:home-assistant"}
           env:
             - name: TZ
               value: Europe/Berlin

kubernetes/apps/immich/machine-learning.yaml

@@ -32,7 +32,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.1.0 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
           ports:
             - name: http
               containerPort: 3003

kubernetes/apps/immich/server.yaml

@@ -35,6 +35,9 @@ metadata:
   namespace: immich
   labels:
     app.kubernetes.io/name: immich-server
+  annotations:
+    # envFrom does not restart the pod when the ConfigMap changes; Reloader does.
+    configmap.reloader.stakater.com/reload: immich-config
 spec:
   # Scaled to 0 alongside immich-machine-learning during single-node
   # maintenance; see the note there. The CNPG Postgres cluster stays running,
@@ -84,7 +87,7 @@ spec:
               mountPath: /ca
       containers:
         - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.1.0 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
           ports:
             - name: http
               containerPort: 3001

kubernetes/apps/monitoring-rules/intel-gpu-exporter-servicemonitor.yaml

@@ -0,0 +1,20 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: intel-gpu-exporter
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: intel-gpu-exporter
+    release: kube-prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: intel-gpu-exporter
+  endpoints:
+    - port: http-metrics
+      path: /metrics
+      interval: 30s
+      relabelings:
+        # The series would otherwise carry only the exporter pod's IP.
+        - sourceLabels: [__meta_kubernetes_pod_node_name]
+          targetLabel: node

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -39,4 +39,5 @@ resources:
   - immich-servicemonitor.yaml
   - loki-servicemonitor.yaml
   - smartctl-exporter-servicemonitor.yaml
+  - intel-gpu-exporter-servicemonitor.yaml
   - probes-blackbox.yaml

kubernetes/apps/monitoring-rules/prometheusrule-ci-openbao.yaml

@@ -34,7 +34,9 @@ spec:
           # query parameter, and an exact match would break silently if the
           # instance label ever encodes it differently.
           expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
-          for: 5m
+          # Two consecutive failed probes: enough to skip a single lost probe
+          # through Traefik, not enough to sit on a sealed store.
+          for: 1m
           labels:
             severity: critical
           annotations:

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -28,24 +28,28 @@ spec:
         # scheduling would also fire this — but both want a human too, so the
         # imprecision costs nothing.
         - alert: OpenBaoReplicaNotReady
-          # A replica that is sealed never recovers on its own, so the
-          # duration is only there to ignore ordinary rescheduling: a healthy
-          # pod moving between nodes is Ready again in well under a minute.
-          expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 3
+          # A replica unseals itself from the static seal, so a restart is
+          # Ready again in well under a minute; the duration only ignores that
+          # ordinary rescheduling. A replica still sealed after it never
+          # recovers on its own: the seal failed (openbao-seal-key Secret
+          # missing or wrong).
+          # max by: kube-state-metrics stamps instance/pod on the series, so a
+          # rescheduled exporter starts a new series and resets the timer.
+          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 3
           for: 10m
           labels:
             severity: warning
           annotations:
             summary: "OpenBao has {{ $value }} of 3 replicas ready"
             description: >-
-              An OpenBao replica has been unavailable for 10 minutes. The usual
-              cause is a sealed replica after a node drain — every restart
-              seals it, and it stays out of the raft cluster until unsealed by
-              hand. The service is still up on the remaining replicas, so
-              nothing is broken yet, but redundancy is reduced and a second
-              drain would cost quorum. Check `kubectl -n openbao get pods` and
-              `bao status` in the not-ready pod; unseal with the key from
-              `pass` (homelab/openbao/unseal-key).
+              An OpenBao replica has been unavailable for 10 minutes. A restart
+              unseals itself from the static seal, so this is not the ordinary
+              drain case. The service is still up on the remaining replicas,
+              so nothing is broken yet, but redundancy is reduced and a second
+              failure would cost quorum. Check `kubectl -n openbao get pods`,
+              the pod log and `bao status` in the not-ready pod. If it is
+              sealed, the static seal failed: verify the openbao-seal-key
+              Secret matches `pass show homelab/openbao/seal-key`.
 
         - alert: OpenBaoQuorumLost
           # Raft quorum of a three-node cluster is two. At one ready replica
@@ -53,11 +57,10 @@ spec:
           # the ClusterSecretStore goes InvalidProviderConfig and no
           # ExternalSecret can refresh.
           #
-          # Shorter window than the warning above because this is an outage,
-          # not a degradation — but not zero, so a rolling restart that
-          # briefly dips does not page.
-          expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 2
-          for: 5m
+          # No window: below quorum the secret store is down and every
+          # ExternalSecret stalls with it, so waiting only delays the page.
+          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
+          for: 0m
           labels:
             severity: critical
           annotations:
@@ -67,9 +70,11 @@ spec:
               quorum and the secret store is unavailable. ExternalSecrets stop
               refreshing; existing Secrets keep their last values, so failures
               surface only when something rotates or a new secret is needed.
-              Unseal the sealed replicas (key in `pass`,
-              homelab/openbao/unseal-key) — each one individually, since a
-              sealed server cannot be a standby.
+              Replicas unseal themselves from the static seal; one that stays
+              sealed could not. Check its log and the openbao-seal-key Secret
+              against `pass show homelab/openbao/seal-key`, fix the Secret and
+              restart the pod. The recovery key cannot unseal an auto-sealed
+              server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
 
         - alert: OpenBaoMetricsMissing
           # Guards the two alerts above. Both are threshold comparisons, so if

kubernetes/apps/monitoring/dashboards/intel-gpu.json

@@ -0,0 +1,368 @@
+{
+  "title": "Intel iGPU",
+  "uid": "intel-gpu",
+  "tags": ["gpu", "intel", "i915"],
+  "schemaVersion": 39,
+  "editable": true,
+  "timezone": "",
+  "refresh": "1m",
+  "time": { "from": "now-6h", "to": "now" },
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": {
+          "query": "label_values(gpumon_power, node)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "definition": "label_values(gpumon_power, node)",
+        "includeAll": true,
+        "multi": true,
+        "refresh": 2,
+        "sort": 1,
+        "current": { "selected": true, "text": ["All"], "value": ["$__all"] }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Render/3D busy now",
+      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 0,
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "fixed", "fixedColor": "text" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "horizontal",
+        "textMode": "value_and_name",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "GPU power now",
+      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "decimals": 1,
+          "color": { "mode": "fixed", "fixedColor": "text" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "horizontal",
+        "textMode": "value_and_name",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "GT frequency now",
+      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "suffix: MHz",
+          "decimals": 0,
+          "color": { "mode": "fixed", "fixedColor": "text" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "horizontal",
+        "textMode": "value_and_name",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Render/3D busy",
+      "description": "Compute and 3D. llama.cpp's Vulkan backend runs here; 0 % during an inference means the model is on the CPU.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Video busy",
+      "description": "Fixed-function decode/encode (QuickSync).",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Video\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "VideoEnhance busy",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"VideoEnhance\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Blitter busy",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Blitter\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "GT frequency (actual)",
+      "description": "0 while the GT sits in RC6.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "suffix: MHz",
+          "min": 0,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU power",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "min": 0,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Package power",
+      "description": "Whole SoC (CPU + GPU + uncore), from the same PMU.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "min": 0,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_power{type=\"pkg\", node=~\"$node\"}",
+          "legendFormat": "{{node}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory controller bandwidth",
+      "description": "Reads solid, writes dashed. The iGPU shares DDR5 with the CPU, so this is the whole package's traffic.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "MiBs",
+          "min": 0,
+          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
+        },
+        "overrides": [
+          { "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] },
+          { "matcher": { "id": "byRegexp", "options": "/write/" }, "properties": [{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] }
+        ]
+      },
+      "options": {
+        "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "expr": "gpumon_imc_read{node=~\"$node\"}",
+          "legendFormat": "{{node}} read",
+          "refId": "A"
+        },
+        {
+          "expr": "gpumon_imc_write{node=~\"$node\"}",
+          "legendFormat": "{{node}} write",
+          "refId": "B"
+        }
+      ]
+    }
+  ]
+}

kubernetes/apps/monitoring/dashboards/kustomization.yaml

@@ -77,6 +77,9 @@ configMapGenerator:
   - name: dashboard-infrastructure-overview
     files:
       - infrastructure-overview.json
+  - name: dashboard-intel-gpu
+    files:
+      - intel-gpu.json
   - name: dashboard-netbird-overview
     files:
       - netbird-overview.json

kubernetes/apps/monitoring/intel-gpu-exporter.yaml

@@ -0,0 +1,81 @@
+# Intel iGPU utilisation, frequency and power for every node (#752).
+#
+# Not privileged: /dev/dri comes from the device plugin's monitoring resource
+# (every GPU on the node, no hostPath), and the i915 PMU that intel_gpu_top
+# reads needs CAP_PERFMON — the one capability granted. No hostPID, no
+# hostNetwork.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-gpu-exporter
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: intel-gpu-exporter
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: intel-gpu-exporter
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: intel-gpu-exporter
+    spec:
+      # Every node is a control plane; without this the exporter covers nothing.
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: intel-gpu-exporter
+          image: ghcr.io/clambin/intel-gpu-exporter:0.7.4@sha256:d78761fef018834c759db154fe9dcf3dd9db5974703fa50ca21f6abc35221be1
+          args:
+            # No -device: intel_gpu_top picks the only GPU the plugin injects.
+            - -interval=5s
+          ports:
+            - name: http-metrics
+              containerPort: 9100
+              protocol: TCP
+          securityContext:
+            # Root so the PERFMON capability is effective without file
+            # capabilities on the binary.
+            runAsUser: 0
+            runAsNonRoot: false
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+              add: ["PERFMON"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+              gpu.intel.com/monitoring: 1
+            limits:
+              memory: 128Mi
+              gpu.intel.com/monitoring: 1
+          livenessProbe:
+            httpGet:
+              path: /metrics
+              port: http-metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: intel-gpu-exporter
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: intel-gpu-exporter
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: intel-gpu-exporter
+  ports:
+    - name: http-metrics
+      port: 9100
+      targetPort: http-metrics
+      protocol: TCP

kubernetes/apps/monitoring/kustomization.yaml

@@ -16,6 +16,7 @@ resources:
   - prometheus-gate-ingress.yaml
   - ci-diagnostics-rbac.yaml
   - smartctl-exporter.yaml
+  - intel-gpu-exporter.yaml
   - dashboards/
 # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
 # separate Flux Kustomization that dependsOn `monitoring` — otherwise the

kubernetes/apps/music-assistant/deployment.yaml

@@ -28,7 +28,7 @@ spec:
         # net1 instead. master: ens19 resolves on every k8s-{1,2,3}/edge
         # worker, though it is no longer the same kind of device everywhere:
         # on the bare-metal control-plane nodes it is a VLAN 30 sub-interface
-        # of enp2s0f0np0 rather than a NIC, which is why the datapath needs
+        # of uplink rather than a NIC, which is why the datapath needs
         # bpf.vlanBypass to carry it (issue #592).
         k8s.v1.cni.cncf.io/networks: music-assistant-iot
     spec:
@@ -55,7 +55,7 @@ spec:
         # stdlib only; Flux keeps the tag in lockstep via the shared image
         # policy) — no extra dependency.
         - name: reconcile-publish-ip
-          image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
           command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
           volumeMounts:
             - name: data
@@ -65,7 +65,7 @@ spec:
               readOnly: true
       containers:
         - name: music-assistant
-          image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
           env:
             - name: LOG_LEVEL
               value: info

kubernetes/apps/paperless/gotenberg.yaml

@@ -22,7 +22,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: gotenberg
-          image: gotenberg/gotenberg:8.36.0 # {"$imagepolicy": "flux-system:gotenberg"}
+          image: gotenberg/gotenberg:8.37.0 # {"$imagepolicy": "flux-system:gotenberg"}
           args:
             - "gotenberg"
             - "--chromium-disable-javascript=true"

kubernetes/apps/trmnl/terminus.yaml

@@ -134,7 +134,7 @@ spec:
         # is a device-bricking class of risk. Bumps are a reviewed, manual
         # act — see kubernetes/infrastructure/image-automation/README.md.
         - name: terminus
-          image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
+          image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
           ports:
             - name: http
               containerPort: 2345
@@ -197,7 +197,7 @@ spec:
             capabilities:
               drop: ["ALL"]
         - name: sidekiq
-          image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
+          image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
           command: ["bundle", "exec", "sidekiq", "-r", "./config/sidekiq.rb"]
           envFrom:
             - configMapRef:

kubernetes/apps/whisper/deployment.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: whisper
-          image: rhasspy/wyoming-whisper:3.7.0 # {"$imagepolicy": "flux-system:wyoming-whisper"}
+          image: rhasspy/wyoming-whisper:3.8.1 # {"$imagepolicy": "flux-system:wyoming-whisper"}
           args:
             - --model=base-int8
             - --language=de

kubernetes/apps/woodpecker/agent.yaml

@@ -34,7 +34,7 @@ spec:
           # Image automation, range 3.x (see image-automation/woodpecker.yaml).
           # MUST stay version-matched with woodpecker-server (gRPC parity) — both
           # track the same range/filter so they bump together on each release.
-          image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
+          image: woodpeckerci/woodpecker-agent:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-agent"}
           env:
             - name: WOODPECKER_SERVER
               value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000

kubernetes/apps/woodpecker/server.yaml

@@ -40,7 +40,7 @@ spec:
           # minor + patch auto-update within 3.x; only the 4.0 major is held for
           # a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
           # -bump hazard is gone). Stays in lockstep with the agent.
-          image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
+          image: woodpeckerci/woodpecker-server:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-server"}
           env:
             - name: WOODPECKER_HOST
               value: https://ci.renner.uno

kubernetes/bootstrap.sh

@@ -179,16 +179,18 @@ phase_runs() {
 OPENBAO_NS="openbao"
 
 # Where the secrets come from. On a workstation, the operator's pass store. In
-# the in-cluster Job there is no pass and no GPG key — and there should not be,
-# because the key that protects OpenBao has no business living in the cluster
-# OpenBao secures. Set OPENBAO_SECRETS_DIR and the run reads them from files
-# instead, which the operator streams in when the run asks for them:
+# the in-cluster Job there is no pass and no GPG key — and there should not be:
+# the pass store opens everything, and it stays on the operator's machine. Set
+# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
+# operator streams in when the run asks for them:
 #
-#     pass show homelab/openbao/unseal-key \
-#       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/unseal-key'
+#     pass show homelab/openbao/seal-key \
+#       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
 #
 # They exist in the pod's filesystem for the length of the run and are never
-# written to a Secret, so they do not land in etcd or in any backup of it.
+# written to a Secret in this cluster, so they do not land in its etcd or in
+# any backup of it. The seal key alone becomes a Secret in the cluster being
+# built, which is what lets that OpenBao unseal itself.
 OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}"
 
 # Every secret the run needs, named once, in one place. The list is here and
@@ -243,7 +245,9 @@ OPENBAO_REQUIRED_SECRETS=(
     "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
     "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
     "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
-    "unseal-key|OpenBao unseal key|pass show homelab/openbao/unseal-key"
+    # The static seal key. The snapshots are encrypted under it, so it has to
+    # be the one the backed-up cluster ran with — a fresh key restores nothing.
+    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
     # Read by the node registry auth (see registry_auth_patch): the images an app
     # needs while the cluster is still being built are mirrored to a private
     # ghcr.io namespace, and a node has to be able to pull them before anything
@@ -277,7 +281,7 @@ OPENBAO_REQUIRED_SECRETS=(
 # That phase used to need nothing but the machines, and `bootstrap.sh talos` is
 # documented as the part that can run unattended. Node registry auth changed
 # that: the machine config now carries a pull credential, so the phase needs
-# exactly one secret. Demanding all six there would drag in the OpenBao unseal
+# exactly one secret. Demanding all six there would drag in the OpenBao seal
 # key, which deliberately never reaches CI — and demanding none, as before,
 # left the phase to discover the missing file halfway through `talosctl gen`.
 require_secrets() {
@@ -408,7 +412,7 @@ read_secret() {
 # the kernel link name enp2s0f1np1, and that name cannot be manufactured here
 # without assuming a Hetzner private network passes VLAN tags. See
 # talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase
-# needs the OpenBao unseal key, which deliberately never reaches CI, and this
+# needs the OpenBao seal key, which deliberately never reaches CI, and this
 # test is meant to run unattended.
 # Talos version and Image Factory schematic, read out of the installer image in
 # cluster.yaml. Both throwaway-cluster phases need them and neither may state
@@ -1121,7 +1125,7 @@ EOF
     # nothing has demanded the secrets yet — the full require_secrets is gated
     # on the flux phase. Ask for the one this needs, and only that one: pulling
     # in the whole list would make an unattended talos run depend on the OpenBao
-    # unseal key, which never reaches CI.
+    # seal key, which never reaches CI.
     require_secrets ghcr-pull
     REGISTRY_PATCH=$(registry_auth_patch)
     [[ -s "$REGISTRY_PATCH" ]] \
@@ -1406,9 +1410,11 @@ fi
 # waits on infrastructure-crds. A rebuild could not reach a reconciled state
 # without someone at a keyboard.
 #
-# The unseal key stays where it was — the operator's `pass` store — and is read
+# The seal key stays where it was — the operator's `pass` store — and is read
 # here at run time. It is never written to the repo and never reaches CI, which
-# is the reason this was left manual in the first place.
+# is the reason this was left manual in the first place. It reaches the cluster
+# as the openbao-seal-key Secret, created by the flux phase before Flux starts
+# OpenBao, and from then on every replica unseals itself.
 #
 # Everything below is idempotent: it is safe to re-run against an OpenBao that
 # is already unsealed and configured, which is what makes it usable as part of
@@ -1498,61 +1504,63 @@ restore_openbao() {
     echo "  extracted $(stat -c%s "$work/restore.snap") bytes"
 
     # A snapshot can only be loaded into a running OpenBao, so the fresh one is
-    # initialised and unsealed with a throwaway key first. That key is used for
-    # the next few seconds and then stops existing: restoring replaces the
-    # keyring with the snapshot's, after which the real key from pass applies.
-    # -force is required for exactly that reason — without it OpenBao refuses a
-    # snapshot whose keys do not match the cluster's own.
-    echo -n "  initialising the empty server with a throwaway key... "
-    local init_json throwaway
-    init_json=$(bao_exec "$pod" bao operator init -key-shares=1 -key-threshold=1 -format=json 2>&1) \
+    # initialised first. Under the static seal, init hands out a recovery key
+    # rather than an unseal key and the server unseals itself; the only thing
+    # used from its output is the root token, for the next few seconds, and
+    # then it stops existing: restoring replaces the keyring with the
+    # snapshot's, after which the root token and recovery key from pass apply.
+    #
+    # No -force. Without it OpenBao checks that the snapshot was written under
+    # the seal key this server runs with, and refuses a mismatch by name
+    # ("possibly the snapshot is using a different autoseal key"). That check
+    # is the one guard against restoring under the wrong seal-key Secret;
+    # -force would skip it and leave a server that can never unseal.
+    #
+    # The init output is not echoed on a parse failure: it carries the root
+    # token and the recovery key of a server that is about to be overwritten,
+    # but a log line is a log line.
+    echo -n "  initialising the empty server (throwaway recovery key)... "
+    local init_json root
+    init_json=$(bao_exec "$pod" bao operator init -recovery-shares=1 -recovery-threshold=1 -format=json 2>&1) \
         || { echo; echo "$init_json" >&2; fail "could not initialise OpenBao"; }
-    throwaway=$(jq -r '.unseal_keys_b64[0]' <<<"$init_json")
-    [[ -n "$throwaway" && "$throwaway" != "null" ]] || fail "could not read the throwaway unseal key from init output"
+    root=$(jq -r '.root_token // empty' <<<"$init_json")
+    [[ -n "$root" ]] || fail "init output carried no root token"
     ok
 
-    echo -n "  unsealing with it... "
-    # The key goes as an argument, which OpenBao's own error message asks for:
-    #
-    #   you are executing outside of a terminal (tty) ... If this is not an
-    #   option, the unseal key can be provided as the first argument
-    #
-    # Two earlier shapes failed. `unseal -` is Vault syntax; OpenBao takes
-    # argv[1] literally, so the dash was sent as the key. Omitting the argument
-    # makes it prompt, and the prompt needs a real tty — a pipe gives "file
-    # descriptor 0 is not a terminal". There is no stdin path at all.
-    #
-    # This does put the key in the pod's process list for the length of the
-    # call, which the code here previously went out of its way to avoid. It is
-    # accepted rather than worked around: reading that list requires exec into
-    # the very pod that holds the key in memory, and the alternative — a
-    # port-forward plus curl against /v1/sys/unseal — puts a background process
-    # in the most critical path of a recovery. The openbao image has no curl,
-    # wget or nc, so it would have to run from here.
-    bao_exec "$pod" bao operator unseal "$throwaway" >/dev/null \
-        || fail "could not unseal with the throwaway key"
-    ok
+    # Sealed after init means the static seal is not working — the Secret is
+    # missing or malformed — and the restore below cannot run against it.
+    echo -n "  waiting for it to unseal itself"
+    for i in $(seq 1 30); do
+        if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
+            | jq -e '.sealed == false' >/dev/null; then
+            echo; break
+        fi
+        printf '.'
+        sleep 2
+        if [[ $i -eq 30 ]]; then
+            echo
+            cat "$OPENBAO_SECRETS_DIR/.baoerr" >&2
+            fail "$pod did not unseal itself after init — check the openbao-seal-key Secret and the pod log"
+        fi
+    done
 
     echo -n "  loading the snapshot... "
     kubectl -n "$OPENBAO_NS" cp "$work/restore.snap" "$pod:/tmp/restore.snap" >/dev/null \
         || fail "could not copy the snapshot into $pod"
-    local root
-    root=$(jq -r '.root_token' <<<"$init_json")
     local out
-    out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore -force /tmp/restore.snap 2>&1) \
+    out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore /tmp/restore.snap 2>&1) \
         || { echo; echo "$out" >&2; fail "snapshot restore rejected"; }
     ok
 
     rm -rf "$work"
-    # The server is now sealed again, against the keyring that came with the
-    # snapshot. unseal_openbao carries on from here with the real key.
-    echo "  restored — the server now expects the original unseal key"
+    # The server seals itself against the keyring that came with the snapshot
+    # and unseals again from the static seal — the same key the snapshot was
+    # written under. bring_up_openbao waits for that on every replica.
+    echo "  restored — the server re-seals and unseals itself from the seal key"
 }
 
-unseal_openbao() {
-    step 0 "Unsealing OpenBao..."
-    [[ -n "$OPENBAO_SECRETS_DIR" ]] || command -v pass >/dev/null \
-        || fail "pass not found — the unseal key lives in the operator's pass store, or set OPENBAO_SECRETS_DIR and stream it in"
+bring_up_openbao() {
+    step 0 "Bringing OpenBao up..."
     command -v jq >/dev/null   || fail "jq not found (needed to read OpenBao's seal status)"
 
     # OpenBao is deployed by the infrastructure Kustomization. What this step
@@ -1658,40 +1666,28 @@ unseal_openbao() {
         restore_openbao "${pods[0]}"
     fi
 
-    local key
-    key=$(read_secret unseal-key) \
-        || fail "could not read the unseal key"
-
-    # Unseal every replica the StatefulSet wants, not the ones that happen to
-    # exist right now. On a cold start only openbao-0 exists when this runs, and
-    # unsealing just that one deadlocks the StatefulSet: podManagementPolicy is
-    # OrderedReady, a sealed pod never passes its readiness probe, so openbao-1
-    # is created only after openbao-0 is unsealed and openbao-2 is never created
-    # at all. The cluster comes up at 1 of 3.
+    # Wait for every replica the StatefulSet wants, not the ones that happen to
+    # exist right now. On a cold start only openbao-0 exists when this runs:
+    # podManagementPolicy is OrderedReady, so openbao-1 is created once
+    # openbao-0 passes its readiness probe and openbao-2 after that. Each
+    # unseals itself from the static seal; nothing here types a key.
     #
-    # That alone would be survivable. What makes it fatal is that the openbao
-    # Service sets publishNotReadyAddresses — the chart does this so replicas can
-    # reach each other to join raft before any of them is unsealed — so the
-    # sealed openbao-1 is a live backend on the service IP. Clients are balanced
-    # onto a node that cannot answer, and the failure surfaces nowhere near here:
-    # external-secrets' ClusterSecretStore flaps between "store validated" and
-    # `dial tcp <svc>:8200: connect: no route to host`, every ExternalSecret in
-    # the cluster reports "could not get secret data from provider", and
-    # infrastructure-crds — which waits on those — never goes Ready, taking all
-    # twenty app Kustomizations with it.
-    #
-    # So this converges instead of iterating once: re-list the pods each round,
-    # unseal whatever is sealed, and finish when spec.replicas of them are
-    # unsealed. A pod that is still starting cannot be unsealed yet and is not an
-    # error — it is retried next round, which is exactly how the OrderedReady
-    # chain is meant to be walked.
+    # Waiting for the whole chain rather than the first pod matters because the
+    # openbao Service sets publishNotReadyAddresses — the chart does this so
+    # replicas can reach each other to join raft before any of them is Ready —
+    # so a replica that is still starting is a live backend on the service IP.
+    # Clients balanced onto it get `dial tcp <svc>:8200: connect: no route to
+    # host`, and the failure surfaces nowhere near here: external-secrets'
+    # ClusterSecretStore flaps, every ExternalSecret reports "could not get
+    # secret data from provider", and infrastructure-crds — which waits on those
+    # — never goes Ready, taking all twenty app Kustomizations with it.
     local want
     want=$(kubectl -n "$OPENBAO_NS" get statefulset openbao -o jsonpath='{.spec.replicas}') \
         || fail "could not read the openbao StatefulSet's replica count"
     [[ "$want" =~ ^[0-9]+$ ]] || fail "openbao StatefulSet reported a non-numeric replica count: '$want'"
-    echo "  unsealing $want replica(s)"
+    echo "  waiting for $want replica(s) to unseal themselves"
 
-    local unsealed=0 round pod err
+    local unsealed=0 round pod
     local -A seen=()
     for round in $(seq 1 180); do
         mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
@@ -1700,29 +1696,21 @@ unseal_openbao() {
         for pod in "${pods[@]}"; do
             if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
                 | jq -e '.sealed == false' >/dev/null; then
-                [[ -n "${seen[$pod]:-}" ]] || { echo "  $pod: already unsealed"; seen[$pod]=1; }
+                [[ -n "${seen[$pod]:-}" ]] || { echo -e "  $pod: ${GREEN}unsealed${NC}"; seen[$pod]=1; }
                 unsealed=$(( unsealed + 1 ))
-                continue
-            fi
-            # Argument, not stdin — see the restore path above for why: OpenBao
-            # has no stdin form, and its prompt requires a tty a Job does not
-            # have. The key is exposed in the pod's process list for the length
-            # of the call.
-            if err=$(bao_exec "$pod" bao operator unseal "$key" 2>&1); then
-                echo -e "  $pod: ${GREEN}unsealed${NC}"
-                seen[$pod]=1
-                unsealed=$(( unsealed + 1 ))
-            else
-                # Not fatal on its own: a container still in Init or still
-                # pulling answers this way, and the next round finds it ready.
-                # Only running out of rounds is a failure.
-                echo "  $pod: not yet (${err##*$'\n'})"
             fi
         done
         (( unsealed >= want )) && break
         sleep 10
         if [[ $round -eq 180 ]]; then
-            fail "only $unsealed of $want OpenBao replicas came unsealed after 30 minutes"
+            # A replica that is running and still sealed did not get a working
+            # seal. Say which, with its own answer, rather than "timed out".
+            for pod in "${pods[@]}"; do
+                [[ -n "${seen[$pod]:-}" ]] && continue
+                echo "  $pod:" >&2
+                bao_exec "$pod" bao status 2>&1 | sed 's/^/    /' >&2 || true
+            done
+            fail "only $unsealed of $want OpenBao replicas unsealed after 30 minutes — a sealed one means the static seal failed (openbao-seal-key Secret, pod log)"
         fi
     done
     ok
@@ -1747,7 +1735,7 @@ if phase_runs flux; then
     command -v flux >/dev/null    || fail "flux not found"
     # In the in-cluster Job neither the SOPS key nor the Gitea token is present,
     # and neither should be baked in. Both are streamed into the secrets
-    # directory when the run asks for them, the same way the unseal key is.
+    # directory when the run asks for them, the same way the seal key is.
     # Unconditional: the same collection step on a laptop and in the Job, so the
     # recovery path is the one that gets exercised.
     require_secrets
@@ -1770,6 +1758,37 @@ if phase_runs flux; then
         --dry-run=client -o yaml | kubectl apply -f -
     ok
 
+    # The static seal key, before Flux brings OpenBao up: a pod whose env
+    # references a Secret that does not exist is never started, and OpenBao
+    # reads the key at start. Left alone when the cluster already holds the
+    # same value. A DIFFERENT value is refused rather than applied: the data
+    # is encrypted under whichever key OpenBao started with, and swapping the
+    # Secret would leave every replica sealed for good at its next restart.
+    # That is a migration, not a bootstrap step.
+    #
+    # --from-literal, not --from-file: `pass show` ends its output with a
+    # newline, the substitution strips it, and OpenBao rejects a key that is
+    # not exactly 32 bytes of base64. The key sits in this process's argument
+    # list for the length of the call, as it did in the pod's before.
+    echo -n "Creating OpenBao seal key secret... "
+    seal_key=$(read_secret seal-key) || fail "could not read the seal key"
+    kubectl create namespace "$OPENBAO_NS" --dry-run=client -o yaml | kubectl apply -f -
+    if existing=$(kubectl -n "$OPENBAO_NS" get secret openbao-seal-key \
+            -o go-template='{{index .data "key" | base64decode}}' 2>"$OPENBAO_SECRETS_DIR/.err"); then
+        [[ "$existing" == "$seal_key" ]] \
+            || fail "openbao-seal-key in $OPENBAO_NS holds a different key than pass — not replacing it (OpenBao's data is written under the key it started with)"
+        echo "already present"
+    elif grep -q NotFound "$OPENBAO_SECRETS_DIR/.err"; then
+        kubectl -n "$OPENBAO_NS" create secret generic openbao-seal-key \
+            --from-literal=key="$seal_key" >/dev/null \
+            || fail "could not create the openbao-seal-key Secret"
+        ok
+    else
+        echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
+        fail "could not read the openbao-seal-key Secret"
+    fi
+    unset seal_key existing
+
     if [[ "$FLUX_SOURCE" == "github" ]]; then
         step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..."
         echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}"
@@ -1820,7 +1839,7 @@ if phase_runs flux; then
             --token-auth --timeout 15m 2>&1
     fi
 
-    unseal_openbao
+    bring_up_openbao
 
     step 0 "Waiting for infrastructure..."
     for i in $(seq 1 60); do

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -180,3 +180,17 @@ spec:
     # the supernets here cover the LAN as well, which is the point.
     ipv4NativeRoutingCIDR: 10.0.0.0/8
     ipv6NativeRoutingCIDR: fd00::/8
+    # Pod east-west leaves the node on pod0 (VLAN 48) rather than the
+    # management VLAN. Native routing hands forwarding to the kernel;
+    # autoDirectNodeRoutes installs the remote PodCIDR routes, which is
+    # allowed because all three nodes share pod0's L2 segment.
+    #
+    # directRoutingDevice is explicit on purpose: with several addressed
+    # devices Cilium falls back to the one carrying the Kubernetes InternalIP,
+    # which is mgmt0.
+    #
+    # Encapsulation is gone, so the datapath MTU goes 1450 -> 1500.
+    routingMode: native
+    autoDirectNodeRoutes: true
+    nodePort:
+      directRoutingDevice: pod0

kubernetes/infrastructure/crowdsec/lapi-ingress.yaml

@@ -31,22 +31,12 @@
 # serves read-only decision streams to a valid bouncer key and nothing else.
 # The admin surface is cscli, in-cluster. If the source address ever becomes
 # deterministic, an ipAllowList can be added on top.
----
+#
 # The LAPI serves HTTPS only once tls.enabled is set — there is no plaintext
 # fallback and no second port; 8080 is the TLS port. Its certificate comes
 # from the chart's own self-signed CA, so Traefik needs that CA to verify the
-# backend. SANs include crowdsec-service.crowdsec.svc.cluster.local, which is
-# what the Service resolves to.
-apiVersion: traefik.io/v1alpha1
-kind: ServersTransport
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-spec:
-  serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAs:
-    - secret: crowdsec-root-secret
----
+# backend; the ServersTransport carrying it is a traefik.io CRD consumer and
+# lives in post-crds/crowdsec-lapi-serverstransport.yaml.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml

@@ -72,6 +72,9 @@ spec:
             # they contend for it. Sized for an inference server plus room for
             # a transcoder and a benchmark pod alongside it.
             - -shared-dev-num=4
+            # Advertises gpu.intel.com/monitoring: one resource that grants a
+            # pod every GPU on the node instead of a slot. The exporter needs it.
+            - -enable-monitoring
           securityContext:
             seLinuxOptions:
               type: "container_device_plugin_t"

kubernetes/infrastructure/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
   - cloudnative-pg/
   - mariadb-operator/
   - netbird-ingress/
+  - netbird-proxy-services/
   - coredns-config/
   - k8s-gateway/
   - external-dns-unifi/

kubernetes/infrastructure/netbird-proxy-services/deployment.yaml

@@ -0,0 +1,108 @@
+# Reconciles NetBird reverse-proxy services from Ingresses that carry
+# dns.renner.uno/public: one TLS-passthrough service per host on :443,
+# targeting Traefik's ClusterIP through a subnet resource over the service
+# CIDR (ci/images/netbird-proxy-services/controller.py).
+#
+# openbao namespace, like the rotators: its default-deny already allows egress
+# to OpenBao, DNS, the apiserver and `world` (netbird.renner.uno), and the
+# NetBird PAT is read from OpenBao on every pass through the
+# netbird-proxy-services role — policies/netbird-proxy-services.hcl, applied
+# once by hand like the rotator roles. Until that role exists the pod logs an
+# auth error each minute and does nothing.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: netbird-proxy-services
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: netbird-proxy-services
+---
+# Read-only, cluster-wide: public Ingresses live in many namespaces, the
+# Traefik Service in ingress/.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: netbird-proxy-services-read
+rules:
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: netbird-proxy-services-read
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: netbird-proxy-services-read
+subjects:
+  - kind: ServiceAccount
+    name: netbird-proxy-services
+    namespace: openbao
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netbird-proxy-services
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: netbird-proxy-services
+spec:
+  replicas: 1
+  # One writer against the management API at a time.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: netbird-proxy-services
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: netbird-proxy-services
+    spec:
+      serviceAccountName: netbird-proxy-services
+      securityContext:
+        runAsNonRoot: true
+        # Numeric, or the kubelet refuses the image's named user with
+        # "cannot verify user is non-root". The script needs no home and no
+        # writable path, so nobody/nogroup is enough.
+        runAsUser: 65534
+        runAsGroup: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: controller
+          # Pinned by digest: the date tag is rebuilt on every push of the
+          # image directory (branch and master alike) and the registry keeps
+          # only the newest manifest per tag, so the digest must come from
+          # the MASTER build of the code change.
+          image: gitea.renner.uno/gitea_admin/netbird-proxy-services:2026-09-07@sha256:caffa308d802c6fcc6aea919ac94657c08b993651baa6b8d18de395bcc40523e
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: NB_MANAGEMENT_URL
+              value: https://netbird.renner.uno
+            - name: BAO_ADDR
+              value: http://openbao.openbao.svc.cluster.local:8200
+            - name: BAO_ROLE
+              value: netbird-proxy-services
+            # Must match cluster.network.serviceSubnets in talos/patches/cluster.yaml.
+            - name: SERVICE_CIDR
+              value: "10.96.0.0/12"
+            - name: TRAEFIK_SERVICE
+              value: ingress/traefik
+            - name: INTERVAL_SECONDS
+              value: "60"
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              memory: 64Mi

kubernetes/infrastructure/netbird-proxy-services/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - deployment.yaml

kubernetes/infrastructure/openbao/helmrelease.yaml

@@ -25,10 +25,10 @@ spec:
         name: openbao
         namespace: openbao
   install:
-    # A fresh OpenBao pod is 0/1 Ready until `bao operator init` + unseal
-    # (readiness probe reports sealed). helm --wait would time out and
-    # remediation would uninstall the release, so don't wait here; the
-    # init/unseal procedure is manual (see kubernetes/MIGRATION-sops-openbao.md).
+    # A fresh OpenBao pod is 0/1 Ready until `bao operator init` (readiness
+    # reports sealed). helm --wait would time out and remediation would
+    # uninstall the release, so don't wait here; init is manual (see
+    # kubernetes/MIGRATION-sops-openbao.md). Unseal is automatic (static seal).
     disableWait: true
     remediation:
       retries: 3
@@ -42,6 +42,11 @@ spec:
       # the agent-injector mutating webhook is unused surface.
       enabled: false
     server:
+      # The static seal key, from the openbao-seal-key Secret (bootstrap.sh).
+      extraSecretEnvironmentVars:
+        - envName: BAO_SEAL_KEY
+          secretName: openbao-seal-key
+          secretKey: key
       # Raft (not standalone/file) for two reasons: `bao operator raft
       # snapshot` as a consistent backup primitive, and real HA across three
       # nodes. TLS terminates at the Traefik ingress; in-cluster traffic is
@@ -53,16 +58,8 @@ spec:
         # the moment one pod is sealed or rescheduled, which is worse than
         # the single replica it replaced. Three tolerates one.
         #
-        # What this buys, and what it does not: a node drain seals the replica
-        # on it, and raft elects a new leader from the remaining standbys
-        # within seconds, so the service does not go down. It does NOT remove
-        # the manual unseal — a sealed server can neither serve requests nor
-        # act as standby, so that replica stays out until someone unseals it.
-        #
-        # The consequence for rolling node maintenance is worth stating: a
-        # Talos upgrade drains all three nodes in turn, sealing each replica
-        # as it goes. Unseal each one as it comes back, or the third drain
-        # takes the last unsealed replica with it and quorum is lost anyway.
+        # A restarted replica unseals itself from the static seal and rejoins
+        # raft, so a rolling reboot only ever costs one replica at a time.
         replicas: 3
         raft:
           enabled: true
@@ -111,6 +108,14 @@ spec:
 
             service_registration "kubernetes" {}
 
+            # Auto-unseal. The key is a 32-byte AES key in the openbao-seal-key
+            # Secret, created from pass at bootstrap before this release. The
+            # former shamir key is the recovery key (#805).
+            seal "static" {
+              current_key_id = "1"
+              current_key    = "env://BAO_SEAL_KEY"
+            }
+
             audit "file" "stdout" {
               description = "audit trail to pod stdout -> Loki"
               options {

kubernetes/infrastructure/openbao/policies/netbird-proxy-services.hcl

@@ -0,0 +1,23 @@
+# OpenBao policy: netbird-proxy-services
+#
+# The reverse-proxy-services controller (Deployment netbird-proxy-services in
+# the openbao namespace) reads the NetBird PAT on every reconcile pass and
+# calls the management API with it. Read-only on that one path; it shares the
+# bootstrap credential with rotator-netbird but not that role's PATCH rights.
+#
+# NOT applied by Flux. Apply ONCE with an admin token:
+#
+#   bao policy write netbird-proxy-services kubernetes/infrastructure/openbao/policies/netbird-proxy-services.hcl
+#   bao write auth/kubernetes/role/netbird-proxy-services \
+#     bound_service_account_names=netbird-proxy-services \
+#     bound_service_account_namespaces=openbao \
+#     policies=netbird-proxy-services ttl=15m
+#
+# The PAT itself is the one rotator-netbird already uses
+# (homelab/infrastructure/secret-rotators-bootstrap/netbird, key `pat`).
+#
+# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
+
+path "homelab/data/infrastructure/secret-rotators-bootstrap/netbird" {
+  capabilities = ["read"]
+}

kubernetes/infrastructure/openbao/snapshot-cronjob.yaml

@@ -11,8 +11,8 @@
 # secret is stored anywhere; each run logs in fresh.
 #
 # A SEALED OpenBao fails the login step and the Job goes Failed — that is
-# deliberate signal, not noise: it means unseal was forgotten after a pod
-# restart and backups have stopped.
+# deliberate signal, not noise: it means the static seal could not unseal a
+# restarted pod and backups have stopped.
 #
 # Retention: last 7 snapshots on the PVC; velero adds 30 d offsite.
 apiVersion: batch/v1

kubernetes/infrastructure/post-crds/cilium-bgp.yaml

@@ -25,13 +25,13 @@ spec:
       stop: "fd99::ff"
 ---
 # Cilium BGPv2 control-plane config peering each node with the UDM over the
-# management LAN — never over v30iot, which is the untrusted IoT segment.
+# ingress VLAN 47 — never over v30iot, which is the untrusted IoT segment.
 # The source address comes from the kernel routing decision, so no explicit
 # localAddress override is needed.
 #
 # Two parallel sessions, one per address family:
-#   - v4 session: peer 10.2.40.1, family IPv4 unicast only
-#   - v6 session: peer fd40::1, family IPv6 unicast only
+#   - v4 session: peer 10.2.47.1, family IPv4 unicast only
+#   - v6 session: peer fd02:47::1, family IPv6 unicast only
 # This avoids the "v6 NLRIs over v4 transport" path, which requires
 # bilateral extended-nexthop capability negotiation (RFC 5549/8950).
 # Cilium 1.19 BGPv2 advertises the capability only for the IPv4 Unicast
@@ -74,14 +74,14 @@ spec:
       peers:
         - name: udm-v6
           peerASN: 64512
-          # ULA — stable across ISP renumber events. UDM's br40 has
-          # fd40::1/64; nodes get fd40::<eui-64> via SLAAC on eth0.
-          peerAddress: fd40::1
+          # ULA — stable across ISP renumber events. UDM's br47 has
+          # fd02:47::1/64; nodes get fd02:47::<eui-64> via SLAAC on ingress0.
+          peerAddress: fd02:47::1
           peerConfigRef:
             name: bgp-peer-udm-v6
         - name: udm-v4
           peerASN: 64512
-          peerAddress: 10.2.40.1
+          peerAddress: 10.2.47.1
           peerConfigRef:
             name: bgp-peer-udm-v4
 ---

kubernetes/infrastructure/post-crds/kustomization.yaml

@@ -31,7 +31,5 @@ resources:
   - traefik-crowdsec-middleware.yaml
   - traefik-rate-limit-middleware.yaml
   - traefik-security-headers-middleware.yaml
-  # Phase 1 of a two-Kustomization move: also still in crowdsec/lapi-ingress.yaml
-  # until this layer has applied it once, so prune never runs ahead of apply.
   - crowdsec-lapi-serverstransport.yaml
   - ../network-policies/

kubernetes/infrastructure/traefik/helmrelease.yaml

@@ -93,13 +93,17 @@ spec:
         # Real client-IP visibility. The netbird-ingress routing peer
         # re-originates every mesh + WAN-via-proxy connection from its own
         # pod IP (rootless userspace WireGuard), so Traefik's TCP peer for
-        # that traffic is always a pod IP (e.g. fd10:244::2de1) — collapsing
-        # all external clients to one source. The NetBird reverse-proxy
-        # (HTTP mode, on the VPS) sets X-Forwarded-For with the real WAN
-        # client IP; trust the pod CIDRs so Traefik reads it instead. Without
-        # this, a CrowdSec bouncer bans the shared pod IP and 403s everyone
-        # (the 2026-06-27 IPv6 outage). Harmless no-op when no XFF is present
-        # (direct LAN/mesh) — those keep their RemoteAddr.
+        # that traffic is always a pod IP — collapsing all external clients
+        # to one source. The NetBird reverse-proxy services are TLS
+        # passthrough and carry the WAN client address in a PROXY protocol
+        # v2 header (options.proxy_protocol, set by netbird-proxy-services);
+        # X-Forwarded-For stays trusted for HTTP-mode services. Both are
+        # accepted from the pod CIDRs only; connections without a header
+        # (direct LAN/mesh) keep their RemoteAddr.
+        proxyProtocol:
+          trustedIPs:
+            - "10.244.0.0/16"
+            - "fd10:244::/56"
         forwardedHeaders:
           trustedIPs:
             - "10.244.0.0/16"

kubernetes/talos/patches/k8s-1.yaml

@@ -12,11 +12,28 @@ machine:
     # Left enabled they autoconfigure a ULA and an IPv6 default route from its
     # RAs, which ties with mgmt0's and costs the cluster IPv6 egress.
     # Disabling v6 on a VLAN parent does not affect the VLAN child.
-    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
-    net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    net.ipv6.conf.uplink.disable_ipv6: "1"
+    net.ipv6.conf.mgmt.disable_ipv6: "1"
+    # pod0 is a closed L2 fabric between the three nodes: statically
+    # addressed, no gateway, nothing to learn from an RA.
+    net.ipv6.conf.pod0.accept_ra: "0"
+    # ingress0 keeps RAs, at the default accept_ra of 2: an IPv6 default
+    # router is a link-local address learned only from the RA, and with
+    # net.ipv6.conf.all.forwarding=1 — which Cilium sets — Linux ignores RAs
+    # entirely below 2. That is how mgmt0 has its IPv6 default today.
+    #
+    # The prefix is announced correctly, so SLAAC addresses this link the same
+    # way mgmt0 gets its fd40:: address: derived from the MAC, stable across
+    # reboots, nothing to allocate. Only IPv4 is declared here, and only
+    # because a second DHCP client is what broke the node's DNS record.
+    #
+    # Just the default route is suppressed, because mgmt0 owns it until the
+    # split flips it and two defaults at one metric tie. Dropping this line is
+    # what hands the IPv6 default over.
+    net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # That network also carries the mgmt VLAN's ARP. Answer ARP only on the
     # link that owns the address, or the UDM learns the node IP against
-    # enp87s0's MAC and delivers IPv4 to a NIC Cilium is not attached to (#793).
+    # mgmt's MAC and delivers IPv4 to a NIC Cilium is not attached to (#793).
     net.ipv4.conf.all.arp_ignore: "1"
     net.ipv4.conf.default.arp_ignore: "1"
     net.ipv4.conf.all.arp_announce: "2"
@@ -36,12 +53,15 @@ machine:
     # stated once, here, by the machine that knows.
     storage.renner.uno/ceph-osd: "true"
   udev:
-    # Renames the SAN port (X710 port 1, matched by its burned-in MAC) to the
-    # role name at boot, before the network stack opens links — a rule applied
-    # to a RUNNING node cannot rename (Resource busy) and takes effect at the
-    # next reboot.
+    # Renames every port this node uses to its role at boot, matched by
+    # burned-in MAC — kernel names derive from PCI topology and shift when
+    # hardware moves. Shared manifests then name roles and only this file
+    # knows the hardware. A rule applied to a RUNNING node cannot rename
+    # (Resource busy) and takes effect at the next reboot.
     rules:
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:70:27", NAME="uplink"
       - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:70:28", NAME="ceph-san"
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:70:29", NAME="mgmt"
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
@@ -83,20 +103,17 @@ provisioning:
   grow: true
 ---
 # X710 port 0 carries the switch trunk and takes no addressing itself — its
-# untagged VLAN is a network this node has no business being on. Management
-# and v30iot ride it as tagged VLANs.
+# untagged VLAN is a network this node has no business being on. Management,
+# pod, ingress and v30iot all ride it as tagged VLANs.
 #
-# Links are named by predictable kernel name, which derives from PCI topology
-# and can still shift when hardware is added.
-#
-# LinkAliasConfig would pin these better, but aliasing the X710 ports by MAC
-# and using the aliases as VLAN parents left the node with no address on any
-# interface — the aliases were never created and every document referencing
-# them silently did nothing. Prove any alias-based revision with
-# `talosctl apply-config --mode=try` first; it rolls back on its own.
+# Named by the udev rule above, not by LinkAliasConfig. Aliases are valid VLAN
+# parents, but a selector matching zero links is skipped silently and Talos
+# disables default DHCP machine-wide as soon as any new-style network document
+# exists — so a silent miss leaves the node with no address at all. udev
+# renames the link itself, so a missed match is visible as a missing link.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: enp2s0f0np0
+name: uplink
 up: true
 ---
 # Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
@@ -105,7 +122,7 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: enp2s0f0np0
+parent: uplink
 up: true
 ---
 # DHCP, and the lease is not reserved in this repo — reservations live on the
@@ -134,9 +151,59 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: enp2s0f0np0
+parent: uplink
 up: true
 ---
+# Pod network, VLAN 48. Node-to-node only: the VXLAN overlay rides it once the
+# kubelet node IP moves here, and nothing on it is routed off the switch.
+#
+# Statically addressed, no gateway, no DHCP. A lease would make the cluster's
+# data path depend on the UDM being up in order to form, on a segment whose
+# traffic never reaches the UDM. Declared per node, the way ceph-san's segment
+# carries a declared range rather than a lease.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: pod0
+vlanID: 48
+parent: uplink
+up: true
+addresses:
+  - address: 10.2.48.11/24
+  - address: fd02:48::11/64
+---
+# Ingress network, VLAN 47. World-to-pod: the BGP session, the next hop for
+# every advertised LoadBalancer VIP, and the masquerade source for pod egress.
+# Publicly routed, which is what separates it from pod0.
+#
+# Statically addressed, not leased. A second DHCP client on the node announced
+# the same hostname as mgmt0's and the UDM registered the node against this
+# segment instead, so mgmt0's `srv.renner.uno` record disappeared and the
+# lease's resolver displaced the management one. The UDM cannot register a
+# network's clients selectively, so the client goes rather than the record.
+# DHCP is off on VLAN 47 for the same reason.
+#
+# Both defaults live in table 47, not in main, so only the pod-source rules in
+# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
+# reply the node sends from its own addresses.
+#
+# The v6 gateway is the UDM's configured ULA on this segment, the same address
+# BGP peers with. The RA-learned link-local would be the alternative and is not
+# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
+# default would land in main, where it does not belong.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ingress0
+vlanID: 47
+parent: uplink
+up: true
+addresses:
+  - address: 10.2.47.11/24
+routes:
+  - gateway: fd02:47::1
+    table: "47"
+  - gateway: 10.2.47.1
+    table: "47"
+---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication and nothing else. The udev rule in the machine
 # section above renames the port to ceph-san at boot, so shared manifests name

kubernetes/talos/patches/k8s-2.yaml

@@ -10,8 +10,13 @@ machine:
     # Same for the physical ports on the untagged fallback network: left
     # enabled they take an IPv6 default route from its RAs and cost the
     # cluster IPv6 egress.
-    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
-    net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    net.ipv6.conf.uplink.disable_ipv6: "1"
+    net.ipv6.conf.mgmt.disable_ipv6: "1"
+    # pod0 is a closed segment with no router, so it ignores RAs outright.
+    # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
+    # and only the default route is held back — see k8s-1.yaml.
+    net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
     net.ipv4.conf.default.arp_ignore: "1"
@@ -29,9 +34,11 @@ machine:
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
   udev:
-    # Renames the SAN port to the role name at boot — see k8s-1.yaml.
+    # Renames every port to its role at boot — see k8s-1.yaml.
     rules:
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:76:6f:73", NAME="uplink"
       - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:76:6f:74", NAME="ceph-san"
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:76:6f:75", NAME="mgmt"
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
@@ -61,17 +68,14 @@ provisioning:
   grow: true
 ---
 # X710 port 0 carries the switch trunk and takes no addressing itself — the
-# port's untagged VLAN is not one this node belongs on. Management and v30iot
-# both ride it as tagged VLANs.
+# port's untagged VLAN is not one this node belongs on. Management, pod,
+# ingress and v30iot all ride it as tagged VLANs.
 #
-# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are
-# valid VLAN parents, but a selector matching zero links is skipped silently
-# and Talos disables default DHCP machine-wide as soon as any new-style network
-# document exists — so a silent miss leaves the node with no address on any
-# interface. See RUNBOOK-baremetal-conversion.md.
+# Named by the udev rule above, not by LinkAliasConfig — see k8s-1.yaml and
+# RUNBOOK-baremetal-conversion.md.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: enp2s0f0np0
+name: uplink
 up: true
 ---
 # Management, VLAN 40. Node IP, etcd, the API and Ceph's public network.
@@ -79,7 +83,7 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: enp2s0f0np0
+parent: uplink
 up: true
 ---
 apiVersion: v1alpha1
@@ -99,9 +103,37 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: enp2s0f0np0
+parent: uplink
 up: true
 ---
+# Pod network, VLAN 48. Node-to-node only, statically addressed, no gateway
+# and no DHCP — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: pod0
+vlanID: 48
+parent: uplink
+up: true
+addresses:
+  - address: 10.2.48.12/24
+  - address: fd02:48::12/64
+---
+# Ingress network, VLAN 47. World-to-pod, publicly routed, statically
+# addressed — see k8s-1.yaml for why it is not leased.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ingress0
+vlanID: 47
+parent: uplink
+up: true
+addresses:
+  - address: 10.2.47.12/24
+routes:
+  - gateway: fd02:47::1
+    table: "47"
+  - gateway: 10.2.47.1
+    table: "47"
+---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.
 # Renamed to the role name by the udev rule above — see k8s-1.yaml.

kubernetes/talos/patches/k8s-3.yaml

@@ -6,8 +6,13 @@ machine:
     net.ipv6.conf.ens19.disable_ipv6: "1"
     # Same for the physical ports on the untagged fallback network — see
     # k8s-1.yaml.
-    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
-    net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    net.ipv6.conf.uplink.disable_ipv6: "1"
+    net.ipv6.conf.mgmt.disable_ipv6: "1"
+    # pod0 is a closed segment with no router, so it ignores RAs outright.
+    # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
+    # and only the default route is held back — see k8s-1.yaml.
+    net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
     net.ipv4.conf.default.arp_ignore: "1"
@@ -25,9 +30,11 @@ machine:
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
   udev:
-    # Renames the SAN port to the role name at boot — see k8s-1.yaml.
+    # Renames every port to its role at boot — see k8s-1.yaml.
     rules:
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:72:d3", NAME="uplink"
       - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:72:d4", NAME="ceph-san"
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:72:d5", NAME="mgmt"
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
@@ -58,14 +65,14 @@ provisioning:
 ---
 apiVersion: v1alpha1
 kind: LinkConfig
-name: enp2s0f0np0
+name: uplink
 up: true
 ---
 apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: enp2s0f0np0
+parent: uplink
 up: true
 ---
 apiVersion: v1alpha1
@@ -83,9 +90,37 @@ apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: enp2s0f0np0
+parent: uplink
 up: true
 ---
+# Pod network, VLAN 48. Node-to-node only, statically addressed, no gateway
+# and no DHCP — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: pod0
+vlanID: 48
+parent: uplink
+up: true
+addresses:
+  - address: 10.2.48.13/24
+  - address: fd02:48::13/64
+---
+# Ingress network, VLAN 47. World-to-pod, publicly routed, statically
+# addressed — see k8s-1.yaml for why it is not leased.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ingress0
+vlanID: 47
+parent: uplink
+up: true
+addresses:
+  - address: 10.2.47.13/24
+routes:
+  - gateway: fd02:47::1
+    table: "47"
+  - gateway: 10.2.47.1
+    table: "47"
+---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
 # here on purpose; the link is handed whole to the OSD pod (host-device).
 apiVersion: v1alpha1

kubernetes/talos/patches/site-homelab.yaml

@@ -20,3 +20,47 @@ machine:
       validSubnets:
         - 10.2.40.0/24
         - fd40::/64
+---
+# Pod egress leaves over ingress0. Matching on the pod source prefix is what
+# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
+# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
+# source onto VLAN 47.
+#
+# The routing decision runs before Cilium's masquerade hook, so the rule sees
+# the pod address and the masquerade then rewrites it to ingress0's.
+#
+# Table 47 holds nothing but the two default routes, so the destination rules
+# below have to run first: without them a pod's packet to a service or to a pod
+# on another node also matches the source rule, finds only the default, and
+# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
+#
+# They send everything inside the cluster and the local networks back to main.
+# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
+# is not foreign traffic.
+#
+# Priorities are below main (32766) and clear of the rule Cilium installs at 9.
+# The two families are separate rule tables, so the priorities need not match;
+# the document name is the priority and must be unique.
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "990"
+dst: fd00::/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "991"
+dst: 10.0.0.0/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "1000"
+src: fd10:244::/48
+table: "47"
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "1001"
+src: 10.244.0.0/16
+table: "47"

kubernetes/vmtest-job.yaml

@@ -10,10 +10,10 @@
 # session. The nodes have ~94G each with about 70G unused.
 #
 # The whole script runs here, flux and OpenBao included, so that what a rebuild
-# rehearses is what a real recovery would execute. The unseal key is not baked
-# in and not stored as a Secret: the operator streams all five secrets into an
-# in-memory emptyDir at the moment the run asks for them, and they die with the
-# Job without ever reaching etcd or a backup of it.
+# rehearses is what a real recovery would execute. The OpenBao seal key is not
+# baked in and not stored as a Secret here: the operator streams the secrets
+# into an in-memory emptyDir at the moment the run asks for them, and they die
+# with the Job without ever reaching this cluster's etcd or a backup of it.
 apiVersion: v1
 kind: Namespace
 metadata:

unifi/bgp.conf

@@ -28,10 +28,19 @@ router bgp 64512
  ! advertise extended-nexthop for the IPv6 Unicast SAFI when v6 routes
  ! are carried over a v4 transport, so we run a native v6 session that
  ! carries v6 routes with v6 next-hops and skip the capability dance
- ! entirely. Dynamic neighbors — k8s nodes are SLAAC-addressed on mgmt
- ! LAN; the listen ranges cover the SLAAC suffix space.
+ ! entirely. Dynamic neighbors — nodes are SLAAC-addressed, so the listen
+ ! ranges cover the SLAAC suffix space rather than naming each peer.
+ !
+ ! Both segments are listed while the ingress split lands (#576). The
+ ! cluster peers from mgmt (40) today and moves to ingress (47), and a
+ ! listen range only accepts a session, it does not open one — so having
+ ! both costs nothing and means the move never has a window where the UDM
+ ! is not listening where the nodes are calling from. Drop the 40 ranges
+ ! once every session is established on 47.
  bgp listen range 10.2.40.0/24 peer-group K8S
  bgp listen range fd40::/64 peer-group K8S
+ bgp listen range 10.2.47.0/24 peer-group K8S
+ bgp listen range fd02:47::/64 peer-group K8S
  !
  address-family ipv4 unicast
   neighbor K8S activate