feat(netbird-proxy-services): roll the PROXY-protocol build (2/2)

Pins the controller image by digest. The registry tag is the build date and pipeline 8452 rebuilt today's tag in place, so the tag alone cannot carry a new build to the cluster; the digest can. Merge only after the Traefik change from 1/2 has reconciled: from this build on every service target sends a PROXY protocol v2 header, and Traefik drops such connections unless websecure trusts the pod CIDRs. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
feat(netbird): carry the WAN client address through the passthrough edge
2026-09-07 13:32:06 +02:00 · 2026-09-07 13:29:48 +02:00 · 2026-09-07 10:48:04 +00:00 · 2026-09-07 12:45:40 +02:00 · 2026-09-07 10:43:56 +00:00 · 2026-09-07 10:43:47 +00:00
63 changed files with 1264 additions and 170 deletions
@@ -19,12 +19,13 @@
 # the admin account could (#330).
 #
 # The token is still OWNED by gitea_admin, because Gitea scopes packages to
-# a user or org namespace and these images live under `gitea_admin/*` — a
-# different account cannot publish there. Moving them to a dedicated org is
-# the remaining cleanup; it is a 38-reference migration across pipelines and
-# Flux-deployed CronJobs, so it is tracked separately in #330 rather than
-# bundled here. Note this pipeline runs no LLM agent, so the exfil path that
-# motivated the other cutovers does not apply to it.
+# a user or org namespace and most of these images live under `gitea_admin/*`.
+# The rebuild-critical ones additionally publish to `konairius/` — an org
+# created for the ghcr mirror path, with gitea_admin as a member so this same
+# token can push there. The full gitea_admin/* -> org migration is a
+# 38-reference change tracked separately in #330 rather than bundled here.
+# Note this pipeline runs no LLM agent, so the exfil path that motivated the
+# other cutovers does not apply to it.
 when:
  - event: push
    path:
@@ -87,6 +88,30 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-k8s:$TAG \
          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest

+  - name: build-vmtest
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images/vmtest
+    depends_on: []
+    when:
+      - event: deployment
+      - event: push
+        path: ["ci/images/vmtest/**", ".woodpecker/build-ci-images.yml"]
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/ci-vmtest:$TAG \
+          --destination gitea.renner.uno/gitea_admin/ci-vmtest:latest
+
  - name: build-tofu
    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
    directory: ci/images/tofu
@@ -147,6 +172,28 @@ steps:
  # not for who uses it — its first consumer is the zigbee2mqtt seed-data init
  # container, but nothing about it is zigbee-specific. Exists so tools come from
  # the image rather than an `apk add` on every container start.
+  #
+  # Published THREE times, and the third one is not a typo. Its consumers — the
+  # zigbee2mqtt seed-data init container and the trmnl renderer — start while
+  # the cluster is still being rebuilt, and at that moment gitea.renner.uno does
+  # not exist, because Gitea runs inside the cluster being built. A cold-rebuild
+  # rehearsal caught both in ImagePullBackOff.
+  #
+  # So there is a copy on ghcr.io. But the manifests must NOT point at it:
+  # normal operation pulls from the local registry, and an external reference
+  # would make every steady-state pull leave the network. The recovery source
+  # and the everyday source are different things, exactly as they are for Flux's
+  # git source.
+  #
+  # Talos resolves that with machine.registries.mirrors, which takes an ordered
+  # endpoint list — gitea first, ghcr second (bootstrap.sh). Containerd keeps
+  # the repository path when it falls through, so the two registries have to
+  # agree on it: `gitea_admin/alpine-jq` cannot be served by ghcr, where the
+  # namespace is fixed to the account name. Hence the second Gitea destination
+  # under konairius/, which the manifests reference and which matches the ghcr
+  # path exactly.
+  #
+  # The gitea_admin/ copy stays until nothing references it any more.
  - name: build-alpine-jq
    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
    directory: ci/images/alpine-jq
@@ -159,23 +206,47 @@ steps:
      GITEA_USER: gitea_admin
      GITEA_TOKEN:
        from_secret: ci_images_token
+      GHCR_USER: konairius
+      GHCR_TOKEN:
+        from_secret: ghcr_mirror_token
    commands:
      - mkdir -p /kaniko/.docker
      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
      - TAG=$(date +%Y-%m-%d)
      - |
        /kaniko/executor \
          --context . \
          --dockerfile Containerfile \
          --destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
-          --destination gitea.renner.uno/gitea_admin/alpine-jq:latest
+          --destination gitea.renner.uno/gitea_admin/alpine-jq:latest \
+          --destination gitea.renner.uno/konairius/alpine-jq:$TAG \
+          --destination gitea.renner.uno/konairius/alpine-jq:latest \
+          --destination ghcr.io/konairius/alpine-jq:$TAG \
+          --destination ghcr.io/konairius/alpine-jq:latest

  # Runtime image, not CI tooling: upstream containernetworking/plugins binaries
  # baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
  # node-side fetch from GitHub that had NO checksum verification and sat on the
  # critical path of every node rebuild. The download now happens here, once,
  # where the sha256 is checked — see the Containerfile for the full rationale.
+  #
+  # This one ALSO publishes to ghcr.io. A cold-rebuild rehearsal proved why: the
+  # DaemonSet could not pull from gitea.renner.uno, because Gitea runs inside the
+  # very cluster being rebuilt. Without the plugins there is no macvlan, so
+  # Multus cannot attach ceph-san, so Rook's network canary never starts, so the
+  # CephCluster sits at "Configuring Ceph Mons" — and OpenBao waits forever for a
+  # PVC that will never bind. Moving the download here fixed the integrity gap
+  # but quietly swapped github.com on the critical path for something strictly
+  # worse: a registry that by definition does not exist during a recovery. The
+  # GHCR copy is public, so a bare node can pull it with no credential and no
+  # Gitea.
+  #
+  # The test that proved this also proved the rule is not specific to this
+  # image. alpine-jq and alpine-openssl mirror for the same reason — see the
+  # note on build-alpine-jq. Anything an app needs while the cluster is still
+  # being built cannot live only in that cluster's own registry.
  - name: build-cni-plugins
    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
    directory: ci/images/cni-plugins
@@ -188,23 +259,36 @@ steps:
      GITEA_USER: gitea_admin
      GITEA_TOKEN:
        from_secret: ci_images_token
+      GHCR_USER: konairius
+      GHCR_TOKEN:
+        from_secret: ghcr_mirror_token
    commands:
      - mkdir -p /kaniko/.docker
      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
      - TAG=$(date +%Y-%m-%d)
      - |
        /kaniko/executor \
          --context . \
          --dockerfile Containerfile \
          --destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
-          --destination gitea.renner.uno/gitea_admin/cni-plugins:latest
+          --destination gitea.renner.uno/gitea_admin/cni-plugins:latest \
+          --destination gitea.renner.uno/konairius/cni-plugins:$TAG \
+          --destination gitea.renner.uno/konairius/cni-plugins:latest \
+          --destination ghcr.io/konairius/cni-plugins:$TAG \
+          --destination ghcr.io/konairius/cni-plugins:latest

  # Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
  # needing s_client/x509. Named for what it is; first consumer is the paperless
  # fetch-bridge-cert init container. Replaces an `apk add openssl` that ran on
  # every paperless pod start. The openssl CLI is genuinely absent from the alpine
  # base, so this one needs an image rather than a busybox substitute.
+  #
+  # Mirrored to ghcr.io for the reason given on build-alpine-jq: it is an init
+  # container, so paperless does not start without it, and during a rebuild
+  # gitea.renner.uno is not there to serve it. A cold-rebuild rehearsal left
+  # paperless in Init:ErrImagePull on exactly this image.
  - name: build-alpine-openssl
    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
    directory: ci/images/alpine-openssl
@@ -217,17 +301,25 @@ steps:
      GITEA_USER: gitea_admin
      GITEA_TOKEN:
        from_secret: ci_images_token
+      GHCR_USER: konairius
+      GHCR_TOKEN:
+        from_secret: ghcr_mirror_token
    commands:
      - mkdir -p /kaniko/.docker
      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
      - TAG=$(date +%Y-%m-%d)
      - |
        /kaniko/executor \
          --context . \
          --dockerfile Containerfile \
          --destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
-          --destination gitea.renner.uno/gitea_admin/alpine-openssl:latest
+          --destination gitea.renner.uno/gitea_admin/alpine-openssl:latest \
+          --destination gitea.renner.uno/konairius/alpine-openssl:$TAG \
+          --destination gitea.renner.uno/konairius/alpine-openssl:latest \
+          --destination ghcr.io/konairius/alpine-openssl:$TAG \
+          --destination ghcr.io/konairius/alpine-openssl:latest

  # Heating CAN readers: open3e (Vitocal/VX3) + E3onCAN (E380 meter), one image
  # run as two services on edge-1. Replaces pip-into-venv on the host, where an
@@ -257,3 +349,30 @@ steps:
          --dockerfile Containerfile \
          --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
          --destination gitea.renner.uno/gitea_admin/heating-can:latest
+
+  # netbird-proxy-services: the controller that reconciles NetBird reverse-proxy
+  # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
+  # Runtime artifact like heating-can; same registry so the audit tracks it.
+  - name: build-netbird-proxy-services
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images/netbird-proxy-services
+    depends_on: []
+    when:
+      - event: deployment
+      - event: push
+        path: ["ci/images/netbird-proxy-services/**", ".woodpecker/build-ci-images.yml"]
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
+          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
@@ -32,8 +32,13 @@ netbird_watchdog_fail_threshold: 3

 # Minimum seconds between watchdog-issued restarts, so a genuinely broken
 # backend can't put the proxy into a restart loop. Shared by both the URL-probe
-# watchdog and the flood watchdog below.
-netbird_watchdog_cooldown_seconds: 600
+# watchdog and the flood watchdog below. One minute: the route-manager wedge
+# recurs on every routing-peer loss, and rolling node reboots lose one peer
+# every few minutes; a suppressed restart there leaves the wedge standing
+# until the URL probe's three rounds fire (3.5 min of 000 on every domain,
+# measured 2026-09-07). A minute still caps a loop at one restart per
+# probe interval.
+netbird_watchdog_cooldown_seconds: 60

 # Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain
 # substring in the proxy's docker logs by the flood watchdog. On this build the
@@ -0,0 +1,12 @@
+# netbird-proxy-services: reconciles NetBird reverse-proxy services (public
+# domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
+# public label. Standard library only, so the image is the interpreter plus one
+# file; runs unprivileged as its own user.
+FROM python:3.13-slim
+
+RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
+COPY controller.py /opt/controller.py
+
+ENV PYTHONUNBUFFERED=1
+USER controller
+ENTRYPOINT ["python3", "/opt/controller.py"]
@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+"""netbird-proxy-services: reconcile NetBird reverse-proxy services from Ingresses.
+
+Source of truth: every Ingress carrying the public label (dns.renner.uno/public).
+For each host on such an Ingress the NetBird management gets one reverse-proxy
+service in TLS-passthrough mode on :443, targeting Traefik's ClusterIP through a
+subnet resource that covers the cluster's service CIDR. SNI and Host reach
+Traefik untouched, the proxy terminates nothing, and no domain resource is
+involved -- the routing peers stay ordinary IP routers. The target sends a
+PROXY protocol v2 header so Traefik (and CrowdSec behind its access log) sees
+the WAN client address instead of the routing peer's pod IP.
+
+Ownership is derived from NetBird itself, not from local state: a service is
+managed iff its mode is "tls" and its target is this controller's subnet
+resource. Services that do not match (the hand-made HTTP/domain ones) are never
+touched; a desired host held by such a service is reported and skipped until
+that service is removed by hand.
+
+Credentials: the NetBird PAT is read from OpenBao on every pass through the
+Kubernetes auth role, exactly like the setup-key rotator; it never lands in a
+Kubernetes Secret or in a log line.
+"""
+
+import json
+import logging
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+LOG = logging.getLogger("netbird-proxy-services")
+
+NB = os.environ.get("NB_MANAGEMENT_URL", "https://netbird.renner.uno").rstrip("/")
+BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
+BAO_ROLE = os.environ.get("BAO_ROLE", "netbird-proxy-services")
+BAO_PAT_PATH = os.environ.get("BAO_PAT_PATH", "homelab/data/infrastructure/secret-rotators-bootstrap/netbird")
+BAO_PAT_KEY = os.environ.get("BAO_PAT_KEY", "pat")
+SERVICE_CIDR = os.environ.get("SERVICE_CIDR", "10.96.0.0/12")
+NETWORK_NAME = os.environ.get("NETWORK_NAME", "k8s public ingress")
+ROUTER_GROUP = os.environ.get("ROUTER_GROUP_NAME", "k8s-ingress-router")
+SOURCE_GROUP = os.environ.get("SOURCE_GROUP_NAME", "All")
+TRAEFIK_NS, TRAEFIK_SVC = os.environ.get("TRAEFIK_SERVICE", "ingress/traefik").split("/", 1)
+PUBLIC_LABEL = os.environ.get("PUBLIC_LABEL", "dns.renner.uno/public")
+LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "443"))
+INTERVAL = int(os.environ.get("INTERVAL_SECONDS", "60"))
+HTTP_TIMEOUT = 20
+
+K8S_API = "https://kubernetes.default.svc"
+K8S_TOKEN_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+K8S_CA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+
+class ApiError(Exception):
+    pass
+
+
+def http(method, url, body=None, headers=None, cafile=None):
+    data = None if body is None else json.dumps(body).encode()
+    req = urllib.request.Request(url, data=data, method=method)
+    req.add_header("Accept", "application/json")
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
+    for k, v in (headers or {}).items():
+        req.add_header(k, v)
+    ctx = None
+    if cafile:
+        import ssl
+        ctx = ssl.create_default_context(cafile=cafile)
+    try:
+        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT, context=ctx) as resp:
+            raw = resp.read()
+            return json.loads(raw) if raw else None
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode(errors="replace")[:300]
+        raise ApiError(f"{method} {url} -> {e.code}: {detail}") from None
+
+
+# --- Kubernetes -------------------------------------------------------------
+
+def k8s(path):
+    with open(K8S_TOKEN_FILE) as f:
+        token = f.read().strip()
+    return http("GET", K8S_API + path, headers={"Authorization": f"Bearer {token}"}, cafile=K8S_CA_FILE)
+
+
+def desired_hosts():
+    sel = urllib.parse.quote(f"{PUBLIC_LABEL}=true")
+    ingresses = k8s(f"/apis/networking.k8s.io/v1/ingresses?labelSelector={sel}")
+    hosts = {}
+    for ing in ingresses.get("items", []):
+        ref = f'{ing["metadata"]["namespace"]}/{ing["metadata"]["name"]}'
+        for rule in ing.get("spec", {}).get("rules", []):
+            host = rule.get("host")
+            if not host or host.startswith("*"):
+                continue
+            hosts.setdefault(host, ref)
+    return hosts
+
+
+def traefik_cluster_ip():
+    svc = k8s(f"/api/v1/namespaces/{TRAEFIK_NS}/services/{TRAEFIK_SVC}")
+    for ip in svc["spec"].get("clusterIPs", []):
+        if ":" not in ip:
+            return ip
+    raise ApiError(f"{TRAEFIK_NS}/{TRAEFIK_SVC} has no IPv4 ClusterIP")
+
+
+# --- OpenBao ----------------------------------------------------------------
+
+def netbird_pat():
+    with open(K8S_TOKEN_FILE) as f:
+        jwt = f.read().strip()
+    login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
+    bao_token = login["auth"]["client_token"]
+    kv = http("GET", f"{BAO_ADDR}/v1/{BAO_PAT_PATH}", headers={"X-Vault-Token": bao_token})
+    pat = kv["data"]["data"].get(BAO_PAT_KEY)
+    if not pat:
+        raise ApiError(f"OpenBao {BAO_PAT_PATH} has no key {BAO_PAT_KEY}")
+    return pat
+
+
+# --- NetBird management -----------------------------------------------------
+
+class NetBird:
+    def __init__(self, pat):
+        self.h = {"Authorization": f"Token {pat}"}
+
+    def get(self, path):
+        return http("GET", f"{NB}/api/{path}", headers=self.h)
+
+    def list(self, path):
+        # An empty collection comes back as an empty body / null, not [].
+        return self.get(path) or []
+
+    def post(self, path, body):
+        return http("POST", f"{NB}/api/{path}", body, headers=self.h)
+
+    def put(self, path, body):
+        return http("PUT", f"{NB}/api/{path}", body, headers=self.h)
+
+    def delete(self, path):
+        return http("DELETE", f"{NB}/api/{path}", headers=self.h)
+
+
+def group_id(nb, name):
+    for g in nb.list("groups"):
+        if g["name"] == name:
+            return g["id"]
+    raise ApiError(f"NetBird group {name!r} not found")
+
+
+def ensure_network(nb):
+    for n in nb.list("networks"):
+        if n["name"] == NETWORK_NAME:
+            return n["id"]
+    n = nb.post("networks", {"name": NETWORK_NAME,
+                             "description": "managed by netbird-proxy-services; public ingress via TLS passthrough"})
+    LOG.info("created network %s (%s)", NETWORK_NAME, n["id"])
+    return n["id"]
+
+
+def ensure_resource(nb, net):
+    for r in nb.list(f"networks/{net}/resources"):
+        if r.get("address") == SERVICE_CIDR:
+            return r["id"]
+    r = nb.post(f"networks/{net}/resources", {"name": "cluster service CIDR", "address": SERVICE_CIDR,
+                                              "description": "managed by netbird-proxy-services",
+                                              "enabled": True, "groups": []})
+    LOG.info("created subnet resource %s (%s)", SERVICE_CIDR, r["id"])
+    return r["id"]
+
+
+def ensure_router(nb, net, router_group):
+    for r in nb.list(f"networks/{net}/routers"):
+        if router_group in (r.get("peer_groups") or []):
+            return r["id"]
+    r = nb.post(f"networks/{net}/routers", {"peer_groups": [router_group], "metric": 9999,
+                                            "masquerade": True, "enabled": True})
+    LOG.info("created router on group %s (%s)", ROUTER_GROUP, r["id"])
+    return r["id"]
+
+
+def ensure_policy(nb, resource, source_group):
+    name = f"{NETWORK_NAME} access"
+    for p in nb.list("policies"):
+        if p["name"] == name:
+            return p["id"]
+    p = nb.post("policies", {"name": name, "enabled": True, "rules": [{
+        "name": name, "enabled": True, "action": "accept", "protocol": "tcp",
+        "ports": [str(LISTEN_PORT)], "bidirectional": False, "sources": [source_group],
+        "destinationResource": {"id": resource, "type": "subnet"}}]})
+    LOG.info("created policy %s (%s)", name, p["id"])
+    return p["id"]
+
+
+def service_body(host, resource, target_ip):
+    # proxy_protocol: PROXY v2 header carries the WAN client address to
+    # Traefik (proxyProtocol.trustedIPs on websecure); TCP/TLS modes only.
+    return {"name": host, "domain": host, "mode": "tls", "listen_port": LISTEN_PORT, "enabled": True,
+            "targets": [{"target_type": "subnet", "target_id": resource, "protocol": "tcp",
+                         "host": target_ip, "port": 443, "enabled": True,
+                         "options": {"proxy_protocol": True}}]}
+
+
+def is_managed(svc, resource):
+    return svc.get("mode") == "tls" and any(t.get("target_id") == resource for t in svc.get("targets") or [])
+
+
+def target_matches(svc, target_ip):
+    if svc.get("listen_port") != LISTEN_PORT or not svc.get("enabled"):
+        return False
+    ts = svc.get("targets") or []
+    return len(ts) == 1 and ts[0].get("host") == target_ip and ts[0].get("port") == 443 \
+        and ts[0].get("protocol") == "tcp" and ts[0].get("enabled") \
+        and (ts[0].get("options") or {}).get("proxy_protocol") is True
+
+
+def reconcile():
+    hosts = desired_hosts()
+    target_ip = traefik_cluster_ip()
+    nb = NetBird(netbird_pat())
+
+    router_group = group_id(nb, ROUTER_GROUP)
+    source_group = group_id(nb, SOURCE_GROUP)
+    net = ensure_network(nb)
+    resource = ensure_resource(nb, net)
+    ensure_router(nb, net, router_group)
+    ensure_policy(nb, resource, source_group)
+
+    by_domain = {s["domain"]: s for s in nb.list("reverse-proxies/services")}
+    created = updated = deleted = held = 0
+
+    for host, ref in sorted(hosts.items()):
+        svc = by_domain.get(host)
+        if svc is None:
+            nb.post("reverse-proxies/services", service_body(host, resource, target_ip))
+            LOG.info("created %s (from %s) -> %s:443", host, ref, target_ip)
+            created += 1
+        elif not is_managed(svc, resource):
+            LOG.warning("%s is held by unmanaged service %s (mode %s); remove it to migrate",
+                        host, svc["id"], svc.get("mode"))
+            held += 1
+        elif not target_matches(svc, target_ip):
+            nb.put(f"reverse-proxies/services/{svc['id']}", service_body(host, resource, target_ip))
+            LOG.info("updated %s -> %s:443", host, target_ip)
+            updated += 1
+
+    for domain, svc in by_domain.items():
+        if is_managed(svc, resource) and domain not in hosts:
+            nb.delete(f"reverse-proxies/services/{svc['id']}")
+            LOG.info("deleted %s (no public Ingress carries it any more)", domain)
+            deleted += 1
+
+    LOG.info("reconciled: %d hosts desired, %d created, %d updated, %d deleted, %d held by unmanaged services",
+             len(hosts), created, updated, deleted, held)
+
+
+def main():
+    logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), stream=sys.stdout,
+                        format="%(asctime)s %(levelname)s %(message)s")
+    if os.environ.get("ONCE") == "1":
+        reconcile()
+        return
+    while True:
+        try:
+            reconcile()
+        except Exception as e:  # keep running; the next pass retries
+            LOG.error("reconcile failed: %s", e)
+        time.sleep(INTERVAL)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,84 @@
+FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+
+# Everything `bootstrap.sh vmtest` needs, so the rebuild test can run as a Job in
+# the cluster instead of on a workstation. The workstation has 31G and the test
+# wants ~14.5G for four VMs — a single run left under 1G headroom, and two
+# overlapping runs OOM-killed the session. The nodes have ~94G with 70G free.
+#
+# Separate from ci-k8s rather than built on it: that image pins talosctl v1.12.6,
+# and this test creates a cluster at the version talos/patches/cluster.yaml
+# installs. A talosctl older than the Talos it provisions is exactly the kind of
+# skew the test exists to catch, so it must not be baked in here.
+#
+# KEEP TALOS_VERSION IN STEP with the installer image tag in
+# kubernetes/talos/patches/cluster.yaml. The phase reads the version from that
+# file at run time and passes it to the provisioner; this pin only decides which
+# client talks to it.
+ARG TALOS_VERSION=v1.13.9
+ARG KUBECTL_VERSION=v1.36.2
+ARG FLUX_VERSION=2.9.2
+ARG KOPIA_VERSION=0.23.1
+
+# qemu-system-x86_64 and qemu-img are the provisioner's actual backend, and ovmf
+# supplies the UEFI firmware those VMs boot from — without it the provisioner
+# stops with "the required flash image was not found in any of the expected
+# paths", having already downloaded the ISO and generated the PKI. Note the
+# firmware comes from ovmf specifically: qemu-system-x86_64 ships an edk2 CODE
+# image but no VARS, which is the half the provisioner needs.
+#
+# iptables is shelled out to by the CNI plugins that build the VM network; the
+# provisioner fails with "error accessing iptables" without it. The rest is
+# what the phase shells out to: helm for the Cilium install, yq to read the
+# HelmRelease and the installer image, jq for OpenBao's seal status.
+#
+# Versions come from the digest-pinned base rather than being pinned one by one,
+# the same way ci-ansible does it.
+RUN apk add --no-cache \
+      bash \
+      ca-certificates \
+      curl \
+      git \
+      helm \
+      iptables \
+      jq \
+      openssl \
+      ovmf \
+      qemu-img \
+      qemu-system-x86_64 \
+      yq
+
+RUN curl -sfL -o /usr/local/bin/talosctl \
+      "https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
+ && echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e  /usr/local/bin/talosctl" | sha256sum -c - \
+ && chmod +x /usr/local/bin/talosctl
+
+RUN curl -sfL -o /usr/local/bin/kubectl \
+      "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
+ && curl -sfL -o /tmp/kubectl.sha256 \
+      "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" \
+ && echo "$(cat /tmp/kubectl.sha256)  /usr/local/bin/kubectl" | sha256sum -c - \
+ && rm -f /tmp/kubectl.sha256 \
+ && chmod +x /usr/local/bin/kubectl
+
+RUN curl -sfL -o /tmp/flux.tar.gz \
+      "https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \
+ && echo "4092b367fd060097976fb7261601f79420ecf7266328417dfd8dfee27de0e6a3  /tmp/flux.tar.gz" | sha256sum -c - \
+ && tar -xzf /tmp/flux.tar.gz -C /usr/local/bin flux \
+ && chmod +x /usr/local/bin/flux \
+ && rm -f /tmp/flux.tar.gz
+
+# kopia reads the Velero backup repository directly, which is what the OpenBao
+# restore does when there is no Velero yet to do it: the credentials Velero
+# needs live in OpenBao, so OpenBao cannot be restored by Velero. Its absence
+# was found by a rebuild rehearsal reaching the restore for the first time and
+# stopping at "kopia not found".
+#
+# Same checksum-verified download as the tools above; the tarball unpacks into
+# a versioned directory, so the binary is pulled out by name.
+RUN curl -sfL -o /tmp/kopia.tar.gz \
+      "https://github.com/kopia/kopia/releases/download/v${KOPIA_VERSION}/kopia-${KOPIA_VERSION}-linux-x64.tar.gz" \
+ && echo "416d0f84a3dbb321a8b2d8f0997b1a0a6e915babe79ee76fa6e4d2bd1e1c5178  /tmp/kopia.tar.gz" | sha256sum -c - \
+ && tar -xzf /tmp/kopia.tar.gz -C /tmp \
+ && mv "/tmp/kopia-${KOPIA_VERSION}-linux-x64/kopia" /usr/local/bin/kopia \
+ && chmod +x /usr/local/bin/kopia \
+ && rm -rf /tmp/kopia.tar.gz "/tmp/kopia-${KOPIA_VERSION}-linux-x64"
@@ -16,7 +16,6 @@ resources:
  - prometheus-gate-egress.yaml
  - wordpress-gate-egress.yaml
  - llama-gate-egress.yaml
-  - outpost-servicemonitors.yaml
  - ldap-outpost-cert.yaml
  - ldap-outpost-lb.yaml
  - ldap-outpost-cnp.yaml
@@ -57,7 +57,14 @@ spec:
          # and leaves this one untouched (tags are not deleted), so this keeps
          # resolving — but the image then came from the branch build rather than
          # master. Re-point it at a master-built tag when convenient.
-          image: gitea.renner.uno/gitea_admin/alpine-jq:2026-07-30
+          #
+          # konairius/, not gitea_admin/: this init container runs during a cold
+          # rebuild, when Gitea does not exist because it runs inside the
+          # cluster being rebuilt. The node then falls back to a ghcr.io copy
+          # (machine.registries.mirrors, applied by bootstrap.sh), and containerd
+          # keeps the repository path when it falls through — so the path has to
+          # exist on both registries. Steady state still pulls from Gitea.
+          image: gitea.renner.uno/konairius/alpine-jq:2026-09-04
          command:
            - sh
            - -c
@@ -0,0 +1,46 @@
+# Cilium's agent ServiceMonitor, declared here rather than by the chart.
+#
+# Reproduces what cilium 1.20.1 renders with prometheus.serviceMonitor.enabled,
+# taken field for field off the live object so scraping is unchanged: same name,
+# same namespace, same selector, same 10s interval and node relabeling.
+#
+# It lives here because the chart's template is gated on monitoring.coreos.com/v1
+# existing at install time, and on an empty cluster it does not — those CRDs
+# arrive with kube-prometheus-stack, which reconciles after infrastructure.
+# Cilium is one of infrastructure's healthChecks, so the chart aborting on the
+# missing CRD held the whole cluster: CNI waiting on monitoring waiting on CNI.
+#
+# velero/helmrelease.yaml already does exactly this for its own SM/PM and says
+# why; this is the same move for the component where it mattered most.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: cilium-agent
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: cilium-agent
+    app.kubernetes.io/part-of: cilium
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cilium-agent
+  namespaceSelector:
+    matchNames:
+      - kube-system
+  # The chart's Service keeps rendering (prometheus.enabled), so the target and
+  # its `metrics` port are there regardless of who declares the ServiceMonitor.
+  targetLabels:
+    - k8s-app
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 10s
+      honorLabels: true
+      relabelings:
+        # Agent metrics are per-node; without this every series lands without a
+        # node label and BPF map pressure cannot be attributed to a machine.
+        - action: replace
+          sourceLabels:
+            - __meta_kubernetes_pod_node_name
+          targetLabel: node
+          replacement: ${1}
@@ -22,8 +22,19 @@ resources:
  # endpoint, which proves reachability, not that anything was written.
  - prometheusrule-cnpg-backup.yaml
  - alloy-servicemonitor.yaml
+  - cilium-servicemonitor.yaml
  - cnpg-podmonitors.yaml
  - crowdsec-servicemonitor.yaml
+  # Velero's own monitors: monitoring.coreos.com CRDs arrive with
+  # kube-prometheus-stack, which reconciles after infrastructure, so these
+  # cannot live beside the Velero HelmRelease.
+  - velero-podmonitor.yaml
+  - velero-servicemonitor.yaml
+  - authentik-outpost-servicemonitors.yaml
+  # Rook's exporter/mgr monitors: the operator aborts its node-daemon reconcile
+  # when it cannot create these, which held the CephCluster in Progressing and
+  # deadlocked the whole apps layer. See the file for the cycle.
+  - rook-servicemonitors.yaml
  - home-assistant-servicemonitor.yaml
  - immich-servicemonitor.yaml
  - loki-servicemonitor.yaml
@@ -0,0 +1,93 @@
+# Rook's two ServiceMonitors, declared here rather than by the operator.
+#
+# Reproduced field for field off the live objects in the production cluster, so
+# scraping is unchanged: same names, same namespace, same selectors, same 10s
+# interval and `cluster` relabeling. The only thing dropped is the CephCluster
+# ownerReference, which Rook set and Flux cannot.
+#
+# They live here because CephCluster.spec.monitoring.enabled is now false, and
+# that is not a monitoring decision — it breaks a deadlock. Rook's node-daemon
+# reconcile creates this ServiceMonitor and, unlike the exporter DaemonSet next
+# to it, treats a failure as fatal (pkg/operator/ceph/cluster/nodedaemon/
+# reconcile.go, v1.20.7):
+#
+#     if !cephCluster.Spec.Monitoring.MetricsDisabled {
+#         // exporter reconciled; its errors are logged and reconcile continues
+#         if cephCluster.Spec.Monitoring.Enabled {
+#             if err := EnableCephExporterServiceMonitor(...); err != nil {
+#                 return errors.Wrap(err, "failed to enable service monitor")
+#             }
+#         }
+#     }
+#
+# On an empty cluster monitoring.coreos.com/v1 does not exist yet, so that call
+# fails with "the server could not find the requested resource (post
+# servicemonitors.monitoring.coreos.com)", the whole node-daemon reconcile
+# aborts, and the CephCluster never leaves status.phase Progressing. That is
+# what infrastructure-ceph-cluster's healthCheckExpr waits on — and the chain
+# closes on itself:
+#
+#     infrastructure-ceph-cluster → CephCluster phase Ready
+#                                 → Rook reconcile settles
+#                                 → monitoring.coreos.com CRDs exist
+#                                 → kube-prometheus-stack (monitoring)
+#                                 → grafana-db
+#                                 → infrastructure-ceph-cluster
+#
+# Nothing outside that loop can break it, so a cold rebuild stopped here with
+# every one of the twenty app Kustomizations behind it.
+#
+# Same treatment velero/helmrelease.yaml and cilium/helmrelease.yaml already
+# get, and for the same reason: the metrics stay where they are, only the
+# scrape wiring moves to the layer where its CRDs exist.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: rook-ceph-exporter
+  namespace: rook-ceph
+  labels:
+    team: rook
+spec:
+  selector:
+    matchLabels:
+      app: rook-ceph-exporter
+      rook_cluster: rook-ceph
+  namespaceSelector:
+    matchNames:
+      - rook-ceph
+  endpoints:
+    - port: ceph-exporter-http-metrics
+      path: /metrics
+      interval: 10s
+      honorLabels: true
+      # Rook stamps the cluster name so series from several Ceph clusters stay
+      # separable. One cluster here, but dropping it would rename every series.
+      relabelings:
+        - action: replace
+          replacement: rook-ceph
+          targetLabel: cluster
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: rook-ceph-mgr
+  namespace: rook-ceph
+  labels:
+    team: rook
+spec:
+  selector:
+    matchLabels:
+      app: rook-ceph-mgr
+      rook_cluster: rook-ceph
+  namespaceSelector:
+    matchNames:
+      - rook-ceph
+  endpoints:
+    - port: http-metrics
+      path: /metrics
+      interval: 10s
+      honorLabels: true
+      relabelings:
+        - action: replace
+          replacement: rook-ceph
+          targetLabel: cluster
@@ -143,7 +143,20 @@ spec:
          # on every paperless pod start: tools belong in the image, not
          # installed at runtime. Dated tag so a change is a reviewable diff
          # here, matching the ci-ansible / alpine-jq convention.
-          image: gitea.renner.uno/gitea_admin/alpine-openssl:2026-07-30
+          #
+          # konairius/, not gitea_admin/, and that is not cosmetic. This is an
+          # INIT container, so paperless does not start without it — and during
+          # a cold rebuild gitea.renner.uno does not exist, because Gitea runs
+          # inside the cluster being rebuilt. A rehearsal left this pod in
+          # Init:ErrImagePull on exactly this image.
+          #
+          # The node falls back to a ghcr.io copy in that window
+          # (machine.registries.mirrors, applied by bootstrap.sh). Containerd
+          # keeps the repository path when it falls through, so the path has to
+          # exist on both registries — hence konairius/, which matches the ghcr
+          # namespace. Steady state still pulls from Gitea and never leaves the
+          # network.
+          image: gitea.renner.uno/konairius/alpine-openssl:2026-09-04
          command:
            - sh
            - -c
@@ -177,7 +177,14 @@ spec:
        # webhook start failing with "expired JWT access token" about
        # thirty minutes after seeding.
        - name: render-schedule
-          image: gitea.renner.uno/gitea_admin/alpine-jq:2026-07-30
+          # konairius/, not gitea_admin/: this container starts during a cold
+          # rebuild, when Gitea does not exist because it runs inside the
+          # cluster being rebuilt — a rehearsal left it in Init:ImagePullBackOff.
+          # The node falls back to a ghcr.io copy then
+          # (machine.registries.mirrors, applied by bootstrap.sh), and containerd
+          # keeps the repository path, so it has to exist on both registries.
+          # Steady state still pulls from Gitea.
+          image: gitea.renner.uno/konairius/alpine-jq:2026-09-04
          command:
            - sh
            - -c
@@ -1,18 +1,3 @@
-# Dedicated namespace where the kubernetes backend spawns pipeline step pods,
-# isolated from the apps namespace.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: woodpecker-ci
-  labels:
-    app.kubernetes.io/name: woodpecker
-    # Image-build steps run a privileged docker/buildx plugin pod (see the
-    # server's WOODPECKER_PLUGINS_PRIVILEGED). Talos enforces Pod Security;
-    # without this the privileged step pods are rejected at admission. The
-    # namespace is dedicated to ephemeral CI step pods, so scoping the
-    # exemption here keeps it off apps/.
-    pod-security.kubernetes.io/enforce: privileged
---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -1379,8 +1379,8 @@ EOF
            vm_addr="${NODE_ADDRS[$idx]}"
            vols=$(talosctl -n "$vm_addr" -e "$vm_addr" get discoveredvolumes 2>&1) \
                || fail "could not read volumes from $vm_node ($vm_addr): ${vols//$'\n'/ | }"
-            awk '$NF == "r-ceph-osd" {found=1} END {exit !found}' <<<"$vols" \
-                || fail "$vm_node has no r-ceph-osd partition — RawVolumeConfig did not take"
+            awk '$NF == "r-osd-data" {found=1} END {exit !found}' <<<"$vols" \
+                || fail "$vm_node has no r-osd-data partition — RawVolumeConfig did not take"
            kubectl get node "$vm_node" \
                -o jsonpath='{.metadata.labels.storage\.renner\.uno/ceph-osd}' \
                | grep -q true \
@@ -28,6 +28,19 @@ spec:
    - name: infrastructure
    - name: infrastructure-crds
    - name: infrastructure-ceph-cluster
+    # ha-pg-objectstore.yaml declares an ObjectStore, whose CRD comes from the
+    # CNPG Barman plugin in infrastructure-cnpg-barman. Without this edge the
+    # kind is unknown when this path is first applied, and a server-side
+    # dry-run rejects the ENTIRE set over the one object it cannot recognise:
+    #
+    #   ObjectStore/home-assistant/ha-pg-store dry-run failed:
+    #   no matches for kind "ObjectStore" in version ...
+    #
+    # Every other ObjectStore user — authentik, immich, paperless, trmnl,
+    # woodpecker — already declares it. This was the only one that did not, and
+    # it took speech-to-phrase, zone-configurator and matter-server down with
+    # it, since all three wait on home-assistant.
+    - name: infrastructure-cnpg-barman
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
@@ -122,8 +122,20 @@ spec:
  path: ./kubernetes/infrastructure/rook-ceph-cluster
  prune: true
  wait: true
-  dependsOn:
-    - name: infrastructure
+  # Deliberately NOT dependsOn: infrastructure. That edge closed a cycle no
+  # existing cluster could ever hit and no new one could ever escape:
+  # infrastructure is Ready only once its openbao healthCheck passes, openbao-0
+  # only runs once its PVC binds, and that PVC wants the ceph-block StorageClass
+  # defined in this very Kustomization. A cold rebuild stopped here for good,
+  # with openbao-0 Pending on "unbound immediate PersistentVolumeClaims".
+  #
+  # The edge was never needed. infrastructure runs wait: false, so it applies
+  # the rook-ceph namespace and the operator immediately and only its readiness
+  # waits on the healthChecks — everything this path needs exists long before
+  # infrastructure reports Ready. On an empty API server this Kustomization
+  # retries on retryInterval until the operator installs its CRDs, the same
+  # self-healing shape post-crds already relies on; on a live cluster the CRDs
+  # are always present and nothing changes.
  # kstatus doesn't know how to read Rook CR readiness, so without these
  # this Kustomization reports Ready as soon as the manifests apply — long
  # before OSDs/mons/MDS are actually serving I/O. Apps depending on this
@@ -70,9 +70,23 @@ spec:
      # map only shows up as "no space left on device" in the log while pod
      # sandbox creation fails (#452). cilium_bpf_map_pressure makes it
      # alertable beforehand.
+      #
+      # That reasoning is about the metrics, which this line provides. The
+      # ServiceMonitor is only the scrape wiring and is declared standalone in
+      # apps/monitoring-rules instead — the same treatment velero/helmrelease
+      # already gives its own, and for the same reason: the chart's template is
+      # gated on monitoring.coreos.com/v1 existing at install time.
+      #
+      # Here it is worse than for velero. Cilium's chart does not skip the
+      # template, it aborts rendering — "Service Monitor requires
+      # monitoring.coreos.com/v1 CRDs" — and cilium is one of the seven
+      # healthChecks the infrastructure Kustomization waits on. So on an empty
+      # cluster the CNI blocked infrastructure, which blocked grafana-db, which
+      # blocked monitoring, which is where those CRDs come from. The lowest
+      # layer in the cluster was waiting on the highest.
      enabled: true
      serviceMonitor:
-        enabled: true
+        enabled: false
    bpf:
      # Per-endpoint policy map size: one entry per (identity, direction,
      # protocol, port) allowed. When it fills, endpoint regeneration fails
@@ -99,7 +99,7 @@ spec:
    # required.
    #
    # NOTE: enabling this makes the LAPI serve HTTPS-only. The Traefik ingress
-    # bouncer is currently INERT (detached, see traefik/crowdsec-middleware.yaml)
+    # bouncer is currently INERT (detached, see post-crds/traefik-crowdsec-middleware.yaml)
    # so there is no live impact, but re-enabling it will require pointing the
    # plugin at https + trusting this CA (or giving it a bouncer-ou client cert).
    tls:
@@ -1,7 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - bouncer-key-externalsecret.yaml
-  - edge-bouncer-key-externalsecret.yaml
  - lapi-ingress.yaml
  - helmrelease.yaml
@@ -31,22 +31,12 @@
 # serves read-only decision streams to a valid bouncer key and nothing else.
 # The admin surface is cscli, in-cluster. If the source address ever becomes
 # deterministic, an ipAllowList can be added on top.
---
+#
 # The LAPI serves HTTPS only once tls.enabled is set — there is no plaintext
 # fallback and no second port; 8080 is the TLS port. Its certificate comes
 # from the chart's own self-signed CA, so Traefik needs that CA to verify the
-# backend. SANs include crowdsec-service.crowdsec.svc.cluster.local, which is
-# what the Service resolves to.
-apiVersion: traefik.io/v1alpha1
-kind: ServersTransport
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-spec:
-  serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAs:
-    - secret: crowdsec-root-secret
---
+# backend; the ServersTransport carrying it is a traefik.io CRD consumer and
+# lives in post-crds/crowdsec-lapi-serverstransport.yaml.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -1,5 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - externalsecret.yaml
  - helmrelease.yaml
@@ -9,9 +9,26 @@ spec:
 ---
 # v-prefix tags (v19.2.3). Upstream also publishes date-suffixed builds
 # (v19.2.3-20250717) and moving major/minor tags (v19, v19.2); the
-# filter pins us to the bare three-component release. Range stays at
-# 19.x — the 19 → 20 (Tentacle) cutover should be a deliberate
-# upgrade tied to the Rook operator version.
+# filter pins us to the bare three-component release.
+#
+# The range was 19.x, to keep the 19 -> 20 (Tentacle) cutover a deliberate
+# upgrade tied to the Rook operator version. That coupling had already come
+# apart without anyone noticing: Rook's floating constraint pulled v1.20 in
+# June, and with it a ceph-csi image whose client libraries are Ceph 20.2.1.
+# The cluster stayed on 19.2.6, so every CSI pod has since been talking to
+# the mons with a client a whole major ahead of them.
+#
+# That is not academic. The 19.2.6 mons mint auth keys as aes256k, and the
+# 20.2.1 client cannot decode them — "auth: failed to decode key 'AgB...'".
+# Nothing broke only because the four CSI users predate the cipher change and
+# still carry legacy aes keys, which Ceph itself flags as insecure. The moment
+# one is recreated — ceph auth rotate, a deleted Secret, Rook resyncing — the
+# provisioner stops and no PVC binds. A rebuild rehearsal hits it immediately,
+# because a fresh cluster has no legacy keys at all.
+#
+# So the cutover happens now, with the version the client already is. Rook
+# v1.20 supports Tentacle from v20.2.1; v20.2.0 is excluded deliberately —
+# upstream reports data corruption there when read affinity is enabled.
 apiVersion: image.toolkit.fluxcd.io/v1
 kind: ImagePolicy
 metadata:
@@ -25,4 +42,4 @@ spec:
    extract: '$ver'
  policy:
    semver:
-      range: "19.x"
+      range: ">=20.2.1 <21.0.0"
@@ -18,6 +18,7 @@ resources:
  - cloudnative-pg/
  - mariadb-operator/
  - netbird-ingress/
+  - netbird-proxy-services/
  - coredns-config/
  - k8s-gateway/
  - external-dns-unifi/
@@ -30,6 +30,11 @@ spec:
      labels:
        app: cni-plugins-installer
    spec:
+      # Same reasoning as multus-daemonset.yaml. This one installs the macvlan
+      # binary the ceph-san attachment is made of, so a node that cannot
+      # schedule it has no plugin to delegate to — the failure the init
+      # container comment below already describes, reached by a different road.
+      priorityClassName: system-node-critical
      hostNetwork: true
      tolerations:
        - operator: Exists
@@ -44,7 +49,32 @@ spec:
          # at all and put github.com on the critical path of every node rebuild.
          # Dated tag so a version change is a reviewable diff here, matching the
          # ci-ansible / alpine-jq convention.
-          image: gitea.renner.uno/gitea_admin/cni-plugins:2026-07-30
+          #
+          # konairius/, not gitea_admin/, and the difference is what makes a
+          # rebuild possible. Gitea runs inside this cluster, so during a rebuild
+          # it does not exist: a rehearsal sat here in ImagePullBackOff with "no
+          # route to host", and because the plugins never installed there was no
+          # macvlan, so Multus could not attach ceph-san, so Rook's network
+          # canary never started, so the CephCluster hung at "Configuring Ceph
+          # Mons" and OpenBao waited forever on a PVC that could not bind. The
+          # failure surfaced four layers from its cause.
+          #
+          # This used to point straight at ghcr.io, which fixed the rebuild and
+          # broke the normal case: every steady-state pull then left the network
+          # for an image the local registry already holds. The fallback belongs
+          # to the node, not to the manifest — machine.registries.mirrors names
+          # Gitea first and ghcr.io second (bootstrap.sh), so nothing here has to
+          # know about the recovery path.
+          #
+          # Containerd keeps the repository path when it falls through, so the
+          # path must resolve on both registries; the build pushes one digest to
+          # konairius/ on each, verified equal. Bumping the tag means building
+          # first — an unpublished tag is unpullable on both.
+          #
+          # No imagePullSecret: the ghcr namespace is private and its credential
+          # is in machine.registries too, because a Secret would have to come
+          # from OpenBao, which needs Ceph, which needs these binaries.
+          image: gitea.renner.uno/konairius/cni-plugins:2026-09-04
          command:
            - sh
            - -c
@@ -154,6 +154,40 @@ spec:
        app: multus
        name: multus
    spec:
+      # Upstream ships this DaemonSet with no priority class, so it runs at 0
+      # and can preempt nothing. Any event that makes the pod need a fresh
+      # scheduling decision — a rollout, a node restart, an eviction — then has
+      # to fit it onto a node that may since have filled up, and if it does not
+      # fit it simply stays Pending. The node has no multus after that, and
+      # nothing reports it: pods still start, they just never get their
+      # secondary attachment, and the annotation everything waits on never
+      # appears.
+      #
+      # Seen on a cold rebuild: two of four nodes sat at 99% memory requests
+      # when the DaemonSet rolled, both multus pods went Pending, and Rook's
+      # network canary hung in Init:0/1 waiting for a network-status annotation
+      # that could never come. The CephCluster stayed in Progressing, which is
+      # what infrastructure-ceph-cluster gates on, so the whole apps layer
+      # stopped behind a scheduling decision three layers away.
+      #
+      # node-critical, the same class cilium and cni-dhcp-daemon carry, because
+      # that is what this is: without multus a node cannot attach a secondary
+      # network, and only that node is affected. Nothing cluster-wide fails.
+      #
+      # This does outrank kube-apiserver, which Talos runs as a static pod at
+      # system-cluster-critical (2000000000, against 2000001000 here), and on a
+      # control plane that has run out of memory the kubelet will pick the API
+      # server as the pod to evict. That was observed twice, at 6GiB and at
+      # 8GiB.
+      #
+      # It is not a reason to demote this DaemonSet. Kubernetes puts the CNI
+      # above the control plane deliberately — a node without its CNI cannot run
+      # anything at all — and a class name is a statement about what the pod is,
+      # not a dial for picking a number. The eviction says the control plane is
+      # starved, and that is where it has to be fixed: enough allocatable
+      # memory, or kubeReserved/systemReserved so the static pods keep what they
+      # need. Demoting this to cluster-critical would only move the victim.
+      priorityClassName: system-node-critical
      hostNetwork: true
      hostPID: true
      tolerations:
@@ -146,6 +146,11 @@ spec:
        app: whereabouts
        name: whereabouts
    spec:
+      # Same reasoning as multus-daemonset.yaml: this is the IPAM half of the
+      # secondary-network path, so a node without it cannot hand out an address
+      # on ceph-san and the attachment fails just as completely as if multus
+      # itself were missing. Upstream leaves the priority class unset.
+      priorityClassName: system-node-critical
      hostNetwork: true
      serviceAccountName: whereabouts
      # Every node here is a control plane node, so the NoSchedule toleration
@@ -190,3 +190,31 @@ apiVersion: v1
 kind: Namespace
 metadata:
  name: ai
+---
+# Where the Woodpecker kubernetes backend spawns pipeline step pods, isolated
+# from `apps`. Distinct from `woodpecker`, which holds the service itself.
+#
+# Here rather than in apps/woodpecker for the reason spelled out above that
+# namespace: post-crds ships this namespace's CiliumNetworkPolicies via
+# ../network-policies/, and infrastructure-crds runs before every app KS. With
+# the namespace declared in the workload KS the order inverted, and a rebuild
+# stopped with
+#
+#   CiliumNetworkPolicy/woodpecker-ci/default-deny-ingress not found:
+#   namespaces "woodpecker-ci" not found
+#
+# which held infrastructure-crds, and with it all twenty app Kustomizations
+# that depend on it. The same reasoning was already written down for
+# `woodpecker` and simply not carried across to its sibling.
+#
+# Image-build steps run a privileged docker/buildx plugin pod (see the server's
+# WOODPECKER_PLUGINS_PRIVILEGED). Talos enforces Pod Security; without the
+# label the privileged step pods are rejected at admission. The namespace holds
+# nothing but ephemeral CI step pods, so the exemption stays off apps/.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: woodpecker-ci
+  labels:
+    app.kubernetes.io/name: woodpecker
+    pod-security.kubernetes.io/enforce: privileged
@@ -1,5 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - externalsecret.yaml
  - daemonset.yaml
@@ -0,0 +1,106 @@
+# Reconciles NetBird reverse-proxy services from Ingresses that carry
+# dns.renner.uno/public: one TLS-passthrough service per host on :443,
+# targeting Traefik's ClusterIP through a subnet resource over the service
+# CIDR (ci/images/netbird-proxy-services/controller.py).
+#
+# openbao namespace, like the rotators: its default-deny already allows egress
+# to OpenBao, DNS, the apiserver and `world` (netbird.renner.uno), and the
+# NetBird PAT is read from OpenBao on every pass through the
+# netbird-proxy-services role — policies/netbird-proxy-services.hcl, applied
+# once by hand like the rotator roles. Until that role exists the pod logs an
+# auth error each minute and does nothing.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: netbird-proxy-services
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: netbird-proxy-services
+---
+# Read-only, cluster-wide: public Ingresses live in many namespaces, the
+# Traefik Service in ingress/.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: netbird-proxy-services-read
+rules:
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: netbird-proxy-services-read
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: netbird-proxy-services-read
+subjects:
+  - kind: ServiceAccount
+    name: netbird-proxy-services
+    namespace: openbao
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netbird-proxy-services
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: netbird-proxy-services
+spec:
+  replicas: 1
+  # One writer against the management API at a time.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: netbird-proxy-services
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: netbird-proxy-services
+    spec:
+      serviceAccountName: netbird-proxy-services
+      securityContext:
+        runAsNonRoot: true
+        # Numeric, or the kubelet refuses the image's named user with
+        # "cannot verify user is non-root". The script needs no home and no
+        # writable path, so nobody/nogroup is enough.
+        runAsUser: 65534
+        runAsGroup: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: controller
+          # Pinned by digest: the date tag is rebuilt on every push of the
+          # image directory, so the tag alone cannot express a new build.
+          image: gitea.renner.uno/gitea_admin/netbird-proxy-services:2026-09-07@sha256:0d266b0288aec692c698ff5c05f76c826e247bfef365e0306a0b9a6541ebf180
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: NB_MANAGEMENT_URL
+              value: https://netbird.renner.uno
+            - name: BAO_ADDR
+              value: http://openbao.openbao.svc.cluster.local:8200
+            - name: BAO_ROLE
+              value: netbird-proxy-services
+            # Must match cluster.network.serviceSubnets in talos/patches/cluster.yaml.
+            - name: SERVICE_CIDR
+              value: "10.96.0.0/12"
+            - name: TRAEFIK_SERVICE
+              value: ingress/traefik
+            - name: INTERVAL_SECONDS
+              value: "60"
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              memory: 64Mi
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - deployment.yaml
@@ -0,0 +1,23 @@
+# OpenBao policy: netbird-proxy-services
+#
+# The reverse-proxy-services controller (Deployment netbird-proxy-services in
+# the openbao namespace) reads the NetBird PAT on every reconcile pass and
+# calls the management API with it. Read-only on that one path; it shares the
+# bootstrap credential with rotator-netbird but not that role's PATCH rights.
+#
+# NOT applied by Flux. Apply ONCE with an admin token:
+#
+#   bao policy write netbird-proxy-services kubernetes/infrastructure/openbao/policies/netbird-proxy-services.hcl
+#   bao write auth/kubernetes/role/netbird-proxy-services \
+#     bound_service_account_names=netbird-proxy-services \
+#     bound_service_account_namespaces=openbao \
+#     policies=netbird-proxy-services ttl=15m
+#
+# The PAT itself is the one rotator-netbird already uses
+# (homelab/infrastructure/secret-rotators-bootstrap/netbird, key `pat`).
+#
+# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
+
+path "homelab/data/infrastructure/secret-rotators-bootstrap/netbird" {
+  capabilities = ["read"]
+}
@@ -10,33 +10,8 @@
 # name hairpins off the ingress LB from a pod. Needs BOTH netpol halves
 # (gitea-allow-flux-rotator + rotator-gitea-flux-egress).
 #
-# The ExternalSecret MERGES into Flux's existing flux-system secret (does not
-# own it) so a sync failure leaves the last-good value in place; Flux's
-# source-controller reads it fresh each fetch, no restart. OpenBao policy/role
-# rotator-gitea-flux + the gitea-flux bootstrap (flux account password) are
-# provisioned by hand (see policies/rotator-gitea-flux.hcl).
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: flux-system-git-auth
-  namespace: flux-system
-spec:
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: flux-system
-    creationPolicy: Merge
-  data:
-    - secretKey: username
-      remoteRef:
-        key: infrastructure/flux-git
-        property: username
-    - secretKey: password
-      remoteRef:
-        key: infrastructure/flux-git
-        property: token
+# The ExternalSecret that publishes the rotated token to Flux lives in
+# ../../post-crds/flux-git-externalsecret.yaml (needs ESO's CRDs).
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -0,0 +1,11 @@
+# traefik.io CRD consumer for the public LAPI endpoint (crowdsec/lapi-ingress.yaml
+# holds the Ingress). The CA it names is the chart's self-signed LAPI CA.
+apiVersion: traefik.io/v1alpha1
+kind: ServersTransport
+metadata:
+  name: crowdsec-lapi
+  namespace: crowdsec
+spec:
+  serverName: crowdsec-service.crowdsec.svc.cluster.local
+  rootCAs:
+    - secret: crowdsec-root-secret
@@ -0,0 +1,60 @@
+# Keeps Flux's flux-system secret supplied with the rotated Gitea credential
+# WITHOUT owning it, so a sync failure leaves the last-good value in place;
+# Flux's source-controller reads it fresh each fetch, no restart. OpenBao
+# policy/role rotator-gitea-flux + the gitea-flux bootstrap (flux account
+# password) are provisioned by hand (see ../openbao/policies/rotator-gitea-flux.hcl).
+#
+# See creationPolicy below for why that is Orphan and not Merge — the
+# distinction is what makes a recovery from the break-glass source possible.
+#
+# Split out of ../openbao/rotation/flux-git.yaml, which still holds the
+# rotation CronJob: external-secrets.io CRDs are installed by the ESO
+# HelmRelease in the infrastructure Kustomization, so this manifest cannot
+# pass server-side dry-run there on a fresh cluster.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: flux-system-git-auth
+  namespace: flux-system
+spec:
+  refreshInterval: 10m
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: flux-system
+    # Orphan, not Merge: create the Secret if it is absent, but never own it.
+    #
+    # Merge cannot create — it only writes into a Secret that already exists,
+    # on the assumption that `flux bootstrap` made one. That assumption breaks
+    # in exactly the situation this cluster is rebuilt from. The break-glass
+    # bootstrap pulls from the GitHub mirror and keeps its token in
+    # flux-system-breakglass (bootstrap.sh), so `flux-system` is never created,
+    # so this ExternalSecret has nothing to merge into — and reports Ready while
+    # producing no Secret at all. The operator then performs the documented
+    # handover back to Gitea and finds the GitRepository pointing at a Secret
+    # that does not exist.
+    #
+    # Owner would create it but attach an ownerReference, so deleting this
+    # object would take Flux's credential with it. Orphan creates and updates
+    # without ownership, which is what the paragraph at the top of this file
+    # actually wanted: a sync failure, or the loss of this object, leaves the
+    # last-good value in place.
+    #
+    # Safe on a live cluster: the Secret holds exactly the two keys written
+    # below (verified against production), so writing it whole is the same
+    # result as merging into it.
+    creationPolicy: Orphan
+    # Explicit rather than relying on the default, because it is the half that
+    # matters here: if the OpenBao entry ever disappears, keep serving the
+    # credential Flux already has rather than leaving it with none.
+    deletionPolicy: Retain
+  data:
+    - secretKey: username
+      remoteRef:
+        key: infrastructure/flux-git
+        property: username
+    - secretKey: password
+      remoteRef:
+        key: infrastructure/flux-git
+        property: token
@@ -10,4 +10,26 @@ resources:
  # configured (2026-07-13): a store that can't turn Ready would block this
  # KS's wait gate and freeze the dependsOn chain.
  - openbao-clustersecretstore.yaml
+  # ExternalSecrets. external-secrets.io CRDs come from the ESO HelmRelease in
+  # the infrastructure Kustomization, so these cannot pass its server-side
+  # dry-run on a fresh cluster — Flux rejects the whole apply set, including
+  # the ESO HelmRelease that would have installed the CRDs. They sat next to
+  # their consumers until a cold-rebuild test hit that deadlock (2026-09-01);
+  # nothing had ever applied this layer against an empty API server before.
+  - crowdsec-bouncer-key-externalsecret.yaml
+  - crowdsec-edge-bouncer-key-externalsecret.yaml
+  - external-dns-unifi-externalsecret.yaml
+  - flux-git-externalsecret.yaml
+  - netbird-ingress-externalsecret.yaml
+  - velero-externalsecret.yaml
+  - velero-hetzner-creds-externalsecret.yaml
+  # traefik.io CRDs, same reasoning — installed by the Traefik HelmRelease in
+  # infrastructure. Traefik is deliberately absent from that Kustomization's
+  # healthChecks, so this KS may dry-run before the chart lands; it retries on
+  # retryInterval rather than deadlocking, because nothing here installs the
+  # CRDs it is waiting for.
+  - traefik-crowdsec-middleware.yaml
+  - traefik-rate-limit-middleware.yaml
+  - traefik-security-headers-middleware.yaml
+  - crowdsec-lapi-serverstransport.yaml
  - ../network-policies/
@@ -5,7 +5,7 @@ metadata:
  namespace: rook-ceph
 spec:
  cephVersion:
-    image: quay.io/ceph/ceph:v19.2.6 # {"$imagepolicy": "flux-system:ceph"}
+    image: quay.io/ceph/ceph:v20.2.4 # {"$imagepolicy": "flux-system:ceph"}
    allowUnsupported: false
  dataDirHostPath: /var/lib/rook
  # Defaults to false. With three OSDs and size 3 there is no spare copy, so
@@ -78,16 +78,35 @@ spec:
    enabled: true
    ssl: false
  monitoring:
-    enabled: true
+    # false, and this is not a decision about metrics. `metricsDisabled` is the
+    # switch that turns metrics off; this one only decides whether Rook creates
+    # the ServiceMonitors — and creating them is fatal to the reconcile when it
+    # fails. On an empty cluster monitoring.coreos.com/v1 does not exist, so
+    # Rook's node-daemon reconcile aborted, the CephCluster never left
+    # status.phase Progressing, and infrastructure-ceph-cluster's healthCheckExpr
+    # waits on exactly that — with grafana-db → monitoring → those same CRDs
+    # behind it, closing the loop.
+    #
+    # The ceph-exporter DaemonSet and the mgr metrics endpoint are untouched;
+    # only the scrape wiring moved, to apps/monitoring-rules/
+    # rook-servicemonitors.yaml, where the CRDs exist by the time it applies.
+    # Both objects are reproduced there field for field off the live cluster.
+    enabled: false
  # Dedicated Ceph cluster network on VLAN 49, attached to the OSDs through
  # Multus. The attachment is san-network-attachment.yaml.
  #
-  # Only `cluster` is selected. Do not add `public`, and do not reach for
-  # `addressRanges` — that is honoured only under provider: host, which puts
-  # every daemon including the mons on host networking. A mon's address then
-  # becomes the node's DHCP lease, and when a lease moves the mon crash-loops
-  # on "Cannot assign requested address" with no node free to fail over to,
-  # because all three already hold the mon host ports.
+  # Only `cluster` is selected. Do not add `public`: under provider: host
+  # that puts every daemon including the mons on host networking. A mon's
+  # address then becomes the node's DHCP lease, and when a lease moves the mon
+  # crash-loops on "Cannot assign requested address" with no node free to
+  # fail over to, because all three already hold the mon host ports.
+  #
+  # addressRanges.cluster is stated, not discovered. Without it Rook runs a
+  # canary pod on an OSD node to read the CIDR off the attachment on EVERY
+  # reconcile; the attachment is host-device, so the OSD on that node already
+  # holds the NIC and the canary can never attach — the reconcile blocks for
+  # 15 minutes, fails, and repeats. The value must equal the whereabouts range
+  # in san-network-attachment.yaml.
  #
  # With `public` unset, client traffic stays on the pod network and VLAN 49
  # carries Ceph-internal traffic only — which is also why Rook's Multus
@@ -104,6 +123,9 @@ spec:
    provider: multus
    selectors:
      cluster: rook-ceph/ceph-san
+    addressRanges:
+      cluster:
+        - 10.99.0.0/29
  # Nodes listed explicitly, not useAllNodes: under useAllNodes the operator
  # re-provisions an OSD on any node whose device still carries a BlueStore
  # signature, so purging an OSD before pulling its drive is impossible — it
@@ -131,10 +153,12 @@ spec:
    useAllDevices: false
    # Two forms while the production drives are migrated:
    #
-    #   by-partlabel/r-ceph-osd  the target. Talos names the drive in the
-    #                            machine config (RawVolumeConfig ceph-osd), so
+    #   by-partlabel/r-osd-data  the target. Talos names the drive in the
+    #                            machine config (RawVolumeConfig osd-data), so
    #                            this file needs to know nothing about models,
-    #                            serials or hostnames.
+    #                            serials or hostnames. The label must not
+    #                            contain "ceph": ceph-volume rejects any such
+    #                            partition as a legacy ceph-disk member.
    #   by-id/nvme-SAMSUNG…      the three MS-01 drives as they are today:
    #                            bluestore written straight onto the raw disk,
    #                            with no partition table and therefore no
@@ -146,7 +170,7 @@ spec:
    #
    # The second half goes when those three are repartitioned, one node at a
    # time. Until then it is what keeps the running OSDs claimed.
-    devicePathFilter: "^/dev/disk/by-(partlabel/r-ceph-osd|id/nvme-SAMSUNG_MZQLB960HAJR)"
+    devicePathFilter: "^/dev/disk/by-(partlabel/r-osd-data|id/nvme-SAMSUNG_MZQLB960HAJR)"
    config:
      osdsPerDevice: "1"
  # Highest built-in class: kubelet evicts these last and the kernel gives
@@ -3,33 +3,26 @@
 # without every Ceph daemon landing on host networking to get there. The
 # alternative and why it was rejected is in cephcluster.yaml.
 #
+# host-device: the SAN NIC is dedicated to Ceph and has exactly one consumer
+# per node — the OSD pod — so it is handed to that pod whole, not virtualised
+# under it. The device keeps its identity (MAC, MTU) and moves back to the
+# host when the pod goes away.
+#
+# `device` names the role, not hardware: every machine renames its own SAN
+# interface to ceph-san at boot (udev rule in its Talos patch). No kernel
+# interface name appears in this file.
+#
 # ISOLATION
 #
-# VLAN 49 carries Ceph-internal traffic and nothing else. Two consequences
-# are deliberate:
+# VLAN 49 carries Ceph-internal traffic and nothing else. No gateway and no
+# routes: there is no router on this segment and there must not be one.
+# whereabouts adds neither unless asked, so nothing here can leak onto or off
+# the SAN.
 #
-#   No gateway and no routes. There is no router on this segment and there
-#   must not be one. whereabouts adds neither unless asked, so nothing here
-#   can leak onto or off the SAN. This also means no `sbr` chaining is
-#   needed — unlike the v30iot attachments, there is no DHCP-supplied default
-#   route to confine, because there is no DHCP and no default route.
-#
-#   The pool is the whole usable /29, .1 through .6. It was .4-.6 while the
-#   migration ran, because the nodes still held .1-.3 for as long as Ceph
-#   needed host networking to reach this link — three addresses for three
-#   OSDs, no spare. The nodes have since given those up (the link is only a
-#   macvlan parent now and needs no address of its own), so the reservation
-#   has nothing left to avoid.
-#
-#   Keeping the pool inside the /29 rather than moving it elsewhere on the
-#   segment is not cosmetic. There is no router here, so an address outside
-#   the /29 would be unreachable from anything still using one inside it —
-#   which is exactly what would break a future rolling change, where migrated
-#   and un-migrated OSDs have to keep talking to each other.
-#
-#   Six addresses for three OSDs is the headroom the migration could not
-#   afford: room for a leaked allocation to sit until the reconciler collects
-#   it, and room to add a second OSD per node without touching this file.
+# The pool is the whole usable /29, .1 through .6 — headroom for a leaked
+# allocation to sit until the reconciler collects it. One address per node is
+# in use; a second OSD on a node would need a second SAN interface, which is a
+# hardware change, not a config one.
 #
 # IPAM
 #
@@ -37,29 +30,18 @@
 # a server that must not exist here, and host-local keeps its state per node
 # with no cluster-wide coordination, so two nodes would hand out the same
 # address on shared L2. See infrastructure/multus/whereabouts-daemonset.yaml.
-#
-# MTU 9000 matches the link. macvlan children inherit the master's MTU, but
-# it is stated so that a change to the parent does not silently halve
-# replication throughput without anything in the tree recording the
-# expectation.
 apiVersion: k8s.cni.cncf.io/v1
 kind: NetworkAttachmentDefinition
 metadata:
  name: ceph-san
  namespace: rook-ceph
 spec:
-  # `master` is an interface name. enp2s0f1np1 is the second X710 SFP+ port,
-  # a busPath-derived predictable name (net.ifnames=1 via the Image Factory
-  # schematic) that is identical on k8s-1/2/3 and cannot shuffle across
-  # reboots.
  config: |
    {
      "cniVersion": "0.3.1",
      "name": "ceph-san",
-      "type": "macvlan",
-      "master": "enp2s0f1np1",
-      "mode": "bridge",
-      "mtu": 9000,
+      "type": "host-device",
+      "device": "ceph-san",
      "ipam": {
        "type": "whereabouts",
        "range": "10.99.0.0/29",
@@ -38,7 +38,7 @@ spec:
      serviceAccountName: rook-ceph-default
      containers:
        - name: rook-ceph-tools
-          image: quay.io/ceph/ceph:v19.2.6 # {"$imagepolicy": "flux-system:ceph"}
+          image: quay.io/ceph/ceph:v20.2.4 # {"$imagepolicy": "flux-system:ceph"}
          command:
            - /bin/bash
            - -c
@@ -93,13 +93,17 @@ spec:
        # Real client-IP visibility. The netbird-ingress routing peer
        # re-originates every mesh + WAN-via-proxy connection from its own
        # pod IP (rootless userspace WireGuard), so Traefik's TCP peer for
-        # that traffic is always a pod IP (e.g. fd10:244::2de1) — collapsing
-        # all external clients to one source. The NetBird reverse-proxy
-        # (HTTP mode, on the VPS) sets X-Forwarded-For with the real WAN
-        # client IP; trust the pod CIDRs so Traefik reads it instead. Without
-        # this, a CrowdSec bouncer bans the shared pod IP and 403s everyone
-        # (the 2026-06-27 IPv6 outage). Harmless no-op when no XFF is present
-        # (direct LAN/mesh) — those keep their RemoteAddr.
+        # that traffic is always a pod IP — collapsing all external clients
+        # to one source. The NetBird reverse-proxy services are TLS
+        # passthrough and carry the WAN client address in a PROXY protocol
+        # v2 header (options.proxy_protocol, set by netbird-proxy-services);
+        # X-Forwarded-For stays trusted for HTTP-mode services. Both are
+        # accepted from the pod CIDRs only; connections without a header
+        # (direct LAN/mesh) keep their RemoteAddr.
+        proxyProtocol:
+          trustedIPs:
+            - "10.244.0.0/16"
+            - "fd10:244::/56"
        forwardedHeaders:
          trustedIPs:
            - "10.244.0.0/16"
@@ -2,6 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - helmrelease.yaml
-  - crowdsec-middleware.yaml
-  - security-headers-middleware.yaml
-  - rate-limit-middleware.yaml
@@ -1,13 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - externalsecret.yaml
-  - hetzner-creds-externalsecret.yaml
  - snapshot-readonly-storageclass.yaml
  - node-agent-config.yaml
  - helmrelease.yaml
-  - servicemonitor.yaml
-  - podmonitor.yaml
  - restore-test-rbac.yaml
  - restore-test-cronjob.yaml
 # Velero Schedule CRs live in kubernetes/infrastructure/post-crds/
@@ -57,7 +57,7 @@ up: true
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
  diskSelector:
    # Two disks: the system disk and the attached volume. Matching on transport
@@ -42,7 +42,7 @@ up: true
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
  diskSelector:
    match: '!system_disk'
@@ -42,7 +42,7 @@ up: true
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
  diskSelector:
    match: '!system_disk'
@@ -14,6 +14,13 @@ machine:
    # Disabling v6 on a VLAN parent does not affect the VLAN child.
    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
    net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    # That network also carries the mgmt VLAN's ARP. Answer ARP only on the
+    # link that owns the address, or the UDM learns the node IP against
+    # enp87s0's MAC and delivers IPv4 to a NIC Cilium is not attached to (#793).
+    net.ipv4.conf.all.arp_ignore: "1"
+    net.ipv4.conf.default.arp_ignore: "1"
+    net.ipv4.conf.all.arp_announce: "2"
+    net.ipv4.conf.default.arp_announce: "2"
  install:
    # Select disks by serial only. A broad selector such as
    # `disk.transport == 'nvme'` matches every drive in the machine, and the
@@ -28,6 +35,13 @@ machine:
    # this label instead of listing hostnames, so which boxes carry storage is
    # stated once, here, by the machine that knows.
    storage.renner.uno/ceph-osd: "true"
+  udev:
+    # Renames the SAN port (X710 port 1, matched by its burned-in MAC) to the
+    # role name at boot, before the network stack opens links — a rule applied
+    # to a RUNNING node cannot rename (Resource busy) and takes effect at the
+    # next reboot.
+    rules:
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:70:28", NAME="ceph-san"
 cluster:
  allowSchedulingOnControlPlanes: true
 ---
@@ -124,14 +138,15 @@ parent: enp2s0f0np0
 up: true
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
-# wire. OSD-to-OSD replication and nothing else.
+# wire. OSD-to-OSD replication and nothing else. The udev rule in the machine
+# section above renames the port to ceph-san at boot, so shared manifests name
+# the role and only this file knows the hardware.
 #
-# The node holds NO address here on purpose. Rook attaches the link to the OSD
-# pods through a Multus macvlan, and a macvlan parent only needs to be up.
-# Giving the node an address would put it on a segment reserved for
-# Ceph-internal traffic.
+# The node holds NO address here on purpose: the link is handed whole to the
+# OSD pod (host-device attachment). Giving the node an address would put it on
+# a segment reserved for Ceph-internal traffic.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: enp2s0f1np1
+name: ceph-san
 up: true
 mtu: 9000
@@ -12,6 +12,11 @@ machine:
    # cluster IPv6 egress.
    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
    net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
+    net.ipv4.conf.all.arp_ignore: "1"
+    net.ipv4.conf.default.arp_ignore: "1"
+    net.ipv4.conf.all.arp_announce: "2"
+    net.ipv4.conf.default.arp_announce: "2"
  install:
    # Select the installer target by serial so it cannot land on the OSD drive
    # or on EPHEMERAL, whatever order the kernel enumerates them in.
@@ -23,6 +28,10 @@ machine:
    topology.kubernetes.io/zone: saturn
    # Holds a Ceph OSD drive — see k8s-1.yaml.
    storage.renner.uno/ceph-osd: "true"
+  udev:
+    # Renames the SAN port to the role name at boot — see k8s-1.yaml.
+    rules:
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:76:6f:74", NAME="ceph-san"
 cluster:
  allowSchedulingOnControlPlanes: true
 ---
@@ -95,11 +104,12 @@ up: true
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.
+# Renamed to the role name by the udev rule above — see k8s-1.yaml.
 #
-# The node holds NO address here on purpose. Rook attaches the link to the OSD
-# pods through a Multus macvlan, and a macvlan parent only needs to be up.
+# The node holds NO address here on purpose: the link is handed whole to the
+# OSD pod (host-device attachment).
 apiVersion: v1alpha1
 kind: LinkConfig
-name: enp2s0f1np1
+name: ceph-san
 up: true
 mtu: 9000
@@ -8,6 +8,11 @@ machine:
    # k8s-1.yaml.
    net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
    net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
+    net.ipv4.conf.all.arp_ignore: "1"
+    net.ipv4.conf.default.arp_ignore: "1"
+    net.ipv4.conf.all.arp_announce: "2"
+    net.ipv4.conf.default.arp_announce: "2"
  install:
    # Select the installer target by serial so it cannot land on the OSD drive
    # or on EPHEMERAL, whatever order the kernel enumerates them in.
@@ -19,6 +24,10 @@ machine:
    topology.kubernetes.io/zone: neptune
    # Holds a Ceph OSD drive — see k8s-1.yaml.
    storage.renner.uno/ceph-osd: "true"
+  udev:
+    # Renames the SAN port to the role name at boot — see k8s-1.yaml.
+    rules:
+      - SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="58:47:ca:74:72:d4", NAME="ceph-san"
 cluster:
  allowSchedulingOnControlPlanes: true
 ---
@@ -78,9 +87,9 @@ parent: enp2s0f0np0
 up: true
 ---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
-# here on purpose; the link is only a macvlan parent for the OSD pods.
+# here on purpose; the link is handed whole to the OSD pod (host-device).
 apiVersion: v1alpha1
 kind: LinkConfig
-name: enp2s0f1np1
+name: ceph-san
 up: true
 mtu: 9000
@@ -43,3 +43,15 @@ apiVersion: v1alpha1
 kind: HostnameConfig
 auto: off
 hostname: vm-1
+---
+# Ceph cluster network — see vm-2.yaml for why this is a VLAN, and why the
+# DHCPv4Config below is load-bearing.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ceph-san
+parent: enp0s2
+vlanID: 30
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: enp0s2
@@ -21,7 +21,7 @@ auto: off
 hostname: vm-2
 ---
 # The OSD device, named by the machine that has it. Ceph then consumes
-# /dev/disk/by-partlabel/r-ceph-osd and needs to know nothing about models,
+# /dev/disk/by-partlabel/r-osd-data and needs to know nothing about models,
 # serials or hostnames — the same CephCluster works here and on the MS-01s.
 #
 # The selector is a CEL expression, so each machine says whatever identifies its
@@ -29,9 +29,33 @@ hostname: vm-2
 # real nodes it will name the Samsung.
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
  diskSelector:
    match: disk.transport == "virtio" && !system_disk
  minSize: 5GiB
  grow: true
+---
+# The Ceph cluster network link. Production gives it a second physical port;
+# this machine has one NIC, so the segment is a VLAN on it.
+#
+# A dummy link would be simpler and worthless — no traffic between nodes, so
+# Ceph would replicate over an interface going nowhere and the test would pass
+# having proven nothing. A VLAN rides the provisioner's bridge that every VM is
+# on, so the SAN carries real frames on its own tag.
+#
+# No MTU, unlike production's 9000: the qemu NIC is created with host_mtu=1500.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ceph-san
+parent: enp0s2
+vlanID: 30
+---
+# Load-bearing. Declaring any network document makes Talos configure this link
+# explicitly, and the implicit DHCP that gives the node its address stops. A
+# first attempt with only the VLANConfig took vm-4 off the network — NotReady,
+# unreachable over the Talos API. Verified on a second throwaway node that this
+# document keeps the parent addressed.
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: enp0s2
@@ -15,9 +15,21 @@ hostname: vm-3
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
  diskSelector:
    match: disk.transport == "virtio" && !system_disk
  minSize: 5GiB
  grow: true
+---
+# Ceph cluster network — see vm-2.yaml for why this is a VLAN, and why the
+# DHCPv4Config below is load-bearing.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ceph-san
+parent: enp0s2
+vlanID: 30
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: enp0s2
@@ -19,9 +19,21 @@ hostname: vm-4
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
  diskSelector:
    match: disk.transport == "virtio" && !system_disk
  minSize: 5GiB
  grow: true
+---
+# Ceph cluster network — see vm-2.yaml for why this is a VLAN, and why the
+# DHCPv4Config below is load-bearing.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ceph-san
+parent: enp0s2
+vlanID: 30
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: enp0s2