fix(llama-server): stable model alias, and thinking off by default

Two problems found by exercising the live endpoint after #727. The model id was the full --hf-repo string, so callers had to ask for "ggml-org/Qwen3.6-35B-A3B-GGUF:Q4_K_M". That name changes whenever the model does, which would break every consumer's config on each swap. --alias chat pins a stable name that survives model changes. Qwen3.6 also enables thinking by default. A request with max_tokens 300 came back with an empty `content` and finish_reason "length" — the entire budget had gone into reasoning_content without ever reaching an answer. At ~11.6 tok/s that is minutes of reasoning before a reply, which is wrong for voice and for short answers generally. Verified against the live endpoint: the same prompt with chat_template_kwargs {"enable_thinking": false} returns a correct two-sentence German answer in 95 tokens at 12.0 tok/s. Off by default; a client that wants reasoning can still ask for it per request. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Merge pull request 'feat(llama-server): split embeddings off, move chat to Qwen3.6-35B-A3B' (#727 ) from llama-split-embed into master
2026-09-01 16:31:30 +02:00 · 2026-09-01 14:26:44 +00:00 · 2026-09-01 16:25:04 +02:00 · 2026-09-01 13:54:39 +00:00 · 2026-09-01 15:51:41 +02:00 · 2026-09-01 12:31:13 +00:00
122 changed files with 2478 additions and 6165 deletions
@@ -32,13 +32,13 @@
 #              mechanism as update-audit.yml):
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name alert-monitor --branch master --schedule '*/15 * * * *'
-#   - manual : woodpecker pipeline create --branch master \
-#                --var PIPELINE=alert-monitor k.renner/homelab
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> alert-monitor
+#              where <n> is an existing pipeline to promote.
 when:
  - event: cron
    cron: alert-monitor
-  - event: manual
-    evaluate: 'PIPELINE == "alert-monitor"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "alert-monitor"'

 steps:
  - name: fetch-alerts
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -31,8 +31,8 @@ when:
      - "ci/images/**"
      - "secret-rotator/**"
      - ".woodpecker/build-ci-images.yml"
-  - event: manual
-    evaluate: 'PIPELINE == "build-ci-images"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-ci-images"'

 steps:
  - name: build-ansible
@@ -44,7 +44,7 @@ steps:
    # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
    # orphaned ~5G of registry layers per run — a main driver of #336.
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -68,7 +68,7 @@ steps:
    directory: ci/images/k8s
    depends_on: []
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -92,7 +92,7 @@ steps:
    directory: ci/images/tofu
    depends_on: []
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -123,7 +123,7 @@ steps:
    directory: secret-rotator
    depends_on: []
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -152,7 +152,7 @@ steps:
    directory: ci/images/alpine-jq
    depends_on: []
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -181,7 +181,7 @@ steps:
    directory: ci/images/cni-plugins
    depends_on: []
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -210,7 +210,7 @@ steps:
    directory: ci/images/alpine-openssl
    depends_on: []
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -239,7 +239,7 @@ steps:
    directory: ci/images/heating-can
    depends_on: []
    when:
-      - event: manual
+      - event: deployment
      - event: push
        path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -54,8 +54,8 @@
 #
 # Triggering:
 #   - cron   : the cron NAMED exactly "ci-claude-refresh" (see above).
-#   - manual : woodpecker pipeline create --branch master \
-#                --var PIPELINE=ci-claude-refresh k.renner/homelab
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> ci-claude-refresh
+#              where <n> is an existing pipeline to promote.
 #   - push   : a change to the Containerfile (or this file) ON master forces a
 #              rebuild of the current latest version so Containerfile fixes ship
 #              promptly. Restricted to master ON PURPOSE — a feature branch must
@@ -65,8 +65,8 @@
 when:
  - event: cron
    cron: ci-claude-refresh
-  - event: manual
-    evaluate: 'PIPELINE == "ci-claude-refresh"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "ci-claude-refresh"'
  - event: push
    branch: master
    path:
@@ -17,6 +17,6 @@ when:

 steps:
  - name: model-selection-tests
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
    commands:
      - sh scripts/ci/test-model-selection.sh
@@ -6,8 +6,8 @@ when:
      - "ansible/roles/claude-workstation/**"
      - "ansible/plays/install-claude-workstation.yml"
      - ".woodpecker/deploy-claude-workstation.yml"
-  - event: manual
-    evaluate: 'PIPELINE == "deploy-claude-workstation"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'

 timeout: 30

@@ -8,8 +8,8 @@ when:
    path:
      - "opentofu/**"
      - ".woodpecker/deploy-infrastructure.yml"
-  - event: manual
-    evaluate: 'PIPELINE == "deploy-infrastructure"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-infrastructure"'

 steps:
  - name: validate-secrets
@@ -84,7 +84,7 @@ steps:
    image: gitea.renner.uno/gitea_admin/ci-tofu:2026-04-29
    directory: opentofu
    when:
-      - event: manual
+      - event: deployment
        evaluate: 'TOFU_APPLY == "true"'
    environment:
      TF_IN_AUTOMATION: "true"
@@ -7,8 +7,8 @@ when:
      - "ansible/plays/install-netbird-client.yml"
      - "ansible/inventory/group_vars/claude_workstation/**"
      - ".woodpecker/deploy-netbird-client.yml"
-  - event: manual
-    evaluate: 'PIPELINE == "deploy-netbird-client"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'

 timeout: 30

@@ -9,8 +9,8 @@ when:
      - "ansible/roles/netbird-proxy-watchdog/**"
      - "ansible/plays/install-netbird-proxy.yml"
      - ".woodpecker/deploy-netbird-proxy.yml"
-  - event: manual
-    evaluate: 'PIPELINE == "deploy-netbird-proxy"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'

 timeout: 30

@@ -9,8 +9,8 @@ when:
      - "ansible/roles/netbird-proxy-watchdog/**"
      - "ansible/plays/install-netbird-server.yml"
      - ".woodpecker/deploy-netbird-server.yml"
-  - event: manual
-    evaluate: 'PIPELINE == "deploy-netbird-server"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'

 timeout: 30

@@ -1,6 +1,6 @@
 when:
-  - event: manual
-    evaluate: 'PIPELINE == "e2e-test"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e-test"'

 steps:
  - name: e2e
@@ -7,7 +7,8 @@
 #                was harmless at daily/weekly cadence — but alert-monitor.yml
 #                fires every 15 minutes, so piggybacking must be opt-in now.
 #                A new cron that SHOULD also run e2e gets its name added here.
-#   - manual   : `woodpecker pipeline create --branch master --var PIPELINE=e2e k.renner/homelab`
+#   - deploy   : `woodpecker-cli pipeline deploy k.renner/homelab <n> e2e`, where
+#                <n> is an existing pipeline to promote.

 when:
  - event: cron
@@ -16,8 +17,8 @@ when:
    cron: update-audit
  - event: cron
    cron: weekly-update
-  - event: manual
-    evaluate: 'PIPELINE == "e2e"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e"'

 steps:
  - name: model-selection-tests
@@ -25,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -1,27 +0,0 @@
-when:
-  - event: manual
-    evaluate: 'PIPELINE == "install-garage-synology"'
-
-steps:
-  - name: run-playbook
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      PROXMOX_TOKEN_ID:
-        from_secret: proxmox_token_id
-      PROXMOX_TOKEN_SECRET:
-        from_secret: proxmox_token_secret
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - 'printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config'
-      - cd ansible
-      - echo "$$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-garage-synology.yml -v
@@ -34,13 +34,13 @@
 #   - cron   : a cron NAMED exactly "issue-fix":
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name issue-fix --branch master --schedule '41 * * * *'
-#   - manual : woodpecker pipeline create --branch master \
-#                --var PIPELINE=issue-fix k.renner/homelab
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> issue-fix
+#              where <n> is an existing pipeline to promote.
 when:
  - event: cron
    cron: issue-fix
-  - event: manual
-    evaluate: 'PIPELINE == "issue-fix"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "issue-fix"'

 steps:
  - name: fetch-diag
@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -53,9 +53,9 @@
 #
 # Triggering: pull_request only — automatic on PR open + push-to-open-PR. To
 # re-review a PR, push a commit (the new head sha clears the dedup marker below
-# and earns a fresh review). There is deliberately no manual PIPELINE trigger:
-# Woodpecker exposes a --var only to YAML variable substitution and `when`
-# filters, NOT to a step's shell environment, so a --var PR=<n> re-review would
+# and earns a fresh review). There is deliberately no on-demand trigger for a
+# specific PR: Woodpecker exposes injected variables to YAML substitution and
+# `when` filters, NOT to a step's shell environment, so a PR=<n> re-review would
 # silently no-op (the step would find no PR number and exit green).
 # (Note: keep literal dollar-brace out of THIS file entirely, even in comments —
 # Woodpecker runs its variable substitution over the whole YAML text, comments
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,8 +1,20 @@
+# Trivy scan of the IaC tree and every image referenced from ansible/, with an
+# agent pass that opens a PR for CRITICAL/HIGH findings.
+#
+# Triggers:
+#   - cron       : the named `daily-security-scan` cron, matched by NAME.
+#                  Crons cannot carry variables — `woodpecker-cli repo cron add`
+#                  has no --var — so a variable gate can never match a cron.
+#   - deployment : woodpecker-cli pipeline deploy k.renner/homelab <n> security-scan
+#                  The target rides in pipeline metadata as
+#                  CI_PIPELINE_DEPLOY_TARGET, which the workflow-root filter
+#                  does see, unlike manual-run variables on Woodpecker 3.18
+#                  (#582).
 when:
  - event: cron
-    evaluate: 'PIPELINE == "security-scan"'
-  - event: manual
-    evaluate: 'PIPELINE == "security-scan"'
+    cron: daily-security-scan
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "security-scan"'

 steps:
  - name: iac-scan
@@ -28,12 +40,28 @@ steps:
        cat /tmp/images.txt
        echo "---"
        i=0
+        failed=0
+        failed_names=""
        while IFS= read -r img; do
          i=$$((i+1))
          out="trivy-image-$$i.json"
          echo "=== [$$i] $$img -> $$out ==="
-          trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img" || true
+          # Keep going on a bad image so one unpullable reference cannot cost
+          # the scan of every image after it, but record that it happened.
+          if ! trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img"; then
+            failed=$$((failed+1))
+            failed_names="$$failed_names $$img"
+          fi
        done < /tmp/images.txt
+        echo "--- scanned $$((i - failed)) of $$i images ---"
+        if [ "$$failed" -gt 0 ]; then
+          echo "NOT SCANNED:$$failed_names"
+          exit 1
+        fi
+    # Non-zero above turns this step red without failing the workflow, so the
+    # fix agent still runs on what was scanned. Coverage is reported, not
+    # enforced: an incomplete scan is visible in the step status instead of
+    # being indistinguishable from a clean one.
    failure: ignore

  - name: security-fix-agent
@@ -42,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -10,14 +10,14 @@
 # report; the pipeline posts it to a single persistent Gitea issue.
 #
 # Triggering:
-#   - cron   : a cron NAMED exactly "update-audit". Unlike the other pipelines
-#              here (which gate on an injected `PIPELINE` var set at cron-create
-#              time in the UI), this uses Woodpecker's documented `cron:` name
-#              filter so the schedule can be created entirely from the CLI:
+#   - cron   : a cron NAMED exactly "update-audit". Crons carry no variables —
+#              `repo cron add` takes name, branch, schedule and enabled — so the
+#              name filter is the only thing that can select a workflow for a
+#              scheduled run:
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name update-audit --branch master --schedule '@weekly'
-#   - manual : woodpecker pipeline create --branch master \
-#                --var PIPELINE=update-audit k.renner/homelab
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> update-audit
+#              where <n> is an existing pipeline to promote.
 #
 # NOTE: e2e.yml runs on an explicit list of cron names that includes this one,
 # so a weekly e2e run piggybacks on the audit (it was a bare `event: cron`
@@ -25,12 +25,12 @@
 when:
  - event: cron
    cron: update-audit
-  - event: manual
-    evaluate: 'PIPELINE == "update-audit"'
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-audit"'

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,37 +0,0 @@
---
-# PBS Client configuration for nas-1
-pbs_backup_paths:
-  - /volume1/Documents
-  - /volume1/homes
-  - /volume1/Homeassistant Backup
-  - /volume1/Games
-  - /volume1/Media
-  - /volume1/Music
-  - /volume1/video
-  - /volume2/Backup
-  - /volume1/Movies
-
-  # Add more paths as needed:
-  # - /volume1/photos
-  # - /volume1/music
-
-# PBS Server connection
-pbs_server_host: "pbs2.renner.uno"
-pbs_server_port: 8007
-pbs_datastore: "pool"
-pbs_backup_id: "nas-1"
-
-# PBS Authentication - credentials in vault.yml
-pbs_username: "root@pam!nas-1"
-
-# Garage S3 (K8s backup target)
-garage_image: "dxflrs/garage:v2.3.0"
-garage_zone: "dsm"
-garage_capacity: "2T"
-
-# SSH connection - use password auth (key not supported by DSM)
-ansible_ssh_pass: "{{ nas_ssh_password }}"
-ansible_become_pass: "{{ nas_ssh_password }}"
-
-# Python 3.9+ from Synology package
-ansible_python_interpreter: /usr/local/bin/python3.9
@@ -1,27 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-34663462636437633437613863366266306334666635643839313737343739643864613136613938
-3138393966306561656366396465383539303137333036300a376237376636356230343834386362
-36636535396330333732383233323263613132333166343739396137636538653435336662353138
-6565643866623235610a316266383933303830323936303735623338623165386339383733373435
-62366461363833616438373237383132366438326364363462333265366533323861376630643365
-39343036343838333639363432643738666630363063323636646661313539643465366532363438
-33303837623234613434633563666562346536346435353461373536323737376233386163326537
-61383963343737656438356136613537623538333163343466343739346135333439616432663763
-36663962633037376561326434303035333938646436663832303432346365353439666435613337
-34363636326530333233313231343566306432376532396666643761633136316333376337666135
-65363338623035663438323338663863366362653131336434323865326530326534623631336263
-36633063383031386161363131333636393364623436613331353733313235383062656131313164
-63393665633361313961363637663237616364643236383835393736643937393564656238376362
-31393465383361326264376133393233616264663636383337363962356532613438663637616536
-33653137656335323763313461323633616635346232303062323062333766383931313965346361
-66656461313239633366636463633465636463663937393932636337643861383937663662306466
-32353166643832303234643264633062616330393963333034623432613730366339306637613239
-34626161663666316339323762373639353335653663353635366664643063393432623039306237
-38363161343930363763393364663364353735343236313665313461626165636135303839363164
-37336638396564326166376233353730346563656266323535666265323062646232323938353732
-32636235333839623166633761376334316434613235333730353439343262393537363461333238
-32653435623362666662336532643832356163326332636436346132643237643030343761663332
-36663530356539303636663438646430363335353135323932613863666264363533326137353164
-31663032346236616263343637396439663334396166656364623637666535386530383535663435
-66303037303265616634373763616437333835333136336563323339666563663931396133633131
-3963653266633535303630653335666166313336653831623362
@@ -8,6 +8,8 @@
 #   pve_cluster (jupiter/neptune/saturn)  run Talos on bare metal; no SSH at all
 #   authentik_db / authentik_app          authentik runs in-cluster (CNPG)
 #   grafana                               runs in-cluster
+#   synology_nas (nas-1)                  powered off; Garage and the PBS client
+#                                         it hosted are gone with it
 # Their hosts no longer resolve, except the three that resolve to Talos nodes.

 [netbird_server]
@@ -17,10 +19,6 @@ netbird.renner.uno ansible_user=root
 proxy-1.renner.uno ansible_user=root
 proxy-2.renner.uno ansible_user=root

-
-[synology_nas]
-nas-1.renner.uno ansible_user=ladmin ansible_python_interpreter=/usr/bin/python3
-
 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN
 # sub-interface. Nothing here assumes any peripheral is attached.
@@ -1,313 +0,0 @@
---
-# Synology-specific playbook using raw module (Python 3.8 workaround)
-# Deploys Garage (https://garagehq.deuxfleurs.fr) as a single-node S3 target
-# for K8s backups (Velero + CNPG Barman Cloud Plugin).
-#
-# Required secrets in host_vars/nas-1.renner.uno/vault.yml:
-#   garage_rpc_secret:     "<openssl rand -hex 32>"
-#   garage_admin_token:    "<openssl rand -base64 32>"
-#   garage_metrics_token:  "<openssl rand -base64 32>"
-#
-# Post-install bootstrap (run once, record the returned secret keys):
-#   docker exec garaged /garage status
-#   docker exec garaged /garage bucket create k8s-velero
-#   docker exec garaged /garage bucket create k8s-cnpg
-#   docker exec garaged /garage bucket create k8s-secrets
-#   docker exec garaged /garage bucket create k8s-migration
-#   docker exec garaged /garage key create velero-key
-#   docker exec garaged /garage key create cnpg-key
-#   docker exec garaged /garage key create secrets-key
-#   docker exec garaged /garage key create migration-key
-#   docker exec garaged /garage bucket allow --read --write --owner k8s-velero    --key velero-key
-#   docker exec garaged /garage bucket allow --read --write --owner k8s-cnpg      --key cnpg-key
-#   docker exec garaged /garage bucket allow --read --write --owner k8s-secrets   --key secrets-key
-#   docker exec garaged /garage bucket allow --read --write --owner k8s-migration --key migration-key
- name: Install Garage S3 on Synology NAS
-  hosts: synology_nas
-  gather_facts: false
-
-  vars:
-    garage_data_dir: "/volume1/docker/garage"
-
-  tasks:
-    - name: Create Garage directory tree
-      ansible.builtin.raw: |
-        mkdir -p {{ garage_data_dir }}/config \
-                 {{ garage_data_dir }}/meta \
-                 {{ garage_data_dir }}/data
-      changed_when: true
-
-    # Secret writes use the same compare-then-write pattern as garage.toml so
-    # rotation is idempotent AND a real change can drive force-recreate below.
-    # Garage reads these files once at startup; without container restart, a
-    # rotated token leaves the running process holding the previous value.
-    - name: Write rpc_secret
-      ansible.builtin.raw: |
-        cat > {{ garage_data_dir }}/config/rpc_secret.new << 'SECRET_EOF'
-        {{ garage_rpc_secret }}
-        SECRET_EOF
-        chmod 600 {{ garage_data_dir }}/config/rpc_secret.new
-        if [ ! -f {{ garage_data_dir }}/config/rpc_secret ] \
-        || ! cmp -s {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret; then
-          mv {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret
-          echo RPC_SECRET_CHANGED
-        else
-          rm {{ garage_data_dir }}/config/rpc_secret.new
-        fi
-      register: rpc_secret_write
-      changed_when: "'RPC_SECRET_CHANGED' in rpc_secret_write.stdout"
-      no_log: true
-
-    - name: Write admin_token
-      ansible.builtin.raw: |
-        cat > {{ garage_data_dir }}/config/admin_token.new << 'SECRET_EOF'
-        {{ garage_admin_token }}
-        SECRET_EOF
-        chmod 600 {{ garage_data_dir }}/config/admin_token.new
-        if [ ! -f {{ garage_data_dir }}/config/admin_token ] \
-        || ! cmp -s {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token; then
-          mv {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token
-          echo ADMIN_TOKEN_CHANGED
-        else
-          rm {{ garage_data_dir }}/config/admin_token.new
-        fi
-      register: admin_token_write
-      changed_when: "'ADMIN_TOKEN_CHANGED' in admin_token_write.stdout"
-      no_log: true
-
-    - name: Write metrics_token
-      ansible.builtin.raw: |
-        cat > {{ garage_data_dir }}/config/metrics_token.new << 'SECRET_EOF'
-        {{ garage_metrics_token }}
-        SECRET_EOF
-        chmod 600 {{ garage_data_dir }}/config/metrics_token.new
-        if [ ! -f {{ garage_data_dir }}/config/metrics_token ] \
-        || ! cmp -s {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token; then
-          mv {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token
-          echo METRICS_TOKEN_CHANGED
-        else
-          rm {{ garage_data_dir }}/config/metrics_token.new
-        fi
-      register: metrics_token_write
-      changed_when: "'METRICS_TOKEN_CHANGED' in metrics_token_write.stdout"
-      no_log: true
-
-    - name: Write garage.toml
-      ansible.builtin.raw: |
-        cat > {{ garage_data_dir }}/config/garage.toml.new << 'TOML_EOF'
-        metadata_dir = "/var/lib/garage/meta"
-        data_dir     = "/var/lib/garage/data"
-
-        # SQLite chosen over LMDB: single-node deployment has no peers to
-        # rebuild metadata from on unclean-shutdown corruption. SQLite is
-        # slower but portable and has no LMDB corruption window.
-        db_engine = "sqlite"
-
-        # 6-hour auto-snapshot of the metadata DB as a backstop against
-        # corruption; last two snapshots retained.
-        metadata_auto_snapshot_interval = "6h"
-
-        # Single-zone, single-node: only coherent replication factor.
-        replication_factor = 1
-
-        rpc_bind_addr    = "[::]:3901"
-        rpc_secret_file  = "/etc/garage/rpc_secret"
-
-        compression_level = 2
-
-        # 64 MB blocks (default 1 MB; previously 20 MB). On the 5 × 7 TB
-        # Btrfs RAID 5 volume the per-block fixed cost (SQLite lookup,
-        # dir traversal, hash verify, zstd decompress, nginx buffering)
-        # is the dominant overhead per object; larger blocks amortize
-        # it. block_size is a maximum, so 16 MiB CNPG WAL segments
-        # still upload as one block each. Only applies to NEW uploads;
-        # existing objects keep their original chunking until re-uploaded.
-        block_size = "64M"
-
-        [s3_api]
-        api_bind_addr = "[::]:3900"
-        s3_region     = "garage"
-        root_domain   = ".s3.{{ inventory_hostname }}"
-
-        [s3_web]
-        bind_addr   = "[::]:3902"
-        root_domain = ".web.{{ inventory_hostname }}"
-
-        [admin]
-        api_bind_addr      = "[::]:3903"
-        admin_token_file   = "/etc/garage/admin_token"
-        metrics_token_file = "/etc/garage/metrics_token"
-        TOML_EOF
-        if [ ! -f {{ garage_data_dir }}/config/garage.toml ] \
-        || ! cmp -s {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml; then
-          mv {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml
-          echo GARAGE_TOML_CHANGED
-        else
-          rm {{ garage_data_dir }}/config/garage.toml.new
-        fi
-      register: garage_toml_write
-      changed_when: "'GARAGE_TOML_CHANGED' in garage_toml_write.stdout"
-
-    - name: Write docker-compose.yml
-      ansible.builtin.raw: |
-        cat > {{ garage_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
-        services:
-          garage:
-            image: {{ garage_image }}
-            container_name: garaged
-            restart: unless-stopped
-            # HTTP services (3900/3902/3903) bind to loopback; DSM
-            # reverse proxy fronts and TLS-terminates them. RPC 3901
-            # is exposed directly — the Noise handshake is raw TCP,
-            # so DSM RP cannot front it. rpc_secret gates auth.
-            ports:
-              - "3901:3901"
-              - "127.0.0.1:3900:3900"
-              - "127.0.0.1:3902:3902"
-              - "127.0.0.1:3903:3903"
-            volumes:
-              - {{ garage_data_dir }}/config/garage.toml:/etc/garage.toml:ro
-              - {{ garage_data_dir }}/config/rpc_secret:/etc/garage/rpc_secret:ro
-              - {{ garage_data_dir }}/config/admin_token:/etc/garage/admin_token:ro
-              - {{ garage_data_dir }}/config/metrics_token:/etc/garage/metrics_token:ro
-              - {{ garage_data_dir }}/meta:/var/lib/garage/meta
-              - {{ garage_data_dir }}/data:/var/lib/garage/data
-        COMPOSE_EOF
-      changed_when: true
-
-    - name: Pull image and start Garage container
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        cd {{ garage_data_dir }} && docker compose pull && docker compose up -d
-      become: true
-      changed_when: true
-
-    # Garage does not reload config on SIGHUP and compose does not notice
-    # when a bind-mounted host file changes. Force-recreate the container
-    # when garage.toml OR any of the three secret files changed so the new
-    # config / rotated tokens are picked up.
-    - name: Force-recreate garaged when garage.toml or secrets changed
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        cd {{ garage_data_dir }} && docker compose up -d --force-recreate garage
-      become: true
-      when: garage_toml_write.changed
-            or rpc_secret_write.changed
-            or admin_token_write.changed
-            or metrics_token_write.changed
-      changed_when: true
-
-    - name: Wait for Garage to accept CLI calls
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        for i in $(seq 1 30); do
-          if docker exec garaged /garage status >/dev/null 2>&1; then
-            exit 0
-          fi
-          sleep 2
-        done
-        exit 1
-      become: true
-      changed_when: false
-
-    - name: Check whether cluster layout is already applied
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        docker exec garaged /garage status
-      become: true
-      register: garage_status
-      changed_when: false
-
-    - name: Apply single-node layout (idempotent — only on first run)
-      when: "'NO ROLE ASSIGNED' in garage_status.stdout"
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        NODE_ID=$(docker exec garaged /garage node id | head -n 1 | cut -d@ -f1)
-        docker exec garaged /garage layout assign "$NODE_ID" \
-          -z {{ garage_zone }} \
-          -c {{ garage_capacity }} \
-          -t {{ inventory_hostname_short }}
-        docker exec garaged /garage layout apply --version 1
-      become: true
-      changed_when: true
-
-    - name: Report Garage status
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        docker exec garaged /garage status
-      become: true
-      register: garage_status_final
-      changed_when: false
-
-    - name: Show Garage status
-      ansible.builtin.debug:
-        var: garage_status_final.stdout_lines
-
-    # Bootstrap buckets + keys. Idempotent: bucket_info / key_info return
-    # non-zero when missing, so we only create on first run. Access-key
-    # secrets are captured at create time and persisted to chmod-600 files
-    # under {{ garage_data_dir }}/config/keys/<key>.creds on nas-1. These
-    # files are the only post-install record of the secrets — Garage does
-    # not expose them again.
-    - name: Bootstrap Garage buckets, keys, and grants
-      ansible.builtin.shell: |
-        set -e
-        export PATH=$PATH:/usr/local/bin
-        KEYS_DIR={{ garage_data_dir }}/config/keys
-        mkdir -p "$KEYS_DIR"
-        chmod 700 "$KEYS_DIR"
-
-        bootstrap_pair() {
-          local BUCKET="$1"
-          local KEY="$2"
-          local CREDS_FILE="$KEYS_DIR/$KEY.creds"
-
-          if ! docker exec garaged /garage bucket info "$BUCKET" >/dev/null 2>&1; then
-            docker exec garaged /garage bucket create "$BUCKET"
-          fi
-
-          if ! docker exec garaged /garage key info "$KEY" >/dev/null 2>&1; then
-            CREATE_OUT=$(docker exec garaged /garage key create "$KEY")
-            KEY_ID=$(echo "$CREATE_OUT"   | awk '/^Key ID:/     {print $3}')
-            SECRET=$(echo "$CREATE_OUT"   | awk '/^Secret key:/ {print $3}')
-            umask 177
-            printf 'KEY_ID=%s\nSECRET_KEY=%s\n' "$KEY_ID" "$SECRET" > "$CREDS_FILE"
-          fi
-
-          docker exec garaged /garage bucket allow \
-            --read --write --owner \
-            "$BUCKET" --key "$KEY"
-        }
-
-        bootstrap_pair k8s-velero    velero-key
-        bootstrap_pair k8s-cnpg      cnpg-key
-        bootstrap_pair k8s-secrets   secrets-key
-        bootstrap_pair k8s-migration migration-key
-        bootstrap_pair tfstate       tfstate-key
-      become: true
-      changed_when: true
-      no_log: true
-
-    - name: Report bucket + key state (no secrets)
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        docker exec garaged /garage bucket list
-        echo "---"
-        docker exec garaged /garage key list
-        echo "---"
-        ls -l {{ garage_data_dir }}/config/keys/ 2>&1
-      become: true
-      register: garage_bootstrap_state
-      changed_when: false
-
-    - name: Show bucket + key state
-      ansible.builtin.debug:
-        var: garage_bootstrap_state.stdout_lines
-
-    - name: Post-install note
-      ansible.builtin.debug:
-        msg:
-          - "Buckets: k8s-velero, k8s-cnpg, k8s-secrets, k8s-migration, tfstate"
-          - "Keys:    velero-key, cnpg-key, secrets-key, migration-key, tfstate-key"
-          - "Secrets are written to {{ garage_data_dir }}/config/keys/<key>.creds on nas-1 (chmod 600, root-owned)."
-          - "SSH to nas-1 and `sudo cat` the file to retrieve KEY_ID + SECRET_KEY for Velero / CNPG / migration / tfstate config."
@@ -1,104 +0,0 @@
---
-# Synology-specific playbook using raw module (Python 3.8 workaround)
- name: Install PBS Client on Synology NAS
-  hosts: synology_nas
-  gather_facts: false
-
-  vars:
-    pbs_client_data_dir: "/volume1/docker/pbs-client"
-
-  tasks:
-    - name: Create PBS client directory
-      ansible.builtin.raw: |
-        mkdir -p {{ pbs_client_data_dir }}/state
-      changed_when: true
-
-    - name: Create docker-compose.yml
-      ansible.builtin.raw: |
-        cat > {{ pbs_client_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
-        services:
-          pbs-client:
-            image: proxmox/proxmox-backup-client:latest
-            container_name: pbs-client
-            restart: unless-stopped
-            hostname: {{ pbs_backup_id }}
-            env_file:
-              - .env
-            volumes:
-              - {{ pbs_client_data_dir }}/state:/root/.proxmox-backup
-        {% for path in pbs_backup_paths %}
-              - "{{ path }}:{{ path }}:ro"
-        {% endfor %}
-            mem_limit: 2g
-            command: ["sleep", "infinity"]
-        COMPOSE_EOF
-      changed_when: true
-
-    - name: Create .env file
-      ansible.builtin.raw: |
-        cat > {{ pbs_client_data_dir }}/.env << 'ENV_EOF'
-        PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
-        PBS_PASSWORD={{ pbs_password }}
-        ENV_EOF
-        chmod 600 {{ pbs_client_data_dir }}/.env
-      changed_when: true
-
-    - name: Create backup script
-      ansible.builtin.raw: |
-        cat > {{ pbs_client_data_dir }}/backup.sh << 'SCRIPT_EOF'
-        #!/bin/bash
-        set -e
-        source {{ pbs_client_data_dir }}/.env
-
-        BACKUP_SOURCES=""
-        {% for path in pbs_backup_paths %}
-        BACKUP_SOURCES="$BACKUP_SOURCES {{ path | basename }}.pxar:{{ path }}"
-        {% endfor %}
-
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup for {{ pbs_backup_id }}"
-        docker exec pbs-client proxmox-backup-client backup \
-            $BACKUP_SOURCES \
-            --backup-id {{ pbs_backup_id }} \
-            --repository "${PBS_REPOSITORY}"
-
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Backup completed, pruning..."
-        docker exec pbs-client proxmox-backup-client prune \
-            --repository "${PBS_REPOSITORY}" \
-            --backup-id {{ pbs_backup_id }} \
-            --keep-last {{ pbs_keep_last | default(3) }} \
-            --keep-daily {{ pbs_keep_daily | default(7) }} \
-            --keep-weekly {{ pbs_keep_weekly | default(4) }} \
-            --keep-monthly {{ pbs_keep_monthly | default(6) }}
-
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Done"
-        SCRIPT_EOF
-        chmod 755 {{ pbs_client_data_dir }}/backup.sh
-      changed_when: true
-
-    - name: Start PBS client container
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        cd {{ pbs_client_data_dir }} && docker compose pull && docker compose up -d
-      become: true
-      changed_when: true
-
-    - name: Setup cron job for daily backup at 2 AM (root crontab)
-      ansible.builtin.shell: |
-        # Remove existing entry if present
-        crontab -l 2>/dev/null | grep -v "pbs-client/backup.sh" | crontab - 2>/dev/null || true
-        # Add new entry
-        (crontab -l 2>/dev/null; echo "0 2 * * * {{ pbs_client_data_dir }}/backup.sh >> /var/log/pbs-backup.log 2>&1") | crontab -
-      become: true
-      changed_when: true
-
-    - name: Verify container is running
-      ansible.builtin.shell: |
-        export PATH=$PATH:/usr/local/bin
-        docker ps --filter name=pbs-client --format "{{ '{{' }}.Status{{ '}}' }}"
-      become: true
-      register: container_status
-      changed_when: false
-
-    - name: Show container status
-      ansible.builtin.debug:
-        msg: "PBS client container status: {{ container_status.stdout }}"
@@ -1,11 +0,0 @@
---
- name: Install PBS Client on Synology NAS
-  hosts: synology_nas
-  become: true
-  # Variables are set in:
-  # - Role defaults: ansible/roles/pbs-client/defaults/main.yml
-  # - Host overrides: ansible/inventory/host_vars/<hostname>/vars.yml
-  # - Secrets: ansible/inventory/host_vars/<hostname>/vault.yml
-
-  roles:
-    - pbs-client
@@ -1,10 +1,10 @@
 #!/bin/sh
 # Forced-command target for the netbird-backup SSH key (see the
 # "Authorize the cluster backup key" task): takes a CONSISTENT snapshot
-# of the management SQLite store via the online-backup API (safe against
-# a live writer, unlike copying store.db) and streams a gzipped tar to
-# stdout. The key's authorized_keys entry is restricted to exactly this
-# script — the in-cluster puller cannot run anything else.
+# of the management SQLite store (safe against a live writer, unlike
+# copying store.db) and streams a gzipped tar to stdout. The key's
+# authorized_keys entry is restricted to exactly this script — the
+# in-cluster puller cannot run anything else.
 #
 # Deliberately store.db ONLY: /opt/netbird/config is fully regenerable
 # from the netbird-server role's templates + vault, and including it
@@ -14,5 +14,23 @@
 set -eu
 tmp=$(mktemp -d)
 trap 'rm -rf "$tmp"' EXIT
-sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'"
+# NetBird sets no journal_mode on the store (its DSN carries only
+# _busy_timeout and cache=shared), so store.db uses the default rollback
+# journal, where the management container's write lock blocks a reader
+# outright rather than letting it through as WAL would. Two things follow.
+#
+# .timeout arms a busy handler on the source connection, which is the
+# contended side; the destination is a fresh file in $tmp that nothing else
+# touches. Without it sqlite3 uses its 0 ms default and gives up the moment
+# it cannot take the lock. 30 s sits well inside the puller's 600 s
+# activeDeadlineSeconds.
+#
+# VACUUM INTO rather than .backup: the shell's .backup copies 100 pages per
+# sqlite3_backup_step(), and SQLite restarts the copy whenever the source is
+# written between steps, so a steady writer drags it out for as long as the
+# writes keep coming. VACUUM INTO reads one transaction in a single pass and
+# is documented as an alternative to the backup API for exactly this. It
+# also compacts, so the dump is smaller than the live file.
+sqlite3 -cmd '.timeout 30000' /opt/netbird/management/store.db \
+    "VACUUM INTO '$tmp/store.db'"
 tar -C "$tmp" -czf - store.db
@@ -1,32 +0,0 @@
---
-# PBS Client configuration
-pbs_client_version: "latest"
-pbs_client_data_dir: "/opt/pbs-client"
-
-# PBS Server connection (override in host_vars)
-pbs_server_host: "pbs.obg.renner.uno"
-pbs_server_port: 8007
-pbs_datastore: "backups"
-
-# Authentication (override in vault!)
-pbs_username: "backup@pbs"
-pbs_password: ""
-# pbs_fingerprint: ""  # Optional - omit for Let's Encrypt certs
-
-# Backup configuration
-pbs_backup_id: "nas-1"
-pbs_backup_paths:
-  - /volume1/docker
-  - /volume1/homes
-
-# Schedule (cron format)
-pbs_backup_schedule: "0 2 * * *"  # 2 AM daily
-
-# Retention policy (prune settings)
-pbs_keep_last: 3
-pbs_keep_daily: 7
-pbs_keep_weekly: 4
-pbs_keep_monthly: 6
-
-# Container resource limits
-pbs_memory_limit: "2g"
@@ -1,6 +0,0 @@
---
- name: Restart PBS client
-  community.docker.docker_compose_v2:
-    project_src: "{{ pbs_client_data_dir }}"
-    state: restarted
-    docker_cli: "{{ docker_cli | default('/usr/local/bin/docker') }}"
@@ -1,23 +0,0 @@
---
- name: Create PBS client directories
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: directory
-    mode: '0755'
-  loop:
-    - "{{ pbs_client_data_dir }}"
-    - "{{ pbs_client_data_dir }}/state"
-
- name: Copy PBS client docker-compose file
-  ansible.builtin.template:
-    src: docker-compose.yml.j2
-    dest: "{{ pbs_client_data_dir }}/docker-compose.yml"
-    mode: '0644'
-  notify: Restart PBS client
-
- name: Start PBS client via Docker Compose
-  community.docker.docker_compose_v2:
-    project_src: "{{ pbs_client_data_dir }}"
-    state: present
-    pull: always
-    docker_cli: "{{ docker_cli | default('/usr/local/bin/docker') }}"
@@ -1,6 +0,0 @@
-# PBS Client Environment Configuration
-PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
-PBS_PASSWORD={{ pbs_password }}
-{% if pbs_fingerprint | default('') != '' %}
-PBS_FINGERPRINT={{ pbs_fingerprint }}
-{% endif %}
@@ -1,36 +0,0 @@
-#!/bin/bash
-# PBS Backup Script for {{ pbs_backup_id }}
-# Managed by Ansible - do not edit manually
-
-set -e
-
-# Load environment
-source {{ pbs_client_data_dir }}/.env
-
-# Build backup command with all paths
-BACKUP_SOURCES=""
-{% for path in pbs_backup_paths %}
-BACKUP_SOURCES="$BACKUP_SOURCES {{ path | basename }}.pxar:{{ path }}"
-{% endfor %}
-
-echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup for {{ pbs_backup_id }}"
-
-# Run backup via Docker
-docker exec pbs-client proxmox-backup-client backup \
-    $BACKUP_SOURCES \
-    --backup-id {{ pbs_backup_id }} \
-    --repository "${PBS_REPOSITORY}"
-
-echo "[$(date '+%Y-%m-%d %H:%M:%S')] Backup completed"
-
-# Prune old backups
-echo "[$(date '+%Y-%m-%d %H:%M:%S')] Pruning old backups"
-docker exec pbs-client proxmox-backup-client prune \
-    --repository "${PBS_REPOSITORY}" \
-    --backup-id {{ pbs_backup_id }} \
-    --keep-last {{ pbs_keep_last }} \
-    --keep-daily {{ pbs_keep_daily }} \
-    --keep-weekly {{ pbs_keep_weekly }} \
-    --keep-monthly {{ pbs_keep_monthly }}
-
-echo "[$(date '+%Y-%m-%d %H:%M:%S')] Prune completed"
@@ -1,30 +0,0 @@
-services:
-  pbs-client:
-    image: aterfax/pbs-client:{{ pbs_client_version }}
-    container_name: pbs-client
-    restart: unless-stopped
-    hostname: {{ pbs_backup_id }}
-    environment:
-      - PBS_ENDPOINT={{ pbs_server_host }}:{{ pbs_server_port }}
-      - PBS_DATASTORE={{ pbs_datastore }}
-      - PBS_API_KEY_NAME={{ pbs_username }}
-      - PBS_API_KEY_SECRET={{ pbs_password }}
-      - PBS_PASSWORD={{ pbs_password }}
-      - PBS_ENCRYPTION_PASSWORD={{ pbs_encryption_password | default(pbs_password) }}
-      - PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
-      - PBS_BACKUP_ID={{ pbs_backup_id }}
-      - CRON_SCHEDULE={{ pbs_backup_schedule }}
-      - PBS_BACKUP_CMD_APPEND_EXTRA_OPTS=--crypt-mode=none
-      - PRUNE_KEEP_LAST={{ pbs_keep_last | default(3) }}
-      - PRUNE_KEEP_DAILY={{ pbs_keep_daily | default(7) }}
-      - PRUNE_KEEP_WEEKLY={{ pbs_keep_weekly | default(4) }}
-      - PRUNE_KEEP_MONTHLY={{ pbs_keep_monthly | default(6) }}
-      - CRON_BACKUP_ONLY=1
-    volumes:
-      - {{ pbs_client_data_dir }}/state:/root/.config/proxmox-backup
-{% for path in pbs_backup_paths %}
-      - "{{ path }}:/backups/{{ path | basename | replace(' ', '_') }}:ro"
-{% endfor %}
-    tmpfs:
-      - /tmp
-    mem_limit: {{ pbs_memory_limit }}
@@ -0,0 +1,111 @@
+# Cilium: one owner
+
+## The rule
+
+Cilium is owned by **Flux**, through `infrastructure/cilium/helmrelease.yaml`.
+Talos provides the conditions for a CNI and nothing else. Never both.
+
+Talos-only would also be a valid answer — it is the simpler one, and it keeps a
+cold rebuild self-forming. It is rejected here only because it costs Flux's
+reconciliation, image automation and value ergonomics for the one component
+most likely to need a careful, reviewed change.
+
+## Where we are
+
+Both own it, which is the state this document exists to end.
+
+- 22 Cilium objects are listed in the `talos-bootstrap-manifests-inventory`
+  ConfigMap: both DaemonSets, the operator Deployment, both ConfigMaps, the
+  ServiceAccounts, the ClusterRoles/Bindings and the `cilium-secrets`
+  namespace. That is the whole installation, not a fragment.
+- The same objects carry `meta.helm.sh/release-name: cilium`.
+- `cilium-config` has three field managers: `helm-controller`, `talos` and
+  `kubectl-patch`.
+
+The consequences are not theoretical. The seed sat at Cilium 1.19.2 for twelve
+days while the chart was on 1.20.1, and `upgrade-k8s --dry-run` shows that a
+Kubernetes upgrade in that state downgrades the CNI and switches off socket
+load balancing. Separately, `enable-bgp-control-plane` and the two
+native-routing CIDRs exist in no file at all — applied by hand, so a cold
+rebuild would have come up with BGP off while the peering CRDs in git made it
+look configured.
+
+Nothing fails when these drift. That is the whole problem.
+
+## Target state
+
+**Talos** keeps exactly this, in `talos/patches/cluster.yaml`, and gains nothing
+else:
+
+```yaml
+cluster:
+  network:
+    cni:
+      name: none
+  proxy:
+    disabled: true
+```
+
+No `inlineManifests` entry for cilium. `talos/patches/cilium-cni.yaml` is
+deleted.
+
+**Flux** owns everything else. `helmrelease.yaml` is the single source for
+every Cilium setting, including the three that were hand-patched.
+
+**A cold rebuild** installs Cilium once, by hand, between `talosctl bootstrap`
+and the Flux phase, from the same HelmRelease values. This is the cost of the
+decision and it is deliberate: the cluster no longer forms itself from the
+machine config alone. Given `bootstrap.sh talos` is already dead and Step 2 of
+RUNBOOK-cold-rebuild.md is already a manual procedure, the marginal cost is one
+documented command.
+
+## Transition
+
+The order matters, because Talos prunes what its inventory used to list and no
+longer does — removing the manifest naively deletes the CNI.
+
+1. **Track the untracked settings first.** `bgpControlPlane.enabled`,
+   `ipv4NativeRoutingCIDR`, `ipv6NativeRoutingCIDR` into the HelmRelease. Note
+   this is not a pure change of ownership: enabling bgpControlPlane through the
+   chart also emits five companion keys the hand-patch never set, so Cilium has
+   been running on its own defaults for router-id allocation. Verify the BGP
+   sessions after Flux applies it.
+
+2. **Remove cilium from `cluster.inlineManifests`.** This has no immediate
+   effect. Evidence: Talos was upgraded to 1.13.9 four days ago and Cilium
+   stayed at 1.20.1 — an OS upgrade does not reconcile bootstrap manifests.
+   Only `upgrade-k8s` does.
+
+3. **Prove the prune before running it.** `upgrade-k8s --dry-run
+   --manifests-no-prune` and confirm the 22 objects are not listed for
+   deletion. `NoPrune` is honoured by the dry run as well as the apply
+   (`pkg/cluster/kubernetes/talos_managed.go`), so this is a real test and not
+   a reading of the flag description.
+
+4. **Run `upgrade-k8s --manifests-no-prune`.** The inventory is rewritten
+   without Cilium and the objects survive. Helm is then sole owner. This is the
+   same run that takes Kubernetes 1.35.2 → 1.36.3.
+
+5. **Verify** the inventory lists no Cilium, the agents did not restart, and
+   the BGP sessions are still established.
+
+6. **Delete `talos/patches/cilium-cni.yaml`** and add the install step to
+   RUNBOOK-cold-rebuild.md.
+
+## Not part of this, and not a defect either
+
+The cluster runs `routingMode: tunnel` (the chart default, set nowhere) while
+also advertising pod CIDRs over BGP. That is not one layer too many — the two
+carry different directions:
+
+- **VXLAN** carries pod-to-pod between nodes, directly, node to node.
+- **BGP** gives the UDM routes into the pod CIDRs so traffic from the LAN can
+  reach pods. That path goes through the router by definition.
+
+Dropping the encapsulation in favour of native routing would not simplify
+anything; it would push inter-node pod traffic onto the underlay path and
+through the UDM, replacing a direct node-to-node hop with a router hop.
+
+`routingMode` being unset is still worth knowing — the current behaviour is a
+default nobody chose rather than a decision anyone recorded — but the default
+is the right one here, and this document is not the place to revisit it.
@@ -198,8 +198,7 @@ talosctl gen secrets --from-controlplane-config cp.yaml -o secrets.yaml
 talosctl gen config homelab https://k8s.renner.uno:6443 \
  --with-secrets secrets.yaml --output-types controlplane \
  --config-patch @talos/patches/cluster.yaml \
-  --config-patch @<cilium additive half> \
-  --config-patch @talos/patches/registry-mirror.yaml \
+  --config-patch @talos/patches/containerd-layer-retention.yaml \
  --config-patch @talos/patches/k8s-N.yaml -o install.yaml
 ```

@@ -208,9 +207,11 @@ Confirm `cluster.id` matches the running cluster before applying.
 Strip the generated `HostnameConfig` document — it collides with
 `machine.network.hostname` from the per-node patch. `bootstrap.sh` does the same.

-`cilium-cni.yaml` cannot be passed whole: its `$patch: delete` document fails
-against a base with no `inlineManifests` (issue #581). Use only the additive
-second document.
+There is no CNI patch to pass any more. Talos supplies only `cni: none` and
+`proxy: disabled`; Cilium is installed from its HelmRelease values, which is
+Step 2b of RUNBOOK-cold-rebuild.md. A node converted into an existing cluster
+gets its CNI from the Cilium DaemonSet already running there, so nothing extra
+is needed here.

 Validate:

@@ -119,9 +119,11 @@ against do not exist, and the node ends up with no address on any interface.
 Beyond that it carries:

 - `siderolabs/intel-ucode` — no hypervisor supplies microcode here
- `siderolabs/i915` — the Alder Lake-P Iris Xe. Nothing requests `/dev/dri`
-  today; present so wanting hardware transcoding later is a config change
-  rather than another rolling reinstall
+- `siderolabs/i915` — the Alder Lake-P Iris Xe. The Intel GPU device plugin
+  (`infrastructure/intel-gpu-plugin`) turns this into the schedulable resource
+  `gpu.intel.com/i915`, which `apps/llama-server` requests for LLM inference
+  and which any hardware-transcoding workload would use too. Without the
+  extension the plugin finds no `/dev/dri` and advertises nothing
 - `siderolabs/thunderbolt` — inert until `machine.kernel.modules` requests
  `thunderbolt`/`thunderbolt_net`, present for the same reason. Note its udev
  rule auto-authorises any Thunderbolt device attached once those modules load
@@ -136,11 +138,18 @@ in `talos/patches/k8s-N.yaml`.

 ## Step 2 — talos: form the cluster + get kubeconfig

-**Do not use `kubernetes/bootstrap.sh talos`.** That phase cannot generate a
-config: it passes `talos/patches/cilium-cni.yaml` whole, and that file's
-`$patch: delete` document fails against a base with no `inlineManifests`
-(#581). Its other phases — `flux`, `wait-apps` — are still used, in Steps 3, 5
-and 6; only config generation is broken.
+`kubernetes/bootstrap.sh talos` does this, including the CNI install that used
+to be a hand-typed step. It was unusable for months because it passed
+`cilium-cni.yaml` whole and the file's `$patch: delete` document failed against
+a fresh config (#581); that file no longer exists, so the failure is gone with
+it.
+
+Verified as far as it can be without wiping a cluster: `talosctl gen config`
+with exactly the patches the script passes produces a config carrying
+`cni: none` and `proxy: disabled` that passes `talosctl validate --mode metal`.
+The apply-and-bootstrap half has not been exercised since the rewrite. Read the
+manual procedure below before running the script, so you know what it is doing
+and can take over at any step if it stops.

 The procedure below is what actually works, and is the same one
 `RUNBOOK-baremetal-conversion.md` uses.
@@ -160,17 +169,17 @@ for N in 1 2 3; do
  talosctl gen config homelab https://k8s.renner.uno:6443 \
    --with-secrets secrets.yaml --output-types controlplane \
    --config-patch @talos/patches/cluster.yaml \
-    --config-patch @<cilium additive half> \
-    --config-patch @talos/patches/registry-mirror.yaml \
+    --config-patch @talos/patches/containerd-layer-retention.yaml \
    --config-patch @talos/patches/k8s-$N.yaml -o install-k8s-$N.yaml
 done
 ```

-Two things the generator gets wrong for this repo, both of which will bite
-silently:
+There is no CNI patch here any more. Talos supplies `cni: none` and
+`proxy: disabled` from `cluster.yaml` and nothing else; Cilium is installed in
+Step 2b below. See PROPOSAL-cilium-single-owner.md.
+
+One thing the generator gets wrong for this repo, and it bites silently:

- **`cilium-cni.yaml` cannot be passed whole** — extract only its additive
-  second document. See issue #581.
 - **Strip the generated `HostnameConfig`** — it collides with
  `machine.network.hostname` from the per-node patch.

@@ -209,8 +218,53 @@ talosctl -n k8s-1.srv.renner.uno bootstrap
 talosctl -n k8s-1.srv.renner.uno kubeconfig .
 ```

-Ends when all three nodes are `Ready`. `scripts/import-cluster-access.sh` is the
-supported way to get the kubeconfig into place afterwards.
+`scripts/import-cluster-access.sh` is the supported way to get the kubeconfig
+into place afterwards.
+
+The nodes will **not** reach `Ready` here. Talos is configured with `cni: none`,
+so there is no pod network until Step 2b — expect `NotReady` with
+`container runtime network not ready` and carry on.
+
+## Step 2b — cilium: install the CNI
+
+Nothing schedules without it, Flux included, so this comes before Step 3. Talos
+no longer carries a rendered copy of Cilium; `helmrelease.yaml` is the only
+place it is configured, and this installs from those same values so the cluster
+comes up on what git describes rather than on a second, drifting copy. See
+PROPOSAL-cilium-single-owner.md for why it works this way.
+
+`bootstrap.sh talos` already did this. What follows is the same thing by hand,
+for when the script stopped partway or you are not using it.
+
+Every input is read out of `helmrelease.yaml`. Nothing here is a version or a
+URL you have to keep in step with anything — copy the block as it stands:
+
+```sh
+HR=kubernetes/infrastructure/cilium/helmrelease.yaml
+
+helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$HR")"
+helm repo update cilium
+
+yq 'select(.kind == "HelmRelease") | .spec.values' "$HR" > /tmp/cilium-values.yaml
+
+helm install cilium cilium/cilium \
+  --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$HR")" \
+  --namespace kube-system \
+  -f /tmp/cilium-values.yaml \
+  --set prometheus.serviceMonitor.enabled=false
+```
+
+`--version` takes the HelmRelease's range verbatim; helm resolves it to the same
+release Flux would pick, so the two cannot diverge. That is the point of writing
+it this way — a pinned version in a runbook is a copy nobody maintains, and the
+next rebuild installs something the HelmRelease will immediately fight.
+
+The one literal is `serviceMonitor`, disabled for this install only: its CRD
+arrives with the monitoring stack much later and the object would fail to apply.
+Flux adds the real one once it owns the release. Nothing else is overridden,
+which is what lets helm-controller adopt this release rather than replace it.
+
+Ends when all three nodes are `Ready`.

 ## Step 3 — flux: bootstrap GitOps **from the GitHub mirror** (break-glass)

@@ -1,18 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: barman-cloud-creds
-  namespace: authentik
-  annotations:
-    # Prune-disabled — a standing guard kept from the seed->authentik-KS handoff (barman creds).
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: seed/authentik/barman-creds
@@ -25,20 +25,13 @@ data:
        attrs:
          local: true

-      # Retired OIDC layer. Paperless's URL-level access is gated by the
-      # paperless-gate ProxyProvider (below); user identity is forwarded
-      # from the gate to paperless via the X-authentik-* request headers
-      # and consumed by paperless via PAPERLESS_ENABLE_HTTP_REMOTE_USER
-      # (see kubernetes/apps/paperless/paperless.yaml). One Authentik
-      # Application, one Provider — the dual-OAuth2-flow setup with a
-      # separate paperless OIDC client + django-allauth callback was
-      # structurally fine but added a second consent screen on first
-      # login for no real defense-in-depth gain.
+      # Retired OIDC layer. Access is gated by the paperless-gate
+      # ProxyProvider below, which forwards identity as X-authentik-* headers
+      # consumed via PAPERLESS_ENABLE_HTTP_REMOTE_USER.
      #
-      # Order matters: delete the Application BEFORE the OAuth2Provider
-      # because Application.provider is on_delete=PROTECT, so the
-      # OAuth2Provider can't be removed while the Application still
-      # references it.
+      # ORDER MATTERS: delete the Application BEFORE the OAuth2Provider.
+      # Application.provider is on_delete=PROTECT, so the provider cannot be
+      # removed while an Application still references it.
      - model: authentik_core.application
        identifiers:
          slug: paperless
@@ -49,22 +42,15 @@ data:
          name: Paperless
        state: absent

-      # Also clean up the very-early orphan `paperless` ProxyProvider
-      # left behind from this session's first attempts. It's not bound
-      # to anything any more, just leftover state in the DB.
+      # Orphan ProxyProvider, unbound — leftover DB state.
      - model: authentik_providers_proxy.proxyprovider
        identifiers:
          name: paperless
        state: absent

-      # Dedicated forward-auth gate for paperless. This is a SEPARATE
-      # outpost (its own Deployment + Service `ak-outpost-paperless-
-      # gate`) running in front of paperless so every request is
-      # authenticated and authorized by Authentik before it reaches
-      # paperless's own attack surface. The gate sits on top of, and
-      # separate from, the OIDC OAuth2Provider above (which is what
-      # paperless's django-allauth uses for SSO once a request makes
-      # it through the gate).
+      # Dedicated forward-auth gate for paperless: its own outpost
+      # Deployment + Service, so every request is authenticated before it
+      # reaches paperless.
      - id: paperless-gate-provider
        model: authentik_providers_proxy.proxyprovider
        identifiers:
@@ -73,16 +59,13 @@ data:
        attrs:
          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          # Proxy (not forward_domain) so the outpost itself terminates
-          # the request and forwards to paperless — moving the trust
-          # boundary off ingress-nginx and onto paperless-gate. ingress-
-          # nginx now only routes paperless.renner.uno to the outpost
-          # service; it never talks to paperless directly. The outpost
-          # is what populates X-Authentik-Username on the upstream
-          # request, and the NetworkPolicy in the paperless namespace
-          # only allows traffic to paperless from the gate outpost pod
-          # itself, so ingress-nginx (or any other in-cluster workload)
-          # cannot bypass the gate to inject headers.
+          # Proxy, NOT forward_domain: the outpost terminates the request
+          # itself, so the trust boundary sits here rather than at the ingress
+          # controller, which only routes to the outpost Service and never
+          # reaches paperless directly. The outpost populates
+          # X-Authentik-Username, and the paperless NetworkPolicy admits
+          # traffic only from this outpost — so nothing else in the cluster
+          # can bypass the gate and inject those headers.
          mode: proxy
          internal_host: http://paperless.paperless.svc.cluster.local
          internal_host_ssl_validation: false
@@ -91,19 +74,12 @@ data:
          access_token_validity: hours=24
          refresh_token_validity: days=30
          intercept_header_auth: true
-          # Pass paperless's entire static-asset tree through anonymously.
-          # Two reasons:
-          # 1. <link rel="manifest"> is fetched with `credentials: 'omit'`
-          #    (W3C spec, no crossorigin attr) so the cookie is never sent
-          #    — every page load and every SPA navigation turned into a
-          #    302 → auth.renner.uno round-trip with an orphan session JWT.
-          #    Same root cause as the retired music-assistant-gate (see
-          #    comment further below); paperless can't retire the gate so
-          #    it carves out the offending paths instead.
-          # 2. Every CSS/JS/icon fetch otherwise validates the cookie at
-          #    the outpost — pure overhead on assets that carry no
-          #    authentication value. Django's STATIC tree is public by
-          #    design; MEDIA (under /api/documents/*) stays gated.
+          # The static-asset tree passes anonymously, and must. A
+          # <link rel="manifest"> is fetched with `credentials: 'omit'` per
+          # the W3C spec, so the session cookie is never sent and every page
+          # load turns into a redirect loop through the login origin. Django's
+          # STATIC tree is public by design; MEDIA under /api/documents/*
+          # stays gated.
          skip_path_regex: ^/static/.*

      - model: authentik_core.application
@@ -130,18 +106,14 @@ data:
          negate: false
          failure_result: false

-      # Retired: the music-assistant-gate ProxyProvider, Application,
-      # and dedicated Outpost. The gate fundamentally conflicted with
-      # MA's PWA model — Chrome fetches manifest.webmanifest / sw.js /
-      # icons with `credentials: 'omit'` (per W3C spec, no crossorigin
-      # attribute on MA's <link rel="manifest">), and the cached
-      # service worker bypasses fresh navigations on PWA cold-start so
-      # the Authentik cookie never lands in the WebAPK's cookie jar.
-      # MA has its own auth (delegated to Home Assistant); the gate's
-      # marginal value didn't justify the recurring breakage.
-      # Order matters: Application before Provider before Outpost.
-      # Application.provider is on_delete=PROTECT, so the Provider
-      # can't be removed while the Application still references it.
+      # Retired: the music-assistant gate. A forward-auth gate cannot work in
+      # front of a PWA — the manifest and service worker are fetched with
+      # `credentials: 'omit'`, and a cached service worker bypasses fresh
+      # navigations, so the session cookie never reaches the installed app.
+      # Do not reintroduce one there; MA delegates auth to Home Assistant.
+      #
+      # ORDER MATTERS: Application before Provider before Outpost, because
+      # Application.provider is on_delete=PROTECT.
      - model: authentik_core.application
        identifiers:
          slug: music-assistant-gate
@@ -157,18 +129,13 @@ data:
          name: music-assistant-gate
        state: absent

-      # Dedicated outpost: its ONLY job is to serve the in-cluster
-      # /outpost.goauthentik.io/auth/nginx endpoint that nginx
-      # auth_request hits for every request to paperless. The
-      # browser-facing legs of the OAuth flow (/start, /callback, the
-      # `/if/...` login UI, /application/o/authorize, /api, etc.) all
-      # need to happen on auth.renner.uno because that's the origin
-      # WebAuthn/FIDO/TOTP credentials are bound to. So we attach the
-      # ProxyProvider to BOTH outposts (see embedded outpost below) and
-      # set authentik_host_browser to auth.renner.uno here too — the
-      # nginx auth_request happens server-to-server, but if the
-      # dedicated outpost ever does emit a redirect URL, it points at
-      # the right host.
+      # Dedicated outpost, serving the in-cluster auth endpoint only.
+      #
+      # The browser-facing legs of the OAuth flow must happen on
+      # auth.renner.uno, because that is the origin WebAuthn credentials are
+      # bound to. Hence the ProxyProvider is attached to BOTH this outpost and
+      # the embedded one, and authentik_host_browser points at auth.renner.uno
+      # here too — so any redirect this outpost emits names the right host.
      - model: authentik_outposts.outpost
        identifiers:
          name: paperless-gate
@@ -196,12 +163,7 @@ data:
            kubernetes_ingress_secret_name: "authentik-outpost-tls"
            kubernetes_json_patches: null

-      # Tear down the leftover paperless-proxy Application from an
-      # earlier experiment. The orphan `paperless` ProxyProvider that
-      # used to back paperless.app is left in place — it's PROTECTed
-      # by Application.provider as long as anything references it, and
-      # leftover orphans don't break anything. Delete via the Authentik
-      # UI once everything's confirmed working.
+      # Leftover Application from an earlier experiment.
      - model: authentik_policies.policybinding
        identifiers:
          target: !Find [authentik_core.application, [slug, paperless-proxy]]
@@ -217,15 +179,11 @@ data:
      # ------------------------------------------------------------------
      # vscode config-editor gate
      #
-      # code-server (the HA /config editor sidecar) runs with --auth=none,
-      # as root, mounting the whole HA /config incl. plaintext secrets.yaml
-      # and .storage. This gate moves it behind Authentik with a passkey
-      # step-up, mirroring the paperless-gate proxy-outpost pattern above:
-      # the dedicated ak-outpost-vscode-gate terminates the request,
-      # enforces the session, and forwards to the code-server Service.
-      # Traefik only ever routes vscode.home.renner.uno to the outpost; the
-      # home-assistant CNP is tightened so ONLY the gate outpost may reach
-      # :8080 (see home-assistant-ns.yaml + vscode-gate-egress.yaml).
+      # code-server runs with --auth=none, as root, mounting the whole HA
+      # /config including plaintext secrets.yaml and .storage. This gate is
+      # the only thing in front of it: the outpost terminates the request and
+      # enforces a passkey step-up, Traefik routes only to the outpost, and
+      # the home-assistant CNP admits :8080 from the gate alone.
      # ------------------------------------------------------------------
      - id: vscode-config-editors-group
        model: authentik_core.group
@@ -265,17 +223,14 @@ data:
          access_token_validity: hours=24
          refresh_token_validity: days=30
          intercept_header_auth: true
-          # code-server is a service-worker app: <link rel="manifest"> and
-          # the service worker are fetched with credentials:'omit', which
-          # turned into a /authorize redirect loop and forced the RETIREMENT
-          # of the music-assistant gate (see comment above). Carve its
-          # static/manifest/SW tree out so those cookieless fetches pass
-          # through instead of 302-looping; the editor's actual content
-          # (the /config tree) is served over authenticated websocket/API
-          # paths, which stay gated.
-          # NOTE: the exact skip set is code-server-version-specific and
-          # MUST be confirmed post-deploy against the real redirect-loop
-          # behaviour (see the PR validation checklist).
+          # code-server is a service-worker app, so its manifest and SW are
+          # fetched with credentials:'omit' and 302-loop unless carved out.
+          # The editor's actual content rides authenticated websocket/API
+          # paths and stays gated.
+          #
+          # This skip set is version-specific: re-check it after a
+          # code-server bump, because a renamed asset path reintroduces the
+          # loop silently.
          skip_path_regex: ^/(_static/|manifest\.json|.*\.webmanifest|.*serviceWorker\.js).*

      - model: authentik_core.application
@@ -331,16 +286,11 @@ data:
      # ------------------------------------------------------------------
      # shelly-manager gate
      #
-      # shelly-manager has no authentication of its own: anything that can
-      # reach :8080 can rewrite Shelly device config, push scripts, and read
-      # the stored per-device AdminPassword values. Until now the only
-      # control was reachability of shelly-manager.home.renner.uno.
-      #
-      # Same shape as the vscode-gate above: a dedicated outpost terminates
-      # the request, enforces a passkey step-up, and forwards to the
-      # shelly-manager Service. Traefik only ever routes the hostname to the
-      # outpost; the home-assistant CNP allows ONLY this outpost to reach
-      # :8080 (see home-assistant-ns.yaml + shelly-manager-gate-egress.yaml).
+      # shelly-manager has no authentication of its own: anything that reaches
+      # :8080 can rewrite device config, push scripts and read the stored
+      # per-device AdminPassword values. Same shape as the vscode gate — the
+      # outpost is the only control, Traefik routes only to it, and the CNP
+      # admits :8080 from this outpost alone.
      # ------------------------------------------------------------------
      - id: shelly-managers-group
        model: authentik_core.group
@@ -15,4 +15,4 @@ spec:
    name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
  dataFrom:
    - extract:
-        key: seed/authentik/hetzner-creds
+        key: infrastructure/hetzner-s3
@@ -27,5 +27,4 @@ resources:
  - cluster.yaml
  - objectstore.yaml
  - scheduled-backup.yaml
-  - barman-creds-externalsecret.yaml
  - hetzner-creds-externalsecret.yaml
@@ -1,50 +1,10 @@
 ---
-# The goauthentik.io operator (inside authentik-worker) auto-creates a
-# ServiceMonitor per outpost, but it does not stamp the
-# `release: kube-prometheus-stack` label that this cluster's Prometheus
-# requires on serviceMonitorSelector. Those operator-owned SMs are
-# silently ignored. Declare standalone SMs here with the correct label
-# so the outposts actually get scraped.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: authentik-outpost-ldap
-  namespace: authentik
-  labels:
-    release: kube-prometheus-stack
-spec:
-  namespaceSelector:
-    matchNames:
-      - authentik
-  selector:
-    matchLabels:
-      goauthentik.io/outpost-name: ldap-outpost
-      goauthentik.io/outpost-type: ldap
-  endpoints:
-    - port: http-metrics
-      path: /metrics
-      interval: 30s
---
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: authentik-outpost-paperless-gate
-  namespace: authentik
-  labels:
-    release: kube-prometheus-stack
-spec:
-  namespaceSelector:
-    matchNames:
-      - authentik
-  selector:
-    matchLabels:
-      goauthentik.io/outpost-name: paperless-gate
-      goauthentik.io/outpost-type: proxy
-  endpoints:
-    - port: http-metrics
-      path: /metrics
-      interval: 30s
---
+# Only radius needs a hand-written ServiceMonitor. The goauthentik.io operator
+# creates one per outpost and Prometheus discovers those directly, because
+# serviceMonitorSelector here is empty (see monitoring/helmrelease-prometheus.yaml).
+# Standalone copies for ldap and paperless-gate used to live here as well; they
+# duplicated the operator's and were removed.
+#
 # Radius requires extra plumbing. The radius binary listens on :9300 just
 # like ldap/proxy (hardcoded in authentik/lib/default.yml), but authentik's
 # Python operator (`providers/radius/controllers/kubernetes.py`) omits the
@@ -53,6 +13,11 @@ spec:
 # it, radius does not. Result: the operator-managed `ak-outpost-local-radius`
 # Service exposes only UDP 1812. Add a sibling Service here that selects the
 # same pod labels and exposes 9300/TCP, then a ServiceMonitor for it.
+#
+# The operator's own `ak-outpost-local-radius` ServiceMonitor cannot cover
+# this: it selects on `app.kubernetes.io/managed-by: goauthentik.io` together
+# with `goauthentik.io/service-type: metrics`, and no Service carries both —
+# the operator's has the first, the one below has the second.
 apiVersion: v1
 kind: Service
 metadata:
@@ -86,6 +51,8 @@ spec:
    matchNames:
      - authentik
  selector:
+    # Pinned to the Service above. Without managed-by this would also match the
+    # operator's Service and scrape the same endpoint twice.
    matchLabels:
      app.kubernetes.io/managed-by: homelab
      goauthentik.io/outpost-name: local-radius
@@ -1,31 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: file
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: file
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-cluster
-    # Backend serves HTTPS with a self-signed cert; re-encrypt + skip
-    # verify via the shared insecure-backend ServersTransport (see
-    # apps/insecure-backend-serverstransport.yaml).
-    traefik.ingress.kubernetes.io/service.serversscheme: https
-    traefik.ingress.kubernetes.io/service.serverstransport: insecure-backend@kubernetescrd
-spec:
-  ingressClassName: traefik
-  rules:
-    - host: file.renner.uno
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: file
-                port:
-                  number: 7001
-  tls:
-    - secretName: file-tls
-      hosts:
-        - file.renner.uno
@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: file
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: file
-spec:
-  # Caddy reverse-proxied https://nas.renner.uno:7001. nas.renner.uno
-  # is a CNAME -> nas-1.renner.uno owned by UniFi DNS, k8s-gateway
-  # forwards srv lookups so this resolves cluster-side.
-  type: ExternalName
-  externalName: nas.renner.uno
-  ports:
-    - name: https
-      port: 7001
-      targetPort: 7001
-      protocol: TCP
@@ -18,15 +18,18 @@ metadata:
 #
 # Opaque-label map (maintainer reference only — NOT served publicly):
 #   services group                   infrastructure group
-#     Web Service 1 = Authentik        Infrastructure 1 = PBS 2 (pbs2.renner.uno)
+#     Web Service 1 = Authentik        Infrastructure 1 = retired (was PBS 2)
 #     Web Service 2 = Gitea            Infrastructure 2 = retired (was Proxmox VE)
 #     Web Service 3 = Immich           Infrastructure 3 = UDM DNS (10.2.40.1)
 #     Web Service 4 = Woodpecker CI    Infrastructure 4 = Mosquitto MQTT broker
 #     Web Service 5 = Grafana
 #     Web Service 6 = Home Assistant
-#     Web Service 7 = Paperless
+#     Web Service 7 = retired (was the NAS file UI; never Paperless, which
+#                     this map claimed and which nothing here checks)
 #     Web Service 8 = Audiobookshelf
 #     Web Service 9 = Music Assistant
+#     Web Service 10 = TRMNL
+#     Web Service 11 = Paperless (gate-level only, see its entry)
 data:
  config.yaml: |
    storage:
@@ -121,18 +124,10 @@ data:
        conditions:
          - "[STATUS] == any(200, 301, 302)"

-      - name: Web Service 7
-        group: services
-        url: https://file.renner.uno
-        interval: 60s
-        ui:
-          hide-hostname: true
-          hide-url: true
-          hide-port: true
-          hide-conditions: true
-          hide-errors: true
-        conditions:
-          - "[STATUS] == any(200, 301, 302)"
+      # Removed 2026-08-28: the NAS file UI (file.renner.uno). Its Ingress and
+      # ExternalName Service were deleted in #696 because they pointed at
+      # nas-1, which is powered off; this check outlived them and had been
+      # reporting failure on the public page since.

      - name: Web Service 8
        group: services
@@ -178,6 +173,29 @@ data:
        conditions:
          - "[STATUS] == any(200, 301, 302)"

+      # Paperless sits entirely behind the Authentik gate — every path,
+      # including /api/, answers 302 to unauthenticated callers, so there is no
+      # endpoint that proves the app rather than the gate. This check is
+      # therefore reachability only: DNS, TLS, Traefik and the outpost. It
+      # stays green if paperless itself is dead behind a live gate.
+      #
+      # Closing that blind spot needs an in-cluster probe of the paperless
+      # ClusterIP, which crosses a namespace boundary and so needs a
+      # CiliumNetworkPolicy first — the monitoring probes get away without one
+      # only because their targets share a namespace with the prober.
+      - name: Web Service 11
+        group: services
+        url: https://paperless.renner.uno
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == any(200, 301, 302)"
+
      # Removed 2026-05-21: Jellyfin (media.renner.uno), Wiki (wiki.renner.uno),
      # InfluxDB (influx.renner.uno) — hostnames NXDOMAIN, no Ingress; services
      # no longer deployed. Re-add when/if they come back.
@@ -191,20 +209,11 @@ data:
      # NoData). Monitor the offsite PBS from the offsite side or via a
      # check that doesn't depend on this cluster's DNS path.

-      - name: Infrastructure 1
-        group: infrastructure
-        url: https://pbs2.renner.uno:8007
-        interval: 60s
-        client:
-          insecure-skip-verify: true
-        ui:
-          hide-hostname: true
-          hide-url: true
-          hide-port: true
-          hide-conditions: true
-          hide-errors: true
-        conditions:
-          - "[STATUS] == any(200, 301, 302)"
+      # Removed 2026-08-28: PBS 2 (pbs2.renner.uno:8007) — it has no clients
+      # left. Its two were the PVE cluster, retired to bare-metal Talos, and
+      # nas-1, powered off and dropped from the repo. Checking the liveness of
+      # a backup server that nothing backs up to reports on nothing. Re-add
+      # alongside the first host that is declared to back up to it.

      - name: Infrastructure 3
        group: infrastructure
@@ -5,6 +5,12 @@ metadata:
  namespace: apps
  labels:
    app: gatus
+  annotations:
+    # Gatus reads its config once at startup and never re-reads the file, so
+    # without this a ConfigMap change reconciles cleanly and then does nothing:
+    # git and the running checks drift apart silently, with no failed resource
+    # to notice. Adding a check would not take effect either.
+    configmap.reloader.stakater.com/reload: gatus-config
 spec:
  replicas: 1
  # RWO PVC (`gatus-data`, ceph-block) — RollingUpdate's default 25% surge
@@ -22,9 +28,30 @@ spec:
      labels:
        app: gatus
    spec:
+      # The upstream image is FROM scratch with no USER, so without this the
+      # pod runs as root and trips PodSecurity restricted on four counts.
+      #
+      # The uid is numeric because a scratch image carries no /etc/passwd for
+      # the kubelet to resolve a name against. fsGroup is what makes the switch
+      # survivable: the only write path is the sqlite store on the RWO volume
+      # (/data/gatus.db, see the storage stanza in the ConfigMap), and fsGroup
+      # both chowns the existing volume and leaves the directory group-writable
+      # so sqlite can create its -wal and -journal siblings next to the db.
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
      containers:
        - name: gatus
          image: twinproduction/gatus:v5.36.0 # {"$imagepolicy": "flux-system:gatus"}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
          ports:
            - containerPort: 8080
              name: http
@@ -210,6 +210,28 @@ spec:
            - name: vscode
              containerPort: 8080
              protocol: TCP
+          # /healthz answers 200 whenever the HTTP server is up; its body
+          # reports session heartbeat state, which is not health — an idle
+          # editor reports "expired" and is perfectly fine. So this asserts
+          # that code-server is serving, not that anyone is using it.
+          #
+          # Liveness matters here more than elsewhere in this pod: code-server
+          # shares the Home Assistant config volume, and a wedged editor is
+          # otherwise invisible until someone tries to open it.
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: vscode
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: vscode
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
          volumeMounts:
            - name: config
              mountPath: /config
@@ -1,21 +1,8 @@
 ---
 # Barman S3 credentials for ha-pg.
 #
-# FOLLOW-UP (#397): sourced from seed/paperless/hetzner-creds, which is not where
-# they belong. woodpecker/hetzner-creds-externalsecret.yaml has the identical
-# problem and the same note. The values are the same today, so this is correct
-# but coupled: rotating paperless's Hetzner credentials would silently change
-# the backup credentials for Home Assistant and Woodpecker too.
-#
-# The fix is one dedicated path per consumer (seed/home-assistant/hetzner-creds)
-# and a one-line change to the `key:` below. Creating it needs an OpenBao write:
-#
-#   bao kv put homelab/seed/home-assistant/hetzner-creds \
-#     @<(bao kv get -format=json homelab/seed/paperless/hetzner-creds | jq '.data.data')
-#
-# Note the mount is `homelab`, so the path is homelab/seed/... — #397 records a
-# 403 on `bao kv list seed`, which was a wrong-path error rather than a
-# permissions problem.
+# One shared path for one shared credential — see the note in
+# ../paperless/hetzner-creds-externalsecret.yaml.
 #
 # The namespace allowlist on the openbao ClusterSecretStore
 # (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
@@ -35,4 +22,4 @@ spec:
    name: hetzner-s3-creds
  dataFrom:
    - extract:
-        key: seed/paperless/hetzner-creds
+        key: infrastructure/hetzner-s3
@@ -1,80 +1,22 @@
 ---
-# mDNS relay: gives Home Assistant sight of the IoT VLAN's mDNS announcements
-# without putting a macvlan on the HA pod itself.
+# mDNS relay: holds the multicast membership on the IoT VLAN through its own
+# macvlan and forwards each datagram to Home Assistant as ordinary UNICAST UDP.
 #
-# THE PROBLEM
+# It must stay unicast. An avahi-style reflector cannot work here — Cilium runs
+# in tunnel mode without enable-multicast, so re-emitted multicast is dropped by
+# the CNI. Unicast is carried, and python-zeroconf accepts it: its datagram
+# handler checks size and duplicates but never the destination or source
+# address. RFC 6762's QU bit makes this protocol-legal rather than a trick.
 #
-# HA's zeroconf listener joins both mDNS groups on eth0 — 224.0.0.251 and
-# ff02::fb — and never hears anything on either.
-# Multicast is link-local, so IoT-VLAN announcements do not reach the pod
-# CIDR; and Cilium runs in tunnel mode (VXLAN) without `enable-multicast`, so
-# it does not replicate multicast between pods either. Both halves of the path
-# are dead, which is why an avahi-style reflector — receive on one interface,
-# re-emit as multicast on the other — cannot work here: the re-emitted packets
-# would be dropped by the CNI.
+# This restores passive reception only — announcements, including the
+# re-announce a device sends when its address changes. HA-initiated browsing
+# still dies on eth0; that would need a querier on the VLAN side, which this
+# deliberately is not.
 #
-# Consequence for HA: the ESPHome integration dials devices at the address
-# cached in its config entry. When a lease rotates, every attempt fails with
-# `[Errno 113] Connect call failed` and the device stays unreachable, because
-# the mDNS announcement that would carry the new address never arrives. New
-# devices are never discovered at all.
-#
-# THE MECHANISM
-#
-# This relay holds the multicast membership on the IoT VLAN through its own
-# macvlan (mdns-relay-network-attachment.yaml) and forwards each datagram it
-# receives to HA as ordinary UNICAST UDP. That is the one direction of travel
-# the CNI will carry.
-#
-# It works because python-zeroconf does not filter on how a packet arrived:
-# its datagram handler validates size, deduplicates, and parses, but performs
-# no check on destination address, source address, or source port. A unicast
-# datagram delivered to :5353 is processed exactly like a multicast one. (mDNS
-# unicast responses are part of RFC 6762 — the QU bit — so this is within the
-# protocol, not a trick.)
-#
-# WHAT THIS DOES AND DOES NOT RESTORE
-#
-# Restored: passive reception. RFC 6762 requires a responder to re-announce
-# when its address changes, so a device that moves to a new lease announces
-# the move, HA sees it, and the ESPHome integration updates the config entry.
-# That is the failure this exists to fix. New devices announcing themselves
-# are likewise seen.
-#
-# NOT restored: HA-initiated browsing. Queries HA sends leave on eth0 as
-# multicast and die there, so HA cannot actively solicit a response. Anything
-# depending on a query rather than an announcement stays broken. Making that
-# work needs a querier on the VLAN side, which this relay deliberately is not.
-#
-# EXPOSURE
-#
-# The receive sockets carry so-bindtodevice=net1, so only datagrams that
-# actually arrived on the IoT VLAN interface are delivered. Binding to net1's
-# address instead is not possible — it is DHCP-leased and unknown at exec time
-# — but binding to the device achieves the same confinement without needing to
-# know the address, and needs no capability: SO_BINDTODEVICE succeeds as an
-# unprivileged uid here.
-#
-# Without it socat would bind the wildcard, and since the namespace CNP grants
-# every pod in home-assistant unrestricted access to every other pod on every
-# port, any sibling pod could reach :5353 over the cluster network and have
-# records forwarded into HA's zeroconf cache.
-#
-# What remains, and cannot be closed here: any host on the IoT VLAN can send
-# this port a datagram. That is the same trust already extended to genuine
-# mDNS from that segment, which any device there can equally forge — the relay
-# widens who can reach HA's listener, not what can be claimed. macvlan traffic
-# bypasses Cilium policy entirely, so the namespace CNP does not constrain
-# this interface either way.
-#
-# Note that the relay is not what makes HA's zeroconf reachable from inside
-# the cluster: HA binds :5353 on the pod address (v4) and the wildcard (v6),
-# and the namespace CNP already permits any sibling pod to reach it directly.
-# Confining the relay removes a second route to that listener; it does not
-# remove the first.
-#
-# Nothing else is exposed: the containers run socat and only socat, with no
-# second listener to reach.
+# so-bindtodevice=net1 on both receive sockets is load-bearing: without it
+# socat binds the wildcard, and the namespace CNP lets any sibling pod reach
+# :5353 and inject records into HA's zeroconf cache. Binding net1's address
+# instead is impossible — it is DHCP-leased and unknown at exec time.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -6,32 +6,23 @@ metadata:
  labels:
    app.kubernetes.io/name: mosquitto
  annotations:
-    # REQUIRED for credential rotation to take effect at all. The startup shim
-    # rebuilds /mosquitto/auth/passwd from these Secrets on every pod start, so
-    # a rotated Secret is inert until the pod restarts — without this the new
-    # password would sit unused until some unrelated restart, while the old one
-    # stayed valid indefinitely.
+    # REQUIRED for credential rotation to take effect. The startup shim rebuilds
+    # the password file only on pod start, so without this a rotated Secret sits
+    # inert while the old password stays valid indefinitely.
    #
-    # Listed per-Secret rather than using the blanket `reloader.stakater.com/
-    # auto: "true"`, so that enrolling each remaining broker user in rotation is
-    # a visible, reviewable edit to this line rather than something that starts
-    # happening implicitly.
+    # Listed per-Secret rather than with the blanket auto annotation, so that
+    # enrolling another broker user is a visible edit to this line.
    secret.reloader.stakater.com/reload: mosquitto-health-credentials
 spec:
  replicas: 1
  strategy:
-    # No PVC anymore, so nothing to fence: the new broker can come up and pass
-    # its readiness probe before the old one is torn down. maxUnavailable:0 keeps
-    # a broker serving throughout an image bump / reschedule (clients on the old
-    # pod just reconnect to the new one); maxSurge:1 permits the brief two-pod
-    # overlap — two independent stateless brokers behind the LB, fine since there
-    # is no shared state.
+    # No PVC, so nothing to fence — the new broker can serve before the old one
+    # is torn down. Two stateless brokers overlapping behind the LB is fine.
    #
-    # NOTE: connectivity survives the roll, but RETAINED STATE DOES NOT — the
-    # new pod starts with an empty retained store, so a roll is equivalent to a
-    # broker restart for every client that depends on retention. Zigbee2MQTT
-    # does not recover from that on its own; see mosquitto-config.yaml and
-    # gitea issue #381.
+    # Connectivity survives a roll but RETAINED STATE DOES NOT: the new pod
+    # starts with an empty retained store, so every roll is a broker restart
+    # for clients that depend on retention. Zigbee2MQTT does not recover from
+    # that on its own (#381).
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
@@ -44,14 +35,10 @@ spec:
      labels:
        app.kubernetes.io/name: mosquitto
    spec:
-      # Runs on any node (no nodeSelector); the mosquitto-iot LB forwards
-      # IoT-VLAN traffic to the broker wherever it lands.
-      # Availability: home-critical priority (never preempted, reschedules
-      # first) + short fault tolerations so a node failure reschedules the
-      # broker in ~30s instead of the default ~5min. With persistence off there
-      # is no PVC to fence/detach/reattach, so the broker can (re)start anywhere
-      # instantly, no longer depends on Ceph being healthy, and rolls without a
-      # tear-down-first gap (see the RollingUpdate strategy above).
+      # Runs anywhere; the mosquitto-iot LB finds it. Short fault tolerations
+      # reschedule the broker in ~30s rather than the default ~5min, which is
+      # only possible because there is no PVC to fence — so it also does not
+      # depend on Ceph being healthy.
      priorityClassName: home-critical
      terminationGracePeriodSeconds: 10
      tolerations:
@@ -6,12 +6,9 @@ metadata:
  labels:
    app.kubernetes.io/name: zigbee2mqtt
 spec:
-  # CUTOVER DONE (2026-07-23): ZHA disabled, the coordinator-backup Secret is
-  # populated in OpenBao, and Z2M is now the live Zigbee stack — so replicas is
-  # 1. It shipped at 0 as a safety interlock (Z2M contends with ZHA for the
-  # coordinator's single TCP session, and starting without the restored backup
-  # would form a NEW network and drop every device); that interlock is spent
-  # once the migration is committed. To roll back, set 0 and re-enable ZHA.
+  # Z2M is the live Zigbee stack. Never run it alongside ZHA: they contend for
+  # the coordinator's single TCP session, and a Z2M start without the restored
+  # coordinator backup forms a NEW network and drops every paired device.
  replicas: 1
  # RWO PVC → single writer; never run two pods against the same /app/data.
  strategy:
@@ -39,14 +36,10 @@ spec:
        # network_key, the latter extracted from the OpenBao coordinator backup
        # as a natural-byte-order int array.
        - name: seed-data
-          # alpine + jq, built by .woodpecker/build-ci-images.yml from
-          # ci/images/alpine-jq/. Replaces `apk add --no-cache jq` on every pod
-          # start: tools belong in the image, not installed at runtime.
-          # Referenced by tag rather than digest — these are our own images.
-          # DATED tag, not :latest, matching the ci-ansible:2026-04-29 precedent
-          # in kubernetes/infrastructure/netbird-backup/cronjob.yaml. With
-          # :latest, rebuilding this base for some unrelated consumer would
-          # silently change what this init container runs on its next pod
+          # Our own alpine + jq image, built by build-ci-images.yml. DATED tag,
+          # never :latest — rebuilding this base for an unrelated consumer
+          # would otherwise silently change what this init container runs on
+          # its next pod
          # restart, with no diff here to review — and this is the container that
          # renders the Zigbee network key. Bumping the tag is a deliberate,
          # reviewable act; the cost is remembering to bump it.
@@ -1,18 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: barman-cloud-creds
-  namespace: immich
-  annotations:
-    # Prune-disabled — a standing guard kept from the seed->immich-KS handoff (barman creds).
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: seed/immich/barman-creds
@@ -15,4 +15,4 @@ spec:
    name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
  dataFrom:
    - extract:
-        key: seed/immich/hetzner-creds
+        key: infrastructure/hetzner-s3
@@ -13,6 +13,5 @@ resources:
  - cluster.yaml
  - objectstore.yaml
  - scheduled-backup.yaml
-  - barman-creds-externalsecret.yaml
  - hetzner-creds-externalsecret.yaml
  - pvcs.yaml
@@ -1,20 +0,0 @@
-# Shared Traefik ServersTransport for ingresses that re-encrypt to a
-# backend serving HTTPS with a self-signed / hostname-mismatched cert:
-#
-#   apps/file       — NAS web UI on a self-signed cert
-#
-# Replaces the nginx `backend-protocol: HTTPS` + `proxy-ssl-verify: off`
-# annotation pair. Each Ingress that wants this behaviour sets
-#   traefik.ingress.kubernetes.io/service.serversscheme: https
-#   traefik.ingress.kubernetes.io/service.serverstransport: insecure-backend
-#
-# (bare name — kubernetesIngress provider resolves ServersTransport in
-# the same namespace as the Ingress; the `<ns>-<name>@kubernetescrd`
-# qualified form is for IngressRoute CRDs only).
-apiVersion: traefik.io/v1alpha1
-kind: ServersTransport
-metadata:
-  name: insecure-backend
-  namespace: apps
-spec:
-  insecureSkipVerify: true
@@ -1,10 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - insecure-backend-serverstransport.yaml
  - gatus/
  - external-dns/
-  - file/
  - gitea/
  - ical-proxy/
  # woodpecker/ moved to its own dedicated `woodpecker` Kustomization
@@ -0,0 +1,178 @@
+---
+# The chat endpoint, and the only one behind the ingress.
+#
+# Single-model rather than router mode. Router mode served chat and embeddings
+# from one port, but two things made that wrong here:
+#
+#   1. Its /health answers 200 as soon as the router process listens, while the
+#      models are still downloading behind it. Kubernetes marked the pod Ready
+#      within seconds of start, so traffic arrived at a server that could not
+#      answer. In single-model mode /health returns 503 until the model is
+#      resident, which is what the probes below actually need.
+#   2. Every model on the port is selectable in the built-in WebUI, including
+#      the embedding model. Asking an embedding model to chat returns fluent
+#      nonsense ("capital capital capital ...") rather than an error, and
+#      `--embeddings` does not prevent it. There is no flag to hide a model
+#      from the list — `--tags` is informational and `--ui-config` only sets UI
+#      preferences — so the only way to keep it out of the picker is to keep it
+#      off this endpoint. It runs in embed.yaml instead.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llama-server
+  namespace: ai
+  labels:
+    app.kubernetes.io/name: llama-server
+spec:
+  replicas: 1
+  # A rolling update would load a second copy of the weights onto the same
+  # iGPU and into the same node's RAM before releasing the first.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: llama-server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: llama-server
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        # Makes the CephFS volume group-writable so the weights can be cached.
+        fsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: llama-server
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          args:
+            # The one place this model is named; llama-server resolves the repo
+            # and downloads it itself, so there is nothing to keep in sync.
+            #
+            # A 35B mixture-of-experts with only ~3B parameters active per
+            # token. Decode on this hardware is bound by how many bytes are read
+            # per token rather than by parameter count, and this model reads
+            # ~1.85 GB/token where a dense 27B reads ~16.5 GB — so it is both
+            # far larger and faster than any dense model that would fit.
+            #
+            # Measured here (llama-bench, k8s-1, this image): tg128 11.59 tok/s,
+            # against 10.78 for a dense 8B. Prefill 120.9 t/s at 512 tokens and
+            # 69.8 at 16384 — a sublinear falloff, so the Gated Delta Net
+            # prefill collapse of llama.cpp#27638 does not occur on this stack.
+            - --hf-repo
+            - ggml-org/Qwen3.6-35B-A3B-GGUF:Q4_K_M
+            # Every layer on the Intel iGPU. Full offload matches CPU throughput
+            # while leaving the node's 14 cores free for the rest of the
+            # cluster; the iGPU is here for that isolation, not for a speed-up.
+            - --n-gpu-layers
+            - "99"
+            # Decode runs on the GPU, so extra threads buy nothing. At 11.59
+            # tok/s this model moves 21.4 GB/s against the 53.9 GB/s a dense
+            # model sustains here, so it is the iGPU's 96 EUs working through
+            # many small per-expert matmuls that sets the pace, not bandwidth.
+            - --threads
+            - "4"
+            # 16k rather than 8k because the measured prefill falloff is mild
+            # and this model's KV cache is unusually cheap — only 10 of its 40
+            # layers use full attention.
+            - --ctx-size
+            - "16384"
+            # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
+            # with ErrorDeviceLost.
+            - --ubatch-size
+            - "1024"
+            # The server otherwise allows every CORS origin and warns about it
+            # at startup. Without this, any page a LAN browser visits could call
+            # this unauthenticated API from the user's machine. Consumers talk
+            # to it server-side; the WebUI is same-origin and unaffected.
+            - --cors-origins
+            - localhost
+            # Stable name for the API, so consumers ask for "chat" rather than
+            # the full repo string. Without it the model id is the value of
+            # --hf-repo, and every consumer's config would break the next time
+            # the model is swapped.
+            - --alias
+            - chat
+            # Qwen3.6 enables thinking by default. At ~11.6 tok/s that spends
+            # minutes reasoning before answering, and a request with a modest
+            # max_tokens returns an empty `content` with finish_reason "length"
+            # because the whole budget went to reasoning_content. Off by default
+            # keeps answers usable; a client that wants it can send
+            # chat_template_kwargs per request.
+            - --chat-template-kwargs
+            - '{"enable_thinking": false}'
+          env:
+            # Where --hf-repo caches the weights. Pointing it at the PVC makes
+            # the download a one-time cost, shared with llama-embed, and keeps
+            # the server off a home directory it cannot write as UID 1000.
+            - name: LLAMA_CACHE
+              value: /models
+          ports:
+            - name: http
+              containerPort: 8080
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          # Vulkan allocates the iGPU's working set from system RAM, and that
+          # counts against the pod's cgroup: a 5.2 GiB model measured 7.54 GiB
+          # RSS. Sized as the 19 GiB of weights plus that overhead, plus the KV
+          # cache — which is unusually cheap here, since only 10 of the 40
+          # layers use full attention.
+          resources:
+            requests:
+              cpu: 500m
+              memory: 24Gi
+              gpu.intel.com/i915: "1"
+            limits:
+              cpu: "6"
+              memory: 32Gi
+              gpu.intel.com/i915: "1"
+          # /health is 503 while the model loads and 200 once it is resident, so
+          # the pod only goes Ready when it can actually answer. On a cold PVC
+          # that includes pulling ~19 GiB from Hugging Face, hence the long
+          # startup budget (40 min); liveness takes over only after that.
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+            failureThreshold: 240
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 6
+          volumeMounts:
+            - name: models
+              mountPath: /models
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: llama-server-models
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: llama-server
+  namespace: ai
+  labels:
+    app.kubernetes.io/name: llama-server
+spec:
+  selector:
+    app.kubernetes.io/name: llama-server
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
@@ -0,0 +1,119 @@
+---
+# The embedding endpoint, for RAG and semantic search.
+#
+# Deliberately separate from llama-server and deliberately not behind the
+# ingress: an embedding model has no generative training, and when it shares a
+# port with the chat model the built-in WebUI lists it as a choice. Selecting it
+# returns fluent nonsense rather than an error, because `--embeddings` does not
+# actually refuse chat completions. Keeping it on its own ClusterIP-only Service
+# is what makes it unselectable, since llama.cpp has no way to hide a model from
+# the picker.
+#
+# Reachable in-cluster at http://llama-embed.ai.svc.cluster.local:8080/v1/embeddings.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llama-embed
+  namespace: ai
+  labels:
+    app.kubernetes.io/name: llama-embed
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: llama-embed
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: llama-embed
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: llama-embed
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          args:
+            - --hf-repo
+            - Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0
+            # Serves POST /v1/embeddings.
+            - --embeddings
+            - --n-gpu-layers
+            - "99"
+            - --threads
+            - "2"
+            - --ctx-size
+            - "2048"
+            # No browser ever talks to this Service, so the WebUI is dead weight
+            # and one more unauthenticated surface.
+            - --no-webui
+          env:
+            # Same cache as llama-server, so the two share one copy of anything
+            # they have in common and neither re-downloads the other's weights.
+            - name: LLAMA_CACHE
+              value: /models
+          ports:
+            - name: http
+              containerPort: 8080
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 250m
+              memory: 2Gi
+              gpu.intel.com/i915: "1"
+            limits:
+              cpu: "2"
+              memory: 4Gi
+              gpu.intel.com/i915: "1"
+          # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
+          # model needs, but still sized for a cold download.
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+            failureThreshold: 60
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 6
+          volumeMounts:
+            - name: models
+              mountPath: /models
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: llama-server-models
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: llama-embed
+  namespace: ai
+  labels:
+    app.kubernetes.io/name: llama-embed
+spec:
+  selector:
+    app.kubernetes.io/name: llama-embed
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
@@ -0,0 +1,28 @@
+---
+# LAN-only: no `dns.renner.uno/public` label, so the Cloudflare external-dns
+# instance skips it and only the UDM instance publishes the record. The API is
+# unauthenticated, so it must not be reachable from the internet.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llama-server
+  namespace: ai
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - llm.renner.uno
+      secretName: llama-server-tls
+  rules:
+    - host: llm.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: llama-server
+                port:
+                  number: 8080
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pvc.yaml
+  - deployment.yaml
+  - embed.yaml
+  - ingress.yaml
@@ -0,0 +1,21 @@
+---
+# Holds the GGUF weights, downloaded once by the Deployment's initContainer.
+# Prune-disabled so retiring this Kustomization does not throw away several
+# GiB of model files that would have to be pulled from Hugging Face again.
+#
+# ReadWriteMany so a second replica or a one-off benchmark Job can mount the
+# same weights; the models are read-only at inference time.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: llama-server-models
+  namespace: ai
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ceph-filesystem
+  resources:
+    requests:
+      storage: 40Gi
@@ -5,9 +5,8 @@ metadata:
  namespace: monitoring
  labels:
    app.kubernetes.io/name: alloy
-    # kube-prometheus-stack's Prometheus resource selects ServiceMonitors
-    # by this label; without it the resource exists but is never
-    # discovered.
+    # Not required for discovery — serviceMonitorSelector is empty — but kept
+    # for consistency with the other monitors.
    release: kube-prometheus-stack
 spec:
  selector:
@@ -1,7 +1,8 @@
 # Scrape the CNPG Postgres instances (port 9187, built-in exporter).
-# CNPG's own enablePodMonitor would generate PodMonitors without the
-# `release: kube-prometheus-stack` label, which this Prometheus requires
-# (podMonitorSelector), so we define them here instead.
+# Defined here rather than via CNPG's own enablePodMonitor so the scrape
+# config lives with the other monitors. The `release` label below is not
+# required for discovery — podMonitorSelector is empty — but kept for
+# consistency.
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -5,9 +5,8 @@ metadata:
  namespace: home-assistant
  labels:
    app.kubernetes.io/name: home-assistant
-    # kube-prometheus-stack's Prometheus resource selects ServiceMonitors
-    # by this label; without it the resource exists but is never
-    # discovered. Same trap the garage PrometheusRule documents.
+    # Not required for discovery — serviceMonitorSelector is empty — but kept
+    # for consistency with the other monitors.
    release: kube-prometheus-stack
 spec:
  selector:
@@ -4,7 +4,9 @@ kind: Kustomization
 # come from the kube-prometheus-stack chart. Flux applies this Kustomization
 # only after `monitoring` is Ready (chart installed → CRDs present).
 resources:
+  - prometheusrule-blackbox.yaml
  - prometheusrule-ceph.yaml
+  - prometheusrule-ci-openbao.yaml
  - prometheusrule-cilium.yaml
  - prometheusrule-disk-wear.yaml
  - prometheusrule-hetzner-backup.yaml
@@ -5,9 +5,8 @@ metadata:
  namespace: monitoring
  labels:
    app.kubernetes.io/name: loki
-    # kube-prometheus-stack's Prometheus resource selects ServiceMonitors
-    # by this label; without it the resource exists but is never
-    # discovered.
+    # Not required for discovery — serviceMonitorSelector is empty — but kept
+    # for consistency with the other monitors.
    release: kube-prometheus-stack
 spec:
  # Matches the `loki` (single-binary) and `loki-canary` services, but
@@ -1,5 +1,9 @@
 # HTTP probes — fan out to blackbox-exporter. Probe targets mirror the
 # legacy Ansible role's blackbox_http_targets list.
+#
+# The `release` label on every Probe here IS load-bearing: probeSelector still
+# matches on it, unlike serviceMonitorSelector and podMonitorSelector, which
+# are empty. Removing it silently stops the probe from being discovered.
 apiVersion: monitoring.coreos.com/v1
 kind: Probe
 metadata:
@@ -45,6 +49,28 @@ spec:
        # ports outside the hairpin's 80/443 — real WAN path.
        - http://proxy-1.renner.uno:9000/health
        - http://proxy-2.renner.uno:9000/health
+        # Woodpecker. Ungated, root returns 200 with no redirect. Every
+        # scheduled job in this repo runs here — the alert-monitor cron, the
+        # nightly scans, the deploy targets — so an outage here is silent by
+        # construction: the thing that would have told you is the thing that
+        # is down.
+        - https://ci.renner.uno
+        # OpenBao. Probes the health API rather than the UI on purpose: a
+        # SEALED instance still serves the UI (307 -> /ui/ -> 200) while every
+        # ExternalSecret in the cluster fails, which is exactly the
+        # green-while-broken shape this file warns about elsewhere.
+        #
+        # standbyok=true because the endpoint answers 200 only on the ACTIVE
+        # node and 429 on a standby — the external route currently lands on a
+        # standby, so the bare path would be permanently red. With the
+        # parameter, 200 means initialized, unsealed and serving, on whichever
+        # node answers; 503 sealed and 501 uninitialized both stay red.
+        - https://openbao.renner.uno/v1/sys/health?standbyok=true
+        # ical-proxy serves an unauthenticated /health that answers 200 with
+        # {"status":"healthy"}. Unlike the gated hostnames this is real app
+        # health, not reachability — the root path returns 404 by design, since
+        # the service only answers on calendar paths.
+        - https://ical-proxy.renner.uno/health
        # APP-HEALTH probes for the gated monitoring UIs, from the correct
        # in-cluster vantage (monitoring-vantage-point-rule): the gate is
        # only on the ingress, so the ClusterIP /-/healthy endpoints are
@@ -57,6 +83,11 @@ spec:
        # external reachability probe above; it is not the health vantage.)
        - http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/-/healthy
        - http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090/-/healthy
+        # Loki has no external vantage at all: its ingress requires auth on
+        # every path, so an outside probe can only ever report 401. In-cluster
+        # /ready answers 200 unauthenticated and is in this namespace, so the
+        # same intra-namespace allowance the two above rely on covers it.
+        - http://loki.monitoring.svc.cluster.local:3100/ready
 # WAN-path probes of the public edge BY PROXY IP. The name-based
 # home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway)
 # and never leaves the LAN — the vantage-point blind spot that let the
@@ -0,0 +1,59 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: blackbox
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: blackbox.rules
+      rules:
+        # Catch-all for every blackbox-http target that has no rule of its own.
+        # Before this existed, a target could be probed, graphed and completely
+        # unalerted — which is how file.renner.uno served 502 for months.
+        #
+        # THE EXCLUSION LIST AND THE DEDICATED RULES ARE ONE INVARIANT: an
+        # instance belongs here only while a rule elsewhere covers it, and
+        # adding a dedicated rule means adding its instance here. Get it wrong
+        # in one direction and the alert fires twice; wrong in the other and it
+        # never fires at all. Currently excluded:
+        #
+        #   fsn1.your-objectstorage.com   HetznerBackupEndpointDown (critical)
+        #   netbird.renner.uno            prometheusrule-netbird-availability
+        #   proxy-1/2:9000/health         same file, relay redundancy
+        #   ci.renner.uno                 WoodpeckerDown
+        #   openbao .../sys/health        OpenBaoUnavailable
+        #
+        # Warning, not critical, on purpose: this rule cannot know what a given
+        # target means. A target whose loss deserves a critical page gets its
+        # own rule — and lands in the list above. That is what keeps the
+        # exclusion list from being arbitrary.
+        - alert: BlackboxProbeFailing
+          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
+          for: 10m
+          annotations:
+            summary: "Blackbox probe of {{ $labels.instance }} has been failing for 10 minutes"
+            description: >-
+              The endpoint is not answering as expected. For the Authentik-gated
+              hostnames this is a reachability check covering DNS, TLS, Traefik
+              and the gate outpost — it says nothing about the app behind the
+              gate, which the internal /-/healthy probes cover separately.
+          labels:
+            severity: warning
+        # The check that the checks are running. Every rule above reads
+        # probe_success; if the series stop being produced at all, none of them
+        # can fire and the silence looks exactly like health.
+        - alert: BlackboxProbesAbsent
+          expr: absent(probe_success{job="blackbox-http"})
+          for: 15m
+          annotations:
+            summary: "No blackbox-http probe results are reaching Prometheus"
+            description: >-
+              Either the blackbox exporter is down or the Probe objects stopped
+              being discovered. Every endpoint alert depends on this job, so
+              they are all blind until it returns.
+          labels:
+            severity: critical
@@ -0,0 +1,47 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ci-openbao
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: ci-openbao.rules
+      rules:
+        - alert: WoodpeckerDown
+          # Probe target defined in probes-blackbox.yaml. Root, ungated, 200.
+          expr: probe_success{job="blackbox-http", instance="https://ci.renner.uno"} == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Woodpecker is not answering on ci.renner.uno"
+            description: >-
+              Every scheduled job in this repo runs here: the alert-monitor
+              cron that files these issues, the nightly scans, and every deploy
+              target. Nothing else reports this outage, because the component
+              that would have reported it is the one that is down. Alertmanager
+              itself is unaffected, which is why this alert exists.
+        - alert: OpenBaoUnavailable
+          # Probes /v1/sys/health?standbyok=true — 200 on any unsealed node,
+          # 503 sealed, 501 uninitialized. The UI stays up when sealed, so a
+          # root probe would not catch this.
+          #
+          # Matched by prefix rather than the full string: the target carries a
+          # query parameter, and an exact match would break silently if the
+          # instance label ever encodes it differently.
+          expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "OpenBao is sealed, uninitialized or unreachable"
+            description: >-
+              Every ExternalSecret in the cluster resolves through OpenBao.
+              Existing Secrets keep working, so nothing breaks at once — the
+              damage shows up at the next refresh interval or the next pod that
+              needs a fresh credential. Check whether it is sealed before
+              assuming a network fault.
@@ -11,25 +11,22 @@ spec:
  groups:
    - name: cnpg-backup.rules
      rules:
-        # WHY THIS FILE EXISTS
-        # Seven Postgres clusters back up to Hetzner and NOTHING alerted if that
-        # stopped. The only related rule was HetznerBackupEndpointDown, which
-        # probes the S3 endpoint — it proves the bucket is reachable, not that
-        # anything was written to it. Same failure shape as the WordPress backup
-        # that failed nightly for three days against a bucket that was declared
-        # in tofu but never applied: the status surface looked fine because
-        # nothing checked the thing that mattered.
+        # The Postgres clusters back up to Hetzner. HetznerBackupEndpointDown
+        # probes that S3 endpoint, which proves the bucket is reachable and
+        # nothing more — these two rules check that something is actually being
+        # written to it.

        - alert: CNPGWalArchiveBacklog
          # `ready` counts WAL segments sitting in pg_wal/archive_status waiting
-          # to be archived. Steady state is 0 across every cluster (verified
-          # 2026-08-19). A sustained backlog means the archiver is failing:
-          # point-in-time recovery is silently broken from that moment, and
-          # pg_wal keeps growing until it fills the PVC and takes Postgres down.
+          # to be archived. Steady state is 0. A sustained backlog means the
+          # archiver is failing: point-in-time recovery is silently broken from
+          # that moment, and pg_wal keeps growing until it fills the PVC and
+          # takes Postgres down.
          #
-          # This is the load-bearing alert of the two. It depends only on the
-          # instance-level collector, so unlike the backup-age metrics below it
-          # is unaffected by the missing Backup CRs (#522).
+          # This is the load-bearing alert of the two. It reads the
+          # instance-level collector directly, so unlike the backup-age metric
+          # discussed below it does not depend on Cluster status being
+          # populated.
          #
          # Threshold: >5 segments sustained for 30m. Brief non-zero values are
          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
@@ -63,16 +60,17 @@ spec:
        #
        # The natural expression is
        #   (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600
-        # and it CANNOT be used today: that metric reads exactly 0 on every
-        # instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2,
-        # pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any
-        # successful backup anywhere, because the Backup CRs it derives that
-        # from are being deleted within a minute of completing — see #522.
+        # and it cannot be used, because that metric reads exactly 0 on every
+        # instance. The collector derives it from `Cluster.status
+        # .lastSuccessfulBackup`, and that field is absent on every Cluster
+        # here: all of them back up with `method: plugin` (barman-cloud), and
+        # the plugin path does not write it back.
        #
-        # Backups themselves DO work; that was verified independently by
-        # listing the bucket with barman-cloud-backup-list and restoring a WAL
-        # segment back out of S3. The gap is in the record, not the data.
+        # The backups themselves run and are retained — completed Backup CRs
+        # going back weeks exist for every scheduled cluster. The gap is in the
+        # status field the metric reads, not in the data.
        #
-        # Adding the rule now would fire permanently for all seven clusters and
-        # be silenced within a day, which is worse than not having it. Add it as
-        # part of resolving #522, once the metric reflects reality.
+        # Adding the rule against that metric would fire permanently for every
+        # cluster, which is worse than not having it. Either the plugin starts
+        # populating the field, or the alert has to be built on the Backup CRs
+        # directly, which Prometheus does not see today.
@@ -11,38 +11,55 @@ spec:
  groups:
    - name: disk-wear.rules
      rules:
-        # THE GAP THESE RULES CLOSE
+        # Wear percentages are self-reported: 100 means the rated endurance is
+        # spent, not that the drive stops.
        #
-        # Drive wear is invisible in every other signal this cluster has.
-        # `ceph -s` reads active+clean at any wear level, no Ceph health check
-        # fires until a device actually fails, and the block-layer metrics show
-        # throughput and latency but never endurance. That blind spot is how a
-        # set of OSD drives reached 81-87% before anyone looked (#549, #434).
+        # Wear alone is not a signal. It only ever rises, so a plain threshold
+        # starts firing the day a drive crosses it and never stops — a status
+        # dressed as an alert (#673, #675, #682, #686 were all one drive set
+        # crossing 80%). What matters is wear that is high AND still climbing,
+        # so the alert below is a projection: at the write rate of the last
+        # week, when does this drive reach 95%?
        #
-        # The numbers below are wear percentages reported by the drive itself:
-        # 100 means the manufacturer's rated endurance is used up. Drives do
-        # not stop at 100 — they typically keep working and start consuming
-        # spare blocks — so treat these as planning thresholds, not failure
-        # predictions.
-        - alert: NvmeDriveWearHigh
-          # 80% with months of runway left is the point where ordering a
-          # replacement is still a calm decision rather than an outage.
-          # `for` is long because wear is monotonic and slow: it cannot
-          # flap, and a short window would only make the alert noisier
-          # after a restart.
-          expr: smartctl_device_percentage_used >= 80
-          for: 1h
+        # The rate is self-calibrating. Each drive reports both its lifetime
+        # bytes written and its own wear percentage, so bytes-per-percent comes
+        # from the drive itself and no rated-endurance table has to be kept in
+        # sync with the hardware. percentage_used is a whole-number percent and
+        # advances only every few weeks here, which is far too coarse to
+        # differentiate; bytes_written moves on every scrape.
+        - alert: NvmeDriveWearProjectedExhaustion
+          # The >= 80 gate is numerical, not editorial: bytes-per-percent is
+          # derived by dividing by an integer percentage, so on a lightly worn
+          # drive a half-percent rounding error swings the projection by tens
+          # of percent. Above 80 the same rounding is under one percent.
+          # Idle drives divide by a zero rate and yield +Inf, which is not < the
+          # threshold, so they stay silent on their own.
+          expr: >-
+            (
+              (95 - smartctl_device_percentage_used)
+              /
+              (
+                rate(smartctl_device_bytes_written[7d]) * 86400
+                  * smartctl_device_percentage_used
+                  / smartctl_device_bytes_written
+              )
+            ) < 180
+            and smartctl_device_percentage_used >= 80
+          # Long, because a projection off a 7d rate should not react to a day.
+          for: 6h
          labels:
            severity: warning
          annotations:
-            summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear"
+            summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} projects to 95% wear in {{ $value | printf \"%.0f\" }} days"
            description: >-
-              The drive has used {{ $value }}% of its rated write endurance.
-              Nothing is failing and no data is at risk yet. Check what the
-              drive is carrying before deciding urgency: an OSD is replaceable
-              with a rebalance, whereas the EPHEMERAL volume has no such
-              escape — a node simply stops when it goes. The Disk Health
-              dashboard resolves which drive holds which role.
+              The drive is past 80% of its rated endurance and the write rate
+              of the last week would spend the rest within six months. This is
+              a lead-time warning, not a fault: decide whether to replace the
+              drive or move what is writing to it. The Disk Health dashboard
+              shows which drives carry which role.
+        # Kept as a flat threshold on purpose: past rated endurance the drive
+        # runs outside what the manufacturer stands behind regardless of how
+        # fast it got there, and the alert clears by replacing it.
        - alert: NvmeDriveWearCritical
          expr: smartctl_device_percentage_used >= 95
          for: 1h
@@ -55,9 +72,7 @@ spec:
              likely keep working for a while, but it is now operating outside
              what the manufacturer stands behind, and replacement should be
              scheduled rather than considered.
-        # A drive that has consumed its spare area is no longer masking media
-        # defects, which is a materially different state from being worn:
-        # wear is a budget, spare exhaustion is damage.
+        # Wear is a budget; spare exhaustion is damage.
        - alert: NvmeDriveSpareExhausted
          expr: smartctl_device_available_spare <= smartctl_device_available_spare_threshold
          for: 15m
@@ -70,10 +85,6 @@ spec:
              drive has run out of reserve blocks to remap failures into.
              Replace it.
        - alert: NvmeDriveCriticalWarning
-          # The NVMe critical warning byte is a bitfield: spare below
-          # threshold, temperature out of range, degraded reliability,
-          # read-only mode, volatile-memory backup failed. Any non-zero value
-          # is the drive itself asking for attention.
          expr: smartctl_device_critical_warning > 0
          for: 5m
          labels:
@@ -85,10 +96,8 @@ spec:
              detail with `smartctl -a` on the node; the value is a bitfield,
              not a count.
        - alert: NvmeDriveMediaErrors
-          # Unlike wear, this counter should never move. Alert on the increase
-          # rather than the absolute value, so a drive that recorded errors
-          # long ago does not alert forever while a drive that starts failing
-          # today does.
+          # On the increase, not the total: pre-existing errors would otherwise
+          # alert forever. Standing counts are visible on the dashboard only.
          expr: increase(smartctl_device_media_errors[6h]) > 0
          for: 5m
          labels:
@@ -99,14 +108,9 @@ spec:
              The drive reported unrecoverable data-integrity errors in the
              last six hours. A healthy drive never increments this. Check
              whether Ceph has logged scrub errors on the same node.
-        # Without this the whole group is silently useless: no series means no
-        # alert fires, and a dashboard of empty panels reads much like a
-        # healthy one. This is the check that the checks are running.
+        # The check that the checks are running: no series fires no alert.
+        # Counted against node-exporter's inventory, not a fixed fleet size.
        - alert: SmartctlExporterIncomplete
-          # Compared against node-exporter's own NVMe inventory rather than a
-          # written-down fleet size. Adding or removing a drive then moves both
-          # sides together, so this cannot rot into a permanent false alarm the
-          # way a hardcoded count would.
          expr: count(smartctl_device_percentage_used) < count(node_nvme_info) or absent(smartctl_device_percentage_used)
          for: 30m
          labels:
@@ -17,11 +17,12 @@ spec:
          # drains on recovery) and Velero backups fail. Replaces the old
          # GarageDown alert now that backups live on Hetzner, not nas-1.
          #
-          # CNPG has no metric-based archiving alert yet — its pod metrics
-          # are not scraped (enablePodMonitor=false). Enabling CNPG
-          # PodMonitors for a granular "archiving falling behind" alert is
-          # a follow-up. Velero's own VeleroBackupValidationFailure /
-          # VeleroBackupStale rules already cover backup-level failures.
+          # The granular counterpart is CNPGWalArchiveBacklog in
+          # prometheusrule-cnpg-backup.yaml, reading the instance collector
+          # scraped by cnpg-podmonitors.yaml: this alert says the target is
+          # unreachable, that one says archiving is falling behind whatever the
+          # cause. Velero's VeleroBackupValidationFailure / VeleroBackupStale
+          # cover backup-level failures.
          expr: probe_success{instance="https://fsn1.your-objectstorage.com"} == 0
          for: 10m
          labels:
@@ -5,25 +5,20 @@ metadata:
  namespace: monitoring
  labels:
    app.kubernetes.io/name: smartctl-exporter
-    # kube-prometheus-stack's Prometheus resource selects ServiceMonitors
-    # by this label; without it the resource exists but is never
-    # discovered.
+    # Not required for discovery — serviceMonitorSelector is empty — but kept
+    # for consistency with the other monitors.
    release: kube-prometheus-stack
 spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: smartctl-exporter
  endpoints:
-    # smartctl itself polls the drives every 60s, so scraping faster than that
-    # returns the same numbers with a new timestamp. Wear moves on a scale of
-    # weeks; 60s is already far finer than the signal.
+    # smartctl polls the drives every 60s; scraping faster gains nothing.
    - port: http-metrics
      path: /metrics
      interval: 60s
      relabelings:
-        # Without this every series is labelled with the exporter pod's IP and
-        # nothing else. A DaemonSet pod's IP says nothing about which machine
-        # the drive is in, and it changes on every reschedule — so the wear
-        # figure would be unattributable, which is most of its value gone.
+        # Without this a series carries only the exporter pod's IP, which does
+        # not identify the machine the drive is in.
        - sourceLabels: [__meta_kubernetes_pod_node_name]
          targetLabel: node
@@ -70,7 +70,7 @@
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Fullest EPHEMERAL volume across the fleet — Talos's /var, holding container images, pod scratch space and kubelet state. A full /var takes the node down, and unlike Ceph nothing rebalances away from it.",
+      "description": "Fullest EPHEMERAL volume across the fleet — Talos's /var. Its contents are meant to be disposable: container images, kubelet state and pod scratch all rebuild. Filling it is still a problem, because a node with no room to write stops being useful long before anything is lost, and nothing rebalances /var the way Ceph rebalances a full OSD.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
@@ -453,7 +453,7 @@
      "id": 8,
      "options": {
        "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
-        "content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Wear is a budget, not a prediction.** The percentage is what the drive reports against its own rated endurance. Drives do not stop at 100 — they usually keep working and start consuming spare blocks — so read it for planning, and read available spare and media errors for whether something is failing now. Those are different questions and the alert rules treat them as such.\n\n**A green top row is not a healthy drive.** Endurance says nothing about the drive that dies of a controller fault tomorrow, and this dashboard cannot see one coming.\n\nCeph keeps its own wear figures for OSDs in the devicehealth module, which is a useful cross-check but covers OSDs only: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. A drive removed from Ceph is purged from that registry and its history goes with it.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, so the drive with the least endurance left can be the one carrying container state.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
+        "content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Wear is a budget, not a prediction.** The percentage is what the drive reports against its own rated endurance. Drives do not stop at 100 — they usually keep working and start consuming spare blocks — so read it for planning, and read available spare and media errors for whether something is failing now. Those are different questions and the alert rules treat them as such.\n\n**A green top row is not a healthy drive.** Endurance says nothing about the drive that dies of a controller fault tomorrow, and this dashboard cannot see one coming.\n\nCeph keeps its own wear figures for OSDs in the devicehealth module, which is a useful cross-check but covers OSDs only: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. A drive removed from Ceph is purged from that registry and its history goes with it.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, and putting a worn one on EPHEMERAL is a sensible destination rather than a hazard — that volume is built to be rebuilt. Check the inventory table for what a given drive holds before reading its wear as urgent.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
        "mode": "markdown"
      },
      "title": "Gaps and caveats",
@@ -24,8 +24,10 @@ spec:
      enabled: true
      defaults:
        labels:
-          # Required for kube-prometheus-stack's Prometheus to pick up
-          # this ServiceMonitor (its selector matches release).
+          # Not required for discovery — serviceMonitorSelector is empty — but
+          # kept for consistency with the other monitors. The Probes in
+          # monitoring-rules/probes-blackbox.yaml do need it; probeSelector
+          # still matches on release.
          release: kube-prometheus-stack
        interval: 30s
    # Helm DOES deep-merge this map with the chart's default (that is why
@@ -56,26 +56,21 @@ spec:
        #
        # By default kube-prometheus-stack sets these selectors to
        # `matchLabels: {release: kube-prometheus-stack}`. ServiceMonitors
-        # created at RUNTIME by third-party operators carry that operator's own
-        # labels instead and can never be made to match — no static manifest or
-        # Kustomize patch can relabel an object the operator creates itself.
+        # created at RUNTIME by third-party operators — Rook, goauthentik.io —
+        # carry that operator's own labels instead and can never be made to
+        # match: no static manifest or Kustomize patch can relabel an object
+        # the operator creates itself. Left strict, those targets are dropped
+        # silently, with no failed resource and no gap visible anywhere except
+        # in the missing series.
        #
-        # This was silently dropping 12 of 30 ServiceMonitors (2026-07-27):
-        #   rook-ceph/rook-ceph-mgr, rook-ceph/rook-ceph-exporter  (team: rook)
-        #   authentik/ak-outpost-*  x10                            (goauthentik.io/...)
-        # Consequence: ZERO `ceph_*` series existed in Prometheus for all 79
-        # days of the cluster's life — the entire storage layer had no
-        # telemetry, no history, and any Ceph alerting rule would have had
-        # nothing to fire on. The Rook side is the reason this was found; the
-        # authentik outpost targets come along with it.
+        # Because these two are empty, the `release` label on this repo's own
+        # ServiceMonitors and PodMonitors is decorative. It is still set for
+        # consistency, and the comments there say so.
        #
-        # This is the ServiceMonitor twin of the trap already documented for
-        # rules in monitoring-rules/prometheusrule-velero.yaml ("without this
-        # label the rule is created but never loaded").
-        #
-        # ruleSelector is deliberately NOT loosened: this repo's PrometheusRules
-        # all carry `release: kube-prometheus-stack` on purpose, and keeping
-        # that selector strict stops a stray third-party rule from firing here.
+        # ruleSelector, probeSelector and scrapeConfigSelector are deliberately
+        # NOT loosened: nothing outside this repo creates those, so keeping the
+        # selectors strict stops a stray third-party rule from firing here. For
+        # PrometheusRules and Probes the label IS required.
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        # Rewrites the in-cluster URL embedded in alert generatorURLs and
@@ -1,37 +1,8 @@
 # SMART/endurance metrics for every NVMe drive in the fleet (#549).
 #
-# WHY THIS HOLDS ELEVATED CAPABILITIES, and why that is a deliberate exception
-# rather than an oversight:
-#
-# NVMe reports health over an admin ioctl on the character device
-# (NVME_IOCTL_ADMIN_CMD), which the kernel gates on CAP_SYS_ADMIN. There is no
-# sysfs path for wear, media errors or written-bytes totals — hwmon publishes
-# the controller temperature and nothing else. Upstream smartctl_exporter
-# documents `privileged: true` and `user: root` for exactly this reason.
-#
-# The alternative that needs no privileges — reading Ceph's devicehealth
-# archive, which does hold full smartctl output — covers OSD drives only. In
-# this cluster that is three of nine, and the six it misses include the drives
-# that were retired from Ceph for wear and now carry EPHEMERAL. Covering the
-# healthy drives and not the worn ones was judged not worth a new service; the
-# owner chose fleet-wide coverage with the privilege exception instead.
-#
-# The exception is `privileged: true`, and narrower settings were tried and
-# measured to fail — see the securityContext below for what was ruled out and
-# how. It is still bounded: no host namespaces, no hostPID, no hostNetwork,
-# read-only root filesystem, and /dev as the only host mount.
-#
-# If this ever needs revisiting, there are exactly two things worth checking,
-# and neither is "does another exporter ask for less":
-#   - whether the kernel has grown a sysfs interface for NVMe health, which
-#     would remove the need for device access altogether
-#   - whether enough workloads need raw device access to justify a device
-#     plugin, which is the only way Kubernetes grants individual devices
-# The requirement comes from the device cgroup and the ioctl gate, not from
-# the choice of tool.
-#
-# Upstream had no release between 2025-04-22 (v0.14.0) and this deployment.
-# The repository is not archived; the pin is simply its newest tag.
+# Runs privileged: host device access is gated by the container device cgroup,
+# and Kubernetes offers no per-device allowlist. Narrower settings do not work;
+# the alternatives that were tried are in the commit history.
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -57,10 +28,8 @@ spec:
      containers:
        - name: smartctl-exporter
          image: quay.io/prometheuscommunity/smartctl-exporter:v0.14.0
-          # No --smartctl.device on purpose: the exporter scans for devices
-          # itself and rescans every 10m by default. Device names are not
-          # stable across these machines, so a hardcoded list would quietly
-          # stop covering a drive the next time one is replaced.
+          # No --smartctl.device: the exporter scans and rescans on its own.
+          # Device names are not stable across these machines.
          args:
            - --web.listen-address=:9633
          ports:
@@ -68,56 +37,18 @@ spec:
              containerPort: 9633
              protocol: TCP
          securityContext:
-            # PRIVILEGED, and there is no narrower setting that works. This
-            # was established by measurement after two narrower attempts
-            # failed, so the reasoning is recorded here rather than rediscovered:
-            #
-            #   Attempt 1 — CAP_SYS_ADMIN + CAP_SYS_RAWIO, everything else
-            #   dropped. The exporter found all three drives per node and
-            #   could open none: "Smartctl open device: /dev/nvme0 failed:
-            #   Operation not permitted".
-            #
-            #   Attempt 2 — the same plus seccompProfile: Unconfined, on the
-            #   theory that EPERM is the default seccomp action for an
-            #   unlisted syscall and the kubelet runs seccompDefault: true.
-            #   /proc/1/status showed CapEff 0x220000 and Seccomp 2, so both
-            #   capabilities were effective and a filter was loaded. Removing
-            #   the filter changed nothing.
-            #
-            #   What actually settled it: `head -c 512 /dev/nvme0n1` inside
-            #   the container fails with EPERM too. A plain read of a block
-            #   device involves no ioctl, no capability and no syscall a
-            #   seccomp profile would filter. The block is the device cgroup —
-            #   containerd denies non-privileged containers access to host
-            #   device nodes whether or not /dev is mounted.
-            #
-            # Kubernetes exposes no per-device allowlist in a Pod spec; there
-            # is no equivalent of `docker --device`. The alternatives are
-            # `privileged: true` or a device plugin advertising the NVMe
-            # drives as a resource. A device plugin is the genuinely narrower
-            # answer and is worth building if more workloads ever need raw
-            # device access; for a single read-only metrics exporter it is a
-            # component to write, deploy and maintain in exchange for the same
-            # access.
            runAsUser: 0
            runAsNonRoot: false
            privileged: true
-            # Stated rather than omitted: the API rejects `privileged: true`
-            # together with `allowPrivilegeEscalation: false`, so leaving it
-            # out would only hide that privileged already implies it.
+            # The API rejects privileged together with
+            # allowPrivilegeEscalation: false.
            allowPrivilegeEscalation: true
            readOnlyRootFilesystem: true
          volumeMounts:
-            # Not read-only. smartmontools opens NVMe devices read-write on
-            # some paths, and once the container is privileged a read-only
-            # flag on this mount is not a boundary anyone should rely on —
-            # keeping it would suggest a protection that is not there.
+            # Not read-only: smartmontools opens NVMe devices read-write.
            - name: dev
              mountPath: /dev
-            # smartctl is not known to need scratch space, but the root
-            # filesystem is read-only and a tool that turns out to want /tmp
-            # would crashloop instead of degrading. Cheaper to provide it than
-            # to find out from a restart counter.
+            # Scratch space, because the root filesystem is read-only.
            - name: tmp
              mountPath: /tmp
          resources:
@@ -55,7 +55,7 @@ spec:
        # stdlib only; Flux keeps the tag in lockstep via the shared image
        # policy) — no extra dependency.
        - name: reconcile-publish-ip
-          image: ghcr.io/music-assistant/server:2.10.0 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.1 # {"$imagepolicy": "flux-system:music-assistant"}
          command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
          volumeMounts:
            - name: data
@@ -65,7 +65,7 @@ spec:
              readOnly: true
      containers:
        - name: music-assistant
-          image: ghcr.io/music-assistant/server:2.10.0 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.1 # {"$imagepolicy": "flux-system:music-assistant"}
          env:
            - name: LOG_LEVEL
              value: info
@@ -24,6 +24,22 @@ spec:
            - name: wyoming
              containerPort: 10400
              protocol: TCP
+          # Wyoming is raw TCP with no HTTP surface and no health verb, so
+          # accepting a connection is all that can be asserted from outside the
+          # process. Weaker than an HTTP probe, but the previous state was none
+          # at all: a dead server stayed Ready and kept being routed to.
+          livenessProbe:
+            tcpSocket:
+              port: wyoming
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            tcpSocket:
+              port: wyoming
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
          resources:
            requests:
              cpu: 100m
@@ -5,25 +5,15 @@ metadata:
  labels:
    app.kubernetes.io/name: otbr
 spec:
-  # Disabled 2026-07-11 (replicas: 0) after otbr broke Matter.
+  # DISABLED (replicas: 0). Running otbr broke Matter: its radio is RF-isolated
+  # enough from the Nest mesh that it forms a rival partition of the same
+  # network, and the two bridge only over Ethernet — fast enough for a ping,
+  # too slow for the fragmented CASE handshake, so devices stay discoverable
+  # while every commissioning attempt times out.
  #
-  # otbr is seeded with the Nest network's dataset so it should join the
-  # existing partition as an additive co-BR. In practice it does not merge
-  # cleanly: its SLZB-06 radio is RF-isolated enough from the Nest mesh that
-  # otbr elects itself leader of a rival partition (or the network re-addresses
-  # under a new OMR prefix, observed fd5f -> fdd5). The two same-network
-  # partitions then bridge only over the Ethernet backbone, which carries a
-  # ping but is too slow for the large fragmented CASE handshake -- every
-  # Matter device stays mDNS-discoverable while CASE times out. Removing otbr
-  # re-merges the mesh onto the Nest hubs and Matter recovers.
-  #
-  # This is NOT a diagnosis of a specific config bug: the DHCP/sbr routing
-  # changes (19a4482, df61407) were investigated and matter-server's IPv6 path
-  # to the mesh works (reaches devices, ping ok); sbr's v6 table 100 is an
-  # inert orphan and otbr's eth0 default is outranked by net1's pref-high RA
-  # default. The instability is at the Thread partition layer. Re-enable only
-  # after the co-BR integration is verified to MERGE (one partition, one OMR,
-  # otbr's router table shows the Nest routers) instead of split.
+  # The instability is at the Thread partition layer, not in the config below.
+  # Re-enable only once co-BR integration is verified to MERGE: one partition,
+  # one OMR prefix, and the Nest routers visible in otbr's router table.
  replicas: 0
  strategy:
    type: Recreate
@@ -46,25 +36,17 @@ spec:
      # (slzb-thread.renner.uno). Dropping the workload-class=pinned
      # toleration lets the edge taint keep it off the tiny edge box.
      containers:
-        # The Thread RCP lives on a SLZB-06 over Ethernet at
-        # slzb-thread.renner.uno:6638 (Serial-over-IP). otbr-agent's
-        # spinel transport supports `spinel+hdlc+forkpty://<exec>?<args>`
-        # which forks an exec whose stdio becomes the spinel-HDLC
-        # transport, but the upstream openthread/otbr image does NOT
-        # ship socat (or any TCP-stream tool), so that URL form ends in
-        # silent ENOENT and otbr-agent dies right after logging the
-        # radio URL.
+        # The Thread RCP is a network-attached radio (Serial-over-IP), so the
+        # spinel transport needs a TCP-to-PTY bridge. Two obvious approaches
+        # do not work and should not be retried:
        #
-        # A sidecar socat won't work either: each container has its own
-        # devpts namespace, so a PTY (e.g. /dev/pts/0) created in a
-        # sibling container is invisible from this container — a shared
-        # emptyDir symlink to /dev/pts/0 is dangling.
+        #   - `spinel+hdlc+forkpty://` needs socat, which the upstream image
+        #     does not ship; the URL fails with a silent ENOENT and otbr-agent
+        #     dies right after logging it.
+        #   - A sidecar cannot provide it either: devpts is per-container, so
+        #     a PTY created next door is invisible here and the symlink dangles.
        #
-        # The bridge has to live in the otbr container's own namespace.
-        # Install socat at startup via apt (Ubuntu 18.04 base, no egress
-        # restrictions on this namespace), run it in the background as
-        # the spinel-HDLC TCP↔PTY bridge, then exec the upstream
-        # entrypoint with RADIO_URL pointing at the PTY symlink.
+        # Hence socat is installed at startup and run inside this container.
        - name: otbr
          image: openthread/otbr:latest
          command:
@@ -1,18 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: barman-cloud-creds
-  namespace: paperless
-  annotations:
-    # Prune-disabled — a standing guard kept from the seed->paperless-KS handoff (barman creds).
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: seed/paperless/barman-creds
@@ -70,6 +70,27 @@ spec:
            - name: consume
              mountPath: /home/scanner
              subPath: scanner
+          # FTP has no health verb, so the control port accepting a connection
+          # is what can be asserted. It is enough for the failure that matters:
+          # if vsftpd dies the scanner cannot deposit anything and nothing else
+          # in the chain notices, because paperless only ever sees an empty
+          # consume directory — which is indistinguishable from nobody having
+          # scanned.
+          #
+          # Only the control port is probed. The passive range is negotiated
+          # per transfer and is not listening at rest.
+          livenessProbe:
+            tcpSocket:
+              port: ftp-ctrl
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            tcpSocket:
+              port: ftp-ctrl
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
          resources:
            requests:
              cpu: 20m
@@ -1,3 +1,9 @@
+# One Hetzner S3 key, one path, shared by every barman ObjectStore. Separate
+# per-app paths would not make rotation independent — that needs one Hetzner
+# key per consumer (#397).
+#
+# velero reads the same key from infrastructure/velero-hetzner, which holds it
+# as a credentials file rather than these three fields.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
@@ -15,4 +21,4 @@ spec:
    name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
  dataFrom:
    - extract:
-        key: seed/paperless/hetzner-creds
+        key: infrastructure/hetzner-s3
@@ -20,7 +20,6 @@ resources:
  - cluster.yaml
  - objectstore.yaml
  - scheduled-backup.yaml
-  - barman-creds-externalsecret.yaml
  - hetzner-creds-externalsecret.yaml
  - pvcs.yaml
 configMapGenerator:
@@ -155,21 +155,33 @@ spec:
            - name: bridge-cert
              mountPath: /shared
              readOnly: true
-          # TCP probes — HTTP probes against /api/ flooded the
-          # `paperless.auth` logger every 10 s with "No authentication
-          # provided from private IP …". With ENABLE_HTTP_REMOTE_USER
-          # the middleware logs every unauth API request, and there's
-          # no public health endpoint to probe. TCP confirms gunicorn
-          # is bound; that's the same liveness signal we'd get from
-          # /api/ minus the noise.
+          # /accounts/login/ rather than /api/ or a TCP connect.
+          #
+          # /api/ is unusable as a probe target: with ENABLE_HTTP_REMOTE_USER
+          # the remote-user middleware logs "No authentication provided from
+          # private IP …" for every unauthenticated API request, so probing it
+          # floods `paperless.auth`. Measured: four requests to /api/ produce
+          # four log lines, four to /accounts/login/ produce none — the
+          # middleware only covers the API surface.
+          #
+          # A TCP connect avoids the noise but only proves gunicorn accepted a
+          # socket. /accounts/login/ renders a real Django view, so URL
+          # routing, settings and templating all have to work. It does not
+          # prove Postgres is healthy; nothing cheap here does.
+          #
+          # Safe against ALLOWED_HOSTS: the kubelet sends the pod IP as Host,
+          # and PAPERLESS_ALLOWED_HOSTS is "*". Narrowing that value later
+          # means giving these probes an explicit Host header.
          livenessProbe:
-            tcpSocket:
+            httpGet:
+              path: /accounts/login/
              port: http
            initialDelaySeconds: 120
            periodSeconds: 30
            timeoutSeconds: 10
          readinessProbe:
-            tcpSocket:
+            httpGet:
+              path: /accounts/login/
              port: http
            initialDelaySeconds: 30
            periodSeconds: 10
@@ -34,6 +34,32 @@ spec:
            - name: wyoming
              containerPort: 10300
              protocol: TCP
+          # Wyoming is raw TCP with no HTTP surface and no health verb, so
+          # accepting a connection is all that can be asserted from outside the
+          # process — weaker than an HTTP probe, but the previous state was
+          # none at all.
+          #
+          # A startupProbe rather than a long liveness delay: this container
+          # runs with --retrain-on-start, and retraining against the Home
+          # Assistant entity list takes as long as it takes. The startup gate
+          # allows up to 10 minutes (60 x 10s) and only then hands over to
+          # liveness, so a slow retrain cannot be mistaken for a hang, and a
+          # genuine hang after startup is still caught within a minute.
+          startupProbe:
+            tcpSocket:
+              port: wyoming
+            periodSeconds: 10
+            failureThreshold: 60
+          livenessProbe:
+            tcpSocket:
+              port: wyoming
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            tcpSocket:
+              port: wyoming
+            periodSeconds: 10
+            timeoutSeconds: 5
          volumeMounts:
            - name: models
              mountPath: /models
@@ -12,4 +12,4 @@ spec:
    name: hetzner-s3-creds
  dataFrom:
    - extract:
-        key: seed/trmnl/hetzner-creds
+        key: infrastructure/hetzner-s3
@@ -1,20 +1,15 @@
 # Headless renderer: screenshots the Home Assistant e-ink dashboard,
 # dithers it to 16 greys and hands it to Terminus.
 #
-# Upstream ships this as a Home Assistant add-on. Our HA is a plain
-# container with no Supervisor, so it runs standalone here instead —
-# viable because the app prefers HOME_ASSISTANT_URL/ACCESS_TOKEN env
-# vars over /data/options.json and authenticates with an ordinary
-# long-lived token. SUPERVISOR_TOKEN is deliberately NOT set: it only
-# flips an internal isAddOn flag, and setting it would switch the app
-# to add-on behaviour (fixed port, /data assumptions).
+# Upstream ships this as a Home Assistant add-on; it runs standalone here
+# because the app prefers the env vars over /data/options.json. Do NOT set
+# SUPERVISOR_TOKEN — it flips an internal isAddOn flag and switches the app to
+# add-on behaviour (fixed port, /data assumptions).
 #
-# DIRECTION OF TRAVEL, which is the non-obvious part: with the
-# byos-hanami webhook in `uri` delivery mode the renderer does NOT
-# push image bytes. It POSTs a URL pointing at its own /output/<file>
-# endpoint, and TERMINUS FETCHES THE IMAGE BACK from it. That is why
-# both live in this namespace — the pull is intra-namespace and is
-# already permitted by the default-deny-ingress rule.
+# DIRECTION OF TRAVEL, the non-obvious part: in `uri` delivery mode the
+# renderer does not push image bytes. It POSTs a URL to its own /output/<file>
+# and Terminus fetches the image back. Both therefore live in this namespace,
+# where that pull is already permitted.
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -27,7 +27,7 @@ spec:
        # Valkey major version tracks upstream's compose.yml (9.x). Sidekiq
        # is the only consumer; a major bump is a separate migration.
        - name: valkey
-          image: valkey/valkey:9.1.1-alpine # {"$imagepolicy": "flux-system:valkey-trmnl"}
+          image: valkey/valkey:9.1.2-alpine # {"$imagepolicy": "flux-system:valkey-trmnl"}
          args:
            - valkey-server
            - --save
@@ -29,6 +29,27 @@ spec:
            - name: wyoming
              containerPort: 10300
              protocol: TCP
+          # Wyoming is a raw TCP protocol with no HTTP surface and no health
+          # verb, so accepting a connection is the only thing that can be
+          # asserted from outside the process. That is weaker than an HTTP
+          # probe but strictly better than the previous state of none at all,
+          # where a dead server stayed Ready and Home Assistant kept routing
+          # speech to it.
+          #
+          # The liveness delay is generous because the model is loaded at
+          # startup from the data volume.
+          livenessProbe:
+            tcpSocket:
+              port: wyoming
+            initialDelaySeconds: 90
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            tcpSocket:
+              port: wyoming
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
          volumeMounts:
            - name: data
              mountPath: /data
@@ -305,7 +305,7 @@ server → agent → step pod → server end to end:

 ```sh
 curl -sS -o /dev/null -w '%{http_code}\n' https://ci.renner.uno/
-woodpecker pipeline create --branch master --var PIPELINE=e2e-test k.renner/homelab
+woodpecker-cli pipeline deploy k.renner/homelab <pipeline-number> e2e-test
 ```

 Step pods must still reach the server on `:8000` while still being unable to
@@ -94,6 +94,31 @@ spec:
          envFrom:
            - secretRef:
                name: woodpecker-secret
+          ports:
+            # WOODPECKER_HEALTHCHECK defaults to true and serves :3000, so this
+            # port already existed; declaring it just lets the probe name it.
+            - name: health
+              containerPort: 3000
+              protocol: TCP
+          # /healthz answers 500 ONLY when a running workflow has passed its own
+          # timeout plus a one-hour buffer (agent/state.go, State.Healthy). An
+          # idle agent always answers 200.
+          #
+          # So this catches a wedged build holding a slot, which is otherwise
+          # invisible — and it does NOT catch an agent that has lost its server
+          # connection, which reports healthy while quietly taking no work. Do
+          # not read a green probe here as "CI is fine".
+          #
+          # Liveness only: the agent fronts no Service, so readiness would gate
+          # nothing. The upstream condition is already debounced by that hour,
+          # which is why the thresholds here are short.
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            timeoutSeconds: 5
          resources:
            requests:
              cpu: 25m
@@ -1,11 +1,8 @@
 ---
 # Barman S3 credentials for wp-pg.
 #
-# Sourced from seed/paperless/hetzner-creds, which is not where they belong.
-# Issue #397 wants woodpecker to own seed/woodpecker/hetzner-creds; creating
-# that path needs OpenBao write access (`bao kv list seed` → 403 for the
-# identities available here), so it stays open. The values are identical, so it
-# is a one-line swap of the `key:` below once the path exists.
+# One shared path for one shared credential — see the note in
+# ../paperless/hetzner-creds-externalsecret.yaml.
 #
 # The namespace allowlist on the openbao ClusterSecretStore
 # (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
@@ -24,4 +21,4 @@ spec:
    name: hetzner-s3-creds
  dataFrom:
    - extract:
-        key: seed/paperless/hetzner-creds
+        key: infrastructure/hetzner-s3
@@ -36,11 +36,9 @@ spec:
  target:
    name: hetzner-s3-creds
  dataFrom:
-    # Reuses the paperless copy of the shared Hetzner S3 credentials (identical
-    # ACCESS_KEY_ID / ACCESS_SECRET_KEY / REGION) to avoid seeding a new OpenBao
-    # path for this migration — same approach woodpecker took. FOLLOW-UP: give
-    # wordpress its own seed/wordpress/hetzner-creds path once OpenBao can be
-    # written, and repoint this here (the value is the same, so it's a drop-in
-    # swap).
+    # One shared path for one shared credential. There is a single Hetzner S3
+    # key behind all of these — see the note in
+    # ../paperless/hetzner-creds-externalsecret.yaml for why per-app paths were
+    # the wrong shape and what would have to change for them to be right.
    - extract:
-        key: seed/paperless/hetzner-creds
+        key: infrastructure/hetzner-s3
@@ -8,6 +8,9 @@ set -euo pipefail
 #   ./bootstrap.sh talos        # Talos apply + K8s bootstrap + kubeconfig (writes access-staging/)
 #   ./bootstrap.sh flux         # reads access-staging/, installs flux, waits for infrastructure
 #   ./bootstrap.sh wait-apps    # waits for apps/authentik/paperless to reconcile
+#   ./bootstrap.sh vmtest       # throwaway QEMU VMs, then the talos phase
+#                               # against them unchanged. Needs root for the
+#                               # provisioner and leaves nothing behind.
 #
 # Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls
 # from is itself Flux-managed, so a cold rebuild cannot pull from it until the
@@ -21,6 +24,13 @@ set -euo pipefail

 TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# Cilium's single source of truth. Talos ships `cni: none` (talos/patches/
+# cluster.yaml) and nothing else, so the CNI has to be installed here, between
+# `talosctl bootstrap` and Flux — Flux's own controllers are pods and cannot
+# reconcile the network they need in order to start. Everything the install
+# needs is read out of this file at run time so there is no second copy of a
+# version or a value to keep in step. See PROPOSAL-cilium-single-owner.md.
+CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml"
 ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging"
 NODES=(k8s-1 k8s-2 k8s-3)
 # No dedicated workers since the k8s-edge VM was retired (2026-07-02) —
@@ -33,6 +43,24 @@ DNS_SERVER="10.2.40.1"  # UDM (old PVE DNS VIP 10.2.40.53 decommissioned)
 # Cluster API endpoint — Talos VIP (10.2.40.60) elected via etcd, fronted by
 # UniFi DNS A record k8s.renner.uno. Defined in talos/patches/k8s-{1,2,3}.yaml.
 K8S_API_HOST="k8s.renner.uno"
+# Which site patch the nodes get: talos/patches/site-$SITE.yaml. The three
+# variables above and this one are everything the vmtest phase has to change in
+# order to run this same code against throwaway VMs — if that list ever grows,
+# the test is drifting away from the thing it claims to exercise.
+SITE="${SITE:-homelab}"
+# Where this script writes the cluster access files. These are the paths
+# talosctl and kubectl read from anyway, so unset they resolve to exactly the
+# old behaviour. Set, the whole run stays out of $HOME — which is what lets the
+# vmtest phase drive a throwaway cluster without replacing the operator's
+# production access with a VM's.
+TALOSCONFIG_PATH="${TALOSCONFIG:-$HOME/.talos/config}"
+KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}"
+case "$KUBECONFIG_PATH" in
+    *:*) echo "KUBECONFIG holds multiple paths; set it to a single file" >&2; exit 1 ;;
+esac
+# Addresses to apply to, in the order of NODES. Empty means resolve the node
+# FQDNs, which is right for real hardware on a network that has DNS for it.
+NODE_ADDRS=()
 GITEA_HOST="gitea.renner.uno"
 GITEA_OWNER="gitea_admin"
 GITEA_REPO="homelab"
@@ -44,6 +72,19 @@ FLUX_SOURCE="${FLUX_SOURCE:-gitea}"
 GITHUB_OWNER="${GITHUB_OWNER:-konairius}"
 GITHUB_REPO="${GITHUB_REPO:-Homelab}"
 FLUX_PATH="kubernetes/clusters/homelab"
+# Branch Flux reconciles from, and that `flux bootstrap` commits its own sync
+# manifests into. master for a real rebuild.
+#
+# Overridable so the phase can be rehearsed without writing to master — point it
+# at a throwaway branch, which has to exist already: `flux bootstrap` clones the
+# branch and fails with `couldn't find remote ref` rather than creating it.
+#
+#     git push origin origin/master:refs/heads/rehearsal-1
+#     FLUX_BRANCH=rehearsal-1 ./bootstrap.sh flux
+#
+# Nothing else about the phase changes, which is the point. Delete the branch
+# afterwards.
+FLUX_BRANCH="${FLUX_BRANCH:-master}"
 SOPS_KEY="$HOME/.config/sops/age/keys.txt"

 PHASE="${1:-all}"
@@ -78,43 +119,145 @@ phase_runs() {
    [[ "$PHASE" == "all" || "$PHASE" == "$1" ]]
 }

+# Provision throwaway QEMU VMs, then run the Talos phase below against
+# them unchanged. Deliberately not a copy of that phase: it sets the site
+# variables and falls through, because a rehearsal that runs different code
+# proves less than one that does not. Everything it changes is listed here — if
+# that list grows, the test is drifting away from what it claims to exercise.
+#
+# What it cannot cover, and what therefore still has to be true on hardware:
+# Ceph (the provisioner gives extra disks to workers only), the mgmt and v30iot
+# VLANs and the Layer2 VIP (one flat NIC per VM), and Flux (its own phase).
+#
+# Needs root for the QEMU provisioner (CNI and KVM), and only for that step.
+if [[ "$PHASE" == "vmtest" ]]; then
+    echo -e "${YELLOW}=== Phase: VM rebuild test ===${NC}"
+    command -v talosctl >/dev/null || fail "talosctl not found"
+    command -v yq >/dev/null       || fail "yq not found"
+    # The node patches are the list; the VM count follows from them rather than
+    # being stated twice. vm-1 is the control plane, the rest are the workers
+    # that carry an OSD device — see the comment on NODES below for why.
+    VM_CP_NODES=(vm-1)
+    VM_WORKER_NODES=(vm-2 vm-3 vm-4)
+    VM_CPS=${#VM_CP_NODES[@]}
+    VM_WORKER_COUNT=${#VM_WORKER_NODES[@]}
+    for n in "${VM_CP_NODES[@]}" "${VM_WORKER_NODES[@]}"; do
+        [[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
+    done
+    [[ -f "$TALOS_DIR/patches/site-qemu.yaml" ]] || fail "site-qemu.yaml not found"
+
+    # Talos version and Image Factory schematic are read out of the installer
+    # image in cluster.yaml. Naming them here would be a second copy to keep in
+    # step, and the day they diverged the test would quietly stop booting the
+    # image a rebuild installs.
+    VM_IMAGE=$(yq -r '.machine.install.image' "$TALOS_DIR/patches/cluster.yaml")
+    VM_SCHEMATIC=${VM_IMAGE#*metal-installer/}
+    VM_SCHEMATIC=${VM_SCHEMATIC%%:*}
+    VM_VERSION=${VM_IMAGE##*:}
+    [[ ${#VM_SCHEMATIC} -eq 64 && "$VM_VERSION" == v* ]] \
+        || fail "could not read a schematic and version out of '$VM_IMAGE'"
+
+    # sun_path caps a unix socket path at 108 bytes and the provisioner puts a
+    # qemu monitor socket under the state directory, so this has to stay short.
+    # A path under the repo or a long mktemp dir silently fails to boot.
+    VM_STATE=/tmp/bootstrap-vmtest
+    VM_CLUSTER=bootstrap-vmtest
+    VM_DIR=$(mktemp -d)
+    TALOSCTL_BIN=$(command -v talosctl)
+
+    # Hard isolation from the production cluster: talosctl and kubectl both
+    # honour these, and the phase below writes to them rather than to $HOME.
+    export TALOSCONFIG="$VM_DIR/talosconfig"
+    export KUBECONFIG="$VM_DIR/kubeconfig"
+    TALOSCONFIG_PATH="$TALOSCONFIG"
+    KUBECONFIG_PATH="$KUBECONFIG"
+
+    cleanup_vmtest() {
+        echo -n "Destroying throwaway VMs... "
+        sudo -E "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
+        # destroy removes the cluster's own directory but leaves the state root.
+        sudo rm -rf "$VM_STATE"
+        rm -rf "$VM_DIR"
+        echo "done"
+    }
+    trap cleanup_vmtest EXIT
+
+    step 0 "Creating $VM_CPS control plane and $VM_WORKER_COUNT workers in maintenance mode..."
+    # maintenance means the provisioner applies no configuration of its own, so
+    # every byte of machine config below comes from talos/patches.
+    sudo -E "$TALOSCTL_BIN" cluster create qemu \
+        --name "$VM_CLUSTER" \
+        --state "$VM_STATE" \
+        --presets iso,maintenance \
+        --controlplanes "$VM_CPS" \
+        --workers "$VM_WORKER_COUNT" \
+        --talos-version "$VM_VERSION" \
+        --schematic-id "$VM_SCHEMATIC" \
+        --memory-controlplanes 2560MiB \
+        --memory-workers 4GiB \
+        --cpus-controlplanes 2 \
+        --cpus-workers 2 \
+        --disks virtio:10GiB,virtio:8GiB \
+        --talosconfig-destination "$TALOSCONFIG" \
+        || fail "could not create the throwaway VMs"
+    sudo chown -R "$(id -u):$(id -g)" "$VM_STATE" "$TALOSCONFIG"
+    ok
+
+    # The provisioner hands out its CIDR sequentially from .2; .1 is the
+    # gateway and the load balancer it puts in front of the API. If that ever
+    # changes, the apply loop below reports which address stopped answering
+    # rather than failing silently.
+    # One control plane and three workers, not three control planes: the
+    # provisioner gives the second disk only to workers ("disks after the first
+    # one are added only to worker machines"), and an OSD needs it. Three of
+    # them because the CephCluster runs three mons with allowMultiplePerNode
+    # false over pools that are replica 3 across failureDomain host.
+    NODES=("${VM_CP_NODES[@]}")
+    WORKERS=("${VM_WORKER_NODES[@]}")
+    ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
+    NODE_ADDRS=()
+    for i in $(seq 0 $(( ${#ALL_NODES[@]} - 1 ))); do
+        NODE_ADDRS+=("10.5.0.$(( i + 2 ))")
+    done
+    SITE=qemu
+    K8S_API_HOST=10.5.0.1
+    VMTEST=1
+
+    # Run the real phase, not a reimplementation of it.
+    PHASE=talos
+fi
+
 if phase_runs talos; then
    echo -e "${YELLOW}=== Phase: Talos bootstrap ===${NC}"
    command -v talosctl >/dev/null || fail "talosctl not found"
    command -v kubectl >/dev/null  || fail "kubectl not found"
+    command -v helm >/dev/null   || fail "helm not found (needed to install the CNI)"
+    command -v yq >/dev/null     || fail "yq not found (needed to read the Cilium HelmRelease)"
    [[ -f "$TALOS_DIR/patches/cluster.yaml"         ]] || fail "cluster.yaml patch not found"
-    [[ -f "$TALOS_DIR/patches/cilium-cni.yaml"      ]] || fail "cilium-cni.yaml patch not found"
-    [[ -f "$TALOS_DIR/patches/registry-mirror.yaml" ]] || fail "registry-mirror.yaml patch not found"
+    [[ -f "$TALOS_DIR/patches/containerd-layer-retention.yaml" ]] || fail "containerd-layer-retention.yaml patch not found"
+    [[ -f "$CILIUM_HR"                              ]] || fail "cilium helmrelease not found at $CILIUM_HR"

    step 0 "Generating Talos configs..."
    TALOS_GEN_DIR=$(mktemp -d)
    talosctl gen config homelab "https://${K8S_API_HOST}:6443" \
        --output "$TALOS_GEN_DIR" \
        --force
-    # Strip the auto-generated HostnameConfig docs (conflicts with machine.network.hostname in per-node patches)
-    for f in "$TALOS_GEN_DIR"/controlplane.yaml "$TALOS_GEN_DIR"/worker.yaml; do
-        awk '
-          BEGIN { doc="" }
-          /^---[ \t]*$/ {
-            if (doc !~ /\nkind: HostnameConfig\n/ && doc !~ /^kind: HostnameConfig\n/) printf "%s", doc
-            doc=$0 "\n"; next
-          }
-          { doc = doc $0 "\n" }
-          END {
-            if (doc !~ /\nkind: HostnameConfig\n/ && doc !~ /^kind: HostnameConfig\n/) printf "%s", doc
-          }
-        ' "$f" > "$f.tmp" && mv "$f.tmp" "$f"
-    done
-    mkdir -p ~/.talos
-    cp "$TALOS_GEN_DIR/talosconfig" ~/.talos/config
+    mkdir -p "$(dirname "$TALOSCONFIG_PATH")"
+    cp "$TALOS_GEN_DIR/talosconfig" "$TALOSCONFIG_PATH"

    step 0 "Applying Talos configs to nodes..."
    apply_node_config() {
-        local node=$1 base_config=$2
+        local node=$1 base_config=$2 fixed_addr=${3:-}
        local fqdn="${node}.${DOMAIN}"
-        echo "  $fqdn:"
+        echo "  ${fixed_addr:-$fqdn}:"
        for i in $(seq 1 60); do
-            node_ip=$(resolve_node_ip "$fqdn")
+            # A fixed address skips DNS. Real hardware has a name to resolve;
+            # a throwaway VM has an address and nothing else.
+            if [[ -n "$fixed_addr" ]]; then
+                node_ip="$fixed_addr"
+            else
+                node_ip=$(resolve_node_ip "$fqdn")
+            fi
            if [[ -z "$node_ip" ]]; then
                echo "    attempt $i/60: DNS returned no IP"
                sleep 10
@@ -123,73 +266,216 @@ if phase_runs talos; then
            if err=$(talosctl apply-config --insecure -n "$node_ip" \
                --file "$base_config" \
                --config-patch "@$TALOS_DIR/patches/cluster.yaml" \
-                --config-patch "@$TALOS_DIR/patches/cilium-cni.yaml" \
-                --config-patch "@$TALOS_DIR/patches/registry-mirror.yaml" \
+                --config-patch "@$TALOS_DIR/patches/site-${SITE}.yaml" \
+                --config-patch "@$TALOS_DIR/patches/containerd-layer-retention.yaml" \
                --config-patch "@$TALOS_DIR/patches/${node}.yaml" 2>&1); then
-                echo "    applied via $node_ip ${GREEN}(OK)${NC}"
+                echo -e "    applied via $node_ip ${GREEN}(OK)${NC}"
                return 0
            fi
            echo "    attempt $i/60: $node_ip - ${err//$'\n'/ | }"
            sleep 10
        done
-        fail "Timeout waiting for $fqdn"
+        fail "Timeout waiting for ${fixed_addr:-$fqdn}"
    }

-    for node in "${NODES[@]}"; do
-        apply_node_config "$node" "$TALOS_GEN_DIR/controlplane.yaml"
-    done
-    for node in "${WORKERS[@]}"; do
-        apply_node_config "$node" "$TALOS_GEN_DIR/worker.yaml"
+    # One loop over ALL_NODES rather than one per role, so NODE_ADDRS can be a
+    # single list in the same order. Control planes come first in ALL_NODES,
+    # which is what decides the base config each node gets.
+    for idx in "${!ALL_NODES[@]}"; do
+        if [[ $idx -lt ${#NODES[@]} ]]; then
+            base="$TALOS_GEN_DIR/controlplane.yaml"
+        else
+            base="$TALOS_GEN_DIR/worker.yaml"
+        fi
+        apply_node_config "${ALL_NODES[$idx]}" "$base" "${NODE_ADDRS[$idx]:-}"
    done

    echo -n "Configuring talosctl... "
-    talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
-    talosctl config node "${NODES[0]}.${DOMAIN}"
+    # `gen config` leaves talosconfig endpoints empty; they are the Talos API
+    # addresses and unrelated to cluster.controlPlane.endpoint.
+    if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
+        talosctl config endpoint "${NODE_ADDRS[@]}"
+        talosctl config node "${NODE_ADDRS[0]}"
+    else
+        talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
+        talosctl config node "${NODES[0]}.${DOMAIN}"
+    fi
    ok

-    step 0 "Waiting for Talos cluster formation..."
-    for i in $(seq 1 30); do
-        members=$(talosctl get members 2>/dev/null || true)
-        missing=0
-        for node in "${ALL_NODES[@]}"; do
-            grep -q "$node" <<<"$members" || missing=1
+    step 0 "Waiting for every node to come back configured..."
+    # Asks each machine directly, by address, and expects it to answer with the
+    # hostname its patch assigns. That is what has to be true before etcd is
+    # bootstrapped: the config took, the node rebooted into it, and the machine
+    # at that address is the one we think it is.
+    #
+    # This used to wait on `talosctl get members`, which was wrong twice over.
+    # It queried through the configured endpoint list rather than each node, so
+    # apid refused it with `PermissionDenied: no request forwarding` — the same
+    # thing that broke the bootstrap call. And membership comes from the
+    # discovery service, because the Kubernetes registry is disabled: a cold
+    # rebuild would have been waiting on discovery.talos.dev being reachable,
+    # before the cluster it is trying to rebuild even exists.
+    #
+    # Each node installs to disk and reboots first, so the window is generous.
+    for idx in "${!ALL_NODES[@]}"; do
+        want="${ALL_NODES[$idx]}"
+        addr="${NODE_ADDRS[$idx]:-${want}.${DOMAIN}}"
+        echo -n "  $want ($addr)... "
+        node_err=""
+        for i in $(seq 1 60); do
+            got=$(talosctl -n "$addr" -e "$addr" get hostname \
+                    -o jsonpath='{.spec.hostname}' 2>"$TALOS_GEN_DIR/hostname.err") || true
+            node_err=$(cat "$TALOS_GEN_DIR/hostname.err")
+            [[ "$got" == "$want" ]] && break
+            sleep 10
+            if [[ $i -eq 60 ]]; then
+                echo
+                [[ -n "$node_err" ]] && echo "  last talosctl error: $node_err" >&2
+                fail "$addr never came back as $want (last answer: '${got:-none}')"
+            fi
        done
-        [[ $missing -eq 0 ]] && break
-        sleep 5
-        [[ $i -eq 30 ]] && fail "Talos cluster didn't form"
+        ok
    done

    echo -n "Bootstrapping Kubernetes... "
-    talosctl bootstrap 2>/dev/null || true
+    # -n AND -e on one control plane, deliberately. etcd bootstrap is the one
+    # call apid refuses to forward: reached through any other node it fails with
+    # `PermissionDenied: no request forwarding`. Without them talosctl picks an
+    # endpoint from the configured list — with more than one control plane, or a
+    # list that includes workers, that is a coin flip.
+    if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
+        bootstrap_target="${NODE_ADDRS[0]}"
+    else
+        bootstrap_target="${NODES[0]}.${DOMAIN}"
+    fi
+    # Re-running against an already-bootstrapped cluster is not an error and is
+    # the reason for `|| true` — but that used to swallow every other reason as
+    # well, and the run carried on to fail somewhere further downstream with no
+    # trace of the actual cause. Anything that is not "already exists" is shown.
+    if ! bootstrap_err=$(talosctl bootstrap -n "$bootstrap_target" -e "$bootstrap_target" 2>&1); then
+        grep -qi 'already\|AlreadyExists' <<<"$bootstrap_err" \
+            || echo -e "\n${YELLOW}bootstrap reported:${NC} ${bootstrap_err//$'\n'/ | }"
+    fi
    ok

    echo -n "Getting kubeconfig... "
+    kubeconfig_err=""
    for i in $(seq 1 30); do
-        if talosctl kubeconfig ~/.kube/config --force 2>/dev/null; then
-            sed -i "s|server:.*|server: https://${K8S_API_HOST}:6443|" ~/.kube/config
+        mkdir -p "$(dirname "$KUBECONFIG_PATH")"
+        if kubeconfig_err=$(talosctl kubeconfig "$KUBECONFIG_PATH" --force 2>&1); then
+            sed -i "s|server:.*|server: https://${K8S_API_HOST}:6443|" "$KUBECONFIG_PATH"
            break
        fi
        sleep 5
-        [[ $i -eq 30 ]] && fail "Couldn't get kubeconfig"
+        # Held back while retrying — the first attempts fail by design — but
+        # printed if it never works, because that message is the diagnosis.
+        [[ $i -eq 30 ]] && { echo; echo "$kubeconfig_err" >&2; fail "Couldn't get kubeconfig"; }
    done
    ok

+    echo -n "Waiting for the Kubernetes API to answer... "
+    # Having a kubeconfig is not the same as having an apiserver. `talosctl
+    # kubeconfig` succeeds as soon as Talos can hand out credentials, which is
+    # before kube-apiserver is serving on the endpoint the file points at — so
+    # the very next command could fail with an EOF against a port nothing is
+    # listening on yet. On hardware the gap usually closes on its own; on a
+    # scratch cluster it does not, which is how this was found.
+    # `get --raw /version` needs the server; `kubectl version` would answer from
+    # the client alone and pass against nothing at all.
+    api_err=""
+    for i in $(seq 1 60); do
+        api_err=$(kubectl get --raw /version 2>&1) && break
+        sleep 5
+        if [[ $i -eq 60 ]]; then
+            echo "$api_err" >&2
+            fail "Kubernetes API at https://${K8S_API_HOST}:6443 never answered"
+        fi
+    done
+    ok
+
+    step 0 "Installing Cilium (CNI)..."
+    # Must run before the node-Ready wait below: with `cni: none` the kubelet
+    # reports NotReady until a CNI is present, so that loop would time out.
+    #
+    # Chart repo, version constraint and values all come out of the HelmRelease
+    # so this cannot drift from what Flux will reconcile minutes later. helm
+    # accepts the semver range verbatim and resolves it the same way Flux does.
+    #
+    # serviceMonitor is the one override, and it is a bootstrap constraint
+    # rather than a value: its CRD arrives with the monitoring stack much
+    # later, and the object would fail to apply. Nothing else is overridden,
+    # which is what lets helm-controller adopt this release instead of
+    # replacing it.
+    CILIUM_VALUES=$(mktemp)
+    yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES"
+    grep -q 'policyMapMax' "$CILIUM_VALUES" \
+        || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR"
+
+    helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null
+    helm repo update cilium >/dev/null
+    helm upgrade --install cilium cilium/cilium \
+        --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \
+        --namespace kube-system \
+        -f "$CILIUM_VALUES" \
+        --set prometheus.serviceMonitor.enabled=false \
+        --wait --timeout 10m
+    rm -f "$CILIUM_VALUES"
+    ok
+
    step 0 "Waiting for K8s nodes..."
    expected=${#ALL_NODES[@]}
+    nodes_err=""
    for i in $(seq 1 60); do
-        ready=$(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready" || true)
+        # The STATUS column, matched whole. `grep -c Ready` counted NotReady
+        # too — the substring is in it — so this waited for nodes to register
+        # rather than to become ready, and passed while every one of them was
+        # still NotReady.
+        nodes_out=$(kubectl get nodes --no-headers 2>"$TALOS_GEN_DIR/nodes.err") || true
+        nodes_err=$(cat "$TALOS_GEN_DIR/nodes.err")
+        ready=$(awk '$2 == "Ready"' <<<"$nodes_out" | grep -c . || true)
        if [[ "$ready" -eq "$expected" ]]; then
            break
        fi
        sleep 5
-        [[ $i -eq 60 ]] && fail "Not all nodes ready (got $ready/$expected)"
+        if [[ $i -eq 60 ]]; then
+            echo
+            [[ -n "$nodes_err" ]] && echo "last kubectl error: $nodes_err" >&2
+            echo "${nodes_out:-  (no nodes listed)}" >&2
+            fail "Not all nodes ready (got $ready/$expected)"
+        fi
    done

-    # Stash configs for subsequent phases / export-access step
-    mkdir -p "$ACCESS_STAGING"
-    cp ~/.talos/config "$ACCESS_STAGING/talosconfig"
-    cp ~/.kube/config  "$ACCESS_STAGING/kubeconfig"
-    echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
+    # Stash configs for subsequent phases / export-access step. Skipped under
+    # vmtest: those are a throwaway cluster's credentials, and writing them here
+    # would leave the flux phase pointed at VMs that no longer exist.
+    if [[ "${VMTEST:-0}" == "1" ]]; then
+        # Nodes being Ready says nothing about storage. The CephCluster picks
+        # OSD nodes by a label out of the machine config and their device by the
+        # partition label RawVolumeConfig creates, so assert both here — running
+        # it by hand afterwards races the teardown, and a test that does not
+        # check its own premise is worth little.
+        step 0 "Checking the OSD prerequisites Ceph selects on..."
+        for idx in $(seq "${#NODES[@]}" $(( ${#ALL_NODES[@]} - 1 ))); do
+            vm_node="${ALL_NODES[$idx]}"
+            vm_addr="${NODE_ADDRS[$idx]}"
+            vols=$(talosctl -n "$vm_addr" -e "$vm_addr" get discoveredvolumes 2>&1) \
+                || fail "could not read volumes from $vm_node ($vm_addr): ${vols//$'\n'/ | }"
+            awk '$NF == "r-ceph-osd" {found=1} END {exit !found}' <<<"$vols" \
+                || fail "$vm_node has no r-ceph-osd partition — RawVolumeConfig did not take"
+            kubectl get node "$vm_node" \
+                -o jsonpath='{.metadata.labels.storage\.renner\.uno/ceph-osd}' \
+                | grep -q true \
+                || fail "$vm_node is missing the storage.renner.uno/ceph-osd label"
+            echo -e "  $vm_node: partition and label ${GREEN}(OK)${NC}"
+        done
+
+        echo -e "${GREEN}VM rebuild test passed.${NC} ${#ALL_NODES[@]} nodes Ready on the repo's own config."
+    else
+        mkdir -p "$ACCESS_STAGING"
+        cp "$TALOSCONFIG_PATH" "$ACCESS_STAGING/talosconfig"
+        cp "$KUBECONFIG_PATH"  "$ACCESS_STAGING/kubeconfig"
+        echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
+    fi
 fi

 # Helper: load staged access files into home dir (for later CI steps)
@@ -228,7 +514,7 @@ if phase_runs flux; then
        flux bootstrap github \
            --owner="$GITHUB_OWNER" \
            --repository="$GITHUB_REPO" \
-            --branch=master \
+            --branch="$FLUX_BRANCH" \
            --path="$FLUX_PATH" \
            --components-extra=image-reflector-controller,image-automation-controller \
            --personal \
@@ -239,7 +525,7 @@ if phase_runs flux; then
            --hostname="$GITEA_HOST" \
            --owner="$GITEA_OWNER" \
            --repository="$GITEA_REPO" \
-            --branch=master \
+            --branch="$FLUX_BRANCH" \
            --path="$FLUX_PATH" \
            --components-extra=image-reflector-controller,image-automation-controller \
            --personal \
@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: llama-server
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 1m
+  # On a cold PVC the chat model pulls ~19 GiB of GGUF weights from Hugging
+  # Face and then loads onto the iGPU before /health goes green, so a first
+  # start is far slower than a normal app's.
+  timeout: 60m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/apps/llama-server
+  prune: true
+  wait: true
+  dependsOn:
+    - name: infrastructure
+    - name: infrastructure-crds
+    - name: infrastructure-ceph-cluster
@@ -33,22 +33,21 @@ spec:
    remediation:
      retries: 3
  values:
-    # Roll the agent DaemonSet when cilium-config changes. Cilium reads
-    # datapath options once, at agent start, and the chart puts no config
-    # checksum on the pod template — so a changed ConfigMap leaves every agent
-    # running the previous datapath until something restarts it. Nothing did.
+    # THIS IS THE ONLY PLACE CILIUM IS CONFIGURED. Talos used to carry a
+    # rendered copy as a bootstrap inline manifest, so every setting existed
+    # two or three times and nothing enforced agreement — see
+    # PROPOSAL-cilium-single-owner.md for what that cost. Talos now provides
+    # only `cni: none` and `proxy: disabled`; a cold rebuild installs Cilium
+    # from these values with helm before Flux takes over.
    #
-    # This is not hypothetical. vlan-bpf-bypass reached cilium-config and every
-    # v30iot macvlan workload stayed broken until the agents were deleted by
-    # hand, one node at a time (issue #592). policyMapMax has the same
-    # exposure, and its comment below says "takes effect on agent restart" —
-    # documentation standing in for automation.
+    # Rolls the agent DaemonSet when cilium-config changes. Cilium reads
+    # datapath options once at agent start and the chart puts no config
+    # checksum on the pod template, so without this a changed ConfigMap leaves
+    # every agent on the previous datapath indefinitely (#592).
    #
-    # The trade: any cilium-config change now rolls the CNI cluster-wide,
-    # node by node. That is disruptive-ish, but it is what a datapath change
-    # already required — the difference is that it now happens on its own
-    # instead of being forgotten. Reloader is already deployed for exactly
-    # this pattern; named per-ConfigMap rather than blanket, as elsewhere.
+    # The trade is that any cilium-config change now rolls the CNI node by
+    # node — which is what a datapath change required anyway, only now it is
+    # not forgotten.
    annotations:
      configmap.reloader.stakater.com/reload: cilium-config
    # Which pod labels feed security-identity computation. Patterns are
@@ -58,8 +57,6 @@ spec:
    # and every identity costs one entry per allowed (direction, proto, port)
    # in every endpoint's policy map, cluster-wide. Nothing selects on these
    # labels in policy, so they buy identity churn and nothing else.
-    # Keep in lockstep with kubernetes/talos/patches/cilium-cni.yaml and
-    # kubernetes/talos/cilium.yaml.
    labels: '!woodpecker-ci\.org/.* !step$'
    ipv6:
      enabled: true
@@ -69,57 +66,38 @@ spec:
    socketLB:
      enabled: true
    prometheus:
-      # The agent exported no metrics at all, so BPF map utilisation was
-      # invisible: when per-endpoint policy maps (cilium_policy_v3_*) run out
-      # of space the agent only says "no space left on device" in its log and
-      # pod sandbox creation fails (issue #452). cilium_bpf_map_pressure makes
-      # that observable — and alertable — before it stalls a rollout.
+      # Without agent metrics, BPF map utilisation is invisible: a full policy
+      # map only shows up as "no space left on device" in the log while pod
+      # sandbox creation fails (#452). cilium_bpf_map_pressure makes it
+      # alertable beforehand.
      enabled: true
      serviceMonitor:
        enabled: true
    bpf:
-      # Size of each endpoint's own cilium_policy_v3_<id> map, holding one
-      # entry per (identity, direction, protocol, port) the endpoint is
-      # allowed. When it fills, endpoint regeneration fails with "no space
-      # left on device", the CNI ADD call never returns, and the pod is stuck
-      # in sandbox creation — so this must stay comfortably above the busiest
-      # endpoint's entry count, tracked by cilium_bpf_map_pressure.
-      # Takes effect on agent restart; the map is sized at creation.
-      # Bootstrap installs Cilium from the inline manifest in
-      # kubernetes/talos/patches/cilium-cni.yaml, so bpf-policy-map-max there
-      # must move with this value or a cold rebuild starts with the old size.
-      # kubernetes/talos/cilium.yaml carries a third copy of the same key.
-      # Nothing enforces any of this; grep bpf-policy-map-max before changing it.
+      # Per-endpoint policy map size: one entry per (identity, direction,
+      # protocol, port) allowed. When it fills, endpoint regeneration fails
+      # with "no space left on device" and the pod hangs in sandbox creation.
+      # Keep it comfortably above the busiest endpoint, watched by
+      # cilium_bpf_map_pressure. Sized at map creation, so it takes an agent
+      # restart.
+      #
      policyMapMax: 65536
      # VLAN ids the eBPF datapath is allowed to pass. Without this it drops
      # VLAN-tagged frames, accounted as "VLAN traffic disallowed by VLAN
      # filter" in cilium_drop_count_total.
      #
-      # 30 is v30iot, the macvlan parent for the shelly-manager, matter-server,
-      # otbr and music-assistant attachments. On bare metal it is a VLAN
-      # sub-interface of enp2s0f0np0; inside the former Proxmox guests it was
-      # an untagged virtio NIC, which is why this was not needed before.
+      # Only VLAN 30 (v30iot) needs listing. Cilium attaches to devices that
+      # carry an address, so it sees mgmt0's frames already untagged by the
+      # 8021q layer — nothing for a VLAN filter to match. ens19 is deliberately
+      # address-less, so its frames only meet the filter on the shared parent,
+      # still tagged. Adding 40 would change nothing and would touch the
+      # datapath carrying etcd and the API.
      #
-      # VLAN 40 (mgmt0) is deliberately NOT listed, and the asymmetry is the
-      # interesting part. Cilium attaches to devices that carry an address —
-      # `cilium-dbg status --verbose` lists enp87s0, enp2s0f0np0, enp2s0f1np1
-      # and mgmt0, but NOT ens19. mgmt0 holds the node IP, so Cilium owns it as
-      # a device and sees frames the 8021q layer has already untagged; nothing
-      # for a VLAN filter to match. ens19 is deliberately address-less (see
-      # talos/patches/k8s-1.yaml — host addresses on that L2 make node replies
-      # bypass the UDM), so Cilium never attaches to it and VLAN 30 frames only
-      # meet the filter on the shared parent, still tagged.
+      # The symptom when this is wrong is not a DHCP timeout: a tc-layer drop
+      # reaches an AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports
+      # "no buffer space available" (#592).
      #
-      # So the fix belongs on 30 alone. Listing 40 would change nothing, and
-      # would touch the datapath carrying etcd and the API for no reason.
-      #
-      # A tc-layer drop surfaces to an AF_PACKET sender as ENOBUFS, so the
-      # symptom is the CNI dhcp plugin reporting "no buffer space available"
-      # rather than a DHCP timeout. See issue #592.
-      #
-      # Same three-copy problem as policyMapMax above: vlan-bpf-bypass in
-      # kubernetes/talos/cilium.yaml and kubernetes/talos/patches/cilium-cni.yaml
-      # must move with this list. Takes effect on agent restart.
+      # Same three-copy problem as policyMapMax above. Takes an agent restart.
      vlanBypass: [30]
      # Let pods reach the cluster's own LoadBalancer external IPs; Cilium
      # redirects the socket directly to the backend pod instead of sending
@@ -163,3 +141,28 @@ spec:
      enabled: true
    externalIPs:
      enabled: true
+    # BGP to the UDM. The peering itself has always been in git as CRDs in
+    # post-crds/cilium-bgp.yaml — but the switch that starts the control plane
+    # was not, and neither were the CIDRs below. They were applied by hand and
+    # held by the `kubectl-patch` field manager on cilium-config. Server-side
+    # apply left them alone, so nothing broke and nothing noticed; a cold
+    # rebuild would have come up with BGP off while the CRDs said otherwise.
+    #
+    # This is not a free change of ownership. Enabling bgpControlPlane through
+    # the chart also emits companion keys the hand-patch never set —
+    # bgp-secrets-namespace, bgp-router-id-allocation-mode and -ip-pool,
+    # enable-bgp-control-plane-status-report, enable-bgp-legacy-origin-attribute
+    # — so Cilium has been running on its own defaults for those. Router-id
+    # allocation is the one to watch: if the chart's default disagrees with
+    # Cilium's, the sessions re-establish.
+    bgpControlPlane:
+      enabled: true
+    # Masquerading exception, not a routing statement: pod traffic to these
+    # ranges leaves unmasqueraded, so LAN services see real pod IPs. That only
+    # works because BGP advertises the pod CIDRs, which is what gives the UDM a
+    # route back. The two settings are one decision and have to move together.
+    #
+    # Pod CIDRs are 10.244.0.0/16 and fd10:244::/48 (talos/patches/cluster.yaml);
+    # the supernets here cover the LAN as well, which is the point.
+    ipv4NativeRoutingCIDR: 10.0.0.0/8
+    ipv6NativeRoutingCIDR: fd00::/8
@@ -36,25 +36,16 @@ spec:
  values:
    container_runtime: containerd

-    # --- mTLS between agents and the LAPI: the durable fix for the recurring
-    # KubeDaemonSetRolloutStuck auth loop (#340) ---
+    # mTLS between agents and the LAPI. Do not switch back to password auth:
+    # the LAPI runs without persistence, so its machine-registration DB is
+    # wiped on every restart, and password-registered agents then get 401
+    # "machine not found" and crashloop until re-registered by hand — which
+    # wedges the whole `infrastructure` Kustomization behind a HelmRelease
+    # that never goes Ready (#340).
    #
-    # The LAPI runs WITHOUT persistence (see persistentVolume note below), so
-    # its sqlite machine-registration DB is ephemeral and every LAPI pod restart
-    # wiped it. Under the old password flow the agents registered exactly ONCE
-    # (init container running `cscli lapi register --token`) and cached a
-    # password the wiped LAPI no longer recognised -> 401 "machine not found"
-    # -> CrashLoopBackOff on all three agents -> DaemonSet never Ready -> the
-    # crowdsec HelmRelease upgrade `--wait` times out forever -> the whole
-    # `infrastructure` Kustomization (and every app depending on it) stays
-    # unhealthy. That is the #340 outage.
-    #
-    # With client-cert auth the agent init container no longer registers at all
-    # (verified via `helm template`: the `cscli lapi register` step is gone).
-    # The agent presents a cert whose OU (agent-ou) the LAPI trusts, and a
-    # freshly restarted LAPI with an EMPTY DB auto-recreates each machine record
-    # from the presented cert on connect. So the system self-heals across LAPI
-    # restarts with no PVC, no Ceph, and no manual re-registration.
+    # With client certs the agent never registers: a restarted LAPI with an
+    # empty DB recreates each machine record from the presented cert, so this
+    # self-heals with no PVC and no intervention.
    #
    # cert-manager (already reconciled earlier in this same `infrastructure`
    # stage) provisions a fully self-contained self-signed CA (selfSigned
@@ -87,13 +78,10 @@ spec:

    lapi:
      replicas: 1
-      # The chart defaults request 500m. Measured steady-state draw is 2m, and
-      # the request is what the scheduler reserves, so the default parks half a
-      # core of unusable capacity per LAPI. That is affordable on three nodes
-      # and is not on two: with one node out for maintenance the remaining pair
-      # sit at 100% of requested CPU while actually using ~22%, and nothing new
-      # can be placed. The limit stays at the chart default so a log flood can
-      # still burst; only the reservation shrinks.
+      # Far below the chart default, which reserves half a core per LAPI for a
+      # workload that draws single-digit millicores — enough to leave the
+      # cluster unschedulable with one node out. Only the reservation shrinks;
+      # the limit stays at the chart default so a log flood can still burst.
      resources:
        requests:
          cpu: 50m
@@ -33,3 +33,16 @@ spec:
    crds: CreateReplace
    remediation:
      retries: 3
+  values:
+    # The chart ships these disabled, so the controller ran with no probe of
+    # any kind: a wedged reconciler stayed Ready and every ExternalSecret would
+    # simply stop refreshing, which surfaces only when a consumer needs a
+    # credential that has since rotated.
+    #
+    # Enabling them also starts the health server — the chart derives
+    # --health-probe-bind-address from these values, and without them the
+    # controller exposes nothing but its metrics port.
+    livenessProbe:
+      enabled: true
+    readinessProbe:
+      enabled: true
@@ -0,0 +1,118 @@
+---
+# Advertises the on-board Intel iGPU as the schedulable resource
+# `gpu.intel.com/i915`, so pods can request it without a /dev/dri hostPath or
+# any privilege escalation — the plugin hands the render node to the container
+# through the kubelet device-plugin API. The same resource covers both GPU
+# compute (LLM inference) and QuickSync video transcoding; it is not
+# compute-specific.
+#
+# All three nodes are MS-01 with an i9-12900H (Alder Lake-P, PCI 8086:46a6,
+# Iris Xe 96 EU) at 0000:00:02.0. Talos carries the driver in the `i915` system
+# extension, already present in the schematic, so no machineconfig change is
+# needed.
+#
+# Vendored from upstream rather than installed via the
+# `intel-device-plugins-gpu` Helm chart: that chart renders a `GpuDevicePlugin`
+# custom resource, which only works alongside the Intel Device Plugin Operator
+# and its CRDs, cert-manager webhook and controller. With one device type on
+# three identical nodes, the operator manages nothing this DaemonSet does not.
+#
+# Source: intel/intel-device-plugins-for-kubernetes, v0.36.0,
+# deployments/gpu_plugin/base/intel-gpu-plugin.yaml
+# Upgrading means re-syncing this file against that path at the new tag and
+# bumping the image tag below.
+#
+# Namespace is kube-system because the plugin mounts hostPaths (/dev/dri,
+# /sys/class/drm, the kubelet device-plugin socket and the CDI path). The
+# cluster enforces PodSecurity `baseline`, which forbids hostPath volumes, and
+# kube-system is the only namespace in the API server's exemption list.
+#
+# Node Feature Discovery is not deployed, so upstream's `nfd_labeled_nodes`
+# overlay is omitted along with its `intel.feature.node.kubernetes.io/gpu`
+# nodeSelector. Every node carries the same iGPU, leaving nothing to discover;
+# the plugin exits cleanly on a node without /dev/dri.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-gpu-plugin
+  namespace: kube-system
+  labels:
+    app: intel-gpu-plugin
+spec:
+  selector:
+    matchLabels:
+      app: intel-gpu-plugin
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app: intel-gpu-plugin
+    spec:
+      containers:
+        - name: intel-gpu-plugin
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+          image: intel/intel-gpu-plugin:0.36.0
+          imagePullPolicy: IfNotPresent
+          args:
+            # Slots the single iGPU is handed out as, letting that many
+            # containers share it concurrently. A slot count, not fractional
+            # accounting — nothing partitions the device between sharers, so
+            # they contend for it. Sized for an inference server plus room for
+            # a transcoder and a benchmark pod alongside it.
+            - -shared-dev-num=4
+          securityContext:
+            seLinuxOptions:
+              type: "container_device_plugin_t"
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests:
+              memory: "45Mi"
+              cpu: 40m
+            limits:
+              memory: "90Mi"
+              cpu: 100m
+          volumeMounts:
+            - name: devfs
+              mountPath: /dev/dri
+              readOnly: true
+            - name: sysfsdrm
+              mountPath: /sys/class/drm
+              readOnly: true
+            - name: kubeletsockets
+              mountPath: /var/lib/kubelet/device-plugins
+            - name: cdipath
+              mountPath: /var/run/cdi
+      volumes:
+        - name: devfs
+          hostPath:
+            path: /dev/dri
+        - name: sysfsdrm
+          hostPath:
+            path: /sys/class/drm
+        - name: kubeletsockets
+          hostPath:
+            path: /var/lib/kubelet/device-plugins
+        - name: cdipath
+          hostPath:
+            path: /var/run/cdi
+            type: DirectoryOrCreate
+      nodeSelector:
+        kubernetes.io/arch: amd64
@@ -1,5 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - service.yaml
-  - ingress.yaml
+  - daemonset.yaml
@@ -10,6 +10,7 @@ resources:
  - cert-manager/
  - snapshot-controller/
  - metrics-server/
+  - intel-gpu-plugin/
  - crowdsec/
  - traefik/
  - rook-ceph/
@@ -183,3 +183,15 @@ apiVersion: v1
 kind: Namespace
 metadata:
  name: woodpecker
+---
+# Local LLM inference (kubernetes/apps/llama-server). Declared here, like
+# `woodpecker` above, so the `infrastructure` KS creates it before
+# `infrastructure-crds` applies the namespace's CiliumNetworkPolicies.
+#
+# No pod-security labels: the workload needs no hostPath or privilege — the
+# Intel GPU device plugin passes the iGPU in through the device-plugin API —
+# so the cluster's default `baseline` enforcement is enough.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ai
@@ -6,19 +6,12 @@
 # fails the route over across all per-node peers (no single point of
 # failure if a node/pod drops).
 #
-# Uses the ROOTLESS image (userspace wireguard-go). The netbird process
-# terminates the WireGuard tunnel in userspace and reaches cluster
-# services by opening its OWN sockets to them — NOT by L3-forwarding
-# packets. That distinction is the whole point:
-#   - Cilium socket-LB DNATs the netbird process's own connections, so
-#     forwarding to ClusterIP / LB VIPs actually works. A kernel L3
-#     routing peer fails here: socket-LB never sees forwarded packets,
-#     so they hit the raw VIP and are blackholed (confirmed via conntrack).
-#   - No NET_ADMIN / no kernel WireGuard → runs fully unprivileged
-#     (drop ALL caps, runAsNonRoot), so it satisfies Pod Security too.
-#
-# Flow: mesh peer (100.96.x.x) --wg--> a per-node pod (userspace netbird)
-#        --own socket (socket-LB DNAT)--> service ClusterIP/LB VIP
+# Must stay on the ROOTLESS image (userspace wireguard-go). The process
+# terminates the tunnel in userspace and reaches services by opening its own
+# sockets, which Cilium's socket-LB can DNAT. A kernel WireGuard peer
+# L3-forwards instead, socket-LB never sees those packets, and they are
+# blackholed at the raw VIP. Rootless also means no NET_ADMIN, so the pod
+# runs fully unprivileged.
 #
 # Egress is locked down by the netbird-ingress-egress CiliumNetworkPolicy
 # (network-policies/ingress-isolation.yaml).
@@ -0,0 +1,96 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: default-deny-ingress
+  namespace: ai
+spec:
+  endpointSelector: {}
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ai
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ingress
+            app.kubernetes.io/name: traefik
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": monitoring
+    - fromEntities:
+        - host
+        - remote-node
+    # ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See
+    # home-assistant-ns.yaml for the rationale (CT-RELATED path not
+    # passing related ICMP empirically; per-namespace shape, mirroring
+    # commit 3c517a0's verified-safe pattern with the 66dbe9f type set).
+    - fromEndpoints:
+        - matchExpressions:
+            - key: io.kubernetes.pod.namespace
+              operator: Exists
+      icmps:
+        - fields:
+            - { type: 0,   family: IPv4 }
+            - { type: 3,   family: IPv4 }
+            - { type: 8,   family: IPv4 }
+            - { type: 11,  family: IPv4 }
+            - { type: 12,  family: IPv4 }
+            - { type: 1,   family: IPv6 }
+            - { type: 2,   family: IPv6 }
+            - { type: 3,   family: IPv6 }
+            - { type: 4,   family: IPv6 }
+            - { type: 128, family: IPv6 }
+            - { type: 129, family: IPv6 }
+            - { type: 133, family: IPv6 }
+            - { type: 134, family: IPv6 }
+            - { type: 135, family: IPv6 }
+            - { type: 136, family: IPv6 }
+            - { type: 143, family: IPv6 }
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ai
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ingress
+            app.kubernetes.io/name: traefik
+    # rook-ceph: the model weights live on a CephFS RWX volume
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": rook-ceph
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+    # host/remote-node: kubelet probes + node-local reach (Ceph mons on host net)
+    # world: image pulls and llama-server's Hugging Face model download
+    - toEntities:
+        - host
+        - remote-node
+        - world
+    # ICMP — egress mirror; see ingress block above.
+    - toEndpoints:
+        - matchExpressions:
+            - key: io.kubernetes.pod.namespace
+              operator: Exists
+      icmps:
+        - fields:
+            - { type: 0,   family: IPv4 }
+            - { type: 3,   family: IPv4 }
+            - { type: 8,   family: IPv4 }
+            - { type: 11,  family: IPv4 }
+            - { type: 12,  family: IPv4 }
+            - { type: 1,   family: IPv6 }
+            - { type: 2,   family: IPv6 }
+            - { type: 3,   family: IPv6 }
+            - { type: 4,   family: IPv6 }
+            - { type: 128, family: IPv6 }
+            - { type: 129, family: IPv6 }
+            - { type: 133, family: IPv6 }
+            - { type: 134, family: IPv6 }
+            - { type: 135, family: IPv6 }
+            - { type: 136, family: IPv6 }
+            - { type: 143, family: IPv6 }
--- a/Show More
+++ b/Show More