fix(monitoring): one value per stat panel

The OSD panel carried two queries, so it rendered two numbers side by side. Both read the same figure in the normal case, which looks like a duplicate and carries no information — a status box has one job, which is to be one colour. It now shows OSDs down alone. An OSD that is up but out was the second number; that is a state rather than a fault, Ceph already raises a health warning for it, and it does not belong in a red-or-green box. Refs #549
2026-08-27 14:34:44 +02:00
174 changed files with 6492 additions and 7299 deletions
@@ -32,13 +32,13 @@
 #              mechanism as update-audit.yml):
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name alert-monitor --branch master --schedule '*/15 * * * *'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> alert-monitor
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=alert-monitor k.renner/homelab
 when:
  - event: cron
    cron: alert-monitor
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "alert-monitor"'
+  - event: manual
+    evaluate: 'PIPELINE == "alert-monitor"'

 steps:
  - name: fetch-alerts
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -31,8 +31,8 @@ when:
      - "ci/images/**"
      - "secret-rotator/**"
      - ".woodpecker/build-ci-images.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-ci-images"'
+  - event: manual
+    evaluate: 'PIPELINE == "build-ci-images"'

 steps:
  - name: build-ansible
@@ -44,7 +44,7 @@ steps:
    # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
    # orphaned ~5G of registry layers per run — a main driver of #336.
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -68,7 +68,7 @@ steps:
    directory: ci/images/k8s
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -92,7 +92,7 @@ steps:
    directory: ci/images/tofu
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -123,7 +123,7 @@ steps:
    directory: secret-rotator
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -152,7 +152,7 @@ steps:
    directory: ci/images/alpine-jq
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -181,7 +181,7 @@ steps:
    directory: ci/images/cni-plugins
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -210,7 +210,7 @@ steps:
    directory: ci/images/alpine-openssl
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -239,7 +239,7 @@ steps:
    directory: ci/images/heating-can
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -54,8 +54,8 @@
 #
 # Triggering:
 #   - cron   : the cron NAMED exactly "ci-claude-refresh" (see above).
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> ci-claude-refresh
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=ci-claude-refresh k.renner/homelab
 #   - push   : a change to the Containerfile (or this file) ON master forces a
 #              rebuild of the current latest version so Containerfile fixes ship
 #              promptly. Restricted to master ON PURPOSE — a feature branch must
@@ -65,8 +65,8 @@
 when:
  - event: cron
    cron: ci-claude-refresh
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "ci-claude-refresh"'
+  - event: manual
+    evaluate: 'PIPELINE == "ci-claude-refresh"'
  - event: push
    branch: master
    path:
@@ -17,6 +17,6 @@ when:

 steps:
  - name: model-selection-tests
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
    commands:
      - sh scripts/ci/test-model-selection.sh
@@ -6,8 +6,8 @@ when:
      - "ansible/roles/claude-workstation/**"
      - "ansible/plays/install-claude-workstation.yml"
      - ".woodpecker/deploy-claude-workstation.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-claude-workstation"'

 timeout: 30

@@ -8,8 +8,8 @@ when:
    path:
      - "opentofu/**"
      - ".woodpecker/deploy-infrastructure.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-infrastructure"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-infrastructure"'

 steps:
  - name: validate-secrets
@@ -84,7 +84,7 @@ steps:
    image: gitea.renner.uno/gitea_admin/ci-tofu:2026-04-29
    directory: opentofu
    when:
-      - event: deployment
+      - event: manual
        evaluate: 'TOFU_APPLY == "true"'
    environment:
      TF_IN_AUTOMATION: "true"
@@ -7,8 +7,8 @@ when:
      - "ansible/plays/install-netbird-client.yml"
      - "ansible/inventory/group_vars/claude_workstation/**"
      - ".woodpecker/deploy-netbird-client.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-client"'

 timeout: 30

@@ -7,24 +7,10 @@ when:
      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
      # so changes to it must redeploy this host too.
      - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
      - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
      - ".woodpecker/deploy-netbird-proxy.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-proxy"'

 timeout: 30

@@ -9,8 +9,8 @@ when:
      - "ansible/roles/netbird-proxy-watchdog/**"
      - "ansible/plays/install-netbird-server.yml"
      - ".woodpecker/deploy-netbird-server.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-server"'

 timeout: 30

@@ -1,6 +1,6 @@
 when:
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e-test"'
+  - event: manual
+    evaluate: 'PIPELINE == "e2e-test"'

 steps:
  - name: e2e
@@ -7,8 +7,7 @@
 #                was harmless at daily/weekly cadence — but alert-monitor.yml
 #                fires every 15 minutes, so piggybacking must be opt-in now.
 #                A new cron that SHOULD also run e2e gets its name added here.
-#   - deploy   : `woodpecker-cli pipeline deploy k.renner/homelab <n> e2e`, where
-#                <n> is an existing pipeline to promote.
+#   - manual   : `woodpecker pipeline create --branch master --var PIPELINE=e2e k.renner/homelab`

 when:
  - event: cron
@@ -17,8 +16,8 @@ when:
    cron: update-audit
  - event: cron
    cron: weekly-update
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e"'
+  - event: manual
+    evaluate: 'PIPELINE == "e2e"'

 steps:
  - name: model-selection-tests
@@ -26,7 +25,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -0,0 +1,27 @@
+when:
+  - event: manual
+    evaluate: 'PIPELINE == "install-garage-synology"'
+
+steps:
+  - name: run-playbook
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+      PROXMOX_TOKEN_ID:
+        from_secret: proxmox_token_id
+      PROXMOX_TOKEN_SECRET:
+        from_secret: proxmox_token_secret
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - 'printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config'
+      - cd ansible
+      - echo "$$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/install-garage-synology.yml -v
@@ -34,13 +34,13 @@
 #   - cron   : a cron NAMED exactly "issue-fix":
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name issue-fix --branch master --schedule '41 * * * *'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> issue-fix
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=issue-fix k.renner/homelab
 when:
  - event: cron
    cron: issue-fix
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "issue-fix"'
+  - event: manual
+    evaluate: 'PIPELINE == "issue-fix"'

 steps:
  - name: fetch-diag
@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -53,9 +53,9 @@
 #
 # Triggering: pull_request only — automatic on PR open + push-to-open-PR. To
 # re-review a PR, push a commit (the new head sha clears the dedup marker below
-# and earns a fresh review). There is deliberately no on-demand trigger for a
-# specific PR: Woodpecker exposes injected variables to YAML substitution and
-# `when` filters, NOT to a step's shell environment, so a PR=<n> re-review would
+# and earns a fresh review). There is deliberately no manual PIPELINE trigger:
+# Woodpecker exposes a --var only to YAML variable substitution and `when`
+# filters, NOT to a step's shell environment, so a --var PR=<n> re-review would
 # silently no-op (the step would find no PR number and exit green).
 # (Note: keep literal dollar-brace out of THIS file entirely, even in comments —
 # Woodpecker runs its variable substitution over the whole YAML text, comments
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,20 +1,8 @@
-# Trivy scan of the IaC tree and every image referenced from ansible/, with an
-# agent pass that opens a PR for CRITICAL/HIGH findings.
-#
-# Triggers:
-#   - cron       : the named `daily-security-scan` cron, matched by NAME.
-#                  Crons cannot carry variables — `woodpecker-cli repo cron add`
-#                  has no --var — so a variable gate can never match a cron.
-#   - deployment : woodpecker-cli pipeline deploy k.renner/homelab <n> security-scan
-#                  The target rides in pipeline metadata as
-#                  CI_PIPELINE_DEPLOY_TARGET, which the workflow-root filter
-#                  does see, unlike manual-run variables on Woodpecker 3.18
-#                  (#582).
 when:
  - event: cron
-    cron: daily-security-scan
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "security-scan"'
+    evaluate: 'PIPELINE == "security-scan"'
+  - event: manual
+    evaluate: 'PIPELINE == "security-scan"'

 steps:
  - name: iac-scan
@@ -40,28 +28,12 @@ steps:
        cat /tmp/images.txt
        echo "---"
        i=0
-        failed=0
-        failed_names=""
        while IFS= read -r img; do
          i=$$((i+1))
          out="trivy-image-$$i.json"
          echo "=== [$$i] $$img -> $$out ==="
-          # Keep going on a bad image so one unpullable reference cannot cost
-          # the scan of every image after it, but record that it happened.
-          if ! trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img"; then
-            failed=$$((failed+1))
-            failed_names="$$failed_names $$img"
-          fi
+          trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img" || true
        done < /tmp/images.txt
-        echo "--- scanned $$((i - failed)) of $$i images ---"
-        if [ "$$failed" -gt 0 ]; then
-          echo "NOT SCANNED:$$failed_names"
-          exit 1
-        fi
-    # Non-zero above turns this step red without failing the workflow, so the
-    # fix agent still runs on what was scanned. Coverage is reported, not
-    # enforced: an incomplete scan is visible in the step status instead of
-    # being indistinguishable from a clean one.
    failure: ignore

  - name: security-fix-agent
@@ -70,7 +42,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -10,14 +10,14 @@
 # report; the pipeline posts it to a single persistent Gitea issue.
 #
 # Triggering:
-#   - cron   : a cron NAMED exactly "update-audit". Crons carry no variables —
-#              `repo cron add` takes name, branch, schedule and enabled — so the
-#              name filter is the only thing that can select a workflow for a
-#              scheduled run:
+#   - cron   : a cron NAMED exactly "update-audit". Unlike the other pipelines
+#              here (which gate on an injected `PIPELINE` var set at cron-create
+#              time in the UI), this uses Woodpecker's documented `cron:` name
+#              filter so the schedule can be created entirely from the CLI:
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name update-audit --branch master --schedule '@weekly'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> update-audit
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=update-audit k.renner/homelab
 #
 # NOTE: e2e.yml runs on an explicit list of cron names that includes this one,
 # so a weekly e2e run piggybacks on the audit (it was a bare `event: cron`
@@ -25,12 +25,12 @@
 when:
  - event: cron
    cron: update-audit
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-audit"'
+  - event: manual
+    evaluate: 'PIPELINE == "update-audit"'

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.247
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,23 +0,0 @@
---
-# Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
-#
-# A single encrypted VALUE rather than a whole-file vault like its siblings in
-# group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
-# an existing whole-file vault means decrypting that file first, which exposes
-# every other secret in it for the sake of appending one line. encrypt_string
-# keeps the blast radius at this one value.
-#
-# Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
-# the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
-# LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
-# one is NOT in the quarterly rotator, because nothing on these hosts re-reads
-# OpenBao on a schedule. Rotating it means updating OpenBao and re-running
-# plays/install-netbird-proxy.yml.
-vault_crowdsec_edge_bouncer_key: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          32653764333536316163303838333839616562663862666637626165303837633866643033393239
-          3363376538656236643161386432326437633232643564630a633635306365323537346139626364
-          36613033343938363366306239363638383466396337623333623136313234656366333366623433
-          6437656632373364340a346566343866366433306533353431383064356362346536383136323663
-          39333864643534326433653233303137373162663066633132623931316135323733333863306330
-          6366623463666435636362376238313639666263323163303733
@@ -0,0 +1,37 @@
+---
+# PBS Client configuration for nas-1
+pbs_backup_paths:
+  - /volume1/Documents
+  - /volume1/homes
+  - /volume1/Homeassistant Backup
+  - /volume1/Games
+  - /volume1/Media
+  - /volume1/Music
+  - /volume1/video
+  - /volume2/Backup
+  - /volume1/Movies
+
+  # Add more paths as needed:
+  # - /volume1/photos
+  # - /volume1/music
+
+# PBS Server connection
+pbs_server_host: "pbs2.renner.uno"
+pbs_server_port: 8007
+pbs_datastore: "pool"
+pbs_backup_id: "nas-1"
+
+# PBS Authentication - credentials in vault.yml
+pbs_username: "root@pam!nas-1"
+
+# Garage S3 (K8s backup target)
+garage_image: "dxflrs/garage:v2.3.0"
+garage_zone: "dsm"
+garage_capacity: "2T"
+
+# SSH connection - use password auth (key not supported by DSM)
+ansible_ssh_pass: "{{ nas_ssh_password }}"
+ansible_become_pass: "{{ nas_ssh_password }}"
+
+# Python 3.9+ from Synology package
+ansible_python_interpreter: /usr/local/bin/python3.9
@@ -0,0 +1,27 @@
+$ANSIBLE_VAULT;1.1;AES256
+34663462636437633437613863366266306334666635643839313737343739643864613136613938
+3138393966306561656366396465383539303137333036300a376237376636356230343834386362
+36636535396330333732383233323263613132333166343739396137636538653435336662353138
+6565643866623235610a316266383933303830323936303735623338623165386339383733373435
+62366461363833616438373237383132366438326364363462333265366533323861376630643365
+39343036343838333639363432643738666630363063323636646661313539643465366532363438
+33303837623234613434633563666562346536346435353461373536323737376233386163326537
+61383963343737656438356136613537623538333163343466343739346135333439616432663763
+36663962633037376561326434303035333938646436663832303432346365353439666435613337
+34363636326530333233313231343566306432376532396666643761633136316333376337666135
+65363338623035663438323338663863366362653131336434323865326530326534623631336263
+36633063383031386161363131333636393364623436613331353733313235383062656131313164
+63393665633361313961363637663237616364643236383835393736643937393564656238376362
+31393465383361326264376133393233616264663636383337363962356532613438663637616536
+33653137656335323763313461323633616635346232303062323062333766383931313965346361
+66656461313239633366636463633465636463663937393932636337643861383937663662306466
+32353166643832303234643264633062616330393963333034623432613730366339306637613239
+34626161663666316339323762373639353335653663353635366664643063393432623039306237
+38363161343930363763393364663364353735343236313665313461626165636135303839363164
+37336638396564326166376233353730346563656266323535666265323062646232323938353732
+32636235333839623166633761376334316434613235333730353439343262393537363461333238
+32653435623362666662336532643832356163326332636436346132643237643030343761663332
+36663530356539303636663438646430363335353135323932613863666264363533326137353164
+31663032346236616263343637396439663334396166656364623637666535386530383535663435
+66303037303265616634373763616437333835333136336563323339666563663931396133633131
+3963653266633535303630653335666166313336653831623362
@@ -8,8 +8,6 @@
 #   pve_cluster (jupiter/neptune/saturn)  run Talos on bare metal; no SSH at all
 #   authentik_db / authentik_app          authentik runs in-cluster (CNPG)
 #   grafana                               runs in-cluster
-#   synology_nas (nas-1)                  powered off; Garage and the PBS client
-#                                         it hosted are gone with it
 # Their hosts no longer resolve, except the three that resolve to Talos nodes.

 [netbird_server]
@@ -19,6 +17,10 @@ netbird.renner.uno ansible_user=root
 proxy-1.renner.uno ansible_user=root
 proxy-2.renner.uno ansible_user=root

+
+[synology_nas]
+nas-1.renner.uno ansible_user=ladmin ansible_python_interpreter=/usr/bin/python3
+
 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN
 # sub-interface. Nothing here assumes any peripheral is attached.
@@ -0,0 +1,313 @@
+---
+# Synology-specific playbook using raw module (Python 3.8 workaround)
+# Deploys Garage (https://garagehq.deuxfleurs.fr) as a single-node S3 target
+# for K8s backups (Velero + CNPG Barman Cloud Plugin).
+#
+# Required secrets in host_vars/nas-1.renner.uno/vault.yml:
+#   garage_rpc_secret:     "<openssl rand -hex 32>"
+#   garage_admin_token:    "<openssl rand -base64 32>"
+#   garage_metrics_token:  "<openssl rand -base64 32>"
+#
+# Post-install bootstrap (run once, record the returned secret keys):
+#   docker exec garaged /garage status
+#   docker exec garaged /garage bucket create k8s-velero
+#   docker exec garaged /garage bucket create k8s-cnpg
+#   docker exec garaged /garage bucket create k8s-secrets
+#   docker exec garaged /garage bucket create k8s-migration
+#   docker exec garaged /garage key create velero-key
+#   docker exec garaged /garage key create cnpg-key
+#   docker exec garaged /garage key create secrets-key
+#   docker exec garaged /garage key create migration-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-velero    --key velero-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-cnpg      --key cnpg-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-secrets   --key secrets-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-migration --key migration-key
+- name: Install Garage S3 on Synology NAS
+  hosts: synology_nas
+  gather_facts: false
+
+  vars:
+    garage_data_dir: "/volume1/docker/garage"
+
+  tasks:
+    - name: Create Garage directory tree
+      ansible.builtin.raw: |
+        mkdir -p {{ garage_data_dir }}/config \
+                 {{ garage_data_dir }}/meta \
+                 {{ garage_data_dir }}/data
+      changed_when: true
+
+    # Secret writes use the same compare-then-write pattern as garage.toml so
+    # rotation is idempotent AND a real change can drive force-recreate below.
+    # Garage reads these files once at startup; without container restart, a
+    # rotated token leaves the running process holding the previous value.
+    - name: Write rpc_secret
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/rpc_secret.new << 'SECRET_EOF'
+        {{ garage_rpc_secret }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/rpc_secret.new
+        if [ ! -f {{ garage_data_dir }}/config/rpc_secret ] \
+        || ! cmp -s {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret; then
+          mv {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret
+          echo RPC_SECRET_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/rpc_secret.new
+        fi
+      register: rpc_secret_write
+      changed_when: "'RPC_SECRET_CHANGED' in rpc_secret_write.stdout"
+      no_log: true
+
+    - name: Write admin_token
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/admin_token.new << 'SECRET_EOF'
+        {{ garage_admin_token }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/admin_token.new
+        if [ ! -f {{ garage_data_dir }}/config/admin_token ] \
+        || ! cmp -s {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token; then
+          mv {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token
+          echo ADMIN_TOKEN_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/admin_token.new
+        fi
+      register: admin_token_write
+      changed_when: "'ADMIN_TOKEN_CHANGED' in admin_token_write.stdout"
+      no_log: true
+
+    - name: Write metrics_token
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/metrics_token.new << 'SECRET_EOF'
+        {{ garage_metrics_token }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/metrics_token.new
+        if [ ! -f {{ garage_data_dir }}/config/metrics_token ] \
+        || ! cmp -s {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token; then
+          mv {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token
+          echo METRICS_TOKEN_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/metrics_token.new
+        fi
+      register: metrics_token_write
+      changed_when: "'METRICS_TOKEN_CHANGED' in metrics_token_write.stdout"
+      no_log: true
+
+    - name: Write garage.toml
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/garage.toml.new << 'TOML_EOF'
+        metadata_dir = "/var/lib/garage/meta"
+        data_dir     = "/var/lib/garage/data"
+
+        # SQLite chosen over LMDB: single-node deployment has no peers to
+        # rebuild metadata from on unclean-shutdown corruption. SQLite is
+        # slower but portable and has no LMDB corruption window.
+        db_engine = "sqlite"
+
+        # 6-hour auto-snapshot of the metadata DB as a backstop against
+        # corruption; last two snapshots retained.
+        metadata_auto_snapshot_interval = "6h"
+
+        # Single-zone, single-node: only coherent replication factor.
+        replication_factor = 1
+
+        rpc_bind_addr    = "[::]:3901"
+        rpc_secret_file  = "/etc/garage/rpc_secret"
+
+        compression_level = 2
+
+        # 64 MB blocks (default 1 MB; previously 20 MB). On the 5 × 7 TB
+        # Btrfs RAID 5 volume the per-block fixed cost (SQLite lookup,
+        # dir traversal, hash verify, zstd decompress, nginx buffering)
+        # is the dominant overhead per object; larger blocks amortize
+        # it. block_size is a maximum, so 16 MiB CNPG WAL segments
+        # still upload as one block each. Only applies to NEW uploads;
+        # existing objects keep their original chunking until re-uploaded.
+        block_size = "64M"
+
+        [s3_api]
+        api_bind_addr = "[::]:3900"
+        s3_region     = "garage"
+        root_domain   = ".s3.{{ inventory_hostname }}"
+
+        [s3_web]
+        bind_addr   = "[::]:3902"
+        root_domain = ".web.{{ inventory_hostname }}"
+
+        [admin]
+        api_bind_addr      = "[::]:3903"
+        admin_token_file   = "/etc/garage/admin_token"
+        metrics_token_file = "/etc/garage/metrics_token"
+        TOML_EOF
+        if [ ! -f {{ garage_data_dir }}/config/garage.toml ] \
+        || ! cmp -s {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml; then
+          mv {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml
+          echo GARAGE_TOML_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/garage.toml.new
+        fi
+      register: garage_toml_write
+      changed_when: "'GARAGE_TOML_CHANGED' in garage_toml_write.stdout"
+
+    - name: Write docker-compose.yml
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
+        services:
+          garage:
+            image: {{ garage_image }}
+            container_name: garaged
+            restart: unless-stopped
+            # HTTP services (3900/3902/3903) bind to loopback; DSM
+            # reverse proxy fronts and TLS-terminates them. RPC 3901
+            # is exposed directly — the Noise handshake is raw TCP,
+            # so DSM RP cannot front it. rpc_secret gates auth.
+            ports:
+              - "3901:3901"
+              - "127.0.0.1:3900:3900"
+              - "127.0.0.1:3902:3902"
+              - "127.0.0.1:3903:3903"
+            volumes:
+              - {{ garage_data_dir }}/config/garage.toml:/etc/garage.toml:ro
+              - {{ garage_data_dir }}/config/rpc_secret:/etc/garage/rpc_secret:ro
+              - {{ garage_data_dir }}/config/admin_token:/etc/garage/admin_token:ro
+              - {{ garage_data_dir }}/config/metrics_token:/etc/garage/metrics_token:ro
+              - {{ garage_data_dir }}/meta:/var/lib/garage/meta
+              - {{ garage_data_dir }}/data:/var/lib/garage/data
+        COMPOSE_EOF
+      changed_when: true
+
+    - name: Pull image and start Garage container
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ garage_data_dir }} && docker compose pull && docker compose up -d
+      become: true
+      changed_when: true
+
+    # Garage does not reload config on SIGHUP and compose does not notice
+    # when a bind-mounted host file changes. Force-recreate the container
+    # when garage.toml OR any of the three secret files changed so the new
+    # config / rotated tokens are picked up.
+    - name: Force-recreate garaged when garage.toml or secrets changed
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ garage_data_dir }} && docker compose up -d --force-recreate garage
+      become: true
+      when: garage_toml_write.changed
+            or rpc_secret_write.changed
+            or admin_token_write.changed
+            or metrics_token_write.changed
+      changed_when: true
+
+    - name: Wait for Garage to accept CLI calls
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        for i in $(seq 1 30); do
+          if docker exec garaged /garage status >/dev/null 2>&1; then
+            exit 0
+          fi
+          sleep 2
+        done
+        exit 1
+      become: true
+      changed_when: false
+
+    - name: Check whether cluster layout is already applied
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage status
+      become: true
+      register: garage_status
+      changed_when: false
+
+    - name: Apply single-node layout (idempotent — only on first run)
+      when: "'NO ROLE ASSIGNED' in garage_status.stdout"
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        NODE_ID=$(docker exec garaged /garage node id | head -n 1 | cut -d@ -f1)
+        docker exec garaged /garage layout assign "$NODE_ID" \
+          -z {{ garage_zone }} \
+          -c {{ garage_capacity }} \
+          -t {{ inventory_hostname_short }}
+        docker exec garaged /garage layout apply --version 1
+      become: true
+      changed_when: true
+
+    - name: Report Garage status
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage status
+      become: true
+      register: garage_status_final
+      changed_when: false
+
+    - name: Show Garage status
+      ansible.builtin.debug:
+        var: garage_status_final.stdout_lines
+
+    # Bootstrap buckets + keys. Idempotent: bucket_info / key_info return
+    # non-zero when missing, so we only create on first run. Access-key
+    # secrets are captured at create time and persisted to chmod-600 files
+    # under {{ garage_data_dir }}/config/keys/<key>.creds on nas-1. These
+    # files are the only post-install record of the secrets — Garage does
+    # not expose them again.
+    - name: Bootstrap Garage buckets, keys, and grants
+      ansible.builtin.shell: |
+        set -e
+        export PATH=$PATH:/usr/local/bin
+        KEYS_DIR={{ garage_data_dir }}/config/keys
+        mkdir -p "$KEYS_DIR"
+        chmod 700 "$KEYS_DIR"
+
+        bootstrap_pair() {
+          local BUCKET="$1"
+          local KEY="$2"
+          local CREDS_FILE="$KEYS_DIR/$KEY.creds"
+
+          if ! docker exec garaged /garage bucket info "$BUCKET" >/dev/null 2>&1; then
+            docker exec garaged /garage bucket create "$BUCKET"
+          fi
+
+          if ! docker exec garaged /garage key info "$KEY" >/dev/null 2>&1; then
+            CREATE_OUT=$(docker exec garaged /garage key create "$KEY")
+            KEY_ID=$(echo "$CREATE_OUT"   | awk '/^Key ID:/     {print $3}')
+            SECRET=$(echo "$CREATE_OUT"   | awk '/^Secret key:/ {print $3}')
+            umask 177
+            printf 'KEY_ID=%s\nSECRET_KEY=%s\n' "$KEY_ID" "$SECRET" > "$CREDS_FILE"
+          fi
+
+          docker exec garaged /garage bucket allow \
+            --read --write --owner \
+            "$BUCKET" --key "$KEY"
+        }
+
+        bootstrap_pair k8s-velero    velero-key
+        bootstrap_pair k8s-cnpg      cnpg-key
+        bootstrap_pair k8s-secrets   secrets-key
+        bootstrap_pair k8s-migration migration-key
+        bootstrap_pair tfstate       tfstate-key
+      become: true
+      changed_when: true
+      no_log: true
+
+    - name: Report bucket + key state (no secrets)
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage bucket list
+        echo "---"
+        docker exec garaged /garage key list
+        echo "---"
+        ls -l {{ garage_data_dir }}/config/keys/ 2>&1
+      become: true
+      register: garage_bootstrap_state
+      changed_when: false
+
+    - name: Show bucket + key state
+      ansible.builtin.debug:
+        var: garage_bootstrap_state.stdout_lines
+
+    - name: Post-install note
+      ansible.builtin.debug:
+        msg:
+          - "Buckets: k8s-velero, k8s-cnpg, k8s-secrets, k8s-migration, tfstate"
+          - "Keys:    velero-key, cnpg-key, secrets-key, migration-key, tfstate-key"
+          - "Secrets are written to {{ garage_data_dir }}/config/keys/<key>.creds on nas-1 (chmod 600, root-owned)."
+          - "SSH to nas-1 and `sudo cat` the file to retrieve KEY_ID + SECRET_KEY for Velero / CNPG / migration / tfstate config."
@@ -26,7 +26,3 @@
    - docker-install
    - netbird-proxy
    - netbird-proxy-watchdog
-    # Enforces the cluster's CrowdSec decisions here, where public traffic
-    # enters. Deployed in dry-run: it logs what it would block and creates no
-    # nftables rules until crowdsec_bouncer_mode is flipped per host.
-    - crowdsec-edge-bouncer
@@ -0,0 +1,104 @@
+---
+# Synology-specific playbook using raw module (Python 3.8 workaround)
+- name: Install PBS Client on Synology NAS
+  hosts: synology_nas
+  gather_facts: false
+
+  vars:
+    pbs_client_data_dir: "/volume1/docker/pbs-client"
+
+  tasks:
+    - name: Create PBS client directory
+      ansible.builtin.raw: |
+        mkdir -p {{ pbs_client_data_dir }}/state
+      changed_when: true
+
+    - name: Create docker-compose.yml
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
+        services:
+          pbs-client:
+            image: proxmox/proxmox-backup-client:latest
+            container_name: pbs-client
+            restart: unless-stopped
+            hostname: {{ pbs_backup_id }}
+            env_file:
+              - .env
+            volumes:
+              - {{ pbs_client_data_dir }}/state:/root/.proxmox-backup
+        {% for path in pbs_backup_paths %}
+              - "{{ path }}:{{ path }}:ro"
+        {% endfor %}
+            mem_limit: 2g
+            command: ["sleep", "infinity"]
+        COMPOSE_EOF
+      changed_when: true
+
+    - name: Create .env file
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/.env << 'ENV_EOF'
+        PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
+        PBS_PASSWORD={{ pbs_password }}
+        ENV_EOF
+        chmod 600 {{ pbs_client_data_dir }}/.env
+      changed_when: true
+
+    - name: Create backup script
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/backup.sh << 'SCRIPT_EOF'
+        #!/bin/bash
+        set -e
+        source {{ pbs_client_data_dir }}/.env
+
+        BACKUP_SOURCES=""
+        {% for path in pbs_backup_paths %}
+        BACKUP_SOURCES="$BACKUP_SOURCES {{ path | basename }}.pxar:{{ path }}"
+        {% endfor %}
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup for {{ pbs_backup_id }}"
+        docker exec pbs-client proxmox-backup-client backup \
+            $BACKUP_SOURCES \
+            --backup-id {{ pbs_backup_id }} \
+            --repository "${PBS_REPOSITORY}"
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Backup completed, pruning..."
+        docker exec pbs-client proxmox-backup-client prune \
+            --repository "${PBS_REPOSITORY}" \
+            --backup-id {{ pbs_backup_id }} \
+            --keep-last {{ pbs_keep_last | default(3) }} \
+            --keep-daily {{ pbs_keep_daily | default(7) }} \
+            --keep-weekly {{ pbs_keep_weekly | default(4) }} \
+            --keep-monthly {{ pbs_keep_monthly | default(6) }}
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Done"
+        SCRIPT_EOF
+        chmod 755 {{ pbs_client_data_dir }}/backup.sh
+      changed_when: true
+
+    - name: Start PBS client container
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ pbs_client_data_dir }} && docker compose pull && docker compose up -d
+      become: true
+      changed_when: true
+
+    - name: Setup cron job for daily backup at 2 AM (root crontab)
+      ansible.builtin.shell: |
+        # Remove existing entry if present
+        crontab -l 2>/dev/null | grep -v "pbs-client/backup.sh" | crontab - 2>/dev/null || true
+        # Add new entry
+        (crontab -l 2>/dev/null; echo "0 2 * * * {{ pbs_client_data_dir }}/backup.sh >> /var/log/pbs-backup.log 2>&1") | crontab -
+      become: true
+      changed_when: true
+
+    - name: Verify container is running
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker ps --filter name=pbs-client --format "{{ '{{' }}.Status{{ '}}' }}"
+      become: true
+      register: container_status
+      changed_when: false
+
+    - name: Show container status
+      ansible.builtin.debug:
+        msg: "PBS client container status: {{ container_status.stdout }}"
@@ -0,0 +1,11 @@
+---
+- name: Install PBS Client on Synology NAS
+  hosts: synology_nas
+  become: true
+  # Variables are set in:
+  # - Role defaults: ansible/roles/pbs-client/defaults/main.yml
+  # - Host overrides: ansible/inventory/host_vars/<hostname>/vars.yml
+  # - Secrets: ansible/inventory/host_vars/<hostname>/vault.yml
+
+  roles:
+    - pbs-client
@@ -1,56 +0,0 @@
---
-# CrowdSec firewall bouncer on the NetBird proxy hosts.
-#
-# Enforces the decisions the in-cluster CrowdSec already detects, at the point
-# where public traffic actually enters: these hosts terminate TLS for every
-# *.renner.uno name. A drop here never reaches the mesh, the relay or the
-# cluster.
-#
-# Detection stays in the cluster. The reverse-proxy produces no parseable
-# access log and there is no CrowdSec collection for it, while Traefik has a
-# working one — and every scenario in use needs HTTP detail (paths, status
-# codes, user agents) that does not exist at this layer.
-
-# Pinned. A release tarball URL is immutable, and this checksum is of the
-# amd64 asset for exactly this tag.
-crowdsec_bouncer_version: v0.0.36
-crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
-
-# The LAPI over the public path these hosts already serve, see
-# kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
-# reach it over the mesh: the reverse-proxy container runs non-root, so its
-# embedded NetBird client terminates the tunnel in userspace and the tunnel
-# belongs to that process rather than to the host netns.
-#
-# No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
-# and the bouncer adds ca_cert_path to the system pool rather than replacing
-# it — the system pool already trusts LE.
-crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
-crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
-
-# Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
-# crowdsec6) holding the decisions it pulled, touches no existing chain and
-# sets no chain policy — SSH and :443 stay open unless the source is banned.
-#
-# `dry-run` is the other value: it pulls and logs what it would block without
-# creating a single rule. That is what a new host should be given before it
-# enforces, and what an existing one can be dropped back to, per host through
-# inventory/host_vars, without disarming the rest.
-#
-# Worth knowing before arming anything new: the first bouncer on a path
-# enforces the whole CAPI set at once — around 15k addresses today — so the
-# false-positive window belongs before the switch, not after it. And because
-# this chain sits on the INPUT hook while these hosts run no other firewall,
-# there is no conntrack rule accepting established flows ahead of it: a banned
-# address is dropped even when it is answering our own outbound connection.
-# That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
-crowdsec_bouncer_mode: nftables
-
-crowdsec_bouncer_update_frequency: 10s
-crowdsec_bouncer_log_level: info
-
-# Bound to loopback deliberately. These hosts are public and have no metrics
-# collection path at all today — opening :60601 to the internet to gain a
-# scrape target is the wrong trade. Revisit when host metrics exist.
-crowdsec_bouncer_metrics_listen: 127.0.0.1
-crowdsec_bouncer_metrics_port: 60601
@@ -1,10 +0,0 @@
---
-# A restart drops both nftables tables and rebuilds them from the next
-# successful LAPI pull, so there is a short window with no blocklist. That is
-# the documented behaviour of a clean shutdown, not a defect — and it is why
-# retry_initial_connect matters.
- name: Restart crowdsec firewall bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    state: restarted
-    daemon_reload: true
@@ -1,111 +0,0 @@
---
-# See defaults/main.yml for why this exists and why it starts inert.
-#
-# The static release tarball, not a distribution package: the AUR package
-# `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
-# whole Security Engine onto hosts that only need the bouncer, the group is
-# mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
-# repo does not otherwise have. The tarball is the same artefact the AUR
-# package downloads.
-
- name: Ensure nftables is present (the enforcing backend)
-  ansible.builtin.package:
-    name: nftables
-    state: present
-
- name: Ensure the bouncer config directory exists
-  ansible.builtin.file:
-    path: /etc/crowdsec/bouncers
-    state: directory
-    mode: "0755"
-
-# Fetched on the CONTROLLER and pushed, not pulled by the host.
-#
-# proxy-1 is IPv6-only, and neither github.com nor objects.githubusercontent.com
-# publishes an AAAA record — a get_url on the host fails with "Network is
-# unreachable" and no amount of retrying fixes it. Mirroring into
-# gitea.renner.uno is no help either: its public records are the internal LB
-# VIP (10.2.99.86 / fd99::86), so it is unreachable from outside the LAN. And
-# upstream ships no container image for this bouncer, which is expected —
-# it manipulates host netfilter.
-#
-# The CI runner has v4, so it does the fetching. The checksum is verified
-# there, before anything is copied to a host.
- name: Fetch the pinned bouncer release on the controller
-  ansible.builtin.get_url:
-    url: >-
-      https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
-      crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
-    dest: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
-    mode: "0644"
-  delegate_to: localhost
-  become: false
-  run_once: true
-
-# Only the download had to move to the controller — the extraction stays on
-# the host. The ci-ansible image ships a tar that ansible rejects ("detected as
-# tar type None. GNU tar required"), while both host distributions have GNU
-# tar. Shipping the archive rather than the unpacked binary also keeps the
-# artefact that the checksum was verified against intact all the way to the
-# host.
- name: Copy the bouncer archive to the host
-  ansible.builtin.copy:
-    src: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    mode: "0644"
-
-# Only the binary. The tarball also carries install.sh / upgrade.sh /
-# uninstall.sh, which manage systemd and config themselves — that is this
-# role's job, and letting them run would leave two owners for the same files.
- name: Unpack the bouncer binary
-  ansible.builtin.unarchive:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt
-    remote_src: true
-    include:
-      - crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-
- name: Install the bouncer binary
-  ansible.builtin.copy:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    dest: /usr/local/bin/crowdsec-firewall-bouncer
-    remote_src: true
-    mode: "0755"
-  notify: Restart crowdsec firewall bouncer
-
-# 0600: this file carries the LAPI key.
- name: Deploy the bouncer configuration
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.yaml.j2
-    dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-    mode: "0600"
-  notify: Restart crowdsec firewall bouncer
-
- name: Deploy the bouncer systemd unit
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.service.j2
-    dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
-    mode: "0644"
-  notify: Restart crowdsec firewall bouncer
-
- name: Enable and start the bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    enabled: true
-    state: started
-    daemon_reload: true
-
- name: Report how to read it
-  ansible.builtin.debug:
-    msg: |
-      crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
-
-      What it would block:  journalctl -u crowdsec-firewall-bouncer -f
-      What it does block:   nft list table ip crowdsec ; nft list table ip6 crowdsec6
-      Metrics:              curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
-
-      Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
-      both nftables tables by itself. Only after a kill -9 do they need
-      removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6
@@ -1,24 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-#
-# Upstream's shipped unit orders itself After=crowdsec.service. There is no
-# local Security Engine on these hosts — the LAPI is in the cluster — so that
-# dependency is dropped; nss-lookup instead, because api_url is a public name
-# that has to resolve before the first connect.
-[Unit]
-Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
-After=network-online.target nss-lookup.target
-Wants=network-online.target
-
-[Service]
-Type=notify
-# -t validates the config and exits non-zero on a bad one, so a broken config
-# fails the start instead of taking the running rules down with it.
-ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
-ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-Restart=always
-RestartSec=10
-LimitNOFILE=65536
-KillMode=mixed
-
-[Install]
-WantedBy=multi-user.target
@@ -1,55 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-mode: {{ crowdsec_bouncer_mode }}
-update_frequency: {{ crowdsec_bouncer_update_frequency }}
-
-# stdout, so journald owns the log and there is no second rotation policy to
-# keep in sync. The shipped example writes to /var/log; that only matters for
-# hosts without journald.
-log_mode: stdout
-log_level: {{ crowdsec_bouncer_log_level }}
-
-api_url: {{ crowdsec_bouncer_api_url }}
-api_key: {{ crowdsec_bouncer_api_key }}
-
-# Without this the process EXITS when the LAPI is unreachable at startup —
-# there is no retry on the first connect. And because a clean shutdown deletes
-# the nftables tables, the systemd restart loop would leave the host with no
-# blocklist at all until the first successful connect. Present in the code,
-# absent from the shipped example config.
-retry_initial_connect: true
-
-disable_ipv4: false
-disable_ipv6: false
-deny_action: DROP
-
-supported_decisions_types:
-  - ban
-
-# Separate tables per family; the bouncer never touches an existing table or
-# chain, and never sets a chain policy. Priority -10 puts it ahead of anything
-# at the default filter priority.
-nftables:
-  ipv4:
-    enabled: true
-    set-only: false
-    table: crowdsec
-    chain: crowdsec-chain
-    priority: -10
-  ipv6:
-    enabled: true
-    set-only: false
-    table: crowdsec6
-    chain: crowdsec6-chain
-    priority: -10
-
-# input is the traffic this exists for: the reverse-proxy is host-networked and
-# owns :443, so its packets traverse INPUT. forward covers the bridge-networked
-# http-redirect sidecar, whose packets never do.
-nftables_hooks:
-  - input
-  - forward
-
-prometheus:
-  enabled: true
-  listen_addr: {{ crowdsec_bouncer_metrics_listen }}
-  listen_port: {{ crowdsec_bouncer_metrics_port }}
@@ -0,0 +1,18 @@
+#!/bin/sh
+# Forced-command target for the netbird-backup SSH key (see the
+# "Authorize the cluster backup key" task): takes a CONSISTENT snapshot
+# of the management SQLite store via the online-backup API (safe against
+# a live writer, unlike copying store.db) and streams a gzipped tar to
+# stdout. The key's authorized_keys entry is restricted to exactly this
+# script — the in-cluster puller cannot run anything else.
+#
+# Deliberately store.db ONLY: /opt/netbird/config is fully regenerable
+# from the netbird-server role's templates + vault, and including it
+# would put the plaintext TURN secret (turnserver.conf) into every
+# nightly tarball. Restore = re-run the role (rebuilds config), then
+# untar this dump into /opt/netbird/management/.
+set -eu
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'"
+tar -C "$tmp" -czf - store.db
@@ -77,22 +77,29 @@
  register: docker_compose_result
  changed_when: "'Started' in docker_compose_result.stdout or 'Creating' in docker_compose_result.stdout"

-# The management store has no cluster-side backup. Pulling one put a root
-# key on this host that the cluster held, and this server's own state is
-# its own business to protect rather than the cluster's. Dropping the two
-# tasks that placed them would only stop ansible managing the key and the
-# script — the pair below revokes them instead, and can go once the role
-# has run against every host that ever carried them.
- name: Remove the backup dump script
-  ansible.builtin.file:
-    path: /usr/local/bin/netbird-backup-dump
-    state: absent
+# Off-VPS backup of the management store (SPOF assessment 2026-07-25:
+# this SQLite DB — peers, setup keys, groups, routes — had zero backup
+# coverage; losing the VPS meant re-enrolling the entire mesh). The VPS
+# cannot reach any PBS (both PBS names resolve to the public proxy,
+# ports closed), so the CLUSTER pulls: a CronJob
+# (kubernetes/infrastructure/netbird-backup) ssh's in with a dedicated
+# key that is locked to the dump script below, and lands the tarball on
+# a Ceph PVC that Velero ships offsite. The dump is store.db ONLY —
+# /opt/netbird/config is regenerable from this role's templates + vault
+# and would otherwise leak the TURN secret into every tarball. Restore:
+# re-run this role (rebuilds config), then untar the dump into
+# /opt/netbird/management/.
+- name: Install the backup dump script
+  ansible.builtin.copy:
+    src: netbird-backup-dump
+    dest: /usr/local/bin/netbird-backup-dump
+    mode: "0755"

- name: Revoke the cluster backup key
+- name: Authorize the cluster backup key (forced command, no shell)
  ansible.posix.authorized_key:
    user: root
    key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeW/L9bHMTSa5/Qb0ClppCf4Z0hL39JwzI6tBmKd2FG netbird-backup@homelab-cluster"
-    state: absent
+    key_options: 'restrict,command="/usr/local/bin/netbird-backup-dump"'

 - name: Wait for management container to be ready
  ansible.builtin.pause:
@@ -0,0 +1,32 @@
+---
+# PBS Client configuration
+pbs_client_version: "latest"
+pbs_client_data_dir: "/opt/pbs-client"
+
+# PBS Server connection (override in host_vars)
+pbs_server_host: "pbs.obg.renner.uno"
+pbs_server_port: 8007
+pbs_datastore: "backups"
+
+# Authentication (override in vault!)
+pbs_username: "backup@pbs"
+pbs_password: ""
+# pbs_fingerprint: ""  # Optional - omit for Let's Encrypt certs
+
+# Backup configuration
+pbs_backup_id: "nas-1"
+pbs_backup_paths:
+  - /volume1/docker
+  - /volume1/homes
+
+# Schedule (cron format)
+pbs_backup_schedule: "0 2 * * *"  # 2 AM daily
+
+# Retention policy (prune settings)
+pbs_keep_last: 3
+pbs_keep_daily: 7
+pbs_keep_weekly: 4
+pbs_keep_monthly: 6
+
+# Container resource limits
+pbs_memory_limit: "2g"
@@ -0,0 +1,6 @@
+---
+- name: Restart PBS client
+  community.docker.docker_compose_v2:
+    project_src: "{{ pbs_client_data_dir }}"
+    state: restarted
+    docker_cli: "{{ docker_cli | default('/usr/local/bin/docker') }}"
@@ -0,0 +1,23 @@
+---
+- name: Create PBS client directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - "{{ pbs_client_data_dir }}"
+    - "{{ pbs_client_data_dir }}/state"
+
+- name: Copy PBS client docker-compose file
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ pbs_client_data_dir }}/docker-compose.yml"
+    mode: '0644'
+  notify: Restart PBS client
+
+- name: Start PBS client via Docker Compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ pbs_client_data_dir }}"
+    state: present
+    pull: always
+    docker_cli: "{{ docker_cli | default('/usr/local/bin/docker') }}"
@@ -0,0 +1,6 @@
+# PBS Client Environment Configuration
+PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
+PBS_PASSWORD={{ pbs_password }}
+{% if pbs_fingerprint | default('') != '' %}
+PBS_FINGERPRINT={{ pbs_fingerprint }}
+{% endif %}
@@ -0,0 +1,36 @@
+#!/bin/bash
+# PBS Backup Script for {{ pbs_backup_id }}
+# Managed by Ansible - do not edit manually
+
+set -e
+
+# Load environment
+source {{ pbs_client_data_dir }}/.env
+
+# Build backup command with all paths
+BACKUP_SOURCES=""
+{% for path in pbs_backup_paths %}
+BACKUP_SOURCES="$BACKUP_SOURCES {{ path | basename }}.pxar:{{ path }}"
+{% endfor %}
+
+echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup for {{ pbs_backup_id }}"
+
+# Run backup via Docker
+docker exec pbs-client proxmox-backup-client backup \
+    $BACKUP_SOURCES \
+    --backup-id {{ pbs_backup_id }} \
+    --repository "${PBS_REPOSITORY}"
+
+echo "[$(date '+%Y-%m-%d %H:%M:%S')] Backup completed"
+
+# Prune old backups
+echo "[$(date '+%Y-%m-%d %H:%M:%S')] Pruning old backups"
+docker exec pbs-client proxmox-backup-client prune \
+    --repository "${PBS_REPOSITORY}" \
+    --backup-id {{ pbs_backup_id }} \
+    --keep-last {{ pbs_keep_last }} \
+    --keep-daily {{ pbs_keep_daily }} \
+    --keep-weekly {{ pbs_keep_weekly }} \
+    --keep-monthly {{ pbs_keep_monthly }}
+
+echo "[$(date '+%Y-%m-%d %H:%M:%S')] Prune completed"
@@ -0,0 +1,30 @@
+services:
+  pbs-client:
+    image: aterfax/pbs-client:{{ pbs_client_version }}
+    container_name: pbs-client
+    restart: unless-stopped
+    hostname: {{ pbs_backup_id }}
+    environment:
+      - PBS_ENDPOINT={{ pbs_server_host }}:{{ pbs_server_port }}
+      - PBS_DATASTORE={{ pbs_datastore }}
+      - PBS_API_KEY_NAME={{ pbs_username }}
+      - PBS_API_KEY_SECRET={{ pbs_password }}
+      - PBS_PASSWORD={{ pbs_password }}
+      - PBS_ENCRYPTION_PASSWORD={{ pbs_encryption_password | default(pbs_password) }}
+      - PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
+      - PBS_BACKUP_ID={{ pbs_backup_id }}
+      - CRON_SCHEDULE={{ pbs_backup_schedule }}
+      - PBS_BACKUP_CMD_APPEND_EXTRA_OPTS=--crypt-mode=none
+      - PRUNE_KEEP_LAST={{ pbs_keep_last | default(3) }}
+      - PRUNE_KEEP_DAILY={{ pbs_keep_daily | default(7) }}
+      - PRUNE_KEEP_WEEKLY={{ pbs_keep_weekly | default(4) }}
+      - PRUNE_KEEP_MONTHLY={{ pbs_keep_monthly | default(6) }}
+      - CRON_BACKUP_ONLY=1
+    volumes:
+      - {{ pbs_client_data_dir }}/state:/root/.config/proxmox-backup
+{% for path in pbs_backup_paths %}
+      - "{{ path }}:/backups/{{ path | basename | replace(' ', '_') }}:ro"
+{% endfor %}
+    tmpfs:
+      - /tmp
+    mem_limit: {{ pbs_memory_limit }}
@@ -1,111 +0,0 @@
-# Cilium: one owner
-
-## The rule
-
-Cilium is owned by **Flux**, through `infrastructure/cilium/helmrelease.yaml`.
-Talos provides the conditions for a CNI and nothing else. Never both.
-
-Talos-only would also be a valid answer — it is the simpler one, and it keeps a
-cold rebuild self-forming. It is rejected here only because it costs Flux's
-reconciliation, image automation and value ergonomics for the one component
-most likely to need a careful, reviewed change.
-
-## Where we are
-
-Both own it, which is the state this document exists to end.
-
- 22 Cilium objects are listed in the `talos-bootstrap-manifests-inventory`
-  ConfigMap: both DaemonSets, the operator Deployment, both ConfigMaps, the
-  ServiceAccounts, the ClusterRoles/Bindings and the `cilium-secrets`
-  namespace. That is the whole installation, not a fragment.
- The same objects carry `meta.helm.sh/release-name: cilium`.
- `cilium-config` has three field managers: `helm-controller`, `talos` and
-  `kubectl-patch`.
-
-The consequences are not theoretical. The seed sat at Cilium 1.19.2 for twelve
-days while the chart was on 1.20.1, and `upgrade-k8s --dry-run` shows that a
-Kubernetes upgrade in that state downgrades the CNI and switches off socket
-load balancing. Separately, `enable-bgp-control-plane` and the two
-native-routing CIDRs exist in no file at all — applied by hand, so a cold
-rebuild would have come up with BGP off while the peering CRDs in git made it
-look configured.
-
-Nothing fails when these drift. That is the whole problem.
-
-## Target state
-
-**Talos** keeps exactly this, in `talos/patches/cluster.yaml`, and gains nothing
-else:
-
-```yaml
-cluster:
-  network:
-    cni:
-      name: none
-  proxy:
-    disabled: true
-```
-
-No `inlineManifests` entry for cilium. `talos/patches/cilium-cni.yaml` is
-deleted.
-
-**Flux** owns everything else. `helmrelease.yaml` is the single source for
-every Cilium setting, including the three that were hand-patched.
-
-**A cold rebuild** installs Cilium once, by hand, between `talosctl bootstrap`
-and the Flux phase, from the same HelmRelease values. This is the cost of the
-decision and it is deliberate: the cluster no longer forms itself from the
-machine config alone. Given `bootstrap.sh talos` is already dead and Step 2 of
-RUNBOOK-cold-rebuild.md is already a manual procedure, the marginal cost is one
-documented command.
-
-## Transition
-
-The order matters, because Talos prunes what its inventory used to list and no
-longer does — removing the manifest naively deletes the CNI.
-
-1. **Track the untracked settings first.** `bgpControlPlane.enabled`,
-   `ipv4NativeRoutingCIDR`, `ipv6NativeRoutingCIDR` into the HelmRelease. Note
-   this is not a pure change of ownership: enabling bgpControlPlane through the
-   chart also emits five companion keys the hand-patch never set, so Cilium has
-   been running on its own defaults for router-id allocation. Verify the BGP
-   sessions after Flux applies it.
-
-2. **Remove cilium from `cluster.inlineManifests`.** This has no immediate
-   effect. Evidence: Talos was upgraded to 1.13.9 four days ago and Cilium
-   stayed at 1.20.1 — an OS upgrade does not reconcile bootstrap manifests.
-   Only `upgrade-k8s` does.
-
-3. **Prove the prune before running it.** `upgrade-k8s --dry-run
-   --manifests-no-prune` and confirm the 22 objects are not listed for
-   deletion. `NoPrune` is honoured by the dry run as well as the apply
-   (`pkg/cluster/kubernetes/talos_managed.go`), so this is a real test and not
-   a reading of the flag description.
-
-4. **Run `upgrade-k8s --manifests-no-prune`.** The inventory is rewritten
-   without Cilium and the objects survive. Helm is then sole owner. This is the
-   same run that takes Kubernetes 1.35.2 → 1.36.3.
-
-5. **Verify** the inventory lists no Cilium, the agents did not restart, and
-   the BGP sessions are still established.
-
-6. **Delete `talos/patches/cilium-cni.yaml`** and add the install step to
-   RUNBOOK-cold-rebuild.md.
-
-## Not part of this, and not a defect either
-
-The cluster runs `routingMode: tunnel` (the chart default, set nowhere) while
-also advertising pod CIDRs over BGP. That is not one layer too many — the two
-carry different directions:
-
- **VXLAN** carries pod-to-pod between nodes, directly, node to node.
- **BGP** gives the UDM routes into the pod CIDRs so traffic from the LAN can
-  reach pods. That path goes through the router by definition.
-
-Dropping the encapsulation in favour of native routing would not simplify
-anything; it would push inter-node pod traffic onto the underlay path and
-through the UDM, replacing a direct node-to-node hop with a router hop.
-
-`routingMode` being unset is still worth knowing — the current behaviour is a
-default nobody chose rather than a decision anyone recorded — but the default
-is the right one here, and this document is not the place to revisit it.
@@ -2,12 +2,7 @@

 Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
 metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
-after doing jupiter on 2026-08-25.
-
-All three nodes are now on bare metal — `talosctl get systeminformation`
-reports Minisforum MS-01 hardware with distinct chassis serials for k8s-1,
-k8s-2 and k8s-3, with no hypervisor in between. Kept as the procedure for a
-rebuild, not as outstanding work.
+after doing jupiter on 2026-08-25; saturn and neptune remain.

 Everything here that reads like an over-specific warning is something that
 actually happened. The conversion itself is straightforward — the failures were
@@ -203,7 +198,8 @@ talosctl gen secrets --from-controlplane-config cp.yaml -o secrets.yaml
 talosctl gen config homelab https://k8s.renner.uno:6443 \
  --with-secrets secrets.yaml --output-types controlplane \
  --config-patch @talos/patches/cluster.yaml \
-  --config-patch @talos/patches/containerd-layer-retention.yaml \
+  --config-patch @<cilium additive half> \
+  --config-patch @talos/patches/registry-mirror.yaml \
  --config-patch @talos/patches/k8s-N.yaml -o install.yaml
 ```

@@ -212,11 +208,9 @@ Confirm `cluster.id` matches the running cluster before applying.
 Strip the generated `HostnameConfig` document — it collides with
 `machine.network.hostname` from the per-node patch. `bootstrap.sh` does the same.

-There is no CNI patch to pass any more. Talos supplies only `cni: none` and
-`proxy: disabled`; Cilium is installed from its HelmRelease values, which is
-Step 2b of RUNBOOK-cold-rebuild.md. A node converted into an existing cluster
-gets its CNI from the Cilium DaemonSet already running there, so nothing extra
-is needed here.
+`cilium-cni.yaml` cannot be passed whole: its `$patch: delete` document fails
+against a base with no `inlineManifests` (issue #581). Use only the additive
+second document.

 Validate:

@@ -119,11 +119,9 @@ against do not exist, and the node ends up with no address on any interface.
 Beyond that it carries:

 - `siderolabs/intel-ucode` — no hypervisor supplies microcode here
- `siderolabs/i915` — the Alder Lake-P Iris Xe. The Intel GPU device plugin
-  (`infrastructure/intel-gpu-plugin`) turns this into the schedulable resource
-  `gpu.intel.com/i915`, which `apps/llama-server` requests for LLM inference
-  and which any hardware-transcoding workload would use too. Without the
-  extension the plugin finds no `/dev/dri` and advertises nothing
+- `siderolabs/i915` — the Alder Lake-P Iris Xe. Nothing requests `/dev/dri`
+  today; present so wanting hardware transcoding later is a config change
+  rather than another rolling reinstall
 - `siderolabs/thunderbolt` — inert until `machine.kernel.modules` requests
  `thunderbolt`/`thunderbolt_net`, present for the same reason. Note its udev
  rule auto-authorises any Thunderbolt device attached once those modules load
@@ -138,18 +136,11 @@ in `talos/patches/k8s-N.yaml`.

 ## Step 2 — talos: form the cluster + get kubeconfig

-`kubernetes/bootstrap.sh talos` does this, including the CNI install that used
-to be a hand-typed step. It was unusable for months because it passed
-`cilium-cni.yaml` whole and the file's `$patch: delete` document failed against
-a fresh config (#581); that file no longer exists, so the failure is gone with
-it.
-
-Verified as far as it can be without wiping a cluster: `talosctl gen config`
-with exactly the patches the script passes produces a config carrying
-`cni: none` and `proxy: disabled` that passes `talosctl validate --mode metal`.
-The apply-and-bootstrap half has not been exercised since the rewrite. Read the
-manual procedure below before running the script, so you know what it is doing
-and can take over at any step if it stops.
+**Do not use `kubernetes/bootstrap.sh talos`.** That phase cannot generate a
+config: it passes `talos/patches/cilium-cni.yaml` whole, and that file's
+`$patch: delete` document fails against a base with no `inlineManifests`
+(#581). Its other phases — `flux`, `wait-apps` — are still used, in Steps 3, 5
+and 6; only config generation is broken.

 The procedure below is what actually works, and is the same one
 `RUNBOOK-baremetal-conversion.md` uses.
@@ -169,17 +160,17 @@ for N in 1 2 3; do
  talosctl gen config homelab https://k8s.renner.uno:6443 \
    --with-secrets secrets.yaml --output-types controlplane \
    --config-patch @talos/patches/cluster.yaml \
-    --config-patch @talos/patches/containerd-layer-retention.yaml \
+    --config-patch @<cilium additive half> \
+    --config-patch @talos/patches/registry-mirror.yaml \
    --config-patch @talos/patches/k8s-$N.yaml -o install-k8s-$N.yaml
 done
 ```

-There is no CNI patch here any more. Talos supplies `cni: none` and
-`proxy: disabled` from `cluster.yaml` and nothing else; Cilium is installed in
-Step 2b below. See PROPOSAL-cilium-single-owner.md.
-
-One thing the generator gets wrong for this repo, and it bites silently:
+Two things the generator gets wrong for this repo, both of which will bite
+silently:

+- **`cilium-cni.yaml` cannot be passed whole** — extract only its additive
+  second document. See issue #581.
 - **Strip the generated `HostnameConfig`** — it collides with
  `machine.network.hostname` from the per-node patch.

@@ -218,53 +209,8 @@ talosctl -n k8s-1.srv.renner.uno bootstrap
 talosctl -n k8s-1.srv.renner.uno kubeconfig .
 ```

-`scripts/import-cluster-access.sh` is the supported way to get the kubeconfig
-into place afterwards.
-
-The nodes will **not** reach `Ready` here. Talos is configured with `cni: none`,
-so there is no pod network until Step 2b — expect `NotReady` with
-`container runtime network not ready` and carry on.
-
-## Step 2b — cilium: install the CNI
-
-Nothing schedules without it, Flux included, so this comes before Step 3. Talos
-no longer carries a rendered copy of Cilium; `helmrelease.yaml` is the only
-place it is configured, and this installs from those same values so the cluster
-comes up on what git describes rather than on a second, drifting copy. See
-PROPOSAL-cilium-single-owner.md for why it works this way.
-
-`bootstrap.sh talos` already did this. What follows is the same thing by hand,
-for when the script stopped partway or you are not using it.
-
-Every input is read out of `helmrelease.yaml`. Nothing here is a version or a
-URL you have to keep in step with anything — copy the block as it stands:
-
-```sh
-HR=kubernetes/infrastructure/cilium/helmrelease.yaml
-
-helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$HR")"
-helm repo update cilium
-
-yq 'select(.kind == "HelmRelease") | .spec.values' "$HR" > /tmp/cilium-values.yaml
-
-helm install cilium cilium/cilium \
-  --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$HR")" \
-  --namespace kube-system \
-  -f /tmp/cilium-values.yaml \
-  --set prometheus.serviceMonitor.enabled=false
-```
-
-`--version` takes the HelmRelease's range verbatim; helm resolves it to the same
-release Flux would pick, so the two cannot diverge. That is the point of writing
-it this way — a pinned version in a runbook is a copy nobody maintains, and the
-next rebuild installs something the HelmRelease will immediately fight.
-
-The one literal is `serviceMonitor`, disabled for this install only: its CRD
-arrives with the monitoring stack much later and the object would fail to apply.
-Flux adds the real one once it owns the release. Nothing else is overridden,
-which is what lets helm-controller adopt this release rather than replace it.
-
-Ends when all three nodes are `Ready`.
+Ends when all three nodes are `Ready`. `scripts/import-cluster-access.sh` is the
+supported way to get the kubeconfig into place afterwards.

 ## Step 3 — flux: bootstrap GitOps **from the GitHub mirror** (break-glass)

@@ -0,0 +1,18 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: barman-cloud-creds
+  namespace: authentik
+  annotations:
+    # Prune-disabled — a standing guard kept from the seed->authentik-KS handoff (barman creds).
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
+  dataFrom:
+    - extract:
+        key: seed/authentik/barman-creds
@@ -25,13 +25,20 @@ data:
        attrs:
          local: true

-      # Retired OIDC layer. Access is gated by the paperless-gate
-      # ProxyProvider below, which forwards identity as X-authentik-* headers
-      # consumed via PAPERLESS_ENABLE_HTTP_REMOTE_USER.
+      # Retired OIDC layer. Paperless's URL-level access is gated by the
+      # paperless-gate ProxyProvider (below); user identity is forwarded
+      # from the gate to paperless via the X-authentik-* request headers
+      # and consumed by paperless via PAPERLESS_ENABLE_HTTP_REMOTE_USER
+      # (see kubernetes/apps/paperless/paperless.yaml). One Authentik
+      # Application, one Provider — the dual-OAuth2-flow setup with a
+      # separate paperless OIDC client + django-allauth callback was
+      # structurally fine but added a second consent screen on first
+      # login for no real defense-in-depth gain.
      #
-      # ORDER MATTERS: delete the Application BEFORE the OAuth2Provider.
-      # Application.provider is on_delete=PROTECT, so the provider cannot be
-      # removed while an Application still references it.
+      # Order matters: delete the Application BEFORE the OAuth2Provider
+      # because Application.provider is on_delete=PROTECT, so the
+      # OAuth2Provider can't be removed while the Application still
+      # references it.
      - model: authentik_core.application
        identifiers:
          slug: paperless
@@ -42,15 +49,22 @@ data:
          name: Paperless
        state: absent

-      # Orphan ProxyProvider, unbound — leftover DB state.
+      # Also clean up the very-early orphan `paperless` ProxyProvider
+      # left behind from this session's first attempts. It's not bound
+      # to anything any more, just leftover state in the DB.
      - model: authentik_providers_proxy.proxyprovider
        identifiers:
          name: paperless
        state: absent

-      # Dedicated forward-auth gate for paperless: its own outpost
-      # Deployment + Service, so every request is authenticated before it
-      # reaches paperless.
+      # Dedicated forward-auth gate for paperless. This is a SEPARATE
+      # outpost (its own Deployment + Service `ak-outpost-paperless-
+      # gate`) running in front of paperless so every request is
+      # authenticated and authorized by Authentik before it reaches
+      # paperless's own attack surface. The gate sits on top of, and
+      # separate from, the OIDC OAuth2Provider above (which is what
+      # paperless's django-allauth uses for SSO once a request makes
+      # it through the gate).
      - id: paperless-gate-provider
        model: authentik_providers_proxy.proxyprovider
        identifiers:
@@ -59,13 +73,16 @@ data:
        attrs:
          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          # Proxy, NOT forward_domain: the outpost terminates the request
-          # itself, so the trust boundary sits here rather than at the ingress
-          # controller, which only routes to the outpost Service and never
-          # reaches paperless directly. The outpost populates
-          # X-Authentik-Username, and the paperless NetworkPolicy admits
-          # traffic only from this outpost — so nothing else in the cluster
-          # can bypass the gate and inject those headers.
+          # Proxy (not forward_domain) so the outpost itself terminates
+          # the request and forwards to paperless — moving the trust
+          # boundary off ingress-nginx and onto paperless-gate. ingress-
+          # nginx now only routes paperless.renner.uno to the outpost
+          # service; it never talks to paperless directly. The outpost
+          # is what populates X-Authentik-Username on the upstream
+          # request, and the NetworkPolicy in the paperless namespace
+          # only allows traffic to paperless from the gate outpost pod
+          # itself, so ingress-nginx (or any other in-cluster workload)
+          # cannot bypass the gate to inject headers.
          mode: proxy
          internal_host: http://paperless.paperless.svc.cluster.local
          internal_host_ssl_validation: false
@@ -74,12 +91,19 @@ data:
          access_token_validity: hours=24
          refresh_token_validity: days=30
          intercept_header_auth: true
-          # The static-asset tree passes anonymously, and must. A
-          # <link rel="manifest"> is fetched with `credentials: 'omit'` per
-          # the W3C spec, so the session cookie is never sent and every page
-          # load turns into a redirect loop through the login origin. Django's
-          # STATIC tree is public by design; MEDIA under /api/documents/*
-          # stays gated.
+          # Pass paperless's entire static-asset tree through anonymously.
+          # Two reasons:
+          # 1. <link rel="manifest"> is fetched with `credentials: 'omit'`
+          #    (W3C spec, no crossorigin attr) so the cookie is never sent
+          #    — every page load and every SPA navigation turned into a
+          #    302 → auth.renner.uno round-trip with an orphan session JWT.
+          #    Same root cause as the retired music-assistant-gate (see
+          #    comment further below); paperless can't retire the gate so
+          #    it carves out the offending paths instead.
+          # 2. Every CSS/JS/icon fetch otherwise validates the cookie at
+          #    the outpost — pure overhead on assets that carry no
+          #    authentication value. Django's STATIC tree is public by
+          #    design; MEDIA (under /api/documents/*) stays gated.
          skip_path_regex: ^/static/.*

      - model: authentik_core.application
@@ -106,14 +130,18 @@ data:
          negate: false
          failure_result: false

-      # Retired: the music-assistant gate. A forward-auth gate cannot work in
-      # front of a PWA — the manifest and service worker are fetched with
-      # `credentials: 'omit'`, and a cached service worker bypasses fresh
-      # navigations, so the session cookie never reaches the installed app.
-      # Do not reintroduce one there; MA delegates auth to Home Assistant.
-      #
-      # ORDER MATTERS: Application before Provider before Outpost, because
-      # Application.provider is on_delete=PROTECT.
+      # Retired: the music-assistant-gate ProxyProvider, Application,
+      # and dedicated Outpost. The gate fundamentally conflicted with
+      # MA's PWA model — Chrome fetches manifest.webmanifest / sw.js /
+      # icons with `credentials: 'omit'` (per W3C spec, no crossorigin
+      # attribute on MA's <link rel="manifest">), and the cached
+      # service worker bypasses fresh navigations on PWA cold-start so
+      # the Authentik cookie never lands in the WebAPK's cookie jar.
+      # MA has its own auth (delegated to Home Assistant); the gate's
+      # marginal value didn't justify the recurring breakage.
+      # Order matters: Application before Provider before Outpost.
+      # Application.provider is on_delete=PROTECT, so the Provider
+      # can't be removed while the Application still references it.
      - model: authentik_core.application
        identifiers:
          slug: music-assistant-gate
@@ -129,13 +157,18 @@ data:
          name: music-assistant-gate
        state: absent

-      # Dedicated outpost, serving the in-cluster auth endpoint only.
-      #
-      # The browser-facing legs of the OAuth flow must happen on
-      # auth.renner.uno, because that is the origin WebAuthn credentials are
-      # bound to. Hence the ProxyProvider is attached to BOTH this outpost and
-      # the embedded one, and authentik_host_browser points at auth.renner.uno
-      # here too — so any redirect this outpost emits names the right host.
+      # Dedicated outpost: its ONLY job is to serve the in-cluster
+      # /outpost.goauthentik.io/auth/nginx endpoint that nginx
+      # auth_request hits for every request to paperless. The
+      # browser-facing legs of the OAuth flow (/start, /callback, the
+      # `/if/...` login UI, /application/o/authorize, /api, etc.) all
+      # need to happen on auth.renner.uno because that's the origin
+      # WebAuthn/FIDO/TOTP credentials are bound to. So we attach the
+      # ProxyProvider to BOTH outposts (see embedded outpost below) and
+      # set authentik_host_browser to auth.renner.uno here too — the
+      # nginx auth_request happens server-to-server, but if the
+      # dedicated outpost ever does emit a redirect URL, it points at
+      # the right host.
      - model: authentik_outposts.outpost
        identifiers:
          name: paperless-gate
@@ -163,7 +196,12 @@ data:
            kubernetes_ingress_secret_name: "authentik-outpost-tls"
            kubernetes_json_patches: null

-      # Leftover Application from an earlier experiment.
+      # Tear down the leftover paperless-proxy Application from an
+      # earlier experiment. The orphan `paperless` ProxyProvider that
+      # used to back paperless.app is left in place — it's PROTECTed
+      # by Application.provider as long as anything references it, and
+      # leftover orphans don't break anything. Delete via the Authentik
+      # UI once everything's confirmed working.
      - model: authentik_policies.policybinding
        identifiers:
          target: !Find [authentik_core.application, [slug, paperless-proxy]]
@@ -179,11 +217,15 @@ data:
      # ------------------------------------------------------------------
      # vscode config-editor gate
      #
-      # code-server runs with --auth=none, as root, mounting the whole HA
-      # /config including plaintext secrets.yaml and .storage. This gate is
-      # the only thing in front of it: the outpost terminates the request and
-      # enforces a passkey step-up, Traefik routes only to the outpost, and
-      # the home-assistant CNP admits :8080 from the gate alone.
+      # code-server (the HA /config editor sidecar) runs with --auth=none,
+      # as root, mounting the whole HA /config incl. plaintext secrets.yaml
+      # and .storage. This gate moves it behind Authentik with a passkey
+      # step-up, mirroring the paperless-gate proxy-outpost pattern above:
+      # the dedicated ak-outpost-vscode-gate terminates the request,
+      # enforces the session, and forwards to the code-server Service.
+      # Traefik only ever routes vscode.home.renner.uno to the outpost; the
+      # home-assistant CNP is tightened so ONLY the gate outpost may reach
+      # :8080 (see home-assistant-ns.yaml + vscode-gate-egress.yaml).
      # ------------------------------------------------------------------
      - id: vscode-config-editors-group
        model: authentik_core.group
@@ -223,14 +265,17 @@ data:
          access_token_validity: hours=24
          refresh_token_validity: days=30
          intercept_header_auth: true
-          # code-server is a service-worker app, so its manifest and SW are
-          # fetched with credentials:'omit' and 302-loop unless carved out.
-          # The editor's actual content rides authenticated websocket/API
-          # paths and stays gated.
-          #
-          # This skip set is version-specific: re-check it after a
-          # code-server bump, because a renamed asset path reintroduces the
-          # loop silently.
+          # code-server is a service-worker app: <link rel="manifest"> and
+          # the service worker are fetched with credentials:'omit', which
+          # turned into a /authorize redirect loop and forced the RETIREMENT
+          # of the music-assistant gate (see comment above). Carve its
+          # static/manifest/SW tree out so those cookieless fetches pass
+          # through instead of 302-looping; the editor's actual content
+          # (the /config tree) is served over authenticated websocket/API
+          # paths, which stay gated.
+          # NOTE: the exact skip set is code-server-version-specific and
+          # MUST be confirmed post-deploy against the real redirect-loop
+          # behaviour (see the PR validation checklist).
          skip_path_regex: ^/(_static/|manifest\.json|.*\.webmanifest|.*serviceWorker\.js).*

      - model: authentik_core.application
@@ -286,11 +331,16 @@ data:
      # ------------------------------------------------------------------
      # shelly-manager gate
      #
-      # shelly-manager has no authentication of its own: anything that reaches
-      # :8080 can rewrite device config, push scripts and read the stored
-      # per-device AdminPassword values. Same shape as the vscode gate — the
-      # outpost is the only control, Traefik routes only to it, and the CNP
-      # admits :8080 from this outpost alone.
+      # shelly-manager has no authentication of its own: anything that can
+      # reach :8080 can rewrite Shelly device config, push scripts, and read
+      # the stored per-device AdminPassword values. Until now the only
+      # control was reachability of shelly-manager.home.renner.uno.
+      #
+      # Same shape as the vscode-gate above: a dedicated outpost terminates
+      # the request, enforces a passkey step-up, and forwards to the
+      # shelly-manager Service. Traefik only ever routes the hostname to the
+      # outpost; the home-assistant CNP allows ONLY this outpost to reach
+      # :8080 (see home-assistant-ns.yaml + shelly-manager-gate-egress.yaml).
      # ------------------------------------------------------------------
      - id: shelly-managers-group
        model: authentik_core.group
@@ -877,105 +927,6 @@ data:
            kubernetes_ingress_secret_name: "authentik-outpost-tls"
            kubernetes_json_patches: null

-      # llama-server WebUI gate
-      #
-      # llama-server has no authentication of its own — it serves its chat UI
-      # and a full OpenAI-compatible API to anyone who can reach the port. That
-      # is why it was LAN-only until now. This gate is what makes a public
-      # hostname defensible: the outpost terminates every request, enforces the
-      # session, and only then forwards to llama-server.
-      #
-      # Same reasoning as vscode-gate, and the same passkey step-up: what sits
-      # behind the gate is entirely unprotected, so the gate is the whole of the
-      # access control rather than a second factor on top of an app login.
-      #
-      # In-cluster consumers (Home Assistant, Paperless, CI) are unaffected:
-      # they address llama-server.ai.svc.cluster.local directly and never pass
-      # through Traefik or this outpost.
-      #
-      # No group and no policy binding: any user who can authenticate gets in,
-      # as with Immich, Audiobookshelf, Home Assistant and Music Assistant. The
-      # gates that pin membership to a named user in git — vscode, esphome,
-      # zigbee2mqtt, shelly-manager, prometheus — are admin interfaces, where
-      # the allowlist is the point. A chat UI is not one of those, and deciding
-      # its membership here would put an access-control choice in a manifest
-      # instead of leaving it where the user directory belongs.
-      # ------------------------------------------------------------------
-      - id: llama-gate-provider
-        model: authentik_providers_proxy.proxyprovider
-        identifiers:
-          name: llama-gate
-        state: present
-        attrs:
-          # Both flows, mirroring the Audiobookshelf provider. Setting only
-          # authorization_flow leaves authentication on Authentik's default
-          # flow, which is what made the first login clumsy — the step-up ran
-          # on top of a password prompt instead of replacing it. With the
-          # passwordless flow named explicitly, signing in is the passkey
-          # itself.
-          #
-          # Referenced by !Find rather than !KeyOf: both flows are defined in
-          # the passkey-stepup document of this ConfigMap, and !KeyOf only
-          # resolves within one document.
-          authentication_flow: !Find [authentik_flows.flow, [slug, passkey-passwordless-authentication]]
-          authorization_flow: !Find [authentik_flows.flow, [slug, passkey-stepup-authorization]]
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          mode: proxy
-          internal_host: http://llama-server.ai.svc.cluster.local:8080
-          internal_host_ssl_validation: false
-          external_host: https://llm.renner.uno
-          cookie_domain: renner.uno
-          access_token_validity: hours=24
-          refresh_token_validity: days=30
-          intercept_header_auth: true
-          # The WebUI ships `<link rel="manifest">` and registers a service
-          # worker. Per the W3C spec a manifest is fetched with
-          # `credentials: 'omit'`, so it carries no session cookie and the gate
-          # answers it with a 302 to auth.renner.uno — which turns every page
-          # load into a redirect round-trip. This is the failure that forced
-          # paperless-gate's own carve-out and that retired music-assistant-gate
-          # outright. Both paths serve no authenticated content, so letting them
-          # through anonymously costs nothing.
-          skip_path_regex: ^/(manifest\.webmanifest|sw\.js)$
-
-      - model: authentik_core.application
-        identifiers:
-          slug: llama-gate
-        state: present
-        attrs:
-          name: Local LLM (Gate)
-          slug: llama-gate
-          provider: !KeyOf llama-gate-provider
-          meta_launch_url: https://llm.renner.uno
-          meta_description: "Forward-auth gate in front of the local llama-server WebUI"
-
-      - model: authentik_outposts.outpost
-        identifiers:
-          name: llama-gate
-        state: present
-        attrs:
-          type: proxy
-          service_connection: !KeyOf k8s-conn
-          providers:
-            - !KeyOf llama-gate-provider
-          config:
-            log_level: info
-            authentik_host: "http://authentik-server.authentik.svc.cluster.local"
-            authentik_host_insecure: false
-            authentik_host_browser: "https://auth.renner.uno/"
-            refresh_interval: "minutes=5"
-            container_image: null
-            object_naming_template: "ak-outpost-%(name)s"
-            kubernetes_namespace: authentik
-            kubernetes_replicas: 1
-            kubernetes_service_type: ClusterIP
-            kubernetes_disabled_components: ["ingress"]
-            kubernetes_image_pull_secrets: []
-            kubernetes_ingress_annotations: {}
-            kubernetes_ingress_class_name: null
-            kubernetes_ingress_secret_name: "authentik-outpost-tls"
-            kubernetes_json_patches: null
-
      - model: authentik_outposts.outpost
        identifiers:
          name: authentik Embedded Outpost
@@ -15,4 +15,4 @@ spec:
    name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
  dataFrom:
    - extract:
-        key: infrastructure/hetzner-s3
+        key: seed/authentik/hetzner-creds
@@ -15,7 +15,6 @@ resources:
  - alertmanager-gate-egress.yaml
  - prometheus-gate-egress.yaml
  - wordpress-gate-egress.yaml
-  - llama-gate-egress.yaml
  - outpost-servicemonitors.yaml
  - ldap-outpost-cert.yaml
  - ldap-outpost-lb.yaml
@@ -28,4 +27,5 @@ resources:
  - cluster.yaml
  - objectstore.yaml
  - scheduled-backup.yaml
+  - barman-creds-externalsecret.yaml
  - hetzner-creds-externalsecret.yaml
@@ -1,30 +0,0 @@
---
-# Lets the llama-gate outpost reach llama-server. The cluster baseline CNP
-# (kubernetes/infrastructure/network-policies/authentik-ns.yaml) otherwise stops
-# any authentik-namespace pod from entering the ai namespace.
-#
-# The outpost is the trust boundary: it forwards to llama-server only after
-# validating the Authentik session, and the matching ingress rule in
-# ai-ns.yaml admits nothing else — so nothing in the cluster can route around
-# the gate to reach the unauthenticated API through the public hostname.
-#
-# Scoped to this outpost's pod label rather than the whole namespace, so the
-# other gates cannot reach llama-server either.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: llama-gate-egress
-  namespace: authentik
-spec:
-  endpointSelector:
-    matchLabels:
-      goauthentik.io/outpost-name: llama-gate
-  egress:
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ai
-            app.kubernetes.io/name: llama-server
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
@@ -1,10 +1,50 @@
 ---
-# Only radius needs a hand-written ServiceMonitor. The goauthentik.io operator
-# creates one per outpost and Prometheus discovers those directly, because
-# serviceMonitorSelector here is empty (see monitoring/helmrelease-prometheus.yaml).
-# Standalone copies for ldap and paperless-gate used to live here as well; they
-# duplicated the operator's and were removed.
-#
+# The goauthentik.io operator (inside authentik-worker) auto-creates a
+# ServiceMonitor per outpost, but it does not stamp the
+# `release: kube-prometheus-stack` label that this cluster's Prometheus
+# requires on serviceMonitorSelector. Those operator-owned SMs are
+# silently ignored. Declare standalone SMs here with the correct label
+# so the outposts actually get scraped.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: authentik-outpost-ldap
+  namespace: authentik
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - authentik
+  selector:
+    matchLabels:
+      goauthentik.io/outpost-name: ldap-outpost
+      goauthentik.io/outpost-type: ldap
+  endpoints:
+    - port: http-metrics
+      path: /metrics
+      interval: 30s
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: authentik-outpost-paperless-gate
+  namespace: authentik
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - authentik
+  selector:
+    matchLabels:
+      goauthentik.io/outpost-name: paperless-gate
+      goauthentik.io/outpost-type: proxy
+  endpoints:
+    - port: http-metrics
+      path: /metrics
+      interval: 30s
+---
 # Radius requires extra plumbing. The radius binary listens on :9300 just
 # like ldap/proxy (hardcoded in authentik/lib/default.yml), but authentik's
 # Python operator (`providers/radius/controllers/kubernetes.py`) omits the
@@ -13,11 +53,6 @@
 # it, radius does not. Result: the operator-managed `ak-outpost-local-radius`
 # Service exposes only UDP 1812. Add a sibling Service here that selects the
 # same pod labels and exposes 9300/TCP, then a ServiceMonitor for it.
-#
-# The operator's own `ak-outpost-local-radius` ServiceMonitor cannot cover
-# this: it selects on `app.kubernetes.io/managed-by: goauthentik.io` together
-# with `goauthentik.io/service-type: metrics`, and no Service carries both —
-# the operator's has the first, the one below has the second.
 apiVersion: v1
 kind: Service
 metadata:
@@ -51,8 +86,6 @@ spec:
    matchNames:
      - authentik
  selector:
-    # Pinned to the Service above. Without managed-by this would also match the
-    # operator's Service and scrape the same endpoint twice.
    matchLabels:
      app.kubernetes.io/managed-by: homelab
      goauthentik.io/outpost-name: local-radius
@@ -0,0 +1,31 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: file
+  namespace: apps
+  labels:
+    app.kubernetes.io/name: file
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+    # Backend serves HTTPS with a self-signed cert; re-encrypt + skip
+    # verify via the shared insecure-backend ServersTransport (see
+    # apps/insecure-backend-serverstransport.yaml).
+    traefik.ingress.kubernetes.io/service.serversscheme: https
+    traefik.ingress.kubernetes.io/service.serverstransport: insecure-backend@kubernetescrd
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: file.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: file
+                port:
+                  number: 7001
+  tls:
+    - secretName: file-tls
+      hosts:
+        - file.renner.uno
@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - daemonset.yaml
+  - service.yaml
+  - ingress.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: file
+  namespace: apps
+  labels:
+    app.kubernetes.io/name: file
+spec:
+  # Caddy reverse-proxied https://nas.renner.uno:7001. nas.renner.uno
+  # is a CNAME -> nas-1.renner.uno owned by UniFi DNS, k8s-gateway
+  # forwards srv lookups so this resolves cluster-side.
+  type: ExternalName
+  externalName: nas.renner.uno
+  ports:
+    - name: https
+      port: 7001
+      targetPort: 7001
+      protocol: TCP
@@ -18,18 +18,15 @@ metadata:
 #
 # Opaque-label map (maintainer reference only — NOT served publicly):
 #   services group                   infrastructure group
-#     Web Service 1 = Authentik        Infrastructure 1 = retired (was PBS 2)
+#     Web Service 1 = Authentik        Infrastructure 1 = PBS 2 (pbs2.renner.uno)
 #     Web Service 2 = Gitea            Infrastructure 2 = retired (was Proxmox VE)
 #     Web Service 3 = Immich           Infrastructure 3 = UDM DNS (10.2.40.1)
 #     Web Service 4 = Woodpecker CI    Infrastructure 4 = Mosquitto MQTT broker
 #     Web Service 5 = Grafana
 #     Web Service 6 = Home Assistant
-#     Web Service 7 = retired (was the NAS file UI; never Paperless, which
-#                     this map claimed and which nothing here checks)
+#     Web Service 7 = Paperless
 #     Web Service 8 = Audiobookshelf
 #     Web Service 9 = Music Assistant
-#     Web Service 10 = TRMNL
-#     Web Service 11 = Paperless (gate-level only, see its entry)
 data:
  config.yaml: |
    storage:
@@ -124,10 +121,18 @@ data:
        conditions:
          - "[STATUS] == any(200, 301, 302)"

-      # Removed 2026-08-28: the NAS file UI (file.renner.uno). Its Ingress and
-      # ExternalName Service were deleted in #696 because they pointed at
-      # nas-1, which is powered off; this check outlived them and had been
-      # reporting failure on the public page since.
+      - name: Web Service 7
+        group: services
+        url: https://file.renner.uno
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == any(200, 301, 302)"

      - name: Web Service 8
        group: services
@@ -173,29 +178,6 @@ data:
        conditions:
          - "[STATUS] == any(200, 301, 302)"

-      # Paperless sits entirely behind the Authentik gate — every path,
-      # including /api/, answers 302 to unauthenticated callers, so there is no
-      # endpoint that proves the app rather than the gate. This check is
-      # therefore reachability only: DNS, TLS, Traefik and the outpost. It
-      # stays green if paperless itself is dead behind a live gate.
-      #
-      # Closing that blind spot needs an in-cluster probe of the paperless
-      # ClusterIP, which crosses a namespace boundary and so needs a
-      # CiliumNetworkPolicy first — the monitoring probes get away without one
-      # only because their targets share a namespace with the prober.
-      - name: Web Service 11
-        group: services
-        url: https://paperless.renner.uno
-        interval: 60s
-        ui:
-          hide-hostname: true
-          hide-url: true
-          hide-port: true
-          hide-conditions: true
-          hide-errors: true
-        conditions:
-          - "[STATUS] == any(200, 301, 302)"
-
      # Removed 2026-05-21: Jellyfin (media.renner.uno), Wiki (wiki.renner.uno),
      # InfluxDB (influx.renner.uno) — hostnames NXDOMAIN, no Ingress; services
      # no longer deployed. Re-add when/if they come back.
@@ -209,11 +191,20 @@ data:
      # NoData). Monitor the offsite PBS from the offsite side or via a
      # check that doesn't depend on this cluster's DNS path.

-      # Removed 2026-08-28: PBS 2 (pbs2.renner.uno:8007) — it has no clients
-      # left. Its two were the PVE cluster, retired to bare-metal Talos, and
-      # nas-1, powered off and dropped from the repo. Checking the liveness of
-      # a backup server that nothing backs up to reports on nothing. Re-add
-      # alongside the first host that is declared to back up to it.
+      - name: Infrastructure 1
+        group: infrastructure
+        url: https://pbs2.renner.uno:8007
+        interval: 60s
+        client:
+          insecure-skip-verify: true
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == any(200, 301, 302)"

      - name: Infrastructure 3
        group: infrastructure
@@ -5,12 +5,6 @@ metadata:
  namespace: apps
  labels:
    app: gatus
-  annotations:
-    # Gatus reads its config once at startup and never re-reads the file, so
-    # without this a ConfigMap change reconciles cleanly and then does nothing:
-    # git and the running checks drift apart silently, with no failed resource
-    # to notice. Adding a check would not take effect either.
-    configmap.reloader.stakater.com/reload: gatus-config
 spec:
  replicas: 1
  # RWO PVC (`gatus-data`, ceph-block) — RollingUpdate's default 25% surge
@@ -28,30 +22,9 @@ spec:
      labels:
        app: gatus
    spec:
-      # The upstream image is FROM scratch with no USER, so without this the
-      # pod runs as root and trips PodSecurity restricted on four counts.
-      #
-      # The uid is numeric because a scratch image carries no /etc/passwd for
-      # the kubelet to resolve a name against. fsGroup is what makes the switch
-      # survivable: the only write path is the sqlite store on the RWO volume
-      # (/data/gatus.db, see the storage stanza in the ConfigMap), and fsGroup
-      # both chowns the existing volume and leaves the directory group-writable
-      # so sqlite can create its -wal and -journal siblings next to the db.
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        fsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
      containers:
        - name: gatus
          image: twinproduction/gatus:v5.36.0 # {"$imagepolicy": "flux-system:gatus"}
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
          ports:
            - containerPort: 8080
              name: http
@@ -111,23 +111,9 @@ statistics             max_id=13323904    max_start_ts=1787162400.0
 statistics_short_term  max_id=159205762   max_start_ts=1787166000.0
 ```

-## Phase 2c — cutover: DONE
+## Phase 2c — cutover (REMAINING)

-Verified 2026-09-02 against the live database:
-
-| table | rows | span |
-|---|---|---|
-| `statistics` | 11,092,586 | 2024-03-13 20:00 → current |
-| `statistics_short_term` | 1,883,709 | rolling ~7 days |
-| `statistics_meta` | 1,480 | |
-
-The 2024-03-13 origin is the proof that matters. It is unchanged from the
-pre-migration count in "What is at risk" above, so the history was copied
-across engines rather than started from zero — which is the failure this
-document exists to prevent. `states` carries current rows, so `db_url` points
-at Postgres.
-
-The procedure below is kept as the record of what was done.
+Do not start until phase 1 is verified.

 **Migrate statistics only.** Copying all 18M `states` rows is slow, needs a long
 HA outage, and buys a table that is discarded within 7 days anyway.
@@ -56,9 +56,10 @@ spec:
        # Re-rendered on every pod start, so a CNPG password rotation takes
        # effect rather than silently leaving a stale URI behind.
        #
-        # Load-bearing: `db_url: !secret recorder_db_url` in the homeassistant
-        # repo's configuration.yaml reads this key, so HA does not start
-        # without it. Removing this init container takes the recorder down.
+        # INERT until the recorder is repointed: nothing reads this key until
+        # `db_url: !secret recorder_db_url` lands in configuration.yaml in the
+        # homeassistant repo. That ordering is required — HA refuses to start
+        # when a referenced secret is missing.
        - name: render-db-url
          image: busybox:1.36
          command:
@@ -95,7 +96,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.0 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -209,28 +210,6 @@ spec:
            - name: vscode
              containerPort: 8080
              protocol: TCP
-          # /healthz answers 200 whenever the HTTP server is up; its body
-          # reports session heartbeat state, which is not health — an idle
-          # editor reports "expired" and is perfectly fine. So this asserts
-          # that code-server is serving, not that anyone is using it.
-          #
-          # Liveness matters here more than elsewhere in this pod: code-server
-          # shares the Home Assistant config volume, and a wedged editor is
-          # otherwise invisible until someone tries to open it.
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: vscode
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            timeoutSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: vscode
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 5
          volumeMounts:
            - name: config
              mountPath: /config
@@ -18,12 +18,12 @@
 # That is the point of the move — it attacks the multiplier rather than trimming
 # what gets recorded, which would only hide the problem.
 #
-# This is the live recorder database. HA writes here; the SQLite file is no
-# longer the system of record. The 2.4 years of long-term statistics were
-# copied across engines rather than regenerated — HA does not migrate data
-# between database engines, so an empty Postgres would have silently started
-# from zero and taken the energy dashboard's history with it.
-# MIGRATION-recorder-postgres.md holds the sequence that was followed.
+# STAGING
+# Phase 1 (this manifest): the cluster comes up EMPTY and HA keeps using SQLite.
+# Nothing depends on it, so a failure here is inert. Verify Ready, WAL archiving
+# to Hetzner, and a successful base backup before going further.
+# Phase 2: migrate the statistics tables and repoint recorder.db_url.
+# See MIGRATION-recorder-postgres.md for the sequence and the rollback.
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
@@ -1,8 +1,21 @@
 ---
 # Barman S3 credentials for ha-pg.
 #
-# One shared path for one shared credential — see the note in
-# ../paperless/hetzner-creds-externalsecret.yaml.
+# FOLLOW-UP (#397): sourced from seed/paperless/hetzner-creds, which is not where
+# they belong. woodpecker/hetzner-creds-externalsecret.yaml has the identical
+# problem and the same note. The values are the same today, so this is correct
+# but coupled: rotating paperless's Hetzner credentials would silently change
+# the backup credentials for Home Assistant and Woodpecker too.
+#
+# The fix is one dedicated path per consumer (seed/home-assistant/hetzner-creds)
+# and a one-line change to the `key:` below. Creating it needs an OpenBao write:
+#
+#   bao kv put homelab/seed/home-assistant/hetzner-creds \
+#     @<(bao kv get -format=json homelab/seed/paperless/hetzner-creds | jq '.data.data')
+#
+# Note the mount is `homelab`, so the path is homelab/seed/... — #397 records a
+# 403 on `bao kv list seed`, which was a wrong-path error rather than a
+# permissions problem.
 #
 # The namespace allowlist on the openbao ClusterSecretStore
 # (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
@@ -22,4 +35,4 @@ spec:
    name: hetzner-s3-creds
  dataFrom:
    - extract:
-        key: infrastructure/hetzner-s3
+        key: seed/paperless/hetzner-creds
@@ -60,8 +60,9 @@ resources:
  - zigbee2mqtt-gate-service.yaml
  - zigbee2mqtt-ingress.yaml
  # Postgres backend for the recorder (MIGRATION-recorder-postgres.md).
-  # Live: recorder.db_url in the homeassistant repo points here, and the
-  # long-term statistics carry their full history from 2024.
+  # PHASE 1: this cluster comes up EMPTY and nothing consumes it — HA is still
+  # on SQLite. Deploying it is therefore inert; phase 2 migrates the statistics
+  # tables and repoints recorder.db_url in the homeassistant repo.
  - ha-pg-hetzner-creds-externalsecret.yaml
  - ha-pg-objectstore.yaml
  - ha-pg-cluster.yaml
@@ -1,22 +1,80 @@
 ---
-# mDNS relay: holds the multicast membership on the IoT VLAN through its own
-# macvlan and forwards each datagram to Home Assistant as ordinary UNICAST UDP.
+# mDNS relay: gives Home Assistant sight of the IoT VLAN's mDNS announcements
+# without putting a macvlan on the HA pod itself.
 #
-# It must stay unicast. An avahi-style reflector cannot work here — Cilium runs
-# in tunnel mode without enable-multicast, so re-emitted multicast is dropped by
-# the CNI. Unicast is carried, and python-zeroconf accepts it: its datagram
-# handler checks size and duplicates but never the destination or source
-# address. RFC 6762's QU bit makes this protocol-legal rather than a trick.
+# THE PROBLEM
 #
-# This restores passive reception only — announcements, including the
-# re-announce a device sends when its address changes. HA-initiated browsing
-# still dies on eth0; that would need a querier on the VLAN side, which this
-# deliberately is not.
+# HA's zeroconf listener joins both mDNS groups on eth0 — 224.0.0.251 and
+# ff02::fb — and never hears anything on either.
+# Multicast is link-local, so IoT-VLAN announcements do not reach the pod
+# CIDR; and Cilium runs in tunnel mode (VXLAN) without `enable-multicast`, so
+# it does not replicate multicast between pods either. Both halves of the path
+# are dead, which is why an avahi-style reflector — receive on one interface,
+# re-emit as multicast on the other — cannot work here: the re-emitted packets
+# would be dropped by the CNI.
 #
-# so-bindtodevice=net1 on both receive sockets is load-bearing: without it
-# socat binds the wildcard, and the namespace CNP lets any sibling pod reach
-# :5353 and inject records into HA's zeroconf cache. Binding net1's address
-# instead is impossible — it is DHCP-leased and unknown at exec time.
+# Consequence for HA: the ESPHome integration dials devices at the address
+# cached in its config entry. When a lease rotates, every attempt fails with
+# `[Errno 113] Connect call failed` and the device stays unreachable, because
+# the mDNS announcement that would carry the new address never arrives. New
+# devices are never discovered at all.
+#
+# THE MECHANISM
+#
+# This relay holds the multicast membership on the IoT VLAN through its own
+# macvlan (mdns-relay-network-attachment.yaml) and forwards each datagram it
+# receives to HA as ordinary UNICAST UDP. That is the one direction of travel
+# the CNI will carry.
+#
+# It works because python-zeroconf does not filter on how a packet arrived:
+# its datagram handler validates size, deduplicates, and parses, but performs
+# no check on destination address, source address, or source port. A unicast
+# datagram delivered to :5353 is processed exactly like a multicast one. (mDNS
+# unicast responses are part of RFC 6762 — the QU bit — so this is within the
+# protocol, not a trick.)
+#
+# WHAT THIS DOES AND DOES NOT RESTORE
+#
+# Restored: passive reception. RFC 6762 requires a responder to re-announce
+# when its address changes, so a device that moves to a new lease announces
+# the move, HA sees it, and the ESPHome integration updates the config entry.
+# That is the failure this exists to fix. New devices announcing themselves
+# are likewise seen.
+#
+# NOT restored: HA-initiated browsing. Queries HA sends leave on eth0 as
+# multicast and die there, so HA cannot actively solicit a response. Anything
+# depending on a query rather than an announcement stays broken. Making that
+# work needs a querier on the VLAN side, which this relay deliberately is not.
+#
+# EXPOSURE
+#
+# The receive sockets carry so-bindtodevice=net1, so only datagrams that
+# actually arrived on the IoT VLAN interface are delivered. Binding to net1's
+# address instead is not possible — it is DHCP-leased and unknown at exec time
+# — but binding to the device achieves the same confinement without needing to
+# know the address, and needs no capability: SO_BINDTODEVICE succeeds as an
+# unprivileged uid here.
+#
+# Without it socat would bind the wildcard, and since the namespace CNP grants
+# every pod in home-assistant unrestricted access to every other pod on every
+# port, any sibling pod could reach :5353 over the cluster network and have
+# records forwarded into HA's zeroconf cache.
+#
+# What remains, and cannot be closed here: any host on the IoT VLAN can send
+# this port a datagram. That is the same trust already extended to genuine
+# mDNS from that segment, which any device there can equally forge — the relay
+# widens who can reach HA's listener, not what can be claimed. macvlan traffic
+# bypasses Cilium policy entirely, so the namespace CNP does not constrain
+# this interface either way.
+#
+# Note that the relay is not what makes HA's zeroconf reachable from inside
+# the cluster: HA binds :5353 on the pod address (v4) and the wildcard (v6),
+# and the namespace CNP already permits any sibling pod to reach it directly.
+# Confining the relay removes a second route to that listener; it does not
+# remove the first.
+#
+# Nothing else is exposed: the containers run socat and only socat, with no
+# second listener to reach.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -6,23 +6,32 @@ metadata:
  labels:
    app.kubernetes.io/name: mosquitto
  annotations:
-    # REQUIRED for credential rotation to take effect. The startup shim rebuilds
-    # the password file only on pod start, so without this a rotated Secret sits
-    # inert while the old password stays valid indefinitely.
+    # REQUIRED for credential rotation to take effect at all. The startup shim
+    # rebuilds /mosquitto/auth/passwd from these Secrets on every pod start, so
+    # a rotated Secret is inert until the pod restarts — without this the new
+    # password would sit unused until some unrelated restart, while the old one
+    # stayed valid indefinitely.
    #
-    # Listed per-Secret rather than with the blanket auto annotation, so that
-    # enrolling another broker user is a visible edit to this line.
+    # Listed per-Secret rather than using the blanket `reloader.stakater.com/
+    # auto: "true"`, so that enrolling each remaining broker user in rotation is
+    # a visible, reviewable edit to this line rather than something that starts
+    # happening implicitly.
    secret.reloader.stakater.com/reload: mosquitto-health-credentials
 spec:
  replicas: 1
  strategy:
-    # No PVC, so nothing to fence — the new broker can serve before the old one
-    # is torn down. Two stateless brokers overlapping behind the LB is fine.
+    # No PVC anymore, so nothing to fence: the new broker can come up and pass
+    # its readiness probe before the old one is torn down. maxUnavailable:0 keeps
+    # a broker serving throughout an image bump / reschedule (clients on the old
+    # pod just reconnect to the new one); maxSurge:1 permits the brief two-pod
+    # overlap — two independent stateless brokers behind the LB, fine since there
+    # is no shared state.
    #
-    # Connectivity survives a roll but RETAINED STATE DOES NOT: the new pod
-    # starts with an empty retained store, so every roll is a broker restart
-    # for clients that depend on retention. Zigbee2MQTT does not recover from
-    # that on its own (#381).
+    # NOTE: connectivity survives the roll, but RETAINED STATE DOES NOT — the
+    # new pod starts with an empty retained store, so a roll is equivalent to a
+    # broker restart for every client that depends on retention. Zigbee2MQTT
+    # does not recover from that on its own; see mosquitto-config.yaml and
+    # gitea issue #381.
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
@@ -35,10 +44,14 @@ spec:
      labels:
        app.kubernetes.io/name: mosquitto
    spec:
-      # Runs anywhere; the mosquitto-iot LB finds it. Short fault tolerations
-      # reschedule the broker in ~30s rather than the default ~5min, which is
-      # only possible because there is no PVC to fence — so it also does not
-      # depend on Ceph being healthy.
+      # Runs on any node (no nodeSelector); the mosquitto-iot LB forwards
+      # IoT-VLAN traffic to the broker wherever it lands.
+      # Availability: home-critical priority (never preempted, reschedules
+      # first) + short fault tolerations so a node failure reschedules the
+      # broker in ~30s instead of the default ~5min. With persistence off there
+      # is no PVC to fence/detach/reattach, so the broker can (re)start anywhere
+      # instantly, no longer depends on Ceph being healthy, and rolls without a
+      # tear-down-first gap (see the RollingUpdate strategy above).
      priorityClassName: home-critical
      terminationGracePeriodSeconds: 10
      tolerations:
@@ -6,9 +6,12 @@ metadata:
  labels:
    app.kubernetes.io/name: zigbee2mqtt
 spec:
-  # Z2M is the live Zigbee stack. Never run it alongside ZHA: they contend for
-  # the coordinator's single TCP session, and a Z2M start without the restored
-  # coordinator backup forms a NEW network and drops every paired device.
+  # CUTOVER DONE (2026-07-23): ZHA disabled, the coordinator-backup Secret is
+  # populated in OpenBao, and Z2M is now the live Zigbee stack — so replicas is
+  # 1. It shipped at 0 as a safety interlock (Z2M contends with ZHA for the
+  # coordinator's single TCP session, and starting without the restored backup
+  # would form a NEW network and drop every device); that interlock is spent
+  # once the migration is committed. To roll back, set 0 and re-enable ZHA.
  replicas: 1
  # RWO PVC → single writer; never run two pods against the same /app/data.
  strategy:
@@ -36,10 +39,14 @@ spec:
        # network_key, the latter extracted from the OpenBao coordinator backup
        # as a natural-byte-order int array.
        - name: seed-data
-          # Our own alpine + jq image, built by build-ci-images.yml. DATED tag,
-          # never :latest — rebuilding this base for an unrelated consumer
-          # would otherwise silently change what this init container runs on
-          # its next pod
+          # alpine + jq, built by .woodpecker/build-ci-images.yml from
+          # ci/images/alpine-jq/. Replaces `apk add --no-cache jq` on every pod
+          # start: tools belong in the image, not installed at runtime.
+          # Referenced by tag rather than digest — these are our own images.
+          # DATED tag, not :latest, matching the ci-ansible:2026-04-29 precedent
+          # in kubernetes/infrastructure/netbird-backup/cronjob.yaml. With
+          # :latest, rebuilding this base for some unrelated consumer would
+          # silently change what this init container runs on its next pod
          # restart, with no diff here to review — and this is the container that
          # renders the Zigbee network key. Bumping the tag is a deliberate,
          # reviewable act; the cost is remembering to bump it.
@@ -0,0 +1,18 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: barman-cloud-creds
+  namespace: immich
+  annotations:
+    # Prune-disabled — a standing guard kept from the seed->immich-KS handoff (barman creds).
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
+  dataFrom:
+    - extract:
+        key: seed/immich/barman-creds
@@ -10,15 +10,15 @@ metadata:
    # Kept permanently as a standing guard.
    kustomize.toolkit.fluxcd.io/prune: disabled
 spec:
-  # A replica is viable because nothing here writes pg_vectors any more. Both
-  # vector indexes run on vchordrq and the vectors extension is gone, so the
-  # long pg_vectors/indexes/<uuid> paths are absent — pg_basebackup's tar
-  # format cannot represent them, and that is what blocks a replica join.
-  instances: 2
-  # See ak-pg cluster.yaml for rationale. The barman-plugin shutdown wedge
-  # (cloudnative-pg#9770) holds an instance up until SIGKILL, so bounding
-  # terminationGracePeriod (stopDelay) is what keeps a failover from waiting
-  # on it. smartShutdownTimeout keeps pod restarts quick.
+  # Single instance during pgvecto.rs → VectorChord migration window;
+  # pg_basebackup tar format chokes on the long pg_vectors/indexes/<uuid>
+  # paths, blocking replica join. Bump back to 2 once Immich's in-app
+  # vchord migration drops the vectors extension.
+  instances: 1
+  # See ak-pg cluster.yaml for rationale. Single-instance, so no failover, but
+  # the barman-plugin shutdown wedge (cloudnative-pg#9770) takes the whole DB
+  # down until SIGKILL — bounding terminationGracePeriod (stopDelay) matters
+  # even without a standby. smartShutdownTimeout keeps pod restarts quick.
  smartShutdownTimeout: 10
  stopDelay: 60
  imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration
@@ -36,10 +36,8 @@ spec:
      database: immich
      owner: immich
  postgresql:
-    # vchord only. The image also ships pgvecto.rs (vectors.so), but its
-    # extension is not installed and preloading it would recreate pg_vectors
-    # in PGDATA for nothing. pgvector needs no preloading.
    shared_preload_libraries:
+      - vectors.so
      - vchord.so
    parameters:
      shared_buffers: "512MB"
@@ -15,4 +15,4 @@ spec:
    name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
  dataFrom:
    - extract:
-        key: infrastructure/hetzner-s3
+        key: seed/immich/hetzner-creds
@@ -13,5 +13,6 @@ resources:
  - cluster.yaml
  - objectstore.yaml
  - scheduled-backup.yaml
+  - barman-creds-externalsecret.yaml
  - hetzner-creds-externalsecret.yaml
  - pvcs.yaml
@@ -0,0 +1,20 @@
+# Shared Traefik ServersTransport for ingresses that re-encrypt to a
+# backend serving HTTPS with a self-signed / hostname-mismatched cert:
+#
+#   apps/file       — NAS web UI on a self-signed cert
+#
+# Replaces the nginx `backend-protocol: HTTPS` + `proxy-ssl-verify: off`
+# annotation pair. Each Ingress that wants this behaviour sets
+#   traefik.ingress.kubernetes.io/service.serversscheme: https
+#   traefik.ingress.kubernetes.io/service.serverstransport: insecure-backend
+#
+# (bare name — kubernetesIngress provider resolves ServersTransport in
+# the same namespace as the Ingress; the `<ns>-<name>@kubernetescrd`
+# qualified form is for IngressRoute CRDs only).
+apiVersion: traefik.io/v1alpha1
+kind: ServersTransport
+metadata:
+  name: insecure-backend
+  namespace: apps
+spec:
+  insecureSkipVerify: true
@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - insecure-backend-serverstransport.yaml
  - gatus/
  - external-dns/
+  - file/
  - gitea/
  - ical-proxy/
  # woodpecker/ moved to its own dedicated `woodpecker` Kustomization
@@ -1,241 +0,0 @@
---
-# The chat endpoint, and the only one behind the ingress.
-#
-# Single-model rather than router mode. Router mode served chat and embeddings
-# from one port, but two things made that wrong here:
-#
-#   1. Its /health answers 200 as soon as the router process listens, while the
-#      models are still downloading behind it. Kubernetes marked the pod Ready
-#      within seconds of start, so traffic arrived at a server that could not
-#      answer. In single-model mode /health returns 503 until the model is
-#      resident, which is what the probes below actually need.
-#   2. Every model on the port is selectable in the built-in WebUI, including
-#      the embedding model. Asking an embedding model to chat returns fluent
-#      nonsense ("capital capital capital ...") rather than an error, and
-#      `--embeddings` does not prevent it. There is no flag to hide a model
-#      from the list — `--tags` is informational and `--ui-config` only sets UI
-#      preferences — so the only way to keep it out of the picker is to keep it
-#      off this endpoint. It runs in embed.yaml instead.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: llama-server
-  namespace: ai
-  labels:
-    app.kubernetes.io/name: llama-server
-spec:
-  replicas: 1
-  # A rolling update would load a second copy of the weights onto the same
-  # iGPU and into the same node's RAM before releasing the first.
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: llama-server
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: llama-server
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
-        # Makes the CephFS volume group-writable so the weights can be cached.
-        fsGroup: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: llama-server
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
-          args:
-            # The one place this model is named; llama-server resolves the repo
-            # and downloads it itself, so there is nothing to keep in sync.
-            #
-            # A 35B mixture-of-experts with only ~3B parameters active per
-            # token. Decode on this hardware is bound by how many bytes are read
-            # per token rather than by parameter count, and this model reads
-            # ~1.85 GB/token where a dense 27B reads ~16.5 GB — so it is both
-            # far larger and faster than any dense model that would fit.
-            #
-            # Measured here (llama-bench, k8s-1, this image): tg128 11.59 tok/s,
-            # against 10.78 for a dense 8B. Prefill 120.9 t/s at 512 tokens and
-            # 69.8 at 16384 — a sublinear falloff, so the Gated Delta Net
-            # prefill collapse of llama.cpp#27638 does not occur on this stack.
-            - --hf-repo
-            - ggml-org/Qwen3.6-35B-A3B-GGUF:Q4_K_M
-            # Every layer on the Intel iGPU. Full offload matches CPU throughput
-            # while leaving the node's 14 cores free for the rest of the
-            # cluster; the iGPU is here for that isolation, not for a speed-up.
-            - --n-gpu-layers
-            - "99"
-            # Decode runs on the GPU, so extra threads buy nothing. At 11.59
-            # tok/s this model moves 21.4 GB/s against the 53.9 GB/s a dense
-            # model sustains here, so it is the iGPU's 96 EUs working through
-            # many small per-expert matmuls that sets the pace, not bandwidth.
-            - --threads
-            - "4"
-            # 16k is room to spare, not a safety limit. The limit that matters
-            # is not set here at all.
-            #
-            # Isolated by bisection on a scratch pod: this stack breaks on
-            # PROMPT length, and the value of --ctx-size makes no difference to
-            # it. Measured at --ctx-size 65536 with a single slot, so with four
-            # times the headroom:
-            #
-            #     13588 prompt tokens   coherent answer
-            #     15086                 one token, immediate EOS, empty content
-            #     16088                 CJK characters and digit runs
-            #
-            # A single request past that boundary then poisons the slot: the
-            # same 13588-token prompt that answered cleanly a minute earlier
-            # returns one empty token afterwards, and stays broken until the
-            # pod restarts. That is why it looked intermittent, and why a
-            # 650-token classification once came back as 1025 tokens of "!" —
-            # it followed Home Assistant's ~19k prompts through the same slot.
-            #
-            # Nothing reports any of it: finish_reason "stop", a well-formed
-            # OpenAI response, HTTP 200, /health green.
-            #
-            # So the operating rule is about callers, not this flag. Paperless
-            # sits near 5k and is safe; Home Assistant's Assist prompt measured
-            # 18910 and never had a chance. An earlier version of this comment
-            # blamed --ctx-size 32768, which the bisection disproves — 32768
-            # and 65536 both answer correctly as long as the prompt stays under
-            # the boundary.
-            #
-            # The same stack also fails hard at --ubatch-size 2048, which is
-            # the loud version of a limit this one keeps quiet about.
-            #
-            - --ctx-size
-            - "16384"
-            # One slot, so the 16384 belongs to one request at a time.
-            #
-            # Left on auto, llama.cpp opens four slots and — because auto is
-            # also what enables --kv-unified — has them share a single buffer
-            # of this size. Four Paperless requests at once wanted 6209, 6472,
-            # 4303 and 1271 tokens against those 16384, and all four died with
-            # "Context size has been exceeded".
-            #
-            # They were concurrent because a client timeout does not stop the
-            # generation behind it: each abandoned request keeps consuming the
-            # shared budget the next one needs, so one slow document is enough
-            # to start the pile-up. Queueing is the behaviour worth having
-            # here — nothing calls this server in parallel on purpose, and
-            # Paperless's celery worker runs at concurrency 1.
-            #
-            # No liveness probe change goes with this. The server recovered on
-            # its own once the batch drained, so a probe that restarted the pod
-            # would have made the incident worse; with a single slot, one that
-            # asked for a real completion would also queue behind live work and
-            # kill healthy pods during a long document.
-            - --parallel
-            - "1"
-            # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
-            # with ErrorDeviceLost.
-            - --ubatch-size
-            - "1024"
-            # The server otherwise allows every CORS origin and warns about it
-            # at startup. Without this, any page a LAN browser visits could call
-            # this unauthenticated API from the user's machine. Consumers talk
-            # to it server-side; the WebUI is same-origin and unaffected.
-            - --cors-origins
-            - localhost
-            # Two names for one model. Without this the id is the whole
-            # --hf-repo string, which is unreadable in a model picker and in
-            # logs.
-            #
-            # Exactly one name, because a second one cannot be had usefully:
-            # llama-server keeps aliases in a sorted set and reports the first as
-            # the model's `id`, so with `chat` alongside this the picker always
-            # showed `chat` — in either flag order, since "chat" sorts first.
-            #
-            # Dropping it costs nothing. A second name would only be worth
-            # having as a stable handle for machine consumers, and there is
-            # nothing to stabilise: a single-model server ignores the request's
-            # `model` field entirely and answers to any string, verified against
-            # this endpoint. So no consumer config can break on a model swap,
-            # with or without an alias. The name exists purely to be advertised
-            # on /v1/models, which is what a client reads to build its picker.
-            - --alias
-            - qwen3.6-35b-a3b
-            # Qwen3.6 enables thinking by default. At ~11.6 tok/s that spends
-            # minutes reasoning before answering, and a request with a modest
-            # max_tokens returns an empty `content` with finish_reason "length"
-            # because the whole budget went to reasoning_content. Off by default
-            # keeps answers usable; a client that wants it can send
-            # chat_template_kwargs per request.
-            - --chat-template-kwargs
-            - '{"enable_thinking": false}'
-          env:
-            # Where --hf-repo caches the weights. Pointing it at the PVC makes
-            # the download a one-time cost, shared with llama-embed, and keeps
-            # the server off a home directory it cannot write as UID 1000.
-            - name: LLAMA_CACHE
-              value: /models
-          ports:
-            - name: http
-              containerPort: 8080
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-          # Vulkan allocates the iGPU's working set from system RAM, and that
-          # counts against the pod's cgroup: a 5.2 GiB model measured 7.54 GiB
-          # RSS. Sized as the 19 GiB of weights plus that overhead, plus the KV
-          # cache — which is unusually cheap here, since only 10 of the 40
-          # layers use full attention.
-          resources:
-            requests:
-              cpu: 500m
-              memory: 24Gi
-              gpu.intel.com/i915: "1"
-            limits:
-              cpu: "6"
-              memory: 32Gi
-              gpu.intel.com/i915: "1"
-          # /health is 503 while the model loads and 200 once it is resident, so
-          # the pod only goes Ready when it can actually answer. On a cold PVC
-          # that includes pulling ~19 GiB from Hugging Face, hence the long
-          # startup budget (40 min); liveness takes over only after that.
-          startupProbe:
-            httpGet:
-              path: /health
-              port: http
-            periodSeconds: 10
-            failureThreshold: 240
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: http
-            periodSeconds: 10
-          livenessProbe:
-            httpGet:
-              path: /health
-              port: http
-            periodSeconds: 30
-            timeoutSeconds: 10
-            failureThreshold: 6
-          volumeMounts:
-            - name: models
-              mountPath: /models
-      volumes:
-        - name: models
-          persistentVolumeClaim:
-            claimName: llama-server-models
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: llama-server
-  namespace: ai
-  labels:
-    app.kubernetes.io/name: llama-server
-spec:
-  selector:
-    app.kubernetes.io/name: llama-server
-  ports:
-    - name: http
-      port: 8080
-      targetPort: http
@@ -1,165 +0,0 @@
---
-# The embedding endpoint, for RAG and semantic search.
-#
-# Deliberately separate from llama-server and deliberately not behind the
-# ingress: an embedding model has no generative training, and when it shares a
-# port with the chat model the built-in WebUI lists it as a choice. Selecting it
-# returns fluent nonsense rather than an error, because `--embeddings` does not
-# actually refuse chat completions. Keeping it on its own ClusterIP-only Service
-# is what makes it unselectable, since llama.cpp has no way to hide a model from
-# the picker.
-#
-# Reachable in-cluster at http://llama-embed.ai.svc.cluster.local:8080/v1/embeddings.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: llama-embed
-  namespace: ai
-  labels:
-    app.kubernetes.io/name: llama-embed
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: llama-embed
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: llama-embed
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: llama-embed
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
-          args:
-            - --hf-repo
-            - gpustack/bge-m3-GGUF:Q8_0
-            # Serves POST /v1/embeddings.
-            - --embeddings
-            # A short, stable id for consumers. Without it the model is
-            # advertised as the whole --hf-repo string, which would end up
-            # verbatim in Paperless's config and break on the next model swap.
-            # One name only: llama-server keeps aliases in a sorted set and
-            # reports the first as the id, so a second name can steal it.
-            - --alias
-            - bge-m3
-            # bge-m3 is trained for CLS pooling — a different type from the
-            # Qwen family this replaces, which uses last-token. Left unspecified
-            # llama-server takes the GGUF default, which is not visible from
-            # outside and would silently produce vectors the model was not
-            # trained to emit. Read from the model's own 1_Pooling/config.json
-            # rather than assumed; of the candidates measured, three different
-            # pooling types were needed.
-            - --pooling
-            - cls
-            - --n-gpu-layers
-            - "99"
-            - --threads
-            - "2"
-            # Paperless chunks at 1024 tokens, but counts them with its own
-            # tokenizer while this server measures the batch in bge-m3's.
-            # Those disagree: XLM-RoBERTa splits German into more pieces than
-            # llama_index's counter does, so a chunk Paperless considers full
-            # at 1024 arrived here as 1069 and the request failed outright —
-            # "input (1069 tokens) is too large to process".
-            #
-            # The headroom therefore has to absorb a tokenizer ratio, not just
-            # a rounding error, and 1.5x covers far more than the 1.044 that
-            # broke it. The defaults are batch 2048 / ubatch 512, and with
-            # --embeddings llama-server clamps both to the smaller rather than
-            # run them mismatched, so both are set.
-            #
-            # 2048 stays avoided: that is where the chat model met
-            # ErrorDeviceLost on this Vulkan stack. That failure came under a
-            # 20 GB model's memory pressure and may not bind a 600 MB one, but
-            # this does not need to find out.
-            - --batch-size
-            - "1536"
-            - --ubatch-size
-            - "1536"
-            # Room for a full chunk plus overhead rather than exactly one.
-            - --ctx-size
-            - "4096"
-            # No browser ever talks to this Service, so the WebUI is dead weight
-            # and one more unauthenticated surface.
-            - --no-webui
-          env:
-            # Same cache as llama-server, so the two share one copy of anything
-            # they have in common and neither re-downloads the other's weights.
-            - name: LLAMA_CACHE
-              value: /models
-          ports:
-            - name: http
-              containerPort: 8080
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-          # Measured, not estimated: embedding 400 real documents peaked at
-          # 1.6 GiB (Prometheus max_over_time on the working set). The limit is
-          # well above that because the footprint is dominated by the four slots
-          # at 4096 context and the Vulkan working set rather than by the
-          # weights — the Qwen model this replaces peaked at 10.6 GiB on the
-          # same settings for the same 600 MiB of weights, and several
-          # candidates were OOMKilled at limits that looked generous against
-          # their model size.
-          resources:
-            requests:
-              cpu: 250m
-              memory: 2Gi
-              gpu.intel.com/i915: "1"
-            limits:
-              cpu: "2"
-              memory: 8Gi
-              gpu.intel.com/i915: "1"
-          # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
-          # model needs, but still sized for a cold download.
-          startupProbe:
-            httpGet:
-              path: /health
-              port: http
-            periodSeconds: 10
-            failureThreshold: 60
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: http
-            periodSeconds: 10
-          livenessProbe:
-            httpGet:
-              path: /health
-              port: http
-            periodSeconds: 30
-            timeoutSeconds: 10
-            failureThreshold: 6
-          volumeMounts:
-            - name: models
-              mountPath: /models
-      volumes:
-        - name: models
-          persistentVolumeClaim:
-            claimName: llama-server-models
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: llama-embed
-  namespace: ai
-  labels:
-    app.kubernetes.io/name: llama-embed
-spec:
-  selector:
-    app.kubernetes.io/name: llama-embed
-  ports:
-    - name: http
-      port: 8080
-      targetPort: http
@@ -1,23 +0,0 @@
---
-# ExternalName mirror so the ai-namespace Ingress can backend to the llama-gate
-# outpost that Authentik deploys into the authentik namespace (cross-namespace
-# Service references are not supported in a standard Ingress).
-#
-# Traffic path:
-#   browser → traefik (llm.renner.uno)
-#           → ak-outpost-llama-gate (ExternalName → authentik ns)
-#           → outpost validates the session + passkey step-up
-#           → http://llama-server.ai.svc.cluster.local:8080
-#
-# In-cluster consumers skip all of this and address llama-server directly.
-apiVersion: v1
-kind: Service
-metadata:
-  name: ak-outpost-llama-gate
-  namespace: ai
-spec:
-  type: ExternalName
-  externalName: ak-outpost-llama-gate.authentik.svc.cluster.local
-  ports:
-    - port: 9000
-      targetPort: 9000
@@ -1,47 +0,0 @@
---
-# llm.renner.uno — the chat WebUI, public and behind Authentik.
-#
-# Traefik no longer reaches llama-server directly. This Ingress backends the
-# llama-gate proxy outpost (ExternalName → authentik ns), which enforces a
-# passkey step-up and forwards only after the session validates. The ai
-# CiliumNetworkPolicy is tightened so ONLY that outpost pod may reach
-# llama-server, so nothing in the cluster can route around the gate. Pattern
-# mirrors kubernetes/apps/home-assistant/vscode-ingress.yaml.
-#
-# The gate is the entire access control here: llama-server has no
-# authentication of its own and serves a full OpenAI-compatible API on the same
-# port as the UI. That is what kept this hostname LAN-only until now.
-#
-# In-cluster consumers — Home Assistant, Paperless, CI — address
-# llama-server.ai.svc.cluster.local:8080 directly and never pass through
-# Traefik or the outpost, so they need no credentials and are unaffected by the
-# gate.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llama-server
-  namespace: ai
-  labels:
-    # Public: Cloudflare wildcard → tunnel → NetBird proxy → Traefik. The UDM
-    # external-dns instance skips anything carrying this, and LAN clients reach
-    # the same hostname through the hairpin NAT.
-    dns.renner.uno/public: "true"
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-cluster
-spec:
-  ingressClassName: traefik
-  tls:
-    - hosts:
-        - llm.renner.uno
-      secretName: llama-server-tls
-  rules:
-    - host: llm.renner.uno
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: ak-outpost-llama-gate
-                port:
-                  number: 9000
@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - pvc.yaml
-  - deployment.yaml
-  - embed.yaml
-  - gate-service.yaml
-  - ingress.yaml
@@ -1,31 +0,0 @@
---
-# Holds the GGUF weights, downloaded once by llama-server itself via --hf-repo.
-# Prune-disabled so retiring this Kustomization does not throw away tens of GiB
-# of model files that would have to be pulled from Hugging Face again.
-#
-# ReadWriteMany so a second replica, a one-off benchmark Job, or a candidate
-# model being evaluated can mount the same weights; the models are read-only at
-# inference time.
-#
-# Sized well beyond the two deployed models. The cache is shared by every
-# candidate under evaluation (scripts/embed-eval-real.sh), and on a slow
-# connection the download is the long pole — a shared cache lets the large
-# candidates pull in the background while the small ones are already being
-# measured, and means a model is never fetched twice across runs.
-#
-# CephFS supports online expansion (allowVolumeExpansion on the storage class),
-# so growing this does not disturb the running pods.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: llama-server-models
-  namespace: ai
-  annotations:
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 120Gi
@@ -5,8 +5,9 @@ metadata:
  namespace: monitoring
  labels:
    app.kubernetes.io/name: alloy
-    # Not required for discovery — serviceMonitorSelector is empty — but kept
-    # for consistency with the other monitors.
+    # kube-prometheus-stack's Prometheus resource selects ServiceMonitors
+    # by this label; without it the resource exists but is never
+    # discovered.
    release: kube-prometheus-stack
 spec:
  selector:
@@ -1,8 +1,7 @@
 # Scrape the CNPG Postgres instances (port 9187, built-in exporter).
-# Defined here rather than via CNPG's own enablePodMonitor so the scrape
-# config lives with the other monitors. The `release` label below is not
-# required for discovery — podMonitorSelector is empty — but kept for
-# consistency.
+# CNPG's own enablePodMonitor would generate PodMonitors without the
+# `release: kube-prometheus-stack` label, which this Prometheus requires
+# (podMonitorSelector), so we define them here instead.
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -1,36 +0,0 @@
-# Scrapes CrowdSec. Until this landed, nothing did: `up{namespace="crowdsec"}`
-# returned zero series and Prometheus held no cs_* metric at all, so the
-# detection side of the ingress path was invisible — including whether the
-# agents were reading logs and whether any bouncer was pulling decisions.
-#
-# One monitor for both services. They carry the same shape (a `metrics` port on
-# 6060) and differ only in `app`, so a matchExpressions selector keeps this to
-# a single object rather than two near-identical ones:
-#
-#   crowdsec-service        the LAPI      — decisions, bouncer + machine requests
-#   crowdsec-agent-service  the agents    — acquisition, parsers, scenarios
-#
-# The crowdsec-ns CiliumNetworkPolicy already admits the monitoring namespace
-# without a port restriction, so no policy change is needed.
-#
-# `release: kube-prometheus-stack` is what the chart's Prometheus selects on.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: crowdsec
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/name: crowdsec
-    release: kube-prometheus-stack
-spec:
-  selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - crowdsec-service
-          - crowdsec-agent-service
-  endpoints:
-    - port: metrics
-      path: /metrics
-      interval: 30s
@@ -5,8 +5,9 @@ metadata:
  namespace: home-assistant
  labels:
    app.kubernetes.io/name: home-assistant
-    # Not required for discovery — serviceMonitorSelector is empty — but kept
-    # for consistency with the other monitors.
+    # kube-prometheus-stack's Prometheus resource selects ServiceMonitors
+    # by this label; without it the resource exists but is never
+    # discovered. Same trap the garage PrometheusRule documents.
    release: kube-prometheus-stack
 spec:
  selector:
@@ -4,15 +4,13 @@ kind: Kustomization
 # come from the kube-prometheus-stack chart. Flux applies this Kustomization
 # only after `monitoring` is Ready (chart installed → CRDs present).
 resources:
-  - prometheusrule-blackbox.yaml
  - prometheusrule-ceph.yaml
-  - prometheusrule-ci-openbao.yaml
  - prometheusrule-cilium.yaml
-  - prometheusrule-disk-wear.yaml
  - prometheusrule-hetzner-backup.yaml
  - prometheusrule-loki.yaml
  - prometheusrule-mariadb-backup.yaml
  - prometheusrule-netbird-availability.yaml
+  - prometheusrule-netbird-backup.yaml
  - prometheusrule-netbird-ingress.yaml
  - prometheusrule-openbao.yaml
  - prometheusrule-velero.yaml
@@ -23,9 +21,7 @@ resources:
  - prometheusrule-cnpg-backup.yaml
  - alloy-servicemonitor.yaml
  - cnpg-podmonitors.yaml
-  - crowdsec-servicemonitor.yaml
  - home-assistant-servicemonitor.yaml
  - immich-servicemonitor.yaml
  - loki-servicemonitor.yaml
-  - smartctl-exporter-servicemonitor.yaml
  - probes-blackbox.yaml
@@ -5,8 +5,9 @@ metadata:
  namespace: monitoring
  labels:
    app.kubernetes.io/name: loki
-    # Not required for discovery — serviceMonitorSelector is empty — but kept
-    # for consistency with the other monitors.
+    # kube-prometheus-stack's Prometheus resource selects ServiceMonitors
+    # by this label; without it the resource exists but is never
+    # discovered.
    release: kube-prometheus-stack
 spec:
  # Matches the `loki` (single-binary) and `loki-canary` services, but
@@ -1,9 +1,5 @@
 # HTTP probes — fan out to blackbox-exporter. Probe targets mirror the
 # legacy Ansible role's blackbox_http_targets list.
-#
-# The `release` label on every Probe here IS load-bearing: probeSelector still
-# matches on it, unlike serviceMonitorSelector and podMonitorSelector, which
-# are empty. Removing it silently stops the probe from being discovered.
 apiVersion: monitoring.coreos.com/v1
 kind: Probe
 metadata:
@@ -49,28 +45,6 @@ spec:
        # ports outside the hairpin's 80/443 — real WAN path.
        - http://proxy-1.renner.uno:9000/health
        - http://proxy-2.renner.uno:9000/health
-        # Woodpecker. Ungated, root returns 200 with no redirect. Every
-        # scheduled job in this repo runs here — the alert-monitor cron, the
-        # nightly scans, the deploy targets — so an outage here is silent by
-        # construction: the thing that would have told you is the thing that
-        # is down.
-        - https://ci.renner.uno
-        # OpenBao. Probes the health API rather than the UI on purpose: a
-        # SEALED instance still serves the UI (307 -> /ui/ -> 200) while every
-        # ExternalSecret in the cluster fails, which is exactly the
-        # green-while-broken shape this file warns about elsewhere.
-        #
-        # standbyok=true because the endpoint answers 200 only on the ACTIVE
-        # node and 429 on a standby — the external route currently lands on a
-        # standby, so the bare path would be permanently red. With the
-        # parameter, 200 means initialized, unsealed and serving, on whichever
-        # node answers; 503 sealed and 501 uninitialized both stay red.
-        - https://openbao.renner.uno/v1/sys/health?standbyok=true
-        # ical-proxy serves an unauthenticated /health that answers 200 with
-        # {"status":"healthy"}. Unlike the gated hostnames this is real app
-        # health, not reachability — the root path returns 404 by design, since
-        # the service only answers on calendar paths.
-        - https://ical-proxy.renner.uno/health
        # APP-HEALTH probes for the gated monitoring UIs, from the correct
        # in-cluster vantage (monitoring-vantage-point-rule): the gate is
        # only on the ingress, so the ClusterIP /-/healthy endpoints are
@@ -83,11 +57,6 @@ spec:
        # external reachability probe above; it is not the health vantage.)
        - http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/-/healthy
        - http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090/-/healthy
-        # Loki has no external vantage at all: its ingress requires auth on
-        # every path, so an outside probe can only ever report 401. In-cluster
-        # /ready answers 200 unauthenticated and is in this namespace, so the
-        # same intra-namespace allowance the two above rely on covers it.
-        - http://loki.monitoring.svc.cluster.local:3100/ready
 # WAN-path probes of the public edge BY PROXY IP. The name-based
 # home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway)
 # and never leaves the LAN — the vantage-point blind spot that let the
@@ -1,59 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: blackbox
-  namespace: monitoring
-  labels:
-    # kube-prometheus-stack's Prometheus resource selects on this label;
-    # without it the rule is created but never loaded into Prometheus.
-    release: kube-prometheus-stack
-spec:
-  groups:
-    - name: blackbox.rules
-      rules:
-        # Catch-all for every blackbox-http target that has no rule of its own.
-        # Before this existed, a target could be probed, graphed and completely
-        # unalerted — which is how file.renner.uno served 502 for months.
-        #
-        # THE EXCLUSION LIST AND THE DEDICATED RULES ARE ONE INVARIANT: an
-        # instance belongs here only while a rule elsewhere covers it, and
-        # adding a dedicated rule means adding its instance here. Get it wrong
-        # in one direction and the alert fires twice; wrong in the other and it
-        # never fires at all. Currently excluded:
-        #
-        #   fsn1.your-objectstorage.com   HetznerBackupEndpointDown (critical)
-        #   netbird.renner.uno            prometheusrule-netbird-availability
-        #   proxy-1/2:9000/health         same file, relay redundancy
-        #   ci.renner.uno                 WoodpeckerDown
-        #   openbao .../sys/health        OpenBaoUnavailable
-        #
-        # Warning, not critical, on purpose: this rule cannot know what a given
-        # target means. A target whose loss deserves a critical page gets its
-        # own rule — and lands in the list above. That is what keeps the
-        # exclusion list from being arbitrary.
-        - alert: BlackboxProbeFailing
-          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
-          for: 10m
-          annotations:
-            summary: "Blackbox probe of {{ $labels.instance }} has been failing for 10 minutes"
-            description: >-
-              The endpoint is not answering as expected. For the Authentik-gated
-              hostnames this is a reachability check covering DNS, TLS, Traefik
-              and the gate outpost — it says nothing about the app behind the
-              gate, which the internal /-/healthy probes cover separately.
-          labels:
-            severity: warning
-        # The check that the checks are running. Every rule above reads
-        # probe_success; if the series stop being produced at all, none of them
-        # can fire and the silence looks exactly like health.
-        - alert: BlackboxProbesAbsent
-          expr: absent(probe_success{job="blackbox-http"})
-          for: 15m
-          annotations:
-            summary: "No blackbox-http probe results are reaching Prometheus"
-            description: >-
-              Either the blackbox exporter is down or the Probe objects stopped
-              being discovered. Every endpoint alert depends on this job, so
-              they are all blind until it returns.
-          labels:
-            severity: critical
@@ -1,47 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: ci-openbao
-  namespace: monitoring
-  labels:
-    # kube-prometheus-stack's Prometheus resource selects on this label;
-    # without it the rule is created but never loaded into Prometheus.
-    release: kube-prometheus-stack
-spec:
-  groups:
-    - name: ci-openbao.rules
-      rules:
-        - alert: WoodpeckerDown
-          # Probe target defined in probes-blackbox.yaml. Root, ungated, 200.
-          expr: probe_success{job="blackbox-http", instance="https://ci.renner.uno"} == 0
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Woodpecker is not answering on ci.renner.uno"
-            description: >-
-              Every scheduled job in this repo runs here: the alert-monitor
-              cron that files these issues, the nightly scans, and every deploy
-              target. Nothing else reports this outage, because the component
-              that would have reported it is the one that is down. Alertmanager
-              itself is unaffected, which is why this alert exists.
-        - alert: OpenBaoUnavailable
-          # Probes /v1/sys/health?standbyok=true — 200 on any unsealed node,
-          # 503 sealed, 501 uninitialized. The UI stays up when sealed, so a
-          # root probe would not catch this.
-          #
-          # Matched by prefix rather than the full string: the target carries a
-          # query parameter, and an exact match would break silently if the
-          # instance label ever encodes it differently.
-          expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
-          for: 5m
-          labels:
-            severity: critical
-          annotations:
-            summary: "OpenBao is sealed, uninitialized or unreachable"
-            description: >-
-              Every ExternalSecret in the cluster resolves through OpenBao.
-              Existing Secrets keep working, so nothing breaks at once — the
-              damage shows up at the next refresh interval or the next pod that
-              needs a fresh credential. Check whether it is sealed before
-              assuming a network fault.
@@ -11,22 +11,25 @@ spec:
  groups:
    - name: cnpg-backup.rules
      rules:
-        # The Postgres clusters back up to Hetzner. HetznerBackupEndpointDown
-        # probes that S3 endpoint, which proves the bucket is reachable and
-        # nothing more — these two rules check that something is actually being
-        # written to it.
+        # WHY THIS FILE EXISTS
+        # Seven Postgres clusters back up to Hetzner and NOTHING alerted if that
+        # stopped. The only related rule was HetznerBackupEndpointDown, which
+        # probes the S3 endpoint — it proves the bucket is reachable, not that
+        # anything was written to it. Same failure shape as the WordPress backup
+        # that failed nightly for three days against a bucket that was declared
+        # in tofu but never applied: the status surface looked fine because
+        # nothing checked the thing that mattered.

        - alert: CNPGWalArchiveBacklog
          # `ready` counts WAL segments sitting in pg_wal/archive_status waiting
-          # to be archived. Steady state is 0. A sustained backlog means the
-          # archiver is failing: point-in-time recovery is silently broken from
-          # that moment, and pg_wal keeps growing until it fills the PVC and
-          # takes Postgres down.
+          # to be archived. Steady state is 0 across every cluster (verified
+          # 2026-08-19). A sustained backlog means the archiver is failing:
+          # point-in-time recovery is silently broken from that moment, and
+          # pg_wal keeps growing until it fills the PVC and takes Postgres down.
          #
-          # This is the load-bearing alert of the two. It reads the
-          # instance-level collector directly, so unlike the backup-age metric
-          # discussed below it does not depend on Cluster status being
-          # populated.
+          # This is the load-bearing alert of the two. It depends only on the
+          # instance-level collector, so unlike the backup-age metrics below it
+          # is unaffected by the missing Backup CRs (#522).
          #
          # Threshold: >5 segments sustained for 30m. Brief non-zero values are
          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
@@ -60,17 +63,16 @@ spec:
        #
        # The natural expression is
        #   (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600
-        # and it cannot be used, because that metric reads exactly 0 on every
-        # instance. The collector derives it from `Cluster.status
-        # .lastSuccessfulBackup`, and that field is absent on every Cluster
-        # here: all of them back up with `method: plugin` (barman-cloud), and
-        # the plugin path does not write it back.
+        # and it CANNOT be used today: that metric reads exactly 0 on every
+        # instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2,
+        # pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any
+        # successful backup anywhere, because the Backup CRs it derives that
+        # from are being deleted within a minute of completing — see #522.
        #
-        # The backups themselves run and are retained — completed Backup CRs
-        # going back weeks exist for every scheduled cluster. The gap is in the
-        # status field the metric reads, not in the data.
+        # Backups themselves DO work; that was verified independently by
+        # listing the bucket with barman-cloud-backup-list and restoring a WAL
+        # segment back out of S3. The gap is in the record, not the data.
        #
-        # Adding the rule against that metric would fire permanently for every
-        # cluster, which is worse than not having it. Either the plugin starts
-        # populating the field, or the alert has to be built on the Backup CRs
-        # directly, which Prometheus does not see today.
+        # Adding the rule now would fire permanently for all seven clusters and
+        # be silenced within a day, which is worse than not having it. Add it as
+        # part of resolving #522, once the metric reflects reality.
@@ -1,125 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: disk-wear
-  namespace: monitoring
-  labels:
-    # kube-prometheus-stack's Prometheus resource selects on this label;
-    # without it the rule is created but never loaded into Prometheus.
-    release: kube-prometheus-stack
-spec:
-  groups:
-    - name: disk-wear.rules
-      rules:
-        # Wear percentages are self-reported: 100 means the rated endurance is
-        # spent, not that the drive stops.
-        #
-        # Wear alone is not a signal. It only ever rises, so a plain threshold
-        # starts firing the day a drive crosses it and never stops — a status
-        # dressed as an alert (#673, #675, #682, #686 were all one drive set
-        # crossing 80%). What matters is wear that is high AND still climbing,
-        # so the alert below is a projection: at the write rate of the last
-        # week, when does this drive reach 95%?
-        #
-        # The rate is self-calibrating. Each drive reports both its lifetime
-        # bytes written and its own wear percentage, so bytes-per-percent comes
-        # from the drive itself and no rated-endurance table has to be kept in
-        # sync with the hardware. percentage_used is a whole-number percent and
-        # advances only every few weeks here, which is far too coarse to
-        # differentiate; bytes_written moves on every scrape.
-        - alert: NvmeDriveWearProjectedExhaustion
-          # The >= 80 gate is numerical, not editorial: bytes-per-percent is
-          # derived by dividing by an integer percentage, so on a lightly worn
-          # drive a half-percent rounding error swings the projection by tens
-          # of percent. Above 80 the same rounding is under one percent.
-          # Idle drives divide by a zero rate and yield +Inf, which is not < the
-          # threshold, so they stay silent on their own.
-          expr: >-
-            (
-              (95 - smartctl_device_percentage_used)
-              /
-              (
-                rate(smartctl_device_bytes_written[7d]) * 86400
-                  * smartctl_device_percentage_used
-                  / smartctl_device_bytes_written
-              )
-            ) < 180
-            and smartctl_device_percentage_used >= 80
-          # Long, because a projection off a 7d rate should not react to a day.
-          for: 6h
-          labels:
-            severity: warning
-          annotations:
-            summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} projects to 95% wear in {{ $value | printf \"%.0f\" }} days"
-            description: >-
-              The drive is past 80% of its rated endurance and the write rate
-              of the last week would spend the rest within six months. This is
-              a lead-time warning, not a fault: decide whether to replace the
-              drive or move what is writing to it. The Disk Health dashboard
-              shows which drives carry which role.
-        # Kept as a flat threshold on purpose: past rated endurance the drive
-        # runs outside what the manufacturer stands behind regardless of how
-        # fast it got there, and the alert clears by replacing it.
-        - alert: NvmeDriveWearCritical
-          expr: smartctl_device_percentage_used >= 95
-          for: 1h
-          labels:
-            severity: critical
-          annotations:
-            summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear"
-            description: >-
-              The drive is at or past the end of its rated endurance. It will
-              likely keep working for a while, but it is now operating outside
-              what the manufacturer stands behind, and replacement should be
-              scheduled rather than considered.
-        # Wear is a budget; spare exhaustion is damage.
-        - alert: NvmeDriveSpareExhausted
-          expr: smartctl_device_available_spare <= smartctl_device_available_spare_threshold
-          for: 15m
-          labels:
-            severity: critical
-          annotations:
-            summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} has exhausted its spare area"
-            description: >-
-              Available spare has fallen to the manufacturer's threshold. The
-              drive has run out of reserve blocks to remap failures into.
-              Replace it.
-        - alert: NvmeDriveCriticalWarning
-          expr: smartctl_device_critical_warning > 0
-          for: 5m
-          labels:
-            severity: critical
-          annotations:
-            summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} reports critical warning {{ $value }}"
-            description: >-
-              The drive has raised its own critical warning flag. Read the
-              detail with `smartctl -a` on the node; the value is a bitfield,
-              not a count.
-        - alert: NvmeDriveMediaErrors
-          # On the increase, not the total: pre-existing errors would otherwise
-          # alert forever. Standing counts are visible on the dashboard only.
-          expr: increase(smartctl_device_media_errors[6h]) > 0
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} recorded {{ $value }} new media errors"
-            description: >-
-              The drive reported unrecoverable data-integrity errors in the
-              last six hours. A healthy drive never increments this. Check
-              whether Ceph has logged scrub errors on the same node.
-        # The check that the checks are running: no series fires no alert.
-        # Counted against node-exporter's inventory, not a fixed fleet size.
-        - alert: SmartctlExporterIncomplete
-          expr: count(smartctl_device_percentage_used) < count(node_nvme_info) or absent(smartctl_device_percentage_used)
-          for: 30m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Fewer NVMe drives report SMART than node-exporter can see"
-            description: >-
-              Some drives visible to node-exporter are not reporting SMART, or
-              none are. Either a DaemonSet pod is down or a drive stopped
-              answering — both mean the wear alerts above have stopped watching
-              part of the fleet, which looks identical to everything being
-              fine.
@@ -17,12 +17,11 @@ spec:
          # drains on recovery) and Velero backups fail. Replaces the old
          # GarageDown alert now that backups live on Hetzner, not nas-1.
          #
-          # The granular counterpart is CNPGWalArchiveBacklog in
-          # prometheusrule-cnpg-backup.yaml, reading the instance collector
-          # scraped by cnpg-podmonitors.yaml: this alert says the target is
-          # unreachable, that one says archiving is falling behind whatever the
-          # cause. Velero's VeleroBackupValidationFailure / VeleroBackupStale
-          # cover backup-level failures.
+          # CNPG has no metric-based archiving alert yet — its pod metrics
+          # are not scraped (enablePodMonitor=false). Enabling CNPG
+          # PodMonitors for a granular "archiving falling behind" alert is
+          # a follow-up. Velero's own VeleroBackupValidationFailure /
+          # VeleroBackupStale rules already cover backup-level failures.
          expr: probe_success{instance="https://fsn1.your-objectstorage.com"} == 0
          for: 10m
          labels:
@@ -0,0 +1,44 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: netbird-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: netbird-backup.rules
+      rules:
+        - alert: NetbirdVpsBackupStale
+          # The nightly pull of the controller VPS's management store
+          # hasn't succeeded in over 26 h (one missed cycle + slack).
+          # This dump is the only copy of the mesh state (peers, setup
+          # keys, routes) that survives losing the Hetzner VPS — a
+          # silently dead backup is exactly the failure mode the SPOF
+          # assessment flagged.
+          expr: |
+            time() - kube_cronjob_status_last_successful_time{namespace="netbird-backup", cronjob="netbird-vps-backup"} > 26 * 3600
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "NetBird VPS backup has not succeeded in 26h+"
+            description: "The netbird-vps-backup CronJob's last successful run is over 26 hours old. Check the job logs in the netbird-backup namespace (ssh reachability of netbird.renner.uno, the netbird-backup-ssh ExternalSecret, PVC space)."
+
+        - alert: NetbirdVpsBackupNeverRan
+          # The CronJob exists but has never succeeded — most likely the
+          # one-time OpenBao provisioning step
+          # (bao kv put homelab/infrastructure/netbird-backup ssh-key=@…)
+          # hasn't been done, so the ExternalSecret cannot sync and the
+          # job pod fails on the missing secret mount.
+          expr: |
+            kube_cronjob_created{namespace="netbird-backup", cronjob="netbird-vps-backup"}
+            unless
+            kube_cronjob_status_last_successful_time{namespace="netbird-backup", cronjob="netbird-vps-backup"}
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            summary: "NetBird VPS backup has never succeeded"
+            description: "The netbird-vps-backup CronJob exists but has no successful run. If this is the initial rollout: the private key must be provisioned once by hand — bao kv put homelab/infrastructure/netbird-backup ssh-key=@<keyfile> — before the ExternalSecret can sync."
@@ -1,24 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: smartctl-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: smartctl-exporter
-    # Not required for discovery — serviceMonitorSelector is empty — but kept
-    # for consistency with the other monitors.
-    release: kube-prometheus-stack
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: smartctl-exporter
-  endpoints:
-    # smartctl polls the drives every 60s; scraping faster gains nothing.
-    - port: http-metrics
-      path: /metrics
-      interval: 60s
-      relabelings:
-        # Without this a series carries only the exporter pod's IP, which does
-        # not identify the machine the drive is in.
-        - sourceLabels: [__meta_kubernetes_pod_node_name]
-          targetLabel: node
@@ -1,163 +0,0 @@
-{
-  "annotations": {"list": []},
-  "description": "CrowdSec: Erkennung im Cluster, Durchsetzung am NetBird-Edge",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Ban-Entscheidungen in der LAPI, ganz überwiegend aus den CAPI-Community-Blocklisten. Das ist die Menge, die ein scharfgeschalteter Bouncer durchsetzt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
-      "id": 1,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(cs_active_decisions)", "refId": "A"}],
-      "title": "Aktive Decisions",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Bouncer, die in den letzten 10 Minuten Decisions abgeholt haben. Fällt das unter die erwartete Zahl, holt ein Bouncer nichts mehr — und setzt damit einen veralteten Stand durch, ohne dass irgendetwas ausfällt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
-      "id": 2,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[10m])) > 0)", "refId": "A"}],
-      "title": "Bouncer, die abholen",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Vom Agent gelesene Logzeilen pro Sekunde. Steht das auf 0, detektiert CrowdSec nichts mehr — der häufigste stille Ausfall, weil alles andere weiterläuft.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 0.01}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
-      "id": 3,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_filesource_hits_total[5m]))", "refId": "A"}],
-      "title": "Gelesene Logzeilen/s",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Szenario-Overflows der letzten 24 h — also lokal erkannte Angriffe, im Gegensatz zu den von CAPI übernommenen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
-      "id": 4,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(increase(cs_bucket_overflowed_total[24h]))", "refId": "A"}],
-      "title": "Lokale Overflows (24 h)",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Abholrate je Bouncer. Der Firewall-Bouncer auf den Proxies fragt alle 10 s. Die LAPI legt bei jeder neuen beobachteten Client-Adresse einen Eintrag `<name>@<ip>` an, und beobachtet wird die Pod-IP des netbird-ingress-Peers — deshalb erscheint ein logischer Bouncer hier mehrfach und nach Pod-Rollen unter neuen Namen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
-      "id": 5,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[5m]))", "legendFormat": "{{bouncer}}", "refId": "A"}],
-      "title": "Bouncer-Abholrate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Decisions nach Grund. `http:scan` dominiert und stammt aus den CAPI-Blocklisten.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
-      "id": 6,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (cs_active_decisions)", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Aktive Decisions nach Grund",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Welche Szenarien lokal auslösen. Das ist das, was CrowdSec selbst aus den Traefik-Logs erkennt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 70, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
-      "id": 7,
-      "options": {"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (name) (increase(cs_bucket_overflowed_total[1h]))", "legendFormat": "{{name}}", "refId": "A"}],
-      "title": "Szenario-Overflows (je Stunde)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Geparste gegen nicht geparste Zeilen. Ein Anstieg bei `ko` heißt meist, dass sich ein Logformat geändert hat und der Parser nicht mehr greift — CrowdSec liest dann weiter, erkennt aber nichts mehr.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
-      "id": 8,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ok_total[5m]))", "legendFormat": "geparst", "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ko_total[5m]))", "legendFormat": "nicht geparst", "refId": "B"}
-      ],
-      "title": "Parser",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Whitelist-Treffer nach Grund. Nützlich beim Scharfschalten: was hier zählt, wäre sonst ein Kandidat für einen Ban gewesen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
-      "id": 9,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (rate(cs_node_wl_hits_ok_total[5m]))", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Whitelist-Treffer",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
-      "id": 10,
-      "options": {
-        "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
-        "content": "## Was dieses Dashboard NICHT zeigt\n\nAlle Panels stammen aus der LAPI und den Agents im Cluster. Die **Bouncer auf proxy-1 und proxy-2 exportieren eigene Metriken**, die hier fehlen:\n\n- `fw_bouncer_active_decisions` — was tatsächlich in den nftables-Sets steht\n- `fw_bouncer_dropped_packets` / `_bytes` — was wirklich verworfen wurde\n- `lapi_requests_total` / `_failures_total` — aus Sicht des Bouncers\n\nSie liegen auf `127.0.0.1:60601` der Proxy-Hosts. Die Hosts sind öffentlich und haben bisher keinerlei Metrik-Pfad; den Port ins Internet zu öffnen, nur um ein Scrape-Ziel zu bekommen, wäre der falsche Tausch.\n\n**Konsequenz:** „Bouncer-Abholrate\" oben zeigt, dass ein Bouncer *fragt* — nicht, dass er die Antwort auch angewandt hat. Solange der Modus `dry-run` ist, wendet er ohnehin nichts an.\n\nEin Scrape-Pfad für die Hosts ist eigene Arbeit und Voraussetzung dafür, dass „was wird wirklich geblockt\" beantwortbar wird."
-      },
-      "title": "Blinder Fleck",
-      "type": "text"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": ["homelab", "crowdsec", "security"],
-  "templating": {"list": []},
-  "time": {"from": "now-6h", "to": "now"},
-  "timepicker": {},
-  "timezone": "",
-  "title": "CrowdSec Overview",
-  "uid": "crowdsec-overview",
-  "version": 1,
-  "weekStart": ""
-}
@@ -1,6 +1,6 @@
 {
  "annotations": {"list": []},
-  "description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance comes from smartctl_exporter, whose series are labelled by node rather than by the exporter pod's IP — see the note at the bottom for what it still cannot tell you.",
+  "description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance/wear is absent; see the note at the bottom and issue #549.",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
@@ -19,7 +19,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 4, "w": 5, "x": 0, "y": 0},
+      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
      "id": 10,
      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_health_status", "legendFormat": "", "refId": "A"}],
@@ -38,7 +38,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 4, "w": 5, "x": 5, "y": 0},
+      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
      "id": 11,
      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [
@@ -61,7 +61,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 4, "w": 5, "x": 10, "y": 0},
+      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
      "id": 12,
      "options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_cluster_total_used_bytes / ceph_cluster_total_bytes", "legendFormat": "", "refId": "A"}],
@@ -70,7 +70,7 @@
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Fullest EPHEMERAL volume across the fleet — Talos's /var. Its contents are meant to be disposable: container images, kubelet state and pod scratch all rebuild. Filling it is still a problem, because a node with no room to write stops being useful long before anything is lost, and nothing rebalances /var the way Ceph rebalances a full OSD.",
+      "description": "Fullest EPHEMERAL volume across the fleet — Talos's /var, holding container images, pod scratch space and kubelet state. A full /var takes the node down, and unlike Ceph nothing rebalances away from it.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
@@ -82,7 +82,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 4, "w": 5, "x": 15, "y": 0},
+      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
      "id": 13,
      "options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(1 - node_filesystem_avail_bytes{mountpoint=\"/var\"} / node_filesystem_size_bytes{mountpoint=\"/var\"})", "legendFormat": "", "refId": "A"}],
@@ -114,7 +114,7 @@
          "refId": "A"
        }
      ],
-      "title": "Drive inventory (identity and role)",
+      "title": "Drive inventory",
      "transformations": [
        {
          "id": "organize",
@@ -127,100 +127,6 @@
      ],
      "type": "table"
    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Rated endurance consumed, worst drive in the fleet. This is the drive's own accounting against what the manufacturer warrants, not a failure prediction — a drive past 100 usually keeps working and starts eating its spare area. Read it for planning; read the SMART table for whether anything is failing now.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
-      "id": 14,
-      "options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(smartctl_device_percentage_used)", "legendFormat": "", "refId": "A"}],
-      "title": "Worst NVMe wear",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Rated endurance consumed per drive. The line is nearly flat by nature — wear moves on a scale of weeks — so the useful reading is the spread between drives and the slope, not today's value. Joined to smartctl_device for the model name, because the device label alone is not comparable across nodes.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "palette-classic"},
-          "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "dashed"}},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
-      "id": 15,
-      "options": {"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [
-        {
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device",
-          "legendFormat": "{{node}} {{model_name}}",
-          "refId": "A"
-        }
-      ],
-      "title": "NVMe wear",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "The failure-now signals, as opposed to the endurance budget above. Spare below its threshold means the drive has run out of blocks to remap defects into; media errors are unrecoverable data-integrity failures and should never increment; the critical warning is a bitfield the drive raises itself. Written bytes is the counter behind the wear percentage, useful for projecting when a drive will reach it.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "custom": {"align": "auto", "cellOptions": {"type": "auto"}, "inspect": false},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}
-        },
-        "overrides": [
-          {"matcher": {"id": "byName", "options": "wear"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]}}]},
-          {"matcher": {"id": "byName", "options": "spare"}, "properties": [{"id": "unit", "value": "percent"}]},
-          {"matcher": {"id": "byName", "options": "media errors"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
-          {"matcher": {"id": "byName", "options": "critical warning"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
-          {"matcher": {"id": "byName", "options": "powered on"}, "properties": [{"id": "unit", "value": "s"}]},
-          {"matcher": {"id": "byName", "options": "written"}, "properties": [{"id": "unit", "value": "bytes"}]}
-        ]
-      },
-      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
-      "id": 16,
-      "options": {"cellHeight": "sm", "footer": {"countRows": false, "fields": "", "reducer": ["sum"], "show": false}, "showHeader": true, "sortBy": [{"desc": true, "displayName": "wear"}]},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_available_spare * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "B"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_media_errors * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "C"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_critical_warning * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "D"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_power_on_seconds * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "E"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_bytes_written * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "F"}
-      ],
-      "title": "SMART health",
-      "transformations": [
-        {"id": "merge", "options": {}},
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {"Time": true, "__name__": true, "container": true, "endpoint": true, "instance": true, "job": true, "namespace": true, "pod": true, "service": true},
-            "indexByName": {},
-            "renameByName": {"node": "node", "device": "device", "model_name": "model", "Value #A": "wear", "Value #B": "spare", "Value #C": "media errors", "Value #D": "critical warning", "Value #E": "powered on", "Value #F": "written"}
-          }
-        }
-      ],
-      "type": "table"
-    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Read and write throughput per drive.",
@@ -234,7 +140,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
+      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
      "id": 2,
      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
@@ -267,7 +173,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
+      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
      "id": 3,
      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
@@ -300,7 +206,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
+      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
      "id": 4,
      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
@@ -338,7 +244,7 @@
          }
        ]
      },
-      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
+      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
      "id": 5,
      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
@@ -371,7 +277,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 40},
+      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
      "id": 6,
      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
@@ -398,7 +304,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 40},
+      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
      "id": 7,
      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
@@ -433,7 +339,7 @@
        },
        "overrides": []
      },
-      "gridPos": {"h": 8, "w": 24, "x": 0, "y": 49},
+      "gridPos": {"h": 8, "w": 24, "x": 0, "y": 40},
      "id": 9,
      "options": {"legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
@@ -449,11 +355,11 @@
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "gridPos": {"h": 11, "w": 24, "x": 0, "y": 57},
+      "gridPos": {"h": 11, "w": 24, "x": 0, "y": 48},
      "id": 8,
      "options": {
        "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
-        "content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Wear is a budget, not a prediction.** The percentage is what the drive reports against its own rated endurance. Drives do not stop at 100 — they usually keep working and start consuming spare blocks — so read it for planning, and read available spare and media errors for whether something is failing now. Those are different questions and the alert rules treat them as such.\n\n**A green top row is not a healthy drive.** Endurance says nothing about the drive that dies of a controller fault tomorrow, and this dashboard cannot see one coming.\n\nCeph keeps its own wear figures for OSDs in the devicehealth module, which is a useful cross-check but covers OSDs only: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. A drive removed from Ceph is purged from that registry and its history goes with it.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, and putting a worn one on EPHEMERAL is a sensible destination rather than a hazard — that volume is built to be rebuilt. Check the inventory table for what a given drive holds before reading its wear as urgent.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
+        "content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Endurance is missing.** No wear, no written-bytes total, no power-on hours, no media-error count — no exporter here produces them. NVMe reports those over an admin ioctl that needs elevated privileges; temperature is the only health value the kernel publishes through sysfs. Tracked as **#549**. Do not read a green top row as a healthy drive.\n\nCeph does record wear, but only inside its own devicehealth module and not in Prometheus: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. It covers OSDs and nothing else, and a drive removed from Ceph is purged from that registry — its history goes with it, so the wear figure of a retired drive is not recoverable after the fact.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, so the drive with the least endurance left can be the one carrying container state.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
        "mode": "markdown"
      },
      "title": "Gaps and caveats",
@@ -38,9 +38,6 @@ configMapGenerator:
  - name: dashboard-blackbox-overview
    files:
      - blackbox-overview.json
-  - name: dashboard-crowdsec-overview
-    files:
-      - crowdsec-overview.json
  - name: dashboard-disk-health
    files:
      - disk-health.json
@@ -24,10 +24,8 @@ spec:
      enabled: true
      defaults:
        labels:
-          # Not required for discovery — serviceMonitorSelector is empty — but
-          # kept for consistency with the other monitors. The Probes in
-          # monitoring-rules/probes-blackbox.yaml do need it; probeSelector
-          # still matches on release.
+          # Required for kube-prometheus-stack's Prometheus to pick up
+          # this ServiceMonitor (its selector matches release).
          release: kube-prometheus-stack
        interval: 30s
    # Helm DOES deep-merge this map with the chart's default (that is why
@@ -56,21 +56,26 @@ spec:
        #
        # By default kube-prometheus-stack sets these selectors to
        # `matchLabels: {release: kube-prometheus-stack}`. ServiceMonitors
-        # created at RUNTIME by third-party operators — Rook, goauthentik.io —
-        # carry that operator's own labels instead and can never be made to
-        # match: no static manifest or Kustomize patch can relabel an object
-        # the operator creates itself. Left strict, those targets are dropped
-        # silently, with no failed resource and no gap visible anywhere except
-        # in the missing series.
+        # created at RUNTIME by third-party operators carry that operator's own
+        # labels instead and can never be made to match — no static manifest or
+        # Kustomize patch can relabel an object the operator creates itself.
        #
-        # Because these two are empty, the `release` label on this repo's own
-        # ServiceMonitors and PodMonitors is decorative. It is still set for
-        # consistency, and the comments there say so.
+        # This was silently dropping 12 of 30 ServiceMonitors (2026-07-27):
+        #   rook-ceph/rook-ceph-mgr, rook-ceph/rook-ceph-exporter  (team: rook)
+        #   authentik/ak-outpost-*  x10                            (goauthentik.io/...)
+        # Consequence: ZERO `ceph_*` series existed in Prometheus for all 79
+        # days of the cluster's life — the entire storage layer had no
+        # telemetry, no history, and any Ceph alerting rule would have had
+        # nothing to fire on. The Rook side is the reason this was found; the
+        # authentik outpost targets come along with it.
        #
-        # ruleSelector, probeSelector and scrapeConfigSelector are deliberately
-        # NOT loosened: nothing outside this repo creates those, so keeping the
-        # selectors strict stops a stray third-party rule from firing here. For
-        # PrometheusRules and Probes the label IS required.
+        # This is the ServiceMonitor twin of the trap already documented for
+        # rules in monitoring-rules/prometheusrule-velero.yaml ("without this
+        # label the rule is created but never loaded").
+        #
+        # ruleSelector is deliberately NOT loosened: this repo's PrometheusRules
+        # all carry `release: kube-prometheus-stack` on purpose, and keeping
+        # that selector strict stops a stray third-party rule from firing here.
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        # Rewrites the in-cluster URL embedded in alert generatorURLs and
@@ -15,7 +15,6 @@ resources:
  - prometheus-gate-service.yaml
  - prometheus-gate-ingress.yaml
  - ci-diagnostics-rbac.yaml
-  - smartctl-exporter.yaml
  - dashboards/
 # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
 # separate Flux Kustomization that dependsOn `monitoring` — otherwise the
@@ -1,89 +0,0 @@
-# SMART/endurance metrics for every NVMe drive in the fleet (#549).
-#
-# Runs privileged: host device access is gated by the container device cgroup,
-# and Kubernetes offers no per-device allowlist. Narrower settings do not work;
-# the alternatives that were tried are in the commit history.
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: smartctl-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: smartctl-exporter
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: smartctl-exporter
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: smartctl-exporter
-    spec:
-      # Control planes are also the storage nodes here, so the exporter has to
-      # tolerate the control-plane taint or it would cover nothing at all.
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      containers:
-        - name: smartctl-exporter
-          image: quay.io/prometheuscommunity/smartctl-exporter:v0.14.0
-          # No --smartctl.device: the exporter scans and rescans on its own.
-          # Device names are not stable across these machines.
-          args:
-            - --web.listen-address=:9633
-          ports:
-            - name: http-metrics
-              containerPort: 9633
-              protocol: TCP
-          securityContext:
-            runAsUser: 0
-            runAsNonRoot: false
-            privileged: true
-            # The API rejects privileged together with
-            # allowPrivilegeEscalation: false.
-            allowPrivilegeEscalation: true
-            readOnlyRootFilesystem: true
-          volumeMounts:
-            # Not read-only: smartmontools opens NVMe devices read-write.
-            - name: dev
-              mountPath: /dev
-            # Scratch space, because the root filesystem is read-only.
-            - name: tmp
-              mountPath: /tmp
-          resources:
-            requests:
-              cpu: 10m
-              memory: 32Mi
-            limits:
-              memory: 128Mi
-          livenessProbe:
-            httpGet:
-              path: /metrics
-              port: http-metrics
-            initialDelaySeconds: 15
-            periodSeconds: 30
-      volumes:
-        - name: dev
-          hostPath:
-            path: /dev
-            type: Directory
-        - name: tmp
-          emptyDir: {}
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: smartctl-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: smartctl-exporter
-spec:
-  clusterIP: None
-  selector:
-    app.kubernetes.io/name: smartctl-exporter
-  ports:
-    - name: http-metrics
-      port: 9633
-      targetPort: http-metrics
-      protocol: TCP
@@ -55,7 +55,7 @@ spec:
        # stdlib only; Flux keeps the tag in lockstep via the shared image
        # policy) — no extra dependency.
        - name: reconcile-publish-ip
-          image: ghcr.io/music-assistant/server:2.10.1 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.0 # {"$imagepolicy": "flux-system:music-assistant"}
          command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
          volumeMounts:
            - name: data
@@ -65,7 +65,7 @@ spec:
              readOnly: true
      containers:
        - name: music-assistant
-          image: ghcr.io/music-assistant/server:2.10.1 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.0 # {"$imagepolicy": "flux-system:music-assistant"}
          env:
            - name: LOG_LEVEL
              value: info
@@ -24,22 +24,6 @@ spec:
            - name: wyoming
              containerPort: 10400
              protocol: TCP
-          # Wyoming is raw TCP with no HTTP surface and no health verb, so
-          # accepting a connection is all that can be asserted from outside the
-          # process. Weaker than an HTTP probe, but the previous state was none
-          # at all: a dead server stayed Ready and kept being routed to.
-          livenessProbe:
-            tcpSocket:
-              port: wyoming
-            initialDelaySeconds: 60
-            periodSeconds: 30
-            timeoutSeconds: 5
-          readinessProbe:
-            tcpSocket:
-              port: wyoming
-            initialDelaySeconds: 15
-            periodSeconds: 10
-            timeoutSeconds: 5
          resources:
            requests:
              cpu: 100m
@@ -5,15 +5,25 @@ metadata:
  labels:
    app.kubernetes.io/name: otbr
 spec:
-  # DISABLED (replicas: 0). Running otbr broke Matter: its radio is RF-isolated
-  # enough from the Nest mesh that it forms a rival partition of the same
-  # network, and the two bridge only over Ethernet — fast enough for a ping,
-  # too slow for the fragmented CASE handshake, so devices stay discoverable
-  # while every commissioning attempt times out.
+  # Disabled 2026-07-11 (replicas: 0) after otbr broke Matter.
  #
-  # The instability is at the Thread partition layer, not in the config below.
-  # Re-enable only once co-BR integration is verified to MERGE: one partition,
-  # one OMR prefix, and the Nest routers visible in otbr's router table.
+  # otbr is seeded with the Nest network's dataset so it should join the
+  # existing partition as an additive co-BR. In practice it does not merge
+  # cleanly: its SLZB-06 radio is RF-isolated enough from the Nest mesh that
+  # otbr elects itself leader of a rival partition (or the network re-addresses
+  # under a new OMR prefix, observed fd5f -> fdd5). The two same-network
+  # partitions then bridge only over the Ethernet backbone, which carries a
+  # ping but is too slow for the large fragmented CASE handshake -- every
+  # Matter device stays mDNS-discoverable while CASE times out. Removing otbr
+  # re-merges the mesh onto the Nest hubs and Matter recovers.
+  #
+  # This is NOT a diagnosis of a specific config bug: the DHCP/sbr routing
+  # changes (19a4482, df61407) were investigated and matter-server's IPv6 path
+  # to the mesh works (reaches devices, ping ok); sbr's v6 table 100 is an
+  # inert orphan and otbr's eth0 default is outranked by net1's pref-high RA
+  # default. The instability is at the Thread partition layer. Re-enable only
+  # after the co-BR integration is verified to MERGE (one partition, one OMR,
+  # otbr's router table shows the Nest routers) instead of split.
  replicas: 0
  strategy:
    type: Recreate
@@ -36,17 +46,25 @@ spec:
      # (slzb-thread.renner.uno). Dropping the workload-class=pinned
      # toleration lets the edge taint keep it off the tiny edge box.
      containers:
-        # The Thread RCP is a network-attached radio (Serial-over-IP), so the
-        # spinel transport needs a TCP-to-PTY bridge. Two obvious approaches
-        # do not work and should not be retried:
+        # The Thread RCP lives on a SLZB-06 over Ethernet at
+        # slzb-thread.renner.uno:6638 (Serial-over-IP). otbr-agent's
+        # spinel transport supports `spinel+hdlc+forkpty://<exec>?<args>`
+        # which forks an exec whose stdio becomes the spinel-HDLC
+        # transport, but the upstream openthread/otbr image does NOT
+        # ship socat (or any TCP-stream tool), so that URL form ends in
+        # silent ENOENT and otbr-agent dies right after logging the
+        # radio URL.
        #
-        #   - `spinel+hdlc+forkpty://` needs socat, which the upstream image
-        #     does not ship; the URL fails with a silent ENOENT and otbr-agent
-        #     dies right after logging it.
-        #   - A sidecar cannot provide it either: devpts is per-container, so
-        #     a PTY created next door is invisible here and the symlink dangles.
+        # A sidecar socat won't work either: each container has its own
+        # devpts namespace, so a PTY (e.g. /dev/pts/0) created in a
+        # sibling container is invisible from this container — a shared
+        # emptyDir symlink to /dev/pts/0 is dangling.
        #
-        # Hence socat is installed at startup and run inside this container.
+        # The bridge has to live in the otbr container's own namespace.
+        # Install socat at startup via apt (Ubuntu 18.04 base, no egress
+        # restrictions on this namespace), run it in the background as
+        # the spinel-HDLC TCP↔PTY bridge, then exec the upstream
+        # entrypoint with RADIO_URL pointing at the PTY symlink.
        - name: otbr
          image: openthread/otbr:latest
          command:
@@ -1,116 +0,0 @@
---
-# A read-only view of paperless's backup store, used as a recovery source and
-# nothing else.
-#
-# It points at the production prefix, which is the whole point — the eval
-# instance is a copy of the real archive's metadata — and that makes what is
-# ABSENT here the safety-critical part:
-#
-#   no retentionPolicy   The production store has "30d". Without one here
-#                        there is no policy to enforce, and enforcement is not
-#                        tied to taking a backup: the plugin runs it from the
-#                        instance sidecar on a timer
-#                        (instanceSidecarConfiguration.retentionPolicyIntervalSeconds,
-#                        default 1800). So "this cluster never backs up" would
-#                        not have been enough on its own.
-#
-# A second property makes it belt-and-braces. Retention runs
-# barman-cloud-backup-delete, which takes a server name and works inside
-# <destinationPath>/<serverName> — the store's own status keys its recovery
-# window by "pprl-pg", confirming that layout. This cluster is pprleval-pg, so
-# even an accidental retention policy would address a prefix that holds nothing
-# rather than production's backups.
-#
-# The cluster below completes the guarantee with isWALArchiver: false, so
-# nothing is written back either. It does load the plugin — that is required
-# for the restore to work at all — and read-only comes from the flag rather
-# than from leaving the plugin out.
-apiVersion: barmancloud.cnpg.io/v1
-kind: ObjectStore
-metadata:
-  name: pprleval-pg-source
-  namespace: paperless-eval
-spec:
-  configuration:
-    destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg
-    endpointURL: https://fsn1.your-objectstorage.com
-    s3Credentials:
-      accessKeyId:
-        name: hetzner-s3-creds
-        key: ACCESS_KEY_ID
-      secretAccessKey:
-        name: hetzner-s3-creds
-        key: ACCESS_SECRET_KEY
-      region:
-        name: hetzner-s3-creds
-        key: REGION
-    wal:
-      compression: gzip
-    data:
-      compression: gzip
---
-# A disposable clone of the paperless database, for measuring how well the
-# local LLM classifies documents without running the experiment against the
-# live archive.
-#
-# Losing this cluster costs a re-restore and nothing else, which is the intent
-# — it is rebuilt whenever a test needs a clean starting point.
-#
-# One instance, not two. Replication buys availability, and an eval instance
-# that is down is simply an eval that waits.
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: pprleval-pg
-  namespace: paperless-eval
-spec:
-  instances: 1
-  imageName: ghcr.io/cloudnative-pg/postgresql:16
-  enableSuperuserAccess: true
-  storage:
-    # Delete-reclaim, unlike the Retain default: this volume is a copy that
-    # gets rebuilt, so it should vanish with the cluster rather than leave a
-    # Released PV and an orphaned RBD image behind (issue #262).
-    storageClass: ceph-block-ephemeral
-    size: 10Gi
-  # Declared with isWALArchiver: false, which is what kubernetes/RECOVERY.md
-  # arrived at after an earlier version of that runbook said to omit it: the
-  # instance manager loads this plugin to replay WAL during the restore, and
-  # without it the instance never starts ("Unknown plugin:
-  # barman-cloud.cloudnative-pg.io").
-  #
-  # externalClusters below only says WHERE the backup is; it does not get the
-  # plugin injected into the pod. So "no plugin" is not the way to keep this
-  # read-only — "plugin, not archiving" is, and it is the combination verified
-  # on im-pg on 2026-07-25.
-  plugins:
-    - name: barman-cloud.cloudnative-pg.io
-      enabled: true
-      isWALArchiver: false
-      parameters:
-        barmanObjectName: pprleval-pg-source
-  bootstrap:
-    recovery:
-      source: pprl-pg
-      database: paperless
-      owner: paperless
-  externalClusters:
-    - name: pprl-pg
-      plugin:
-        name: barman-cloud.cloudnative-pg.io
-        parameters:
-          barmanObjectName: pprleval-pg-source
-          serverName: pprl-pg
-  postgresql:
-    parameters:
-      shared_buffers: "256MB"
-      max_connections: "100"
-      # The production tuning for write reduction is deliberately not copied.
-      # It exists to spare the SSDs under a database that runs for years; this
-      # one is read-only for hours at a time and then deleted.
-  resources:
-    requests:
-      cpu: 200m
-      memory: 512Mi
-    limits:
-      memory: 1Gi
@@ -1,13 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: paperless-eval
-# The instance only. The measurement harness lives in scripts/paperless-eval/
-# and is applied by hand — a Job is immutable, re-running one means deleting and
-# recreating it, and Flux would just put the old one back. Nothing here should
-# converge on a finished measurement.
-resources:
-  - secrets.yaml
-  - cluster.yaml
-  - pvcs.yaml
-  - redis.yaml
-  - paperless.yaml
@@ -1,203 +0,0 @@
---
-# Deliberately much smaller than the production ConfigMap. Everything that
-# exists to serve people is gone: no remote-user auth, no CSRF origins, no
-# barcode or mail consumption, no Tika/Gotenberg. Nothing consumes documents
-# here, and nothing signs in.
-#
-# What remains is the database connection and the AI settings, because those
-# are what the evaluation measures.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: paperless-eval-config
-  namespace: paperless-eval
-data:
-  PAPERLESS_URL: "http://paperless.paperless-eval.svc.cluster.local"
-  PAPERLESS_ALLOWED_HOSTS: "*"
-  PAPERLESS_TIME_ZONE: "Europe/Berlin"
-  PAPERLESS_REDIS: "redis://redis:6379"
-  PAPERLESS_DBENGINE: "postgresql"
-  PAPERLESS_DBHOST: "pprleval-pg-rw"
-  PAPERLESS_DBNAME: "paperless"
-  # Polling off, so a pod that finds an empty consume volume does not spend a
-  # thread looking at it every ten seconds for the length of a run.
-  PAPERLESS_CONSUMER_POLLING_INTERVAL: "0"
-
-  # The AI settings under test. The endpoints are placeholders that each eval
-  # job overrides for the model it is measuring — a job runs its own
-  # llama-server beside paperless in the same pod, so the value that matters at
-  # run time is the one in the job, not the one here.
-  #
-  # Same caveat as production, and sharper here: paperless reads these from an
-  # ApplicationConfiguration row and falls back to the environment only when
-  # that row is empty. The row comes across in the restored database, so
-  # anything configured in the production admin UI arrives with it and silently
-  # wins over this ConfigMap — an eval would then measure production's settings
-  # while appearing to measure the job's.
-  #
-  # Check it before trusting any result, and after every fresh restore:
-  #
-  #   select llm_backend, llm_model, llm_embedding_backend, llm_embedding_model
-  #     from paperless_applicationconfiguration;
-  #
-  # Every column must be null for the environment to apply.
-  PAPERLESS_AI_ENABLED: "true"
-  PAPERLESS_AI_LLM_BACKEND: "openai-like"
-  PAPERLESS_AI_LLM_ENDPOINT: "http://127.0.0.1:8080/v1"
-  PAPERLESS_AI_LLM_MODEL: "qwen3.6-35b-a3b"
-  PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS: "true"
-  PAPERLESS_AI_LLM_CONTEXT_SIZE: "12288"
-  PAPERLESS_AI_LLM_REQUEST_TIMEOUT: "900"
-  PAPERLESS_AI_LLM_OUTPUT_LANGUAGE: "de-de"
-  PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
-  PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
-  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
-  PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
---
-# A base instance that proves the restored database is usable and that the
-# search index and classifier build against it. The evaluation itself does not
-# run here — each job brings its own paperless container so it can point at its
-# own model — but this is what makes a broken clone visible before a job spends
-# hours on it.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless
-  namespace: paperless-eval
-  labels:
-    app.kubernetes.io/name: paperless
-  annotations:
-    configmap.reloader.stakater.com/reload: paperless-eval-config
-    # Both, like production: the secret refreshes on the same 10m cycle, and
-    # without this the pod would keep a rotated key until something else
-    # restarted it — which defeats the point of matching the interval.
-    secret.reloader.stakater.com/reload: paperless-eval-secrets
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless
-    spec:
-      enableServiceLinks: false
-      securityContext:
-        fsGroup: 1000
-        fsGroupChangePolicy: OnRootMismatch
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: paperless
-          # No $imagepolicy marker, unlike apps/paperless: a measurement is
-          # only comparable against a fixed version, so this tag moves when
-          # somebody decides to re-measure, not when a release lands.
-          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
-          ports:
-            - name: http
-              containerPort: 8000
-          envFrom:
-            - configMapRef:
-                name: paperless-eval-config
-          env:
-            - name: PAPERLESS_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-eval-secrets
-                  key: secret-key
-            - name: PAPERLESS_DBUSER
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: username
-            - name: PAPERLESS_DBPASS
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: password
-          volumeMounts:
-            - name: data
-              mountPath: /usr/src/paperless/data
-            - name: media
-              mountPath: /usr/src/paperless/media
-            - name: consume
-              mountPath: /usr/src/paperless/consume
-          # 60 minutes, against production's 10, and matching this app's Flux
-          # Kustomization timeout.
-          #
-          # Production's budget was sized for a from-scratch boot on this same
-          # corpus and held, so 10 would probably do — but there the volume is
-          # Retain and the index survives, making that a one-off. Here the
-          # volume is ephemeral and the index is rebuilt on every re-test, so
-          # the expensive path is the normal one. A killed pod starts the work
-          # over rather than resuming, which turns a slow boot into a
-          # crash-loop that stalls Flux behind wait: true.
-          #
-          # The LLM index is not in this budget: it is a celery task, and boot
-          # only runs init-llmindex-migrate, a migration check.
-          startupProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            periodSeconds: 20
-            timeoutSeconds: 5
-            failureThreshold: 180
-          livenessProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            initialDelaySeconds: 120
-            periodSeconds: 30
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            timeoutSeconds: 5
-          resources:
-            requests:
-              cpu: 200m
-              memory: 512Mi
-            limits:
-              memory: 2Gi
-          # Production's context verbatim, and it has to be: the paperless-ngx
-          # entrypoint starts as root, chowns the mounted data/media/consume
-          # volumes and then drops to uid 1000 via setgid/setuid. Without these
-          # the same image fails that sequence with EPERM.
-          #
-          # It bites harder here than in production, where those volumes were
-          # chowned once years ago and keep their ownership. These are fresh on
-          # every rebuild, so the chown is not a no-op.
-          securityContext:
-            runAsNonRoot: false
-            allowPrivilegeEscalation: true
-            capabilities:
-              drop: ["ALL"]
-              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: paperless-eval-golden
-        - name: media
-          persistentVolumeClaim:
-            claimName: paperless-eval-media
-        - name: consume
-          persistentVolumeClaim:
-            claimName: paperless-eval-consume
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: paperless
-  namespace: paperless-eval
-spec:
-  selector:
-    app.kubernetes.io/name: paperless
-  ports:
-    - port: 80
-      targetPort: http
@@ -1,129 +0,0 @@
---
-# All four use ceph-block-ephemeral rather than the Retain default. Everything
-# here is derived — a restored database, an index built from it, a cache —  so
-# the volumes should go when the namespace does instead of leaving Released PVs
-# and orphaned RBD images to reclaim by hand (issue #262).
-#
-# /data holds the Tantivy search index and the classifier pickle, both rebuilt
-# from the database, and the LLM index — which is the expensive one at roughly
-# three hours for the full corpus.
-#
-# Measured on production for the same ~3900 documents: 36 MiB search index,
-# 125 MiB LLM index. 10Gi is far more than that needs, and deliberately so —
-# an embedding model with wider vectors than bge-m3's 1024 dimensions is
-# exactly what phase two varies, and running out of space mid-run would cost
-# the whole rebuild.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  # Named for its role rather than its mount point: the smoke-test deployment
-  # builds /data here, and every eval job copies it out. Replaces the earlier
-  # RWO paperless-eval-data, whose access mode could not be widened in place —
-  # a bound claim's spec is immutable.
-  name: paperless-eval-golden
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  # RWX, because the eval jobs read it too — each copies it into its own
-  # emptyDir rather than mounting it live. The vector store is SQLite, and
-  # SQLite locking across pods over CephFS is not something to rely on.
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem-ephemeral
-  resources:
-    requests:
-      storage: 10Gi
---
-# Empty, and stays that way. Paperless refuses to start without the path, but
-# the AI path reads title, content and filename out of the database and never
-# opens a file — so the documents themselves are not copied here. That is what
-# keeps this instance cheap and keeps a second copy of the archive off the
-# cluster.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-media
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
---
-# Also empty: nothing is consumed here, but the path has to exist.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-consume
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
---
-# The eval jobs' model cache, RWX so several jobs can read the same weights at
-# once — that is the whole point of measuring two candidates in parallel.
-#
-# Separate from the ai namespace's llama-server-models despite holding some of
-# the same files: PVCs do not cross namespaces. A model therefore downloads
-# once here and is then shared by every job, which is the cost a new candidate
-# carries anyway.
-#
-# NOT ephemeral, unlike everything else in this namespace: re-downloading tens
-# of gigabytes on a slow uplink is a worse outcome than an orphaned volume.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: llama-models
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 120Gi
---
-# Results outlive the jobs that produce them: a Job is deleted and recreated to
-# re-run a configuration, and comparing configurations is the entire exercise.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-results
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 1Gi
---
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-redis
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
@@ -1,81 +0,0 @@
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis
-  namespace: paperless-eval
-  labels:
-    app.kubernetes.io/name: redis
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: redis
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: redis
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 999
-        fsGroup: 999
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: redis
-          # No $imagepolicy marker; see the note on the paperless image.
-          image: redis:8.10.1-alpine
-          # No --save: this holds the celery queue and the suggestion cache,
-          # both of which are regenerated. Persisting them across a restart of
-          # a throwaway instance would only preserve stale suggestions.
-          args:
-            - redis-server
-            - --save
-            - ""
-            - --loglevel
-            - warning
-          ports:
-            - name: redis
-              containerPort: 6379
-          volumeMounts:
-            - name: data
-              mountPath: /data
-          livenessProbe:
-            tcpSocket:
-              port: redis
-            initialDelaySeconds: 10
-          readinessProbe:
-            exec:
-              command: ["redis-cli", "ping"]
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 20m
-              memory: 64Mi
-            limits:
-              memory: 256Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: paperless-eval-redis
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-  namespace: paperless-eval
-spec:
-  selector:
-    app.kubernetes.io/name: redis
-  ports:
-    - port: 6379
-      targetPort: redis
--- a/Show More
+++ b/Show More