fix: [alert] KubePodNotReady (ai) (#880)

.gitignore

@@ -20,5 +20,11 @@ kubernetes/access-staging/
 trivy-iac.json
 trivy-image-*.json
 
+# Alert-monitor pipeline workspace (generated in CI, never commit)
+.ci-alert-monitor/
+
+# CI diagnostic kubeconfig + kubectl for the agent steps (generated, never commit)
+.ci-diag/
+
 # Plaintext secret templates awaiting SOPS encryption (never commit)
 *.plaintext.yaml

.woodpecker/alert-monitor.yml

@@ -0,0 +1,298 @@
+# Alert monitor — every 15 minutes, read the active alerts from Alertmanager.
+# When a NEW alert is firing, an AI agent triages it against the repo and the
+# cluster context gathered here; alerts it confirms as real problems become
+# Gitea issues, which the issue-fix pipeline (issue-fix.yml) later turns into
+# PRs. Alerts it classifies as noise are recorded in a persistent "Alert
+# monitor: noise log" issue so they are never re-triaged (delete a line from
+# that issue to un-suppress an alert).
+#
+# Alertmanager access: the woodpecker-ci namespace cannot reach monitoring
+# ClusterIPs directly (Cilium default-deny-ingress on monitoring, see
+# kubernetes/infrastructure/network-policies/monitoring-ns.yaml), and the
+# public hostname alerts.renner.uno sits behind the Authentik gate. The
+# apiserver *service proxy* path works: apiserver→pod traffic arrives with
+# host/remote-node identity, which that policy explicitly allows. Verified
+# 2026-07-16 with `kubectl get --raw .../services/kube-prometheus-stack-
+# alertmanager:9093/proxy/api/v2/alerts`.
+#
+# Step split: the ci-k8s image has kubectl/flux but NO jq; the ci-claude image
+# has jq but no kubectl. So the fetch step only talks to the cluster and dumps
+# raw files into the workspace, and every JSON decision (heartbeat filter,
+# dedup, gating) lives in the agent step.
+#
+# Dedup contract (what makes 15-minute re-runs cheap): an alert is dropped
+# before the agent ever sees it when EITHER
+#   - an open issue titled exactly "[alert] <alertname> (<namespace|job|cluster>)"
+#     exists (namespace first, job as fallback for aggregate alerts), OR
+#   - its Alertmanager fingerprint appears in the noise-log issue body.
+# Runs where nothing new is firing stop before the agent — no tokens spent.
+#
+# Triggering:
+#   - cron   : a cron NAMED exactly "alert-monitor" (name filter, same
+#              mechanism as update-audit.yml):
+#                woodpecker repo cron add --repo k.renner/homelab \
+#                  --name alert-monitor --branch master --schedule '*/15 * * * *'
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> alert-monitor
+#              where <n> is an existing pipeline to promote.
+when:
+  - event: cron
+    cron: alert-monitor
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "alert-monitor"'
+
+steps:
+  - name: fetch-alerts
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    environment:
+      SOPS_AGE_KEY:
+        from_secret: sops_age_key
+      S3_ACCESS_KEY_ID:
+        from_secret: hetzner_s3_access_key
+      S3_SECRET_ACCESS_KEY:
+        from_secret: hetzner_s3_secret_key
+    commands:
+      - bash scripts/ci/fetch-kubeconfig.sh
+      - |
+        # Active, unsilenced, uninhibited alerts — raw dump; filtering happens
+        # in the agent step (no jq in this image).
+        set -eu
+        mkdir -p .ci-alert-monitor
+        AM="/api/v1/namespaces/monitoring/services/kube-prometheus-stack-alertmanager:9093/proxy"
+        kubectl get --raw "$$AM/api/v2/alerts?active=true&silenced=false&inhibited=false" \
+          > .ci-alert-monitor/alerts-raw.json
+        head -c 400 .ci-alert-monitor/alerts-raw.json; echo
+      - |
+        # Blip lookback: an alert that fires and resolves BETWEEN ticks is
+        # invisible to the live snapshot (a 17-minute KubeHpaMaxedOut escaped
+        # exactly this way on 2026-07-16). Scan Prometheus's ALERTS history
+        # over ~2 tick intervals + jitter; the overlap between runs is
+        # deduped downstream like everything else.
+        set -eu
+        PROM="/api/v1/namespaces/monitoring/services/kube-prometheus-stack-prometheus:9090/proxy"
+        kubectl get --raw "$$PROM/api/v1/query?query=max%20by%20(alertname,namespace,job,severity)%20(max_over_time(ALERTS%7Balertstate%3D%22firing%22,alertname!~%22Watchdog%7CInfoInhibitor%22%7D%5B35m%5D))" \
+          > .ci-alert-monitor/blips-raw.json \
+          || printf '{"data":{"result":[]}}' > .ci-alert-monitor/blips-raw.json
+        # (no jq in this image — counting happens in the agent step)
+        head -c 300 .ci-alert-monitor/blips-raw.json; echo
+      - |
+        # Cluster context for the triage agent — gathered here because the
+        # agent itself gets NO cluster access (no Bash, no kubeconfig). Runs
+        # unconditionally (a handful of read-only calls every 15 min is
+        # cheaper than teaching this jq-less step to gate on the JSON).
+        set -eu
+        {
+          echo "=== nodes ==="
+          kubectl get nodes -o wide
+          echo "=== pods not Running/Succeeded ==="
+          kubectl get pods -A --no-headers | grep -Ev 'Running|Completed|Succeeded' || echo "(none)"
+          echo "=== warning events (most recent last, tail 100) ==="
+          kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -100
+          echo "=== flux kustomizations not ready ==="
+          flux get kustomizations -A --status-selector ready=false 2>&1 || true
+          echo "=== flux helmreleases not ready ==="
+          flux get helmreleases -A --status-selector ready=false 2>&1 || true
+        } > .ci-alert-monitor/context.txt
+        wc -l .ci-alert-monitor/context.txt
+      - |
+        # Least-privilege live-diagnostics kubeconfig (+ kubectl binary — the
+        # ci-claude image ships none) so the triage agent can VERIFY instead
+        # of hypothesize. Degrades to snapshot-only if the ServiceAccount
+        # isn't reconciled yet.
+        bash scripts/ci/make-diag-kubeconfig.sh
+
+  - name: alert-triage-agent
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    environment:
+      CLAUDE_CODE_OAUTH_TOKEN:
+        from_secret: claude_code_oauth_token
+      # Dedicated identity: alert-monitor-bot, PAT scoped to write:issue ONLY.
+      # This step runs an LLM agent over untrusted input (alert annotations,
+      # cluster event text) and its output is published verbatim as a public
+      # issue body, so the credential it holds must not be able to do anything
+      # but write issues. The account is a write collaborator, but the PAT
+      # scope is the real cap: verified 403 on /user/repos and /admin/users.
+      # Replaces the admin-owned gitea_token that used to sit here (#330).
+      GITEA_TOKEN:
+        from_secret: alert_monitor_bot_token
+    commands:
+      - |
+        # Filter + dedup + gate. Drops the always-firing heartbeats
+        # (Watchdog, InfoInhibitor), alerts routed ONLY to the `null`
+        # receiver (the owner already declared those unactionable in the
+        # Alertmanager route config — triaging them re-litigates a settled
+        # decision; found via #252/KubeCPUOvercommit), alerts that already
+        # have an open "[alert] ..." issue, and fingerprints recorded in the
+        # noise log. The repo is public, so these reads need no token. Exits
+        # the step — agent never starts — when nothing new is firing.
+        set -eu
+        jq '[.[]
+             | select(.labels.alertname != "Watchdog" and .labels.alertname != "InfoInhibitor")
+             | select([.receivers[]?.name] | (length == 0) or any(. != "null"))]' \
+          .ci-alert-monitor/alerts-raw.json > .ci-alert-monitor/alerts-active.json
+        echo "Active alerts (after heartbeat filter): $$(jq length .ci-alert-monitor/alerts-active.json)"
+        API="https://gitea.renner.uno/api/v1/repos/k.renner/homelab"
+        curl -sfL "$$API/issues?state=open&type=issues&limit=100" > .ci-alert-monitor/open-issues.json
+        TITLES=$$(jq -c '[.[].title]' .ci-alert-monitor/open-issues.json)
+        NOISE_BODY=$$(jq -r 'map(select(.title == "Alert monitor: noise log")) | .[0].body // ""' \
+          .ci-alert-monitor/open-issues.json)
+        # Suppression entries are exactly the structured lines the write-back
+        # step emits: "- <fp> <ALERTNAME> — <YYYY-MM-DD>: <reason>". Parse THAT
+        # shape rather than grepping the whole body for 16-hex tokens. A reason
+        # routinely cites another alert's fingerprint in prose ("see the issue
+        # filed for fingerprint <fp>"), and the body-wide grep turned every such
+        # citation into a silent, permanent suppression of an alert that was
+        # never classified as noise — verified against the live noise log, which
+        # yielded 22 fingerprints for 21 real entries, the extra one being
+        # 2ddc41edb689fc33, an alert that HAD been filed as a genuine issue.
+        #
+        # Entries also expire after NOISE_TTL_DAYS. A noise verdict says "this
+        # occurrence was benign", not "this alert can never be interesting
+        # again" — but the fingerprint is sha256(alertname|scope) and therefore
+        # stable across recurrences, so without an expiry a single blip mutes
+        # that alertname+scope pair forever. Re-triaging a recurrence weeks
+        # later is cheap; missing a real outage because a July blip muted
+        # openbao's KubePodNotReady is not.
+        #
+        # Lines that fail to parse are simply dropped, i.e. they stop
+        # suppressing — this fails toward MORE alerting, never less.
+        #
+        # date is BusyBox here, not GNU: `date -d '30 days ago'` errors out, so
+        # the cutoff goes through epoch arithmetic and `date -d @<epoch>`, which
+        # this image does support.
+        NOISE_TTL_DAYS=$${NOISE_TTL_DAYS:-30}
+        CUTOFF=$$(date -u -d @$$(( $$(date -u +%s) - NOISE_TTL_DAYS * 86400 )) +%F)
+        FPS=$$(printf '%s\n' "$$NOISE_BODY" \
+          | sed -nE 's/^- ([0-9a-f]{16}) [^ ]+ — ([0-9]{4}-[0-9]{2}-[0-9]{2}):.*/\2 \1/p' \
+          | awk -v c="$$CUTOFF" 'BEGIN{gsub(/-/,"",c)} { d=$$1; gsub(/-/,"",d); if (d+0 >= c+0) print $$2 }' \
+          | jq -R -s -c 'split("\n") | map(select(length > 0))')
+        echo "Noise suppressions in effect: $$(printf '%s' "$${FPS:-[]}" | jq length)" \
+             "(TTL $${NOISE_TTL_DAYS}d, entries dated >= $$CUTOFF)"
+        # Synthesize pseudo-alerts for BLIPS: lookback series whose label set
+        # is not currently firing at all (checked against the PRE-filter raw
+        # snapshot, so live-but-null-routed alerts don't reappear as blips).
+        # Pseudo-fingerprint = sha256(alertname|scope)[:16] — STABLE across
+        # recurrences, so one noise verdict suppresses a flappy alert
+        # permanently via the noise log.
+        : > .ci-alert-monitor/blips-new.jsonl
+        jq -c '.data.result[]?.metric // empty' .ci-alert-monitor/blips-raw.json | while IFS= read -r M; do
+          NAME=$$(printf '%s' "$$M" | jq -r '.alertname')
+          SCOPE=$$(printf '%s' "$$M" | jq -r '.namespace // .job // "cluster"')
+          if jq -e --arg n "$$NAME" --arg s "$$SCOPE" \
+               '[.[] | select(.labels.alertname == $$n and ((.labels.namespace // .labels.job // "cluster") == $$s))] | length > 0' \
+               .ci-alert-monitor/alerts-raw.json > /dev/null; then
+            continue
+          fi
+          FP=$$(printf '%s|%s' "$$NAME" "$$SCOPE" | sha256sum | cut -c1-16)
+          jq -n --arg fp "$$FP" --argjson m "$$M" \
+            '{labels: $$m, fingerprint: $$fp, blip: true,
+              annotations: {summary: "BLIP: fired within the lookback window (~35 min) and already resolved — caught by the ALERTS history scan, not currently active."}}' \
+            >> .ci-alert-monitor/blips-new.jsonl
+        done
+        jq -s '.' .ci-alert-monitor/blips-new.jsonl > .ci-alert-monitor/blips-candidates.json
+        # Title key: namespace, else job, else "cluster". The job fallback
+        # exists because aggregate alerts like TargetDown carry no namespace
+        # — with only "(cluster)" two simultaneous TargetDowns for different
+        # jobs (pve and dns-servers, 2026-07-16) collapsed into ONE issue
+        # and the second root cause was never triaged.
+        jq -s 'add' .ci-alert-monitor/alerts-active.json .ci-alert-monitor/blips-candidates.json \
+          > .ci-alert-monitor/alerts-combined.json
+        jq --argjson titles "$$TITLES" --argjson fps "$${FPS:-[]}" '
+          [ .[]
+            | ("[alert] " + .labels.alertname + " (" + (.labels.namespace // .labels.job // "cluster") + ")") as $$t
+            | select( ($$t | IN($$titles[]) | not) and (.fingerprint | IN($$fps[]) | not) )
+            | . + {issue_title: $$t} ]' \
+          .ci-alert-monitor/alerts-combined.json > .ci-alert-monitor/alerts-new.json
+        NEW=$$(jq length .ci-alert-monitor/alerts-new.json)
+        jq -r '.[] | "\(.fingerprint)\t\(.issue_title)"' .ci-alert-monitor/alerts-new.json
+        if [ "$$NEW" -eq 0 ]; then
+          echo "No new alerts — agent not started."
+          exit 0
+        fi
+      - |
+        # Headless triage run. Same auth model as security-scan.yml
+        # (subscription OAuth, prompt piped via stdin — see the comments
+        # there). Tool allowlist is read-only on the repo plus Write for the
+        # /tmp verdict file, plus EXACTLY ONE live command: the RBAC-scoped
+        # kubectl from make-diag-kubeconfig.sh (read-only ClusterRole, no
+        # Secrets). The prefix match means no pipes/redirects and no other
+        # binaries — the agent verifies cluster state, it cannot mutate it,
+        # fetch the web, or read secrets.
+        export KUBECONFIG="$$PWD/.ci-diag/kubeconfig"
+        # Model chosen live from the catalog (not hardcoded, no family assumed) —
+        # see the header of scripts/ci/pick-model.sh. Empty MODEL -> CLI default.
+        MODEL=$$(sh scripts/ci/pick-model.sh "triage firing Prometheus/Alertmanager alerts against read-only live cluster state and classify each as a real problem or noise; careful diagnostic reasoning, no changes")
+        if [ -n "$$MODEL" ]; then echo "alert-monitor: selected model $$MODEL"; else echo "alert-monitor: no live catalog — using the CLI default model"; fi
+        cat ci/agents/alert-monitor/prompt.md | claude $${MODEL:+--model $$MODEL} \
+          --permission-mode acceptEdits \
+          --allowedTools "Read,Glob,Grep,Write,Bash(./.ci-diag/kubectl:*)" \
+          --settings ci/agents/alert-monitor/settings.json \
+          -p
+      - |
+        # Post the verdicts: one Gitea issue per confirmed problem, and one
+        # appended line in the persistent noise log per noise verdict. Every
+        # write checks its HTTP status explicitly (see update-audit.yml for
+        # why piping into `curl -sf | jq` is not enough).
+        set -eu
+        if [ ! -s /tmp/alert-verdicts.json ]; then
+          echo "Agent wrote no verdicts (/tmp/alert-verdicts.json missing) — nothing to post."
+          exit 0
+        fi
+        jq . /tmp/alert-verdicts.json > /dev/null   # fail loudly on malformed output
+        API="https://gitea.renner.uno/api/v1/repos/k.renner/homelab"
+        STAMP=$$(date -u +%Y-%m-%d)
+
+        gitea_write() {
+          code=$$(curl -s -o /tmp/gitea-resp.json -w '%{http_code}' \
+            -X "$$1" -H "Authorization: token $$GITEA_TOKEN" \
+            -H "Content-Type: application/json" --data @"$$3" "$$2")
+          echo "$$1 $$2 -> HTTP $$code"
+          if [ "$$code" -lt 200 ] || [ "$$code" -ge 300 ]; then
+            echo "Gitea API error body:"; cat /tmp/gitea-resp.json; echo
+            return 1
+          fi
+        }
+
+        N=$$(jq length /tmp/alert-verdicts.json)
+        i=0
+        : > /tmp/noise-lines.txt
+        while [ "$$i" -lt "$$N" ]; do
+          VERDICT=$$(jq -r ".[$$i].verdict" /tmp/alert-verdicts.json)
+          TITLE=$$(jq -r ".[$$i].title" /tmp/alert-verdicts.json)
+          FP=$$(jq -r ".[$$i].fingerprint" /tmp/alert-verdicts.json)
+          NAME=$$(jq -r ".[$$i].alertname" /tmp/alert-verdicts.json)
+          REASON=$$(jq -r ".[$$i].reason" /tmp/alert-verdicts.json)
+          if [ "$$VERDICT" = "problem" ]; then
+            # Safety re-check: never double-file if a parallel run won the race.
+            EXISTS=$$(curl -sfL "$$API/issues?state=open&type=issues&limit=100" \
+              | jq -r --arg t "$$TITLE" 'map(select(.title == $$t)) | length')
+            if [ "$$EXISTS" -gt 0 ]; then
+              echo "Issue already open for '$$TITLE' — skipping."
+            else
+              jq --arg footer "$$(printf '\n\n---\n<!-- alert-monitor:fingerprint=%s -->\n_Filed automatically by the alert-monitor pipeline (%s). The issue-fix pipeline may pick this up and open a PR._' "$$FP" "$$STAMP")" \
+                 --argjson idx "$$i" \
+                 '.[$$idx] | {title: .title, body: (.body + $$footer)}' \
+                 /tmp/alert-verdicts.json > /tmp/gitea-create.json
+              gitea_write POST "$$API/issues" /tmp/gitea-create.json
+              jq '{number, html_url, title}' /tmp/gitea-resp.json
+            fi
+          else
+            printf -- '- %s %s — %s: %s\n' "$$FP" "$$NAME" "$$STAMP" "$$REASON" >> /tmp/noise-lines.txt
+          fi
+          i=$$((i+1))
+        done
+
+        if [ -s /tmp/noise-lines.txt ]; then
+          NOISE_NUM=$$(curl -sfL "$$API/issues?state=open&type=issues&limit=100" \
+            | jq -r 'map(select(.title == "Alert monitor: noise log")) | .[0].number // empty')
+          if [ -n "$$NOISE_NUM" ]; then
+            curl -sfL "$$API/issues/$$NOISE_NUM" | jq -r '.body // ""' > /tmp/noise-old.txt
+            jq -n --rawfile old /tmp/noise-old.txt --rawfile add /tmp/noise-lines.txt \
+              '{body: ($$old + "\n" + $$add)}' > /tmp/gitea-patch.json
+            gitea_write PATCH "$$API/issues/$$NOISE_NUM" /tmp/gitea-patch.json
+          else
+            printf '%s\n\n' "Fingerprints the alert-monitor triage agent classified as noise. The dedup step skips any alert whose fingerprint appears below — delete a line to make that alert eligible for triage again. Keep this issue open; it is the suppression state. Entries EXPIRE after NOISE_TTL_DAYS (default 30) counted from the date in the line, so a one-off blip stops suppressing its alertname+scope pair once the window passes; re-add happens automatically if it is triaged as noise again. Only lines in the exact '- <fingerprint> <ALERTNAME> — <YYYY-MM-DD>: <reason>' form suppress anything — a fingerprint mentioned in prose does not." > /tmp/noise-header.txt
+            jq -n --rawfile hdr /tmp/noise-header.txt --rawfile add /tmp/noise-lines.txt \
+              '{title: "Alert monitor: noise log", body: ($$hdr + $$add)}' > /tmp/gitea-create.json
+            gitea_write POST "$$API/issues" /tmp/gitea-create.json
+          fi
+        fi

.woodpecker/build-ci-images.yml

@@ -26,29 +26,20 @@
 # 38-reference change tracked separately in #330 rather than bundled here.
 # Note this pipeline runs no LLM agent, so the exfil path that motivated the
 # other cutovers does not apply to it.
-#
-# Push builds run on main only. Without the branch filter a feature branch
-# that touches this file started all steps at once: it pushed unreviewed
-# images and used the whole woodpecker-ci-quota, so the PR checks could not
-# get a pod (REN-286).
-#
-# Steps run in sequence (no depends_on): a change to this file rebuilds every
-# image, and in parallel they exceed the woodpecker-ci-quota.
 when:
   - event: push
-    branch: main
     path:
       - "ci/images/**"
       - "secret-rotator/**"
-      - "review-bot/**"
       - ".woodpecker/build-ci-images.yml"
   - event: deployment
     evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-ci-images"'
 
 steps:
   - name: build-ansible
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/ansible
+    depends_on: []
     # Only rebuild when THIS image's source (or the pipeline file) changed, or
     # on a manual run. Without this every step ran on any matching push, so a
     # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
@@ -74,8 +65,9 @@ steps:
           --destination gitea.renner.uno/gitea_admin/ci-ansible:latest
 
   - name: build-k8s
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/k8s
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -97,8 +89,9 @@ steps:
           --destination gitea.renner.uno/gitea_admin/ci-k8s:latest
 
   - name: build-vmtest
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/vmtest
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -120,8 +113,9 @@ steps:
           --destination gitea.renner.uno/gitea_admin/ci-vmtest:latest
 
   - name: build-tofu
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/tofu
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -150,8 +144,9 @@ steps:
   # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
   # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
   - name: build-secret-rotator
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: secret-rotator
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -172,31 +167,6 @@ steps:
           --destination gitea.renner.uno/gitea_admin/secret-rotator:$TAG \
           --destination gitea.renner.uno/gitea_admin/secret-rotator:latest
 
-  # review-bot: the Go service that reviews pull requests with a Mistral agent (webhook plus sweep).
-  # Runs after Gitea is up, so no ghcr mirror.
-  - name: build-review-bot
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: review-bot
-    when:
-      - event: deployment
-      - event: push
-        path: ["review-bot/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d-%H%M)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/review-bot:$TAG \
-          --destination gitea.renner.uno/gitea_admin/review-bot:latest
-
   # Runtime image, not CI tooling: a general-purpose alpine + jq base for
   # workloads that need to read JSON from a shell script. Named for what it IS,
   # not for who uses it — its first consumer is the zigbee2mqtt seed-data init
@@ -225,8 +195,9 @@ steps:
   #
   # The gitea_admin/ copy stays until nothing references it any more.
   - name: build-alpine-jq
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/alpine-jq
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -243,8 +214,7 @@ steps:
       - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
       - GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
       - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
-      # konairius/ gets one tag per build: Gitea's cleanup rule counts tags.
-      - TAG=$(date +%Y-%m-%d-%H%M)
+      - TAG=$(date +%Y-%m-%d)
       - |
         /kaniko/executor \
           --context . \
@@ -252,7 +222,9 @@ steps:
           --destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
           --destination gitea.renner.uno/gitea_admin/alpine-jq:latest \
           --destination gitea.renner.uno/konairius/alpine-jq:$TAG \
-          --destination ghcr.io/konairius/alpine-jq:$TAG
+          --destination gitea.renner.uno/konairius/alpine-jq:latest \
+          --destination ghcr.io/konairius/alpine-jq:$TAG \
+          --destination ghcr.io/konairius/alpine-jq:latest
 
   # Runtime image, not CI tooling: upstream containernetworking/plugins binaries
   # baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
@@ -276,8 +248,9 @@ steps:
   # note on build-alpine-jq. Anything an app needs while the cluster is still
   # being built cannot live only in that cluster's own registry.
   - name: build-cni-plugins
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/cni-plugins
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -294,8 +267,7 @@ steps:
       - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
       - GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
       - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
-      # konairius/ gets one tag per build: Gitea's cleanup rule counts tags.
-      - TAG=$(date +%Y-%m-%d-%H%M)
+      - TAG=$(date +%Y-%m-%d)
       - |
         /kaniko/executor \
           --context . \
@@ -303,7 +275,9 @@ steps:
           --destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
           --destination gitea.renner.uno/gitea_admin/cni-plugins:latest \
           --destination gitea.renner.uno/konairius/cni-plugins:$TAG \
-          --destination ghcr.io/konairius/cni-plugins:$TAG
+          --destination gitea.renner.uno/konairius/cni-plugins:latest \
+          --destination ghcr.io/konairius/cni-plugins:$TAG \
+          --destination ghcr.io/konairius/cni-plugins:latest
 
   # Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
   # needing s_client/x509. Named for what it is; first consumer is the paperless
@@ -316,8 +290,9 @@ steps:
   # gitea.renner.uno is not there to serve it. A cold-rebuild rehearsal left
   # paperless in Init:ErrImagePull on exactly this image.
   - name: build-alpine-openssl
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/alpine-openssl
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -334,8 +309,7 @@ steps:
       - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
       - GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
       - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
-      # konairius/ gets one tag per build: Gitea's cleanup rule counts tags.
-      - TAG=$(date +%Y-%m-%d-%H%M)
+      - TAG=$(date +%Y-%m-%d)
       - |
         /kaniko/executor \
           --context . \
@@ -343,7 +317,9 @@ steps:
           --destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
           --destination gitea.renner.uno/gitea_admin/alpine-openssl:latest \
           --destination gitea.renner.uno/konairius/alpine-openssl:$TAG \
-          --destination ghcr.io/konairius/alpine-openssl:$TAG
+          --destination gitea.renner.uno/konairius/alpine-openssl:latest \
+          --destination ghcr.io/konairius/alpine-openssl:$TAG \
+          --destination ghcr.io/konairius/alpine-openssl:latest
 
   # Heating CAN readers: open3e (Vitocal/VX3) + E3onCAN (E380 meter), one image
   # run as two services on edge-1. Replaces pip-into-venv on the host, where an
@@ -351,8 +327,9 @@ steps:
   # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
   # builds the same way and lands in the same registry so the audit sees it.
   - name: build-heating-can
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/heating-can
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -373,16 +350,17 @@ steps:
           --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
           --destination gitea.renner.uno/gitea_admin/heating-can:latest
 
-  # netbird-service-sync: the controller that reconciles NetBird reverse-proxy
+  # netbird-proxy-services: the controller that reconciles NetBird reverse-proxy
   # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
   # Runtime artifact like heating-can; same registry so the audit tracks it.
-  - name: build-netbird-service-sync
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: ci/images/netbird-service-sync
+  - name: build-netbird-proxy-services
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images/netbird-proxy-services
+    depends_on: []
     when:
       - event: deployment
       - event: push
-        path: ["ci/images/netbird-service-sync/**", ".woodpecker/build-ci-images.yml"]
+        path: ["ci/images/netbird-proxy-services/**", ".woodpecker/build-ci-images.yml"]
     environment:
       GITEA_USER: gitea_admin
       GITEA_TOKEN:
@@ -396,18 +374,19 @@ steps:
         /kaniko/executor \
           --context . \
           --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/netbird-service-sync:$TAG \
-          --destination gitea.renner.uno/gitea_admin/netbird-service-sync:latest
+          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
+          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
 
-  # authentik-gate-sync: the authentik-gate ingress class (Ingress -> Authentik
-  # proxy gate + Traefik route). Its unit tests run as a stage of the image build.
-  - name: build-authentik-gate-sync
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: ci/images/authentik-gate-sync
+  # wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea and
+  # queried by ~/Notes/bin/wiki_rank.py. Runtime artifact like netbird-proxy-services.
+  - name: build-wiki-index
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images/wiki-index
+    depends_on: []
     when:
       - event: deployment
       - event: push
-        path: ["ci/images/authentik-gate-sync/**", ".woodpecker/build-ci-images.yml"]
+        path: ["ci/images/wiki-index/**", ".woodpecker/build-ci-images.yml"]
     environment:
       GITEA_USER: gitea_admin
       GITEA_TOKEN:
@@ -416,72 +395,20 @@ steps:
       - mkdir -p /kaniko/.docker
       - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
       - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      # YYYY-MM-DD-HHMM: the image-automation ImagePolicy picks the newest.
-      - TAG=$(date +%Y-%m-%d-%H%M)
+      - TAG=$(date +%Y-%m-%d)
       - |
         /kaniko/executor \
           --context . \
           --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/authentik-gate-sync:$TAG \
-          --destination gitea.renner.uno/gitea_admin/authentik-gate-sync:latest
-
-  # kuma-sync: reconciles KumaMonitor resources into Uptime Kuma. Its unit
-  # tests run as a stage of the image build.
-  - name: build-kuma-sync
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: ci/images/kuma-sync
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/kuma-sync/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      # YYYY-MM-DD-HHMM: the image-automation ImagePolicy picks the newest.
-      - TAG=$(date +%Y-%m-%d-%H%M)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/kuma-sync:$TAG \
-          --destination gitea.renner.uno/gitea_admin/kuma-sync:latest
-
-  # cloudflare-lb-sync: reconciles CloudflareLoadBalancer resources into
-  # Cloudflare. Its unit tests run as a stage of the image build.
-  - name: build-cloudflare-lb-sync
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: ci/images/cloudflare-lb-sync
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/cloudflare-lb-sync/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      # YYYY-MM-DD-HHMM: the image-automation ImagePolicy picks the newest.
-      - TAG=$(date +%Y-%m-%d-%H%M)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/cloudflare-lb-sync:$TAG \
-          --destination gitea.renner.uno/gitea_admin/cloudflare-lb-sync:latest
+          --destination gitea.renner.uno/gitea_admin/wiki-index:$TAG \
+          --destination gitea.renner.uno/gitea_admin/wiki-index:latest
 
   # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
   # artifact like heating-can; same registry so the audit tracks it.
   - name: build-bird
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/bird
+    depends_on: []
     when:
       - event: deployment
       - event: push
@@ -501,27 +428,3 @@ steps:
           --dockerfile Containerfile \
           --destination gitea.renner.uno/gitea_admin/bird:$TAG \
           --destination gitea.renner.uno/gitea_admin/bird:latest
-
-  # Orchestrator for the monthly backup-verify CronJob (velero ns).
-  - name: build-backup-verify
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: ci/images/backup-verify
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/backup-verify/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-backup-verify:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-backup-verify:latest

.woodpecker/ci-claude-refresh.yml

@@ -14,8 +14,8 @@
 #                 and  ci-claude:latest.  The Containerfile makes this layer
 #                 byte-reproducible, so rebuilding an unchanged version dedups
 #                 in the registry instead of orphaning a fresh ~260 MB copy.
-#   3. bump-pr  — (only when any pin is off the latest version) rewrite every
-#                 consumer pin to  ci-claude:<version>  and open a PR. A human
+#   3. bump-pr  — (only when the version actually changed) rewrite the 5
+#                 consumer pins to  ci-claude:<version>  and open a PR. A human
 #                 reviews and merges it; nothing auto-merges.
 #
 # TAG SCHEME — deliberately version tags (ci-claude:<claude-version>), NOT the dated
@@ -46,20 +46,20 @@
 #                                push/cron/manual events this pipeline fires on.
 #
 # OWNER SETUP — the token + secret above are already provisioned. The one
-# remaining step, to be run AFTER this merges to main (a cron references a
+# remaining step, to be run AFTER this merges to master (a cron references a
 # pipeline that must exist on the target branch), is to register the schedule:
-#   woodpecker repo cron add --repo konairius/homelab \
-#     --name ci-claude-refresh --branch main --schedule '@weekly'
+#   woodpecker repo cron add --repo k.renner/homelab \
+#     --name ci-claude-refresh --branch master --schedule '@weekly'
 # Weekly is plenty; the pipelines tolerate being a few days behind a release.
 #
 # Triggering:
 #   - cron   : the cron NAMED exactly "ci-claude-refresh" (see above).
-#   - deploy : woodpecker-cli pipeline deploy konairius/homelab <n> ci-claude-refresh
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> ci-claude-refresh
 #              where <n> is an existing pipeline to promote.
-#   - push   : a change to the Containerfile (or this file) ON main forces a
+#   - push   : a change to the Containerfile (or this file) ON master forces a
 #              rebuild of the current latest version so Containerfile fixes ship
-#              promptly. Restricted to main ON PURPOSE — a feature branch must
-#              never build/push an image or open a bump PR (only main pushes
+#              promptly. Restricted to master ON PURPOSE — a feature branch must
+#              never build/push an image or open a bump PR (only master pushes
 #              images), and without this filter every PR that edits these paths
 #              would do exactly that from its own branch.
 when:
@@ -68,7 +68,7 @@ when:
   - event: deployment
     evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "ci-claude-refresh"'
   - event: push
-    branch: main
+    branch: master
     path:
       - "ci/images/claude/**"
       - ".woodpecker/ci-claude-refresh.yml"
@@ -107,22 +107,31 @@ steps:
           [0-9]*.[0-9]*.[0-9]*) : ;;
           *) echo "npm returned an implausible version '$$NEW' — aborting."; exit 1 ;;
         esac
-        # Distinct tags the consumers currently reference. Divergent pins are
-        # normal (a pipeline bumped on its own); the bump rewrites all of them.
-        CUR=$$(grep -hoE 'ci-claude:[0-9][0-9A-Za-z._-]*' $$CONSUMERS | sed 's/^ci-claude://' | sort -u | tr '\n' ' ' | sed 's/ $$//')
-        [ -n "$$CUR" ] || { echo "No ci-claude pin found in the consumers — aborting."; exit 1; }
-        echo "latest=$$NEW  current=[$$CUR]  event=$$CI_PIPELINE_EVENT"
+        # The tag every consumer currently references. They are kept identical,
+        # so this must resolve to EXACTLY ONE non-empty tag; anything else
+        # (no match, or a partial/disagreeing set) aborts rather than risk a
+        # corrupting sed later. Counting with `grep -c .` (non-empty lines)
+        # instead of `wc -l` is deliberate: an empty result yields 0 here but
+        # 1 under wc -l, which would slip past the guard.
+        CUR=$$(grep -hoE 'ci-claude:[0-9A-Za-z._-]+' $$CONSUMERS | sed 's/^ci-claude://' | sort -u)
+        NTAGS=$$(printf '%s' "$$CUR" | grep -c . || true)
+        if [ -z "$$CUR" ] || [ "$$NTAGS" -ne 1 ]; then
+          echo "Expected exactly one ci-claude tag across the consumers, got [$$CUR]."
+          echo "Refusing to guess — fix the pins by hand."; exit 1
+        fi
+        echo "latest=$$NEW  current=$$CUR  event=$$CI_PIPELINE_EVENT"
         printf '%s' "$$NEW" > "$$CI_WORKSPACE/.ci-refresh/version"
         printf '%s' "$$CUR" > "$$CI_WORKSPACE/.ci-refresh/current"
-        # Build when any pin is off latest, or on a push (Containerfile edit must
-        # ship even if the version is unchanged). Bump when any pin is off latest.
+        # Build when the version changed, or on a push (Containerfile edit must
+        # ship even if the version is unchanged). Bump the pins only when the
+        # version the consumers reference is not the latest.
         if [ "$$CUR" != "$$NEW" ] || [ "$$CI_PIPELINE_EVENT" = "push" ]; then
           touch "$$CI_WORKSPACE/.ci-refresh/do-build"; echo "will build $$NEW"
         else
           echo "already on $$NEW — nothing to build"
         fi
         if [ "$$CUR" != "$$NEW" ]; then
-          touch "$$CI_WORKSPACE/.ci-refresh/do-bump"; echo "will open bump PR [$$CUR] -> $$NEW"
+          touch "$$CI_WORKSPACE/.ci-refresh/do-bump"; echo "will open bump PR $$CUR -> $$NEW"
         else
           echo "consumers already pin $$NEW — no PR"
         fi
@@ -131,7 +140,7 @@ steps:
   #    --build-arg to the resolved version (not "latest") guarantees the tag and
   #    the image content agree even if a new release lands mid-run.
   - name: build
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
     directory: ci/images/claude
     depends_on: [decide]
     environment:
@@ -166,17 +175,14 @@ steps:
       - |
         [ -f "$$CI_WORKSPACE/.ci-refresh/do-bump" ] || { echo "no version change — no PR"; exit 0; }
         set -eu
-        # Secret values can carry a trailing newline; it breaks the push URL and the auth header.
-        PUSH_TOKEN=$$(printf '%s' "$$PUSH_TOKEN" | tr -d '\r\n')
-        [ -n "$$PUSH_TOKEN" ] || { echo "PUSH_TOKEN is empty — aborting"; exit 1; }
         NEW=$$(cat "$$CI_WORKSPACE/.ci-refresh/version")
         CUR=$$(cat "$$CI_WORKSPACE/.ci-refresh/current")
         # Both were validated non-empty by the decide step; re-assert before a
         # sed that would corrupt every pin if either were blank.
         [ -n "$$NEW" ] && [ -n "$$CUR" ] || { echo "empty version/current sentinel — aborting"; exit 1; }
-        API="https://gitea.renner.uno/api/v1/repos/konairius/homelab"
+        API="https://gitea.renner.uno/api/v1/repos/k.renner/homelab"
         BRANCH="ci-image/claude-$$NEW"
-        if git ls-remote --heads "https://gitea.renner.uno/konairius/homelab.git" "$$BRANCH" | grep -q .; then
+        if git ls-remote --heads "https://gitea.renner.uno/k.renner/homelab.git" "$$BRANCH" | grep -q .; then
           echo "Branch $$BRANCH already exists — a PR for $$NEW is already open. Nothing to do."
           exit 0
         fi
@@ -184,11 +190,9 @@ steps:
         # is not a maintained list.
         CONSUMERS=$$(grep -rlE 'ci-claude:[0-9]' .woodpecker)
         [ -n "$$CONSUMERS" ] || { echo "No ci-claude consumers found — refusing to continue."; exit 1; }
-        # NEW goes into a sed replacement: allow only a plain x.y.z version.
-        printf '%s' "$$NEW" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$$' || { echo "refusing non-x.y.z version '$$NEW'"; exit 1; }
-        # Every version-pinned tag becomes NEW, whatever it was; consumers only.
+        # Exact-string swap of the current tag for the new one, consumers only.
         for f in $$CONSUMERS; do
-          sed -i "s#ci-claude:[0-9][0-9A-Za-z._-]*#ci-claude:$$NEW#g" "$$f"
+          sed -i "s#ci-claude:$$CUR#ci-claude:$$NEW#g" "$$f"
         done
         # Author is claude-bot (the PAT's account); the commit subject makes
         # the automated-bump nature clear.
@@ -197,7 +201,7 @@ steps:
         git checkout -b "$$BRANCH"
         git add $$CONSUMERS
         git commit -m "chore(ci): bump ci-claude image to $$NEW"
-        git push "https://$$PUSH_TOKEN@gitea.renner.uno/konairius/homelab.git" HEAD:"$$BRANCH"
+        git push "https://$$PUSH_TOKEN@gitea.renner.uno/k.renner/homelab.git" HEAD:"$$BRANCH"
         {
           printf 'Automated bump of the ci-claude agent image from `%s` to `%s`.\n\n' "$$CUR" "$$NEW"
           printf 'The refresh pipeline built and pushed `ci-claude:%s` (and `:latest`) from the\n' "$$NEW"
@@ -208,7 +212,7 @@ steps:
         jq -n --arg title "chore(ci): bump ci-claude image to $$NEW" \
               --rawfile body /tmp/pr-body.md \
               --arg head "$$BRANCH" \
-          '{title: $$title, body: $$body, head: $$head, base: "main"}' > /tmp/pr.json
+          '{title: $$title, body: $$body, head: $$head, base: "master"}' > /tmp/pr.json
         code=$$(curl -s -o /tmp/resp.json -w '%{http_code}' \
           -X POST -H "Authorization: token $$PUSH_TOKEN" \
           -H "Content-Type: application/json" --data @/tmp/pr.json "$$API/pulls")

.woodpecker/ci-manifests-test.yml

@@ -17,44 +17,30 @@
 # here is a static check that either passes or fails on the checkout alone, so it
 # can gate PRs without becoming a bottleneck or a flake source.
 when:
-  # Path-filtered on purpose, unlike ci-scripts-test.yml. That pipeline is two
-  # steps on already-cached images, so running it everywhere is free; this one
-  # pulls four images and builds every kustomization in the tree, which is real
+  # Path-filtered on purpose, unlike ci-scripts-test.yml. That pipeline is one
+  # step on one already-cached image, so running it everywhere is free; this one
+  # pulls three images and builds every kustomization in the tree, which is real
   # cost to put on a docs-only PR. The filter covers everything the steps actually read:
-  # kubernetes/** for the manifests, scripts and promtool suites, .woodpecker/**
-  # for the lint (and so an edit to this file re-runs its own checks), and the
-  # promtool driver script by name.
-  #
-  # That last entry is NOT covered by kubernetes/**: promtool-rules.sh lives
-  # under scripts/, so without it a PR that only edits the script would never run
-  # the check it drives. It is listed as one file rather than scripts/ci/**
-  # because the rest of that directory is CI-agent helpers with nothing to do
-  # with this pipeline, and pulling four images for them is the cost this filter
-  # exists to avoid. Their own tests run in ci-scripts-test.yml on every PR.
+  # kubernetes/** for the manifests and scripts, .woodpecker/** for the lint (and
+  # so an edit to this file re-runs its own checks).
   - event: pull_request
     path:
       - "kubernetes/**"
       - ".woodpecker/**"
-      - "scripts/ci/promtool-rules.sh"
-      - "scripts/ci/yaml-citations.sh"
 
-# THE THIRD DISCOVERY CHECK WENT TO A SCRIPT. kustomize-build and shellcheck
-# both hand-roll the same shape — `find`, a floor guard against empty discovery,
-# then a file-fed `while read`. Two copies was deliberate: the two traps being
-# commented around (BusyBox find lacking `-printf`, and a piped loop running in
-# a subshell where `fail` cannot propagate) are different, and both are worth
-# spelling out where they bite. An earlier revision of this comment said a third
-# instance would be the point to stop copying and factor it into `scripts/ci/`,
-# where it would gain unit tests — and that is what `promtool` below does. Its
-# discovery, floors and guards live in scripts/ci/promtool-rules.sh and are
-# tested by scripts/ci/test-promtool-rules.sh on every PR via
-# ci-scripts-test.yml. A FOURTH inline copy should go the same way; do not add
-# one here.
+# ADDING A THIRD DISCOVERY CHECK: kustomize-build and shellcheck both hand-roll
+# the same shape — `find`, a floor guard against empty discovery, then a
+# file-fed `while read`. Two copies is deliberate: the two traps being commented
+# around (BusyBox find lacking `-printf`, and a piped loop running in a subshell
+# where `fail` cannot propagate) are different, and both are worth spelling out
+# where they bite. A third instance is the point to stop copying and factor it
+# into a shared script under `scripts/ci/` — which is already covered by
+# ci-scripts-test.yml, so it would gain unit tests in the process.
 #
-# Every step carries `depends_on: []` so all six run in PARALLEL and every
+# Every step carries `depends_on: []` so all four run in PARALLEL and every
 # failure shows in one run. Woodpecker's default is sequential, which would stop
 # at the first failure and make a contributor fix-and-repush to discover the
-# next — and these six are independent, cheap, offline checks with no ordering
+# next — and these four are independent, cheap, offline checks with no ordering
 # between them. Keep it on every step: `depends_on` is per-step, so dropping it
 # from one silently re-serialises that step behind the others.
 steps:
@@ -66,9 +52,9 @@ steps:
   # Verified green across every kustomization in the tree when this was added,
   # so this gates new breakage rather than pre-existing debt.
   #
-  # ci-k8s TAG SPLIT: this file pins a newer dated tag than e2e.yml and
-  # e2e-test.yml, which still sit on an older one. That is
-  # staleness in those two, not a deliberate split here — everything should be on
+  # ci-k8s TAG SPLIT: this file pins a newer dated tag than e2e.yml, e2e-test.yml,
+  # alert-monitor.yml and issue-fix.yml, which still sit on an older one. That is
+  # staleness in those four, not a deliberate split here — everything should be on
   # the newest dated tag. They are not bumped here because none of them are
   # triggered by this PR, so it would ship untested changes under cover of a green
   # run that never touched them. Tracked in #461.
@@ -172,7 +158,7 @@ steps:
   # whenever it trails the deployed server. If you bump the server, bump here.
   - name: woodpecker-lint
     depends_on: [] # see note above `steps:`
-    image: woodpeckerci/woodpecker-cli:v3.19.0-alpine@sha256:27a77117e38d7bbdc380f0e90ff3ee4871ab1eefa78b5d9136a2c969fbecd9f4
+    image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
     commands:
       - |
         set -eu
@@ -199,7 +185,7 @@ steps:
   # separately rather than one this step quietly implies is covered.
   - name: shellcheck
     depends_on: [] # see note above `steps:`
-    image: koalaman/shellcheck-alpine:v0.11.0@sha256:9955be09ea7f0dbf7ae942ac1f2094355bb30d96fffba0ec09f5432207544002
+    image: koalaman/shellcheck-alpine:v0.11.0
     commands:
       - |
         set -eu
@@ -272,72 +258,3 @@ steps:
           exit 1
         fi
         echo "ok    no digest pins under openbao/rotation/"
-
-  # Machine-check the alert rules: parse every PrometheusRule, run the committed
-  # promtool unit suites, then damage the rules on purpose and require the suites
-  # to notice. Everything it does lives in scripts/ci/promtool-rules.sh; read
-  # that file and kubernetes/apps/monitoring-rules/tests/README.md for the why.
-  #
-  # The gap it closes: the other four steps prove a PrometheusRule assembles into
-  # valid YAML and nothing more — not that an expression parses, not that a label
-  # join matches anything, not that a `for` window does what its comment claims.
-  # On REN-129 the CNPG surge rule was written `for: 2m`, which fires on six of
-  # seven measured nights and would have mailed the owner twice a night at 04:16
-  # and 04:22 local for a backlog that clears itself. Nothing here would have
-  # caught it, and the arithmetic that did catch it was hand-written from scratch
-  # twice, in two runs, and thrown away both times because there was nowhere in
-  # the repo to commit it. REN-152 retired that rule; its `for` margin is now
-  # guarded by mutations.txt:backlog-for-12m.
-  #
-  # Offline, like the rest of this pipeline: no cluster, no registry, no secrets,
-  # no network. promtool reads files and exits.
-  #
-  # IMAGE — prom/prometheus, not a registry-local ci-* image, and pinned to an
-  # exact upstream version rather than a dated tag. The dated-tag convention
-  # guarded by no-digest-pins above is about OUR images, whose digests are
-  # garbage-collected out from under a pin; this is upstream and immutable.
-  #
-  # The version must be 3.x, and that is not a preference. The cluster runs
-  # whatever kube-prometheus-stack >= 72.0.0 ships, which is Prometheus 3, and
-  # alert `for` evaluation is version-sensitive — a 2.x promtool would be
-  # checking a different evaluator than production, so the margins the suites
-  # assert would be claims about the wrong program. promtool-rules.sh asserts the
-  # major version it was handed and fails on 2.x, so a careless bump here is a
-  # red step rather than a quietly wrong check. Keep this at the latest stable
-  # 3.x, per the standing rule that any software uses the latest stable version.
-  #
-  # This image is NOT covered by image-automation (same reason as
-  # woodpecker-lint above: a Woodpecker pipeline file has no apiVersion/kind, so
-  # an $imagepolicy marker here is silently ignored — fluxcd/flux2#2826). The
-  # bump is manual.
-  #
-  # `prom/prometheus` is busybox-based and ships /bin/sh, so it can run a
-  # `commands:` block — unlike the plain woodpecker-cli tag, which cannot. Its
-  # find/grep/awk/sed are BusyBox, which is why promtool-rules.sh avoids
-  # `find -printf` and `grep -r --include`; it was verified against BusyBox
-  # 1.35.0 as well as GNU. The image also declares `USER nobody`, so the script
-  # writes its stripped mirror under mktemp and never into the workspace.
-  - name: promtool
-    depends_on: [] # see note above `steps:`
-    image: prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2
-    commands:
-      - sh scripts/ci/promtool-rules.sh
-
-  # Comments cite other YAML blocks by anchor name, never by line number
-  # (REN-183; the exact syntax is in CONTRIBUTING.md, not repeated here because
-  # an example written in this comment would itself be checked). Line numbers drift on any edit
-  # above them and nothing noticed: three drift fixes were merged by hand
-  # before this step existed. This fails on a new line-number citation and on
-  # a citation whose file or anchor marker does not exist.
-  # Existence only — it never judges whether the anchored block still says
-  # what the citing comment claims, so ordinary edits cannot turn it red.
-  #
-  # It scans every YAML file in the checkout, but this pipeline only runs on
-  # the paths in `when:` above, so an edit confined to ansible/ or esphome/ is
-  # checked on the next PR that does touch kubernetes/ or .woodpecker/.
-  # Self-test: scripts/ci/test-yaml-citations.sh, in ci-scripts-test.yml.
-  - name: yaml-citations
-    depends_on: [] # see note above `steps:`
-    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
-    commands:
-      - sh scripts/ci/yaml-citations.sh

.woodpecker/ci-scripts-test.yml

@@ -8,69 +8,15 @@
 # against adversarial PR input. Running it here fails the offending PR's checks
 # before it can merge.
 #
-# scripts/ci/test-promtool-rules.sh is the same idea for the alert-rule gate.
-# What it tests is not whether promtool evaluates PromQL — ci-manifests-test.yml
-# does that with the real binary against the real rules — but whether
-# promtool-rules.sh REFUSES TO REPORT SUCCESS when it has checked nothing: zero
-# manifests discovered, zero suites discovered, a `rule_files` path that does not
-# resolve (promtool warns, loads zero rules, and passes every assertion), a
-# mutation the suites no longer catch. Those are the states a green pipeline
-# cannot tell apart from real coverage, so they need a test that asserts the
-# failure. It stubs promtool, so it needs neither the binary nor the heavy image.
-#
-# Both are unfiltered by path on purpose: this pipeline is two steps on one
-# already-cached image, so running it everywhere is free, and a guard whose test
-# only runs when someone remembers to touch the right directory is not a guard.
-#
 # Only in homelab (scripts/ci/ lives here); the file is not deployed to the
-# mirrored repos, so it does not run on their PRs. All tests here are offline (no
-# token, no network, no cluster) — the ci-claude image just provides sh + jq, and
-# it is already pulled by pr-review.yml on the same event, so the runner has it
-# cached.
+# mirrored repos, so it does not run on their PRs. The test is offline (no token,
+# no network, no cluster) — the ci-claude image just provides sh + jq, and it is
+# already pulled by pr-review.yml on the same event, so the runner has it cached.
 when:
   - event: pull_request
 
-# `depends_on: []` on every step so they run in parallel and one failure does not hide
-# the other. Same reasoning as the note above `steps:` in ci-manifests-test.yml.
 steps:
   - name: model-selection-tests
-    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
     commands:
       - sh scripts/ci/test-model-selection.sh
-
-  - name: pr-review-mistral-tests
-    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
-    commands:
-      - sh scripts/ci/test-pr-review-mistral.sh
-
-  - name: promtool-rules-guard-tests
-    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
-    commands:
-      - sh scripts/ci/test-promtool-rules.sh
-
-  - name: yaml-citations-guard-tests
-    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
-    commands:
-      - sh scripts/ci/test-yaml-citations.sh
-
-  # Offline unittests for the Hermes skill scripts
-  # (kubernetes/apps/hermes/skills/): the [alert] title, noise-log and blip
-  # fingerprint contract the alert-monitor dedup depends on, the issue->card
-  # bridge, and the secret guard.
-  - name: hermes-skill-tests
-    depends_on: []
-    image: python:3.15-alpine@sha256:d44b1254cb92329418d164b22b20d86bbd3869d48e80e09e476531a409a728bb
-    environment:
-      PYTHONDONTWRITEBYTECODE: "1"
-    commands:
-      - python3 -m unittest discover -s kubernetes/apps/hermes/skills/tests -v
-
-  - name: backup-verify-tests
-    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-backup-verify:2026-10-10
-    commands:
-      - bash kubernetes/infrastructure/velero/backup-verify/tests/run.sh

.woodpecker/deploy-claude-workstation.yml

@@ -0,0 +1,33 @@
+when:
+  # Deploy only on master (merge) + manual — never auto-deploy from a branch.
+  - event: push
+    branch: master
+    path:
+      - "ansible/roles/claude-workstation/**"
+      - "ansible/plays/install-claude-workstation.yml"
+      - ".woodpecker/deploy-claude-workstation.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'
+
+timeout: 30
+
+steps:
+  - name: deploy-claude-workstation
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/install-claude-workstation.yml -v

.woodpecker/deploy-infrastructure.yml

@@ -1,10 +1,10 @@
 when:
-  # Plan only on main (merge) + manual — a push trigger without this gate
+  # Plan only on master (merge) + manual — a push trigger without this gate
   # would run `tofu plan` (which evaluates provider/data-source code) with
   # all proxmox/hetzner/unifi secrets mounted for any pushed branch,
   # including malicious feature-branch .tf.
   - event: push
-    branch: main
+    branch: master
     path:
       - "opentofu/**"
       - ".woodpecker/deploy-infrastructure.yml"
@@ -13,7 +13,7 @@ when:
 
 steps:
   - name: validate-secrets
-    image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a
+    image: alpine:3.24.1
     environment:
       TF_VAR_proxmox_api_url:
         from_secret: proxmox_api_url
@@ -109,6 +109,8 @@ steps:
         from_secret: hetzner_s3_access_key
       TF_VAR_hetzner_s3_secret_key:
         from_secret: hetzner_s3_secret_key
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
       PROXMOX_TOKEN_ID:
         from_secret: proxmox_token_id
       PROXMOX_TOKEN_SECRET:
@@ -119,6 +121,11 @@ steps:
       ANSIBLE_ROLES_PATH: "../roles:./roles"
       ANSIBLE_FORCE_COLOR: "true"
     commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - export TF_VAR_ssh_private_key="$(cat ~/.ssh/id_ed25519)"
+      - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
       - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
       - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
       - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"

.woodpecker/deploy-netbird-client.yml

@@ -0,0 +1,35 @@
+when:
+  # Deploy only on master (merge) + manual — never auto-deploy from a branch.
+  - event: push
+    branch: master
+    path:
+      - "ansible/roles/netbird-client/**"
+      - "ansible/plays/install-netbird-client.yml"
+      - "ansible/inventory/group_vars/claude_workstation/**"
+      - ".woodpecker/deploy-netbird-client.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'
+
+timeout: 30
+
+steps:
+  - name: deploy-netbird-client
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      # Enrol only the off-site Claude workstation into the Netbird mesh.
+      - ansible-playbook -i inventory/ plays/install-netbird-client.yml --limit claude_workstation -v

.woodpecker/deploy-netbird-proxy.yml

@@ -0,0 +1,54 @@
+when:
+  # Deploy only on master (merge) + manual — never auto-deploy from a branch.
+  - event: push
+    branch: master
+    path:
+      - "ansible/roles/netbird-proxy/**"
+      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
+      # so changes to it must redeploy this host too.
+      - "ansible/roles/netbird-proxy-watchdog/**"
+      # install-netbird-proxy.yml also applies the edge bouncer.
+      - "ansible/roles/crowdsec-edge-bouncer/**"
+      - "ansible/plays/install-netbird-proxy.yml"
+      # The variables that steer those roles, not just the roles themselves.
+      # Without these two, a change that only touches inventory never deploys
+      # and nothing says so: arming the bouncer lives in host_vars, and its
+      # LAPI key lives in group_vars, so both would sit in git looking applied
+      # while the hosts kept running the previous configuration.
+      #
+      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
+      # file for the whole fleet, so listing it would redeploy the proxies for
+      # any unrelated host edit. The cost is that a change to [netbird_proxy]
+      # group membership needs a manual deployment run.
+      - "ansible/inventory/host_vars/proxy-*.yml"
+      - "ansible/inventory/group_vars/netbird_proxy/**"
+      - ".woodpecker/deploy-netbird-proxy.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
+
+timeout: 30
+
+steps:
+  - name: deploy-netbird-proxy
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+      PROXMOX_TOKEN_ID:
+        from_secret: proxmox_token_id
+      PROXMOX_TOKEN_SECRET:
+        from_secret: proxmox_token_secret
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/install-netbird-proxy.yml -v

.woodpecker/deploy-netbird-server.yml

@@ -0,0 +1,40 @@
+when:
+  # Deploy only on master (merge) + manual — never auto-deploy from a branch.
+  - event: push
+    branch: master
+    path:
+      - "ansible/roles/netbird-server/**"
+      # install-netbird-server.yml also applies the netbird-proxy-watchdog role,
+      # so changes to it must redeploy this host too.
+      - "ansible/roles/netbird-proxy-watchdog/**"
+      - "ansible/plays/install-netbird-server.yml"
+      - ".woodpecker/deploy-netbird-server.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'
+
+timeout: 30
+
+steps:
+  - name: deploy-netbird-server
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+      PROXMOX_TOKEN_ID:
+        from_secret: proxmox_token_id
+      PROXMOX_TOKEN_SECRET:
+        from_secret: proxmox_token_secret
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/install-netbird-server.yml -v

.woodpecker/e2e.yml

@@ -4,10 +4,10 @@
 # Triggers:
 #   - cron     : scoped to the named daily/weekly crons below. This used to be
 #                a bare `event: cron` (run on EVERY cron of this repo), which
-#                was harmless at daily/weekly cadence but not with hourly
-#                crons, so piggybacking must be opt-in.
+#                was harmless at daily/weekly cadence — but alert-monitor.yml
+#                fires every 15 minutes, so piggybacking must be opt-in now.
 #                A new cron that SHOULD also run e2e gets its name added here.
-#   - deploy   : `woodpecker-cli pipeline deploy konairius/homelab <n> e2e`, where
+#   - deploy   : `woodpecker-cli pipeline deploy k.renner/homelab <n> e2e`, where
 #                <n> is an existing pipeline to promote.
 
 when:
@@ -26,7 +26,7 @@ steps:
     # fetch-models.sh response mapping. No cluster, no secrets, no network — the
     # ci-claude image just provides sh + jq. Keeps the flag-injection regression
     # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
     commands:
       - sh scripts/ci/test-model-selection.sh
 

.woodpecker/hermes-image.yml

@@ -1,28 +0,0 @@
-# Builds ci/images/hermes (nousresearch/hermes-agent plus the agent's CLIs) as
-# gitea.renner.uno/konairius/hermes:YYYY-MM-DD-HHMM, one tag per build.
-when:
-  - event: push
-    branch: main
-    path:
-      - "ci/images/hermes/**"
-      - ".woodpecker/hermes-image.yml"
-  - event: manual
-
-steps:
-  - name: build
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: ci/images/hermes
-    environment:
-      # registry-bot write:package PAT (konairius org secret).
-      GITEA_TOKEN:
-        from_secret: registry_bot_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' registry-bot "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d-%H%M)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/konairius/hermes:$TAG

.woodpecker/issue-fix.yml

@@ -0,0 +1,217 @@
+# Issue fixer — hourly, pick the oldest open "[alert] ..." Gitea issue (filed
+# by the alert-monitor pipeline, alert-monitor.yml) that nobody has worked on
+# yet, hand it to an AI agent, and open a PR with the fix. One issue per run
+# keeps runs bounded and PRs reviewable.
+#
+# "Not worked on yet" means BOTH:
+#   - no alert-fix/issue-<n> branch exists on origin (a branch means a PR was
+#     already opened — or is being opened by a concurrent run), and
+#   - no issue comment contains "<!-- alert-fix:assessed -->" (the marker this
+#     pipeline leaves when the agent concluded the problem is NOT fixable in
+#     the repo, so we don't burn an agent run on it every hour) — UNLESS a
+#     TRUSTED_AUTHORS comment is newer than the latest marker: an owner reply
+#     re-opens the issue for the agent, with the reply passed to it as
+#     pipeline-verified guidance (see below).
+#
+# Trusted owner guidance: comments by TRUSTED_AUTHORS are appended to the
+# agent's issue.md under "Verified owner guidance". The trust anchor is the
+# hardcoded author list in THIS file (reviewed via PR; agents cannot edit
+# .woodpecker/). Deliberately NOT the Gitea permission API: this pipeline's
+# own comments post as issue-fix-bot, which is itself a write collaborator,
+# so "has write access" cannot distinguish the human from the pipeline — and
+# an injected instruction inside an agent-authored comment must never be able
+# to masquerade as owner guidance (the #259 lesson, in both directions: the
+# agent rightly refused unauthenticated body text, but then had no channel
+# for the owner's authentic reply).
+#
+# Tokens: a single dedicated identity, issue-fix-bot, for both the issue
+# comments and the push/PR. Its PAT is scoped to write:issue +
+# write:repository and nothing else. It deliberately does NOT reuse
+# security_bot_token (that identity belongs to the security-review pipeline)
+# nor the admin-owned gitea_token — see #330.
+#
+# Triggering:
+#   - cron   : a cron NAMED exactly "issue-fix":
+#                woodpecker repo cron add --repo k.renner/homelab \
+#                  --name issue-fix --branch master --schedule '41 * * * *'
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> issue-fix
+#              where <n> is an existing pipeline to promote.
+when:
+  - event: cron
+    cron: issue-fix
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "issue-fix"'
+
+steps:
+  - name: fetch-diag
+    # Least-privilege live-diagnostics kubeconfig (+ kubectl binary) for the
+    # fix agent, same mechanism as alert-monitor.yml. Runs unconditionally
+    # (the issue-pick logic needs jq, which only the ci-claude image has);
+    # minting a 1h read-only token per run is cheap. Degrades to
+    # snapshot-only if the ci-diagnostics ServiceAccount isn't reconciled.
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    environment:
+      SOPS_AGE_KEY:
+        from_secret: sops_age_key
+      S3_ACCESS_KEY_ID:
+        from_secret: hetzner_s3_access_key
+      S3_SECRET_ACCESS_KEY:
+        from_secret: hetzner_s3_secret_key
+    commands:
+      - bash scripts/ci/fetch-kubeconfig.sh
+      - bash scripts/ci/make-diag-kubeconfig.sh
+
+  - name: issue-fix-agent
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    environment:
+      CLAUDE_CODE_OAUTH_TOKEN:
+        from_secret: claude_code_oauth_token
+      # Dedicated identity: issue-fix-bot, PAT scoped to write:issue +
+      # write:repository (it comments, pushes a branch, and opens a PR).
+      # One identity for both roles — the previous split borrowed
+      # security_bot_token, which belongs to the security-review pipeline
+      # and has no business in this trust domain (#330).
+      GITEA_TOKEN:
+        from_secret: issue_fix_bot_token
+      PUSH_TOKEN:
+        from_secret: issue_fix_bot_token
+    commands:
+      - |
+        # Pick the oldest unhandled [alert] issue. The repo is public, so the
+        # candidate reads are unauthenticated; only writes need tokens.
+        set -eu
+        TRUSTED_AUTHORS="k.renner"   # space-separated; the trust anchor (see header)
+        mkdir -p /tmp/alert-fix
+        API="https://gitea.renner.uno/api/v1/repos/k.renner/homelab"
+        curl -sfL "$$API/issues?state=open&type=issues&limit=100" \
+          | jq '[.[] | select(.title | startswith("[alert] "))] | sort_by(.number)' \
+          > /tmp/alert-fix/candidates.json
+        COUNT=$$(jq length /tmp/alert-fix/candidates.json)
+        echo "Open [alert] issues: $$COUNT"
+        TRUSTED_JSON=$$(printf '%s\n' $$TRUSTED_AUTHORS | jq -R . | jq -s -c .)
+        i=0
+        while [ "$$i" -lt "$$COUNT" ]; do
+          NUM=$$(jq -r ".[$$i].number" /tmp/alert-fix/candidates.json)
+          curl -sfL "$$API/issues/$$NUM/comments" > /tmp/alert-fix/comments.json
+          # Eligibility in jq (ISO timestamps compare lexically; shell test
+          # has no portable string-< in busybox ash): eligible unless an
+          # assessed marker exists with no NEWER trusted-author comment.
+          STATE=$$(jq -r --argjson t "$$TRUSTED_JSON" '
+            ([.[] | select(.body | contains("alert-fix:assessed")) | .created_at] | max // "") as $$mark
+            | ([.[] | select(.user.login as $$u | $$t | index($$u)) | .created_at] | max // "") as $$trust
+            | if $$mark == "" then "fresh"
+              elif $$trust > $$mark then "retry"
+              else "skip" end' /tmp/alert-fix/comments.json)
+          if git ls-remote --heads origin "alert-fix/issue-$$NUM" | grep -q .; then
+            echo "#$$NUM: branch alert-fix/issue-$$NUM already exists — skipping."
+          elif [ "$$STATE" = "skip" ]; then
+            echo "#$$NUM: assessed and no newer owner comment — skipping."
+          else
+            if [ "$$STATE" = "retry" ]; then echo "#$$NUM: owner replied after assessment — retrying with guidance."; fi
+            echo "Picking issue #$$NUM."
+            jq ".[$$i]" /tmp/alert-fix/candidates.json > /tmp/alert-fix/issue.json
+            {
+              printf '# %s\n\n' "$$(jq -r '.title' /tmp/alert-fix/issue.json)"
+              printf 'Issue number: %s\n\n' "$$NUM"
+              jq -r '.body' /tmp/alert-fix/issue.json
+              # Owner comments, authorship checked against TRUSTED_AUTHORS
+              # above. Everything in the section is authentic owner input;
+              # agent-authored comments never match (they post as
+              # issue-fix-bot) and marker comments are excluded regardless.
+              if [ "$$(jq -r --argjson t "$$TRUSTED_JSON" '[.[] | select((.user.login as $$u | $$t | index($$u)) and (.body | contains("alert-fix:") | not))] | length' /tmp/alert-fix/comments.json)" -gt 0 ]; then
+                printf '\n\n## Verified owner guidance (authorship checked by the pipeline)\n'
+                jq -r --argjson t "$$TRUSTED_JSON" '.[] | select((.user.login as $$u | $$t | index($$u)) and (.body | contains("alert-fix:") | not)) | "\n### \(.user.login), \(.created_at)\n\n\(.body)"' /tmp/alert-fix/comments.json
+              fi
+            } > /tmp/alert-fix/issue.md
+            break
+          fi
+          i=$$((i+1))
+        done
+        [ -f /tmp/alert-fix/issue.json ] || echo "Nothing to fix — all [alert] issues handled."
+      - |
+        # Branch + git identity. The bot user owns the commits so PRs are
+        # visibly machine-authored, and the identity names THIS pipeline —
+        # previously these commits and PRs were attributed to security-bot,
+        # which made alert fixes look like security-review output (#330).
+        [ -f /tmp/alert-fix/issue.json ] || exit 0
+        set -eu
+        NUM=$$(jq -r '.number' /tmp/alert-fix/issue.json)
+        BRANCH="alert-fix/issue-$$NUM"
+        git config user.email "issue-fix-bot@renner.uno"
+        git config user.name "Alert Fix Agent"
+        git checkout -b "$$BRANCH"
+        echo "$$BRANCH" > /tmp/alert-fix/branch
+      - |
+        # Headless fix run. Prompt piped via stdin, subscription OAuth, file
+        # tools plus EXACTLY ONE live command: the RBAC-scoped read-only
+        # kubectl from fetch-diag (no Secrets readable, prefix match forbids
+        # pipes and other binaries). No web (see security-scan.yml).
+        [ -f /tmp/alert-fix/issue.json ] || exit 0
+        export KUBECONFIG="$$PWD/.ci-diag/kubeconfig"
+        # Model chosen live from the catalog (not hardcoded, no family assumed) —
+        # see the header of scripts/ci/pick-model.sh. Empty MODEL -> CLI default.
+        MODEL=$$(sh scripts/ci/pick-model.sh "diagnose an [alert] issue against read-only live cluster state and fix the root cause by editing repo manifests; multi-step reasoning that changes infrastructure code and needs a capable model")
+        if [ -n "$$MODEL" ]; then echo "issue-fix: selected model $$MODEL"; else echo "issue-fix: no live catalog — using the CLI default model"; fi
+        cat ci/agents/issue-fix/prompt.md | claude $${MODEL:+--model $$MODEL} \
+          --permission-mode acceptEdits \
+          --allowedTools "Read,Edit,Write,Glob,Grep,Bash(./.ci-diag/kubectl:*)" \
+          --settings ci/agents/issue-fix/settings.json \
+          -p
+      - |
+        # No edits → the agent judged this not fixable in the repo. Post its
+        # summary (with the assessed-marker) as an issue comment so the next
+        # run skips this issue, and stop cleanly.
+        [ -f /tmp/alert-fix/issue.json ] || exit 0
+        set -eu
+        API="https://gitea.renner.uno/api/v1/repos/k.renner/homelab"
+        NUM=$$(jq -r '.number' /tmp/alert-fix/issue.json)
+        if git diff --quiet && git diff --cached --quiet \
+           && [ -z "$$(git ls-files --others --exclude-standard)" ]; then
+          echo "Agent made no edits — commenting assessment on issue #$$NUM."
+          [ -s /tmp/alert-fix/summary.md ] \
+            || echo "Agent made no edits and wrote no summary." > /tmp/alert-fix/summary.md
+          printf '\n\n<!-- alert-fix:assessed -->\n_Assessed by the issue-fix pipeline: no repo change proposed. Remove this comment to make the pipeline retry._\n' \
+            >> /tmp/alert-fix/summary.md
+          jq -n --rawfile body /tmp/alert-fix/summary.md '{body: $$body}' > /tmp/gitea-comment.json
+          code=$$(curl -s -o /tmp/gitea-resp.json -w '%{http_code}' \
+            -X POST -H "Authorization: token $$GITEA_TOKEN" \
+            -H "Content-Type: application/json" --data @/tmp/gitea-comment.json \
+            "$$API/issues/$$NUM/comments")
+          echo "POST comment -> HTTP $$code"
+          [ "$$code" -ge 200 ] && [ "$$code" -lt 300 ] || { cat /tmp/gitea-resp.json; exit 1; }
+          touch /tmp/alert-fix/no-edits
+        fi
+      - |
+        # Edits → commit, push, open the PR (Closes #<n> so the merge closes
+        # the issue), and link the PR from the issue.
+        [ -f /tmp/alert-fix/issue.json ] || exit 0
+        if [ -f /tmp/alert-fix/no-edits ]; then exit 0; fi
+        set -eu
+        API="https://gitea.renner.uno/api/v1/repos/k.renner/homelab"
+        NUM=$$(jq -r '.number' /tmp/alert-fix/issue.json)
+        BRANCH=$$(cat /tmp/alert-fix/branch)
+        TITLE="fix: $$(jq -r '.title' /tmp/alert-fix/issue.json) (#$$NUM)"
+        [ -s /tmp/alert-fix/summary.md ] \
+          || echo "Agent-proposed fix (no summary written by the agent)." > /tmp/alert-fix/summary.md
+        printf '\n\nCloses #%s\n' "$$NUM" >> /tmp/alert-fix/summary.md
+        git add -A
+        git commit -m "$$TITLE"
+        git push "https://$$PUSH_TOKEN@gitea.renner.uno/k.renner/homelab.git" HEAD:"$$BRANCH"
+        jq -n --arg title "$$TITLE" --rawfile body /tmp/alert-fix/summary.md \
+          --arg head "$$BRANCH" \
+          '{title: $$title, body: $$body, head: $$head, base: "master"}' > /tmp/gitea-pr.json
+        code=$$(curl -s -o /tmp/gitea-resp.json -w '%{http_code}' \
+          -X POST -H "Authorization: token $$PUSH_TOKEN" \
+          -H "Content-Type: application/json" --data @/tmp/gitea-pr.json \
+          "$$API/pulls")
+        echo "POST pull -> HTTP $$code"
+        [ "$$code" -ge 200 ] && [ "$$code" -lt 300 ] || { cat /tmp/gitea-resp.json; exit 1; }
+        jq '{number, html_url, title}' /tmp/gitea-resp.json
+        PR_URL=$$(jq -r '.html_url' /tmp/gitea-resp.json)
+        jq -n --arg body "🤖 Opened $$PR_URL with a proposed fix." '{body: $$body}' > /tmp/gitea-comment.json
+        code=$$(curl -s -o /tmp/gitea-resp.json -w '%{http_code}' \
+          -X POST -H "Authorization: token $$GITEA_TOKEN" \
+          -H "Content-Type: application/json" --data @/tmp/gitea-comment.json \
+          "$$API/issues/$$NUM/comments")
+        echo "POST comment -> HTTP $$code"
+        [ "$$code" -ge 200 ] && [ "$$code" -lt 300 ] || { cat /tmp/gitea-resp.json; exit 1; }

.woodpecker/pr-review.yml

@@ -1,47 +1,301 @@
-# DISABLED: the review-bot service (review-bot/, deployed from kubernetes/apps/review-bot/) reviews
-# pull requests. This pipeline only starts when triggered by hand and is kept for reference.
+# PR review bot — event-driven. Woodpecker fires this on `pull_request` (which
+# covers BOTH "PR opened" and "new commits pushed to an open PR"), so a review
+# is produced the moment a PR appears or is updated — no polling, no cron.
 #
-# PR review bot — per-repo STUB. All logic lives centrally in
-# konairius/homelab: ci/agents/pr-review/run.sh (plus prompt.md,
-# settings.json and scripts/ci/). This file is deployed byte-identical into every
-# reviewed repo (konairius/homelab, shelly-manager, authentik-samba-bridge,
-# homeassistant) and holds NO logic, so the copies cannot drift apart: change the
-# review in homelab and every repo picks it up on its next PR event. Only edit this
-# stub if the trigger, the image or the secrets change — then change all copies in
-# one go.
+# This file is deployed UNCHANGED into every repo the fleet wants reviewed
+# (k.renner/homelab, shelly-manager, authentik-samba-bridge, homeassistant).
+# Woodpecker events are per-repo, so each repo triggers a review of ITS OWN PR
+# only; the triggering PR is identified from the CI event (CI_REPO +
+# CI_COMMIT_PULL_REQUEST) instead of the old hourly cross-repo discovery loop.
+# Because it runs on the PR event, Woodpecker's built-in clone hands the step
+# THIS repo already checked out at the PR head — so, unlike the old cron model,
+# there is no manual clone of the target repo (the source of an earlier token
+# leak); the agents read the checkout in the workspace directly.
 #
-# Triggers on pull_request: PR opened AND new commits pushed to an open PR.
-# Approves only when the review found nothing red/yellow (see run.sh); humans merge.
+# Prompts: the two agent prompts live once, in k.renner/homelab under
+# ci/agents/pr-review/. The homelab checkout already has them; the other repos
+# don't, so the step falls back to a shallow clone of public homelab@master.
+# One source of truth for the review rubric across all repos.
 #
-# The central code is cloned from public homelab@main on every run, never taken
-# from the PR checkout, so a PR cannot alter the rubric or helper scripts it is
-# reviewed with.
+# Model selection: no model id — and no model family — is baked into this file.
+# scripts/ci/fetch-models.sh pulls the LIVE catalog (Anthropic /v1/models, via
+# CLAUDE_CODE_OAUTH_TOKEN), and a triage agent reads the PR diff + stats and
+# picks whichever currently-available model best fits the review, judged on the
+# catalog's own capability metadata. A model released tomorrow is eligible the
+# day it appears. The shell validates the pick against the catalog snapshot taken
+# BEFORE the triage runs; if the fetch fails, the pick is rejected, or the review
+# is complex/security-sensitive (where the injectable pick is deliberately
+# ignored), it leaves the model unset and the review runs on the CLI's default
+# model. A wrong-way miss only over/under-spends one review.
 #
-# Image: ci-claude:latest — the refresh pipeline keeps it on current Claude Code,
-# and a pinned tag would be the one thing in this file that diverges per repo.
+# Posture: posted as a Gitea PR review, event COMMENT normally or
+# REQUEST_CHANGES on a serious defect. It NEVER approves — a bot must not
+# satisfy branch protection. Humans still merge.
 #
-# Secrets (both must be granted the pull_request event):
-#   mistral_api_key         -> MISTRAL_API_KEY (Mistral Conversations API)
-#   review_bot_token        -> REVIEW_TOKEN (review-bot PAT, write:repository)
-# Claude CLI path disabled: claude_code_oauth_token -> CLAUDE_CODE_OAUTH_TOKEN
+# Dedup (guards pipeline retries/restarts, not cron re-runs anymore): the PR is
+# skipped when review-bot already left a review carrying the marker
+# "<!-- pr-review:sha=<head_sha> -->" for the PR's CURRENT head commit. A pushed
+# commit changes the head sha and fires a fresh pull_request event -> fresh
+# review.
 #
-# (Note: keep literal dollar-brace out of THIS file, even in comments — Woodpecker
-# runs its variable substitution over the whole YAML text.)
+# Tokens:
+#   CLAUDE_CODE_OAUTH_TOKEN (claude_code_oauth_token) — Claude subscription
+#     OAuth, how the `claude` CLI authenticates (same as the other agents).
+#   REVIEW_TOKEN (review_bot_token) — a dedicated `review-bot` Gitea account's
+#     PAT, scope write:repository. Reads the PR + diff and POSTs the review, all
+#     over the Gitea API (ephemeral curl Authorization headers — the token is
+#     never written to disk; the repo is NOT cloned). review-bot must be a
+#     collaborator (read) on every reviewed repo.
+#   Both secrets MUST be granted the `pull_request` event, or Woodpecker will
+#     withhold them from this pipeline. Because homelab is PUBLIC, it is also set
+#     to require approval for forked-PR pipelines, so an untrusted fork cannot
+#     rewrite this file to exfiltrate the tokens.
+#
+# Triggering: pull_request only — automatic on PR open + push-to-open-PR. To
+# re-review a PR, push a commit (the new head sha clears the dedup marker below
+# and earns a fresh review). There is deliberately no on-demand trigger for a
+# specific PR: Woodpecker exposes injected variables to YAML substitution and
+# `when` filters, NOT to a step's shell environment, so a PR=<n> re-review would
+# silently no-op (the step would find no PR number and exit green).
+# (Note: keep literal dollar-brace out of THIS file entirely, even in comments —
+# Woodpecker runs its variable substitution over the whole YAML text, comments
+# included, and an unparseable one errors the pipeline before any step runs.)
 when:
-  - event: manual
+  - event: pull_request
 
 steps:
   - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:latest
-    pull: true
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
     environment:
-      # CLAUDE_CODE_OAUTH_TOKEN:
-      #   from_secret: claude_code_oauth_token
-      MISTRAL_API_KEY:
-        from_secret: mistral_api_key
+      CLAUDE_CODE_OAUTH_TOKEN:
+        from_secret: claude_code_oauth_token
       REVIEW_TOKEN:
         from_secret: review_bot_token
     commands:
-      - rm -rf /tmp/pr-review-central
-      - git clone --quiet --depth 1 https://gitea.renner.uno/konairius/homelab /tmp/pr-review-central
-      - sh /tmp/pr-review-central/ci/agents/pr-review/run.sh
+      - |
+        # ---- Resolve the single triggering PR --------------------------------
+        # Woodpecker hands us the repo (CI_REPO, canonical owner/repo) and the PR
+        # number (CI_COMMIT_PULL_REQUEST, a pull_request-event built-in). Build
+        # /tmp/pr-review/worklist.jsonl with 0 or 1 entries; an empty file makes
+        # the next block a no-op (the agent never starts).
+        set -eu
+        API="https://gitea.renner.uno/api/v1"
+        AUTH="Authorization: token $$REVIEW_TOKEN"
+        SLUG="$$CI_REPO"
+        NUM="$${CI_COMMIT_PULL_REQUEST:-}"
+        REVIEW_BOT_LOGIN="review-bot"
+        mkdir -p /tmp/pr-review
+        : > /tmp/pr-review/worklist.jsonl
+
+        if [ -z "$$NUM" ]; then
+          echo "No PR number in CI env (event=$${CI_PIPELINE_EVENT:-?}) — nothing to review."
+          exit 0
+        fi
+        echo "Triggered for $$SLUG#$$NUM (event $${CI_PIPELINE_EVENT:-?})."
+
+        if ! curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM" > /tmp/pr-review/pr.json; then
+          echo "Cannot fetch $$SLUG#$$NUM (review-bot not a collaborator / PR gone?) — skipping."
+          exit 0
+        fi
+        STATE=$$(jq -r '.state'        /tmp/pr-review/pr.json)
+        DRAFT=$$(jq -r '.draft // false' /tmp/pr-review/pr.json)
+        TITLE=$$(jq -r '.title'        /tmp/pr-review/pr.json)
+        HEAD=$$(jq -r '.head.sha'      /tmp/pr-review/pr.json)
+        SHORT=$$(printf '%s' "$$HEAD" | cut -c1-8)
+
+        # Skip closed/merged, drafts, and WIP-titled PRs — not ready for review.
+        case "$$STATE" in open) ;; *) echo "#$$NUM is $$STATE — skip."; exit 0;; esac
+        case "$$DRAFT" in true) echo "#$$NUM draft — skip."; exit 0;; esac
+        case "$$TITLE" in [Ww][Ii][Pp]:*|[Ww][Ii][Pp]\ *) echo "#$$NUM WIP — skip."; exit 0;; esac
+
+        # Dedup: already reviewed at THIS head sha? (guards re-runs / restarts)
+        REVIEWED=$$(curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM/reviews?limit=50" \
+          | jq -r --arg u "$$REVIEW_BOT_LOGIN" --arg m "pr-review:sha=$$HEAD" \
+              '[.[] | select(.user.login == $$u) | select(.body | contains($$m))] | length' \
+          || echo 0)
+        if [ "$${REVIEWED:-0}" -gt 0 ]; then
+          echo "#$$NUM already reviewed at $$SHORT — skip."
+          exit 0
+        fi
+
+        jq -c --arg slug "$$SLUG" \
+          '{slug: $$slug, number: .number, head: .head.sha, base: .base.sha,
+            title: .title, author: .user.login}' \
+          /tmp/pr-review/pr.json >> /tmp/pr-review/worklist.jsonl
+        echo "#$$NUM enqueued (head $$SHORT)."
+      - |
+        # ---- Clone, pick model, review, post ---------------------------------
+        set -eu
+        [ -s /tmp/pr-review/worklist.jsonl ] || { echo "Nothing to review."; exit 0; }
+        API="https://gitea.renner.uno/api/v1"
+        AUTH="Authorization: token $$REVIEW_TOKEN"
+        WS="$$PWD"                     # workspace = this repo, checked out at the PR head by Woodpecker
+        MAX_DIFF_LINES=20000
+
+        # Agent prompts: use the local homelab checkout if present, else clone
+        # public homelab@master for them (one source of truth for the rubric).
+        PROMPTS="$$WS/ci/agents/pr-review"
+        HOMELAB="$$WS"                 # repo root holding ci/ AND scripts/ci/
+        # NB: for a homelab PR, HOMELAB is the PR head checkout, so the
+        # fetch-models.sh run below executes branch-supplied shell. It is wrapped
+        # in `env -u REVIEW_TOKEN`, so the only credential it can see is
+        # CLAUDE_CODE_OAUTH_TOKEN. This is not a new exposure regardless: on
+        # pull_request Woodpecker already takes .woodpecker/*.yml itself from the
+        # PR branch, so a branch can run arbitrary commands in this step anyway.
+        if [ ! -f "$$PROMPTS/prompt.md" ]; then
+          rm -rf /tmp/pr-review/homelab
+          git clone --quiet --depth 1 https://gitea.renner.uno/k.renner/homelab \
+            /tmp/pr-review/homelab
+          PROMPTS=/tmp/pr-review/homelab/ci/agents/pr-review
+          HOMELAB=/tmp/pr-review/homelab
+        fi
+        echo "Using prompts from $$PROMPTS"
+
+        gitea_post() {                 # method url datafile ; echoes+checks HTTP
+          code=$$(curl -s -o /tmp/pr-review/resp.json -w '%{http_code}' \
+            -X "$$1" -H "$$AUTH" -H "Content-Type: application/json" \
+            --data @"$$3" "$$2")
+          echo "$$1 $$2 -> HTTP $$code"
+          if [ "$$code" -lt 200 ] || [ "$$code" -ge 300 ]; then
+            echo "Gitea API error body:"; cat /tmp/pr-review/resp.json; echo; return 1
+          fi
+        }
+
+        while IFS= read -r ITEM; do
+          SLUG=$$(printf '%s' "$$ITEM"  | jq -r '.slug')
+          NUM=$$(printf '%s' "$$ITEM"   | jq -r '.number')
+          HEAD=$$(printf '%s' "$$ITEM"  | jq -r '.head')
+          TITLE=$$(printf '%s' "$$ITEM" | jq -r '.title')
+          SHORT=$$(printf '%s' "$$HEAD" | cut -c1-8)
+          echo "=== Reviewing $$SLUG#$$NUM ($$SHORT) ==="
+          D="/tmp/pr-review/$$NUM"
+          rm -rf "$$D"; mkdir -p "$$D"
+
+          # PR metadata + unified diff (authenticated; private repos need it).
+          curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM" > "$$D/pr.json"
+          curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM.diff" > "$$D/diff.full.patch" \
+            || { echo "Could not fetch diff for $$SLUG#$$NUM — skipping."; continue; }
+          FILES=$$(grep -c '^diff --git ' "$$D/diff.full.patch" || true)
+          ADDED=$$(grep -c '^+[^+]' "$$D/diff.full.patch" || true)
+          DELETED=$$(grep -c '^-[^-]' "$$D/diff.full.patch" || true)
+          TOTAL=$$(wc -l < "$$D/diff.full.patch")
+          if [ "$$TOTAL" -gt "$$MAX_DIFF_LINES" ]; then
+            head -n "$$MAX_DIFF_LINES" "$$D/diff.full.patch" > "$$D/diff.patch"
+            printf '\n... [diff truncated at %s of %s lines] ...\n' \
+              "$$MAX_DIFF_LINES" "$$TOTAL" >> "$$D/diff.patch"
+          else
+            cp "$$D/diff.full.patch" "$$D/diff.patch"
+          fi
+          printf 'repo: %s\nPR: #%s\nfiles_changed: %s\nlines_added: %s\nlines_deleted: %s\ndiff_total_lines: %s\n' \
+            "$$SLUG" "$$NUM" "$$FILES" "$$ADDED" "$$DELETED" "$$TOTAL" > "$$D/stats.txt"
+
+          # Cheaper inputs for the TRIAGE agent (which now runs on a capable
+          # default model): the COMPLETE changed-file list, plus a diff capped far
+          # smaller than the review's. Triage only gauges size/shape/sensitivity,
+          # so it never needs the full (up to 20k-line) review diff — this keeps
+          # "triage is cheap" true now that its model is no longer pinned to Haiku.
+          grep '^diff --git ' "$$D/diff.full.patch" | sed 's|^diff --git a/||; s| b/.*$$||' \
+            | sort -u > "$$D/files.txt"
+          head -n 4000 "$$D/diff.patch" > "$$D/diff.triage.patch"
+
+          # No manual clone of the target repo: this runs on a pull_request
+          # event, so Woodpecker's built-in clone already checked THIS repo out
+          # at the PR head in $$WS — the agents (which cd into $$WS) read it
+          # directly for context. The old cron model had to clone each target
+          # repo by hand (its workspace was homelab@master), and that clone
+          # embedded REVIEW_TOKEN in the URL, persisting a write-scoped PAT in
+          # work/target/.git/config — readable by the review agent whose report
+          # is posted publicly. Per-repo pull_request triggering makes the manual
+          # clone redundant AND keeps the token off disk (it now lives only in
+          # the ephemeral curl Authorization headers used for the API calls).
+
+          # Inputs the agents read live at ./.pr-review (symlink into $$D).
+          rm -f "$$WS/.pr-review"; ln -s "$$D" "$$WS/.pr-review"
+
+          # --- Triage: an agent picks the model from the LIVE catalog, so no
+          #     model id and no model family is baked into this file.
+          #     fetch-models.sh asks the Anthropic /v1/models list (via the same
+          #     CLAUDE_CODE_OAUTH_TOKEN) and we hand the triage the result at
+          #     ./.pr-review/models.txt. If the fetch fails the file is empty and
+          #     MODEL stays empty below, so the review runs on the CLI default.
+          if MODELS_TABLE=$$(env -u REVIEW_TOKEN sh "$$HOMELAB/scripts/ci/fetch-models.sh" --table); then
+            printf '%s\n' "$$MODELS_TABLE" > "$$D/models.txt"
+            echo "Live model catalog ($$(wc -l < "$$D/models.txt") models) written for triage."
+          else
+            : > "$$D/models.txt"
+            echo "Model catalog fetch failed — triage/review will use the CLI default model."
+          fi
+          # env -u REVIEW_TOKEN: the agents never call Gitea (the shell does, via
+          # $$AUTH), so the write:repository PAT must not sit in the agent's env
+          # where a prompt-injected diff could read /proc/self/environ. Defence in
+          # depth alongside the settings.json Read-sandbox. Triage itself runs on
+          # the CLI default model (no --model) — we don't name a family even to
+          # bootstrap the pick.
+          cat "$$PROMPTS/triage-prompt.md" | env -u REVIEW_TOKEN claude \
+            --permission-mode acceptEdits \
+            --allowedTools "Read,Glob,Grep,Write" \
+            --settings "$$PROMPTS/settings.json" \
+            -p || echo "(triage agent exited non-zero)"
+          # Use the triage's pick only if it is a currently-available id;
+          # otherwise (hallucinated id, or empty catalog) leave MODEL empty so
+          # the review runs on the CLI default — a run never dies on selection.
+          MODEL=$$(jq -r '.model // empty' "$$D/triage.json" 2>/dev/null || true)
+          CPLX=$$(jq -r '.complexity // "unknown"' "$$D/triage.json" 2>/dev/null || echo unknown)
+          # triage.json is UNTRUSTED (the triage reads the attacker-controlled
+          # diff and its Write is not path-scoped). valid-model.sh is the single,
+          # unit-tested gate (scripts/ci/test-model-selection.sh): it echoes the id
+          # only if it is a safe [A-Za-z0-9._-] token present in the catalog, else
+          # nothing — so a value like "claude-x\n--dangerously-skip-permissions"
+          # can never reach the unquoted --model expansion below. Empty -> default.
+          # CRITICAL: validate against $$MODELS_TABLE — the catalog captured in a
+          # shell variable BEFORE the triage ran — NOT the on-disk ./.pr-review/
+          # models.txt, which the triage's unscoped Write could overwrite to smuggle
+          # an attacker-chosen token past the membership check (the triage reads the
+          # file, but only $$MODELS_TABLE is trusted). This mirrors pick-model.sh,
+          # which validates its pick against its in-memory table for the same reason.
+          # env -u REVIEW_TOKEN: like fetch-models.sh above, this runs a script from
+          # $$HOMELAB (branch-supplied for a homelab PR) — it must not see the PAT.
+          # `|| true`: keep the "a run never dies on selection" invariant even if
+          # the script is absent (e.g. a PR that moves scripts/ci/).
+          MODEL=$$(printf '%s\n' "$$MODELS_TABLE" | env -u REVIEW_TOKEN sh "$$HOMELAB/scripts/ci/valid-model.sh" "$$MODEL" || true)
+          [ -n "$$MODEL" ] || echo "Triage model rejected or not in live catalog — using the CLI default model."
+          # Defence in depth: for a complex / security-sensitive review, do NOT
+          # trust the triage's pick (it read the attacker-controlled diff and could
+          # be injected into naming the weakest catalog entry to get a shallow
+          # review). Fall back to the CLI default — the strongest configured model.
+          if [ "$$CPLX" = complex ] && [ -n "$$MODEL" ]; then
+            echo "complex review — ignoring triage pick '$$MODEL', using the CLI default (strongest) model."
+            MODEL=
+          fi
+          if [ -n "$$MODEL" ]; then echo "Triage: complexity=$$CPLX -> model=$$MODEL"; else echo "Triage: complexity=$$CPLX -> CLI default model"; fi
+
+          # --- Review with the chosen model ---
+          ( cd "$$WS" && cat "$$PROMPTS/prompt.md" | env -u REVIEW_TOKEN claude $${MODEL:+--model $$MODEL} \
+              --permission-mode acceptEdits \
+              --allowedTools "Read,Glob,Grep,Write" \
+              --settings "$$PROMPTS/settings.json" \
+              -p ) || echo "(review agent exited non-zero)"
+
+          if [ ! -s "$$D/report.md" ]; then
+            echo "Agent wrote no report for $$SLUG#$$NUM — skipping post."
+            continue
+          fi
+          EVENT=$$(jq -r '.event // "COMMENT"' "$$D/verdict.json" 2>/dev/null || echo COMMENT)
+          case "$$EVENT" in REQUEST_CHANGES|COMMENT) ;; *) EVENT=COMMENT;; esac   # never APPROVE
+
+          # Assemble the review body: header + report + marker footer.
+          {
+            printf '🤖 **Automated review** — model `%s` (complexity: %s), commit `%s`.\n\n' \
+              "$${MODEL:-CLI default}" "$$CPLX" "$${HEAD}"
+            printf '> Advisory review by the `review-bot` pipeline. It never approves; a human merges. Push new commits to get a fresh review.\n\n---\n\n'
+            cat "$$D/report.md"
+            printf '\n\n<!-- pr-review:sha=%s -->\n' "$$HEAD"
+          } > "$$D/body.md"
+          jq -n --rawfile b "$$D/body.md" --arg e "$$EVENT" --arg c "$$HEAD" \
+            '{body: $$b, event: $$e, commit_id: $$c}' > "$$D/review.json"
+          gitea_post POST "$$API/repos/$$SLUG/pulls/$$NUM/reviews" "$$D/review.json" \
+            && echo "Posted $$EVENT review on $$SLUG#$$NUM." \
+            || echo "FAILED to post review on $$SLUG#$$NUM (continuing)."
+          rm -f "$$WS/.pr-review"
+        done < /tmp/pr-review/worklist.jsonl
+        echo "PR review run complete."

.woodpecker/review-bot-test.yml

@@ -1,13 +0,0 @@
-# Tests for the review-bot Go service (review-bot/). Offline: the tests run against fake Gitea and
-# Mistral servers, so the step needs no secret and no network beyond the module cache.
-when:
-  - event: pull_request
-    path: ["review-bot/**", ".woodpecker/review-bot-test.yml"]
-
-steps:
-  - name: go-test
-    image: golang:1.27.3-bookworm@sha256:972b4110ba0f47539a975c31e52c64765e1ff19f2d74469ae4a595c6676e80a5
-    commands:
-      - test -z "$(gofmt -l review-bot)"
-      - go -C review-bot vet ./...
-      - go -C review-bot test -race ./...

.woodpecker/security-scan.yml

@@ -5,7 +5,7 @@
 #   - cron       : the named `daily-security-scan` cron, matched by NAME.
 #                  Crons cannot carry variables — `woodpecker-cli repo cron add`
 #                  has no --var — so a variable gate can never match a cron.
-#   - deployment : woodpecker-cli pipeline deploy konairius/homelab <n> security-scan
+#   - deployment : woodpecker-cli pipeline deploy k.renner/homelab <n> security-scan
 #                  The target rides in pipeline metadata as
 #                  CI_PIPELINE_DEPLOY_TARGET, which the workflow-root filter
 #                  does see, unlike manual-run variables on Woodpecker 3.18
@@ -18,12 +18,12 @@ when:
 
 steps:
   - name: iac-scan
-    image: aquasec/trivy:latest@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c
+    image: aquasec/trivy:latest
     commands:
       - trivy fs --format json -o trivy-iac.json --severity CRITICAL,HIGH --ignore-unfixed .
 
   - name: image-scans
-    image: aquasec/trivy:latest@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c
+    image: aquasec/trivy:latest
     commands:
       - trivy image --download-db-only
       - |
@@ -65,12 +65,12 @@ steps:
     failure: ignore
 
   - name: security-fix-agent
-    # Auto-fix run: only on main (cron / manual / direct push). On the
+    # Auto-fix run: only on master (cron / manual / direct push). On the
     # bot's own PR branches, the scans re-run as a re-validation gate but
     # this step is skipped — otherwise we'd fork-bomb PRs.
     when:
-      - branch: main
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
+      - branch: master
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
     environment:
       CLAUDE_CODE_OAUTH_TOKEN:
         from_secret: claude_code_oauth_token
@@ -147,15 +147,15 @@ steps:
         TITLE="security: agent-proposed fixes ($$(date +%Y-%m-%d))"
         git add -A
         git commit -m "$$TITLE"
-        git push "https://$$GITEA_TOKEN@gitea.renner.uno/konairius/homelab.git" HEAD:"$$BRANCH"
+        git push "https://$$GITEA_TOKEN@gitea.renner.uno/k.renner/homelab.git" HEAD:"$$BRANCH"
         jq -n \
           --arg title "$$TITLE" \
           --arg body  "$$BODY"  \
           --arg head  "$$BRANCH" \
-          '{title: $$title, body: $$body, head: $$head, base: "main"}' \
+          '{title: $$title, body: $$body, head: $$head, base: "master"}' \
         | curl -sfL -X POST \
             -H "Authorization: token $$GITEA_TOKEN" \
             -H "Content-Type: application/json" \
             --data @- \
-            https://gitea.renner.uno/api/v1/repos/konairius/homelab/pulls \
+            https://gitea.renner.uno/api/v1/repos/k.renner/homelab/pulls \
         | jq '{number, html_url, title}'

.woodpecker/update-audit.yml

@@ -14,9 +14,9 @@
 #              `repo cron add` takes name, branch, schedule and enabled — so the
 #              name filter is the only thing that can select a workflow for a
 #              scheduled run:
-#                woodpecker repo cron add --repo konairius/homelab \
-#                  --name update-audit --branch main --schedule '@weekly'
-#   - deploy : woodpecker-cli pipeline deploy konairius/homelab <n> update-audit
+#                woodpecker repo cron add --repo k.renner/homelab \
+#                  --name update-audit --branch master --schedule '@weekly'
+#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> update-audit
 #              where <n> is an existing pipeline to promote.
 #
 # NOTE: e2e.yml runs on an explicit list of cron names that includes this one,
@@ -30,7 +30,7 @@ when:
 
 steps:
   - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
     environment:
       CLAUDE_CODE_OAUTH_TOKEN:
         from_secret: claude_code_oauth_token
@@ -111,7 +111,7 @@ steps:
         set -eu
         TITLE="Infrastructure update audit"
         STAMP=$$(date -u +%Y-%m-%dT%H:%M:%SZ)
-        API="https://gitea.renner.uno/api/v1/repos/konairius/homelab"
+        API="https://gitea.renner.uno/api/v1/repos/k.renner/homelab"
 
         # write <method> <url> <payload-file>  -> prints HTTP code, fails non-2xx
         gitea_write() {

CONTRIBUTING.md

@@ -1,7 +0,0 @@
-# Contributing
-
-## Citing other YAML from a comment
-
-- Mark the block you cite with a comment line `# anchor: <name>` (kebab-case, unique in that file). If the block is inside a `|` block scalar, put the marker just above the key, never inside the scalar.
-- Cite it as `path.yaml#<name>`, with the path relative to the citing file's directory or to the repo root. Never cite `path.yaml:<line>`, because line numbers go stale on any edit above them.
-- `scripts/ci/yaml-citations.sh` enforces this in `ci-manifests-test.yml`: it fails on line-number citations and on citations whose file or anchor does not exist.

README.md

@@ -1,55 +0,0 @@
-# homelab
-
-GitOps source for the Renner homelab cluster. Flux applies `main` to
-production, so every merge is a production change. Changes go in by pull
-request only; see [CONTRIBUTING.md](CONTRIBUTING.md) for YAML conventions.
-
-## Change rules that protect Home Assistant
-
-Owner decision (REN-240, REN-243). Home Assistant runs the house, so it
-gets two rules on top of normal review.
-
-### 1. Disruptive work runs only 02:00–05:00 Europe/Berlin
-
-Planned work that can restart Home Assistant or disrupt the cluster is
-merged and applied only between **02:00 and 05:00 Europe/Berlin**. That
-covers:
-
-- node drain, node reboot, Talos upgrade
-- CNI (Cilium, Multus) and cluster-wide network policy
-- storage (Rook/Ceph, CSI, snapshot controller, CNPG operator)
-- Flux core (`kubernetes/clusters/*/flux-system/`, Flux controllers)
-- Authentik
-- OpenBao, including secret rotation that restarts consumers
-- anything in `kubernetes/apps/home-assistant/`
-
-A PR in this list states in its description that it waits for the window.
-Flux applies on merge, so "wait for the window" means **merge** in the
-window, not just apply. If unsure whether a change is disruptive, treat it
-as disruptive.
-
-Not covered: Home Assistant image bumps by Flux image automation. They stay
-fully automatic by owner decision. If one breaks Home Assistant, roll it
-back with [RUNBOOK-home-assistant-image-rollback.md](kubernetes/RUNBOOK-home-assistant-image-rollback.md).
-An incident rollback is not planned work; the owner decides its timing.
-
-Also not covered, by owner decision: the Hermes `ha-updates` job
-(`kubernetes/apps/hermes/skills/ha-updates/`) installs HACS and device-firmware
-updates that Home Assistant offers, and restarts Home Assistant for them, on
-its own inside the 02:00–05:00 window (UniFi and SMLIGHT firmware excluded;
-reported only). The window is enforced by its `ha.py`, not only by the prompt.
-
-### 2. HA-impact review before merge
-
-Every PR that touches `kubernetes/apps/home-assistant/`, or makes a
-cluster-wide disruptive change from the list above, needs an **HA-impact
-review** before merge: a named reviewer states in the PR what Home
-Assistant will notice (restart, lost integration, lost history, none) and
-approves.
-
-- Reviewer: Hestia, the Home Assistant maintainer.
-- Until the owner hands this review to Hestia (the `hestia` Hermes profile, which today only states the expected impact in its PRs): the owner does the HA-impact review.
-
-This is in addition to the normal review and merge by Argus. Argus does
-not merge such a PR without the HA-impact approval.
-

ansible/ansible.cfg

@@ -112,7 +112,7 @@
 
 # (string) This setting controls the default policy of fact gathering (facts discovered about remote systems).
 # This option can be useful for those wishing to save fact gathering time. Both 'smart' and 'explicit' will use the cache plugin.
-gathering=smart
+;gathering=implicit
 
 # (list) Set the `gather_subset` option for the :ref:`ansible_collections.ansible.builtin.setup_module` task in the implicit fact gathering. See the module documentation for specifics.
 # It does **not** apply to user defined ``ansible.builtin.setup`` tasks.
@@ -134,7 +134,7 @@ gathering=smart
 ;hash_behaviour=replace
 
 # (pathlist) Comma separated list of Ansible inventory sources
-inventory=inventory
+inventory=~/source/homelab/ansible/inventory
 
 # (pathspec) Colon separated paths in which Ansible will search for HttpApi Plugins.
 ;httpapi_plugins={{ ANSIBLE_HOME ~ "/plugins/httpapi:/usr/share/ansible/plugins/httpapi" }}
@@ -206,7 +206,7 @@ inventory=inventory
 ;poll_interval=15
 
 # (path) Option for connections using a certificate or key file to authenticate, rather than an agent or passwords, you can set the default value here to avoid re-specifying --private-key with every invocation.
-;private_key_file=
+private_key_file=~/.ssh/id_ed25519
 
 # (boolean) By default, imported roles publish their variables to the play and other roles, this setting can avoid that.
 # This was introduced as a way to reset role variables to default values if a role is used more than once in a playbook.
@@ -274,7 +274,7 @@ roles_path=../roles:./roles
 
 # (path) The vault password file to use. Equivalent to --vault-password-file or --vault-id
 # If executable, it will be run and the resulting stdout will be used as the password.
-vault_password_file=vault-password.sh
+vault_password_file=~/source/homelab/ansible/vault-password.sh
 
 # (integer) Sets the default verbosity, equivalent to the number of ``-v`` passed in the command line.
 ;verbosity=0
@@ -484,7 +484,7 @@ vault_password_file=vault-password.sh
 # It can result in a very significant performance improvement when enabled.
 # However this conflicts with privilege escalation (become). For example, when using 'sudo:' operations you must first disable 'requiretty' in /etc/sudoers on all managed hosts, which is why it is disabled by default.
 # This setting will be disabled if ``ANSIBLE_KEEP_REMOTE_FILES`` is enabled.
-pipelining=True
+;pipelining=False
 
 
 [colors]
@@ -694,8 +694,3 @@ pipelining=True
 # (list) default list of tags to skip in your plays, has precedence over Run Tags
 ;skip=
 
-
-[ssh_connection]
-# Connections stay open across the plays of a run, so each host costs one
-# YubiKey PIN; plays/close-connections.yml closes them at the end of a run.
-ssh_args=-C -o ControlMaster=auto -o ControlPersist=30m

ansible/docs/edge-host.md

@@ -41,11 +41,13 @@ Build and boot the dedicated installer image — see
 4. Add the host to `[edge_host]` in `inventory/pve_hosts.yml`, by name.
    That group is the base: every edge thin client belongs to it, and it
    assumes no peripherals.
-5. From the workstation, in `ansible/`:
+5. From the workstation:
    ```
-   ansible-playbook site.yml
+   ansible-playbook -i inventory/pve_hosts.yml plays/install-edge-host.yml
    ```
-   One unrestricted run, no `--limit`. It converges every host, which is
+   One unrestricted run over the whole group. The play is self-contained --
+   it carries `certificates` and `common` itself, so there is no second
+   command and no `--limit`. Running it converges every edge host, which is
    the point: a run that touches only the new box tells you nothing about
    the others.
 
@@ -60,6 +62,7 @@ inventory change.
 |---|---|---|
 | Innomaker USB2CAN (Vitocal/VX3 + E380) | `heating_can` | `plays/install-heating-can.yml` |
 | Nabu Casa ZWA-2 Z-Wave radio | `zwave_gateway` | `plays/install-zwave-gateway.yml` |
+| FTDI serial power-meter head | `power_meter` | `plays/install-power-meter.yml` |
 
 Add the host to the matching group, then run that playbook. They depend on
 the base play having run: all three need the local MQTT bridge, and two of
@@ -70,20 +73,21 @@ none of these commands.
 
 ## Secrets (before first play run)
 
-Create `inventory/group_vars/edge_host/vault.yml` with ansible-vault. The
-cluster broker has no `open3e` user: edge-1 logs in only as the register
-account `edge-1` (see "MQTT bridge" below), so
-`vault_edge_open3e_mqtt_password` reaches only the loopback broker, which
-allows anonymous clients; any random value works (`openssl rand -hex 24`).
-The zwave-js value comes out of the cluster:
+Create `inventory/group_vars/edge_host/vault.yml` with ansible-vault; the
+current live values come straight out of the cluster:
 
 ```
+# sml2mqtt broker password
+kubectl -n home-assistant get secret mosquitto-sml2mqtt-credentials -o jsonpath='{.data.password}' | base64 -d
+# open3e broker password
+kubectl -n home-assistant get secret mosquitto-open3e-credentials -o jsonpath='{.data.password}' | base64 -d
 # zwave-js session secret
 kubectl -n home-assistant get secret zwave-js-session -o jsonpath='{.data.session-secret}' | base64 -d
 ```
 
 ```yaml
 # vault.yml
+vault_edge_sml2mqtt_mqtt_password: "..."
 vault_edge_open3e_mqtt_password: "..."
 vault_edge_zwavejs_session_secret: "..."
 vault_edge_loki_password: "..."   # dedicated edge-1 Loki push cred (see "Logging & eMMC")
@@ -228,24 +232,11 @@ They connect to a **local mosquitto on loopback** (`127.0.0.1:1883`,
   broker is briefly unreachable, and flushes on reconnect — edge-1 no longer
   hard-depends on cluster/network uptime for every publish.
 
-The local broker is `allow_anonymous` on loopback — the trust boundary is the
-host; nothing off-host can reach it.
-
-**Bridge credential (REN-1307).** One bridge `connection` logs in as the
-register account `edge-1` (`kubernetes/apps/home-assistant/mqtt-accounts.yaml`)
-and carries `open3e/#` out and `open3e/cmnd` back. The cluster broker does not
-accept the vault-password bridges in `bridge.d/legacy.conf`, which exists only
-until the first push. Its password is pushed
-by mqtt-account-controller to `edge-mqtt-cred` (`:8443`, the controller's
-esp32 API over HTTPS). That endpoint accepts only a client certificate from
-the MQTT CA with CN `mqtt-account-controller`; edge-1 holds the public CA and
-its own server key, nothing else. On a push it writes
-`/etc/mosquitto/bridge.d/edge-1.conf`, deletes `bridge.d/legacy.conf` and
-restarts the local mosquitto. The controller sets a new password whenever the
-bridge reports "not connected" (also after a reinstall), so no play run is
-needed for a rotation. Diagnosis: `journalctl -u
-edge-mqtt-cred`, and `mosquitto_sub -t '$SYS/broker/connection/edge-1/state'`
-(retained `1` = connected).
+Two bridge `connection`s preserve the existing cluster identities, so **no
+cluster-side change was needed**: one authenticates as the `open3e` user
+(carries `open3e/#` out + `open3e/cmnd` back), one as `sml2mqtt` (`sml2mqtt/#`
+out). The local broker is `allow_anonymous` on loopback — the trust boundary
+is the host; nothing off-host can reach it.
 
 **eMMC:** the local broker writes **nothing** to flash — `persistence false`
 (retained/queued messages in RAM) and `log_dest stdout` (to the volatile

ansible/files/discover_shelly.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""
+Discover Shelly devices on the network via HTTP API
+Returns JSON array of discovered devices
+"""
+
+import sys
+import json
+import ipaddress
+import concurrent.futures
+import urllib.request
+import urllib.error
+
+def check_shelly(ip):
+    """Check if IP is a Shelly device and return info"""
+    try:
+        url = f"http://{ip}/shelly"
+        req = urllib.request.Request(url, headers={'User-Agent': 'Ansible-Shelly-Discovery'})
+        with urllib.request.urlopen(req, timeout=2) as response:
+            data = json.loads(response.read().decode())
+            # Gen1 devices don't have 'gen' field, default to 1
+            gen = data.get('gen', 1)
+            if gen is None:
+                gen = 1
+            return {
+                'ip': str(ip),
+                'type': data.get('type', 'unknown'),
+                'mac': data.get('mac', ''),
+                'gen': gen,
+                'name': data.get('name', data.get('id', str(ip))),
+                'fw_id': data.get('fw_id', data.get('fw', 'unknown'))
+            }
+    except (urllib.error.URLError, urllib.error.HTTPError, OSError, json.JSONDecodeError):
+        return None
+    except Exception:
+        return None
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: discover_shelly.py <network1> [network2] ...")
+        print("Example: discover_shelly.py 10.2.30.0/24 10.2.20.0/24")
+        sys.exit(1)
+
+    networks = sys.argv[1:]
+    devices = []
+
+    # Collect all IPs to scan
+    all_ips = []
+    for network in networks:
+        try:
+            all_ips.extend(ipaddress.IPv4Network(network).hosts())
+        except ValueError as e:
+            print(f"Invalid network: {network} - {e}", file=sys.stderr)
+            continue
+
+    # Scan in parallel
+    with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+        results = executor.map(check_shelly, all_ips)
+        for result in results:
+            if result:
+                devices.append(result)
+
+    # Output JSON
+    print(json.dumps(devices, indent=2))
+
+if __name__ == "__main__":
+    main()

ansible/inventory/group_vars/all/vars.yml

@@ -5,10 +5,27 @@ pve_token_id: "ansible"
 ansible_user: "ladmin"
 ansible_python_interpreter: /usr/bin/python3
 
-# The only plain key the admin logins accept on [ssh_keys_managed] hosts
-# (ssh-ca role, tasks/authorized_keys.yml). People log in with OpenBao SSH CA
-# certificates; this one is for when that path is unavailable.
-#   breakglass-neo: PIV key (slot 9a, PIN) on the offline YubiKey NEO.
-#     ssh -I /usr/lib/opensc-pkcs11.so root@<host>
-ssh_admin_authorized_keys:
-  - key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348"
+# SSH Public Keys for Cloud-Init
+# ansible-runner@ci-new removed (issue #184): dead key, no such host exists.
+common_ssh_public_keys: |
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
+
+# Network-of-origin restriction for the woodpecker-ci@homelab CI key
+# (issue #184). Consumed by plays/configure-ssh-keys.yml as the authorized_key
+# `from=` option so the CI key only authenticates from within the cluster's
+# pod network — NOT from any host-pinned IP.
+#
+# The in-cluster Woodpecker step pods SSH to fleet hosts using their POD IP as
+# source, not a node IP: Cilium runs `ipv4-native-routing-cidr=10.0.0.0/8` +
+# `ipv6-native-routing-cidr=fd00::/8` with masquerade enabled, so pod egress to
+# any 10.x / fd.. LAN destination is NOT SNAT'd, and each node's PodCIDR is
+# advertised to the UDM over BGP (kubernetes/infrastructure/post-crds/
+# cilium-bgp.yaml) so return traffic L3-routes back to the pod. Hosts therefore
+# see the real pod source IP.
+#
+# The cluster pod subnets (kubernetes/talos/patches/cluster.yaml podSubnets)
+# are the stable boundary — they survive node DHCP-lease churn and pod
+# rescheduling across k8s-1/2/3, unlike pinning any node's current IP.
+# v6 is included because some SSH targets (edge IoT hosts) are IPv6-only.
+ci_ssh_from_cidrs: '10.244.0.0/16,fd10:244::/48'

ansible/inventory/group_vars/claude_workstation/vault.yml

@@ -0,0 +1,16 @@
+$ANSIBLE_VAULT;1.1;AES256
+35646538363164363061366433343963623166336630376233333033646637336266343337343230
+3233356135323762636439336466663430366238363338650a633762623366643137303333383634
+62623739363132303335636261626263333332623066303161633466306431623930666166323833
+6439663964353338350a656435626664646666363831363164393335386666356637613037323838
+63653461656366613331353064656161346263356131346437666632386166623037373363333534
+37323435623366643534633333303534313434613135653237303966656162633266623433363934
+36343133393834393765353338383764303664363431653865633764306138376366623832346262
+36303766623331666632613639366139333661636665363631326565653938343637323861653064
+37393465383263376337376639613162306563653365623036376662303030613264653534353332
+66363030356363373439306164386561363734353631363738656236633164633062613437663366
+62386430396262366336316166353366383233316333333234666538643962336361363630363762
+31653963303032373531386566336632336130383034343430303964396538663237346434613737
+39643765623766373639306132316635376434643232336163303066393736313130323065666232
+39393038616162666666383837326137376233613437636361636238663566336365653863643933
+353338646134383038323134323034653564

ansible/inventory/group_vars/edge_host/vars.yml

@@ -2,6 +2,7 @@
 # Secret indirection — real values live in vault.yml (ansible-vault) next
 # to this file; see docs/edge-host.md for how to populate it from
 # the cluster secrets.
+edge_sml2mqtt_mqtt_password: "{{ vault_edge_sml2mqtt_mqtt_password }}"
 edge_open3e_mqtt_password: "{{ vault_edge_open3e_mqtt_password }}"
 edge_zwavejs_session_secret: "{{ vault_edge_zwavejs_session_secret }}"
 # Loki push credential (dedicated 'edge-1' user, Traefik basic-auth on the
@@ -38,8 +39,8 @@ chrony_hwtimestamp: false
 
 # --- Local MQTT contract -------------------------------------------------
 # Shared between the broker and its clients, so it belongs to the inventory
-# rather than to either side's role. mqtt-bridge listens here; heating-can
-# publishes here; and the bridge authenticates to the cluster
+# rather than to either side's role. mqtt-bridge listens here; heating-can and
+# power-meter-sml publish here; and the bridge authenticates to the cluster
 # broker AS the open3e user, so it needs that identity too.
 #
 # Keeping these in one role's defaults only worked while a single play loaded
@@ -50,11 +51,3 @@ edge_mqtt_host: 127.0.0.1
 edge_mqtt_port: 1883
 edge_open3e_mqtt_user: open3e
 edge_open3e_mqtt_topic: open3e
-
-# Units claude-bot may restart (ssh-ca role); the containers go through docker.
-ssh_ca_claude_sudo_units:
-  - mosquitto
-  - edge-mqtt-cred
-  - edge-alloy
-  - keepalived
-  - chronyd

ansible/inventory/group_vars/netbird_proxy/vars.yml

@@ -1,9 +0,0 @@
----
-# The host NetBird client (wt0) must not take over DNS: resolv.conf is pinned to
-# the systemd-resolved stub by the netbird-proxy role.
-netbird_disable_dns: true
-
-ssh_ca_claude_sudo_units:
-  - netbird
-  - netbird-proxy-flood-watchdog
-  - crowdsec-firewall-bouncer

ansible/inventory/host_vars/netbird.renner.uno.yml

@@ -2,6 +2,3 @@
 # NetBird server - VPS not managed by Proxmox
 # Enable Docker monitoring (cAdvisor) for container metrics
 docker_monitoring_enabled: true
-
-ssh_ca_claude_sudo_units:
-  - netbird

ansible/inventory/pve_hosts.yml

@@ -13,11 +13,11 @@
 # Their hosts no longer resolve, except the three that resolve to Talos nodes.
 
 [netbird_server]
-netbird.renner.uno ansible_host=netbird.nb.renner.uno ansible_user=root
+netbird.renner.uno ansible_user=root
 
 [netbird_proxy]
-proxy-1.renner.uno ansible_host=proxy-1.nb.renner.uno ansible_user=root
-proxy-2.renner.uno ansible_host=proxy-2.nb.renner.uno ansible_user=root
+proxy-1.renner.uno ansible_user=root
+proxy-2.renner.uno ansible_user=root
 
 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN
@@ -41,9 +41,13 @@ edge-1.iot.renner.uno
 [heating_can]
 edge-1.iot.renner.uno
 
+[power_meter]
+edge-1.iot.renner.uno
+
 [edge_host:children]
 zwave_gateway
 heating_can
+power_meter
 
 # NTP servers. edge-1 was added here so the role would survive the PVE hosts
 # being retired for bare-metal Talos, which cannot run chrony or keepalived
@@ -62,28 +66,10 @@ heating_can
 [ntp_cluster:children]
 edge_host
 
-# Hosts that trust the OpenBao SSH user CA (plays/configure-ssh-ca.yml).
-# Rolled out one host at a time: add the next host after the previous one
-# was verified.
-[ssh_ca]
-proxy-1.renner.uno
-proxy-2.renner.uno
-netbird.renner.uno
-edge-1.iot.renner.uno
-
-# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass);
-# everything else needs a CA certificate. Members must be in [ssh_ca].
-# Added one host at a time after the [ssh_ca] report was reviewed.
-[ssh_keys_managed]
-proxy-1.renner.uno
-proxy-2.renner.uno
-netbird.renner.uno
-edge-1.iot.renner.uno
-
-# Internet hosts whose sshd listens on the NetBird interface only
-# (plays/configure-ssh-mesh-only.yml). Members need the host NetBird client.
-# Added one host at a time after the previous one was verified.
-[ssh_mesh_only]
-proxy-1.renner.uno
-proxy-2.renner.uno
-netbird.renner.uno
+# Off-site Debian VM (Synology VMM on nas-2, 10.3.40.0/24) running Claude Code.
+# Reached over the site-to-site tunnel at its current DHCP lease. Switch
+# ansible_host to the stable Netbird mesh IP once the host is enrolled — the
+# obg.renner.uno FQDN is split-horizon and resolves to the public proxy from
+# home, so it must NOT be used as ansible_host.
+[claude_workstation]
+claude-ws ansible_host=10.3.40.78 ansible_user=root

ansible/plays/close-connections.yml

@@ -1,11 +0,0 @@
----
-# Last play of every playbook that is run directly (site.yml and the one-off
-# actions). SSH connections stay open for 30 minutes (ansible.cfg) so a run
-# costs one YubiKey PIN per host; closing them at the end leaves nothing that
-# another process of the same user could reuse.
-- name: Close the SSH connections
-  hosts: all
-  gather_facts: false
-  tasks:
-    - name: Close the connection
-      ansible.builtin.meta: reset_connection

ansible/plays/common.yml

@@ -13,5 +13,4 @@
 
   roles:
     - common
-    - role: qemu-guest-agent
-      when: ansible_facts['virtualization_role'] == 'guest'
+    - qemu-guest-agent

ansible/plays/configure-shelly-ntp.yml

@@ -0,0 +1,125 @@
+---
+# Configure NTP server on all Shelly devices
+# Discovers Shelly devices via network scan and configures them to use local NTP server
+#
+# Usage:
+#   ansible-playbook plays/configure-shelly-ntp.yml
+#   ansible-playbook plays/configure-shelly-ntp.yml -e "shelly_ntp_server=10.2.40.4"
+#   ansible-playbook plays/configure-shelly-ntp.yml -e "shelly_networks=10.2.30.0/24,10.2.20.0/24"
+
+- name: Configure Shelly NTP Settings
+  hosts: localhost
+  connection: local
+  gather_facts: no
+
+  vars:
+    shelly_ntp_server: "ntp.renner.uno"
+    shelly_networks: "10.2.30.0/24 10.2.20.0/24"
+    shelly_timeout: 5
+
+  tasks:
+    - name: Discover Shelly devices on network
+      script: "{{ playbook_dir }}/../files/discover_shelly.py {{ shelly_networks }}"
+      args:
+        executable: python3
+      register: discovered_devices
+      changed_when: false
+
+    - name: Parse discovered devices
+      set_fact:
+        shelly_devices: "{{ discovered_devices.stdout | from_json }}"
+
+    - name: Display discovered Shelly devices
+      debug:
+        msg: "Found {{ shelly_devices | length }} Shelly devices"
+
+    - name: Show device list
+      debug:
+        msg: "{{ item.name }} ({{ item.type }}) - {{ item.ip }} - Gen{{ item.gen }}"
+      loop: "{{ shelly_devices }}"
+      when: shelly_devices | length > 0
+
+    - name: Configure NTP on Gen1 devices
+      uri:
+        url: "http://{{ item.ip }}/settings?sntp_server={{ shelly_ntp_server }}"
+        method: GET
+        timeout: "{{ shelly_timeout }}"
+        return_content: yes
+      loop: "{{ shelly_devices | selectattr('gen', 'equalto', 1) | list }}"
+      register: gen1_results
+      ignore_errors: yes
+      when: shelly_devices | length > 0
+
+    - name: Configure NTP on Gen2+ devices
+      uri:
+        url: "http://{{ item.ip }}/rpc/Sys.SetConfig"
+        method: POST
+        body_format: json
+        body:
+          config:
+            sntp:
+              server: "{{ shelly_ntp_server }}"
+        timeout: "{{ shelly_timeout }}"
+        return_content: yes
+      loop: "{{ shelly_devices | selectattr('gen', 'greaterthan', 1) | list }}"
+      register: gen2_results
+      ignore_errors: yes
+      when: shelly_devices | length > 0
+
+    - name: Verify Gen1 NTP configuration
+      uri:
+        url: "http://{{ item.ip }}/settings"
+        method: GET
+        timeout: "{{ shelly_timeout }}"
+        return_content: yes
+      loop: "{{ shelly_devices | selectattr('gen', 'equalto', 1) | list }}"
+      register: gen1_verify
+      ignore_errors: yes
+      when: shelly_devices | length > 0
+
+    - name: Verify Gen2+ NTP configuration
+      uri:
+        url: "http://{{ item.ip }}/rpc/Sys.GetConfig"
+        method: GET
+        timeout: "{{ shelly_timeout }}"
+        return_content: yes
+      loop: "{{ shelly_devices | selectattr('gen', 'greaterthan', 1) | list }}"
+      register: gen2_verify
+      ignore_errors: yes
+      when: shelly_devices | length > 0
+
+    - name: Summary - Gen1 devices configured
+      debug:
+        msg: "{{ item.item.name }} ({{ item.item.ip }}): NTP={{ item.json.sntp.server | default('unknown') }}"
+      loop: "{{ gen1_verify.results | default([]) }}"
+      when:
+        - gen1_verify is defined
+        - gen1_verify.results is defined
+        - item.json is defined
+        - item.json.sntp is defined
+      loop_control:
+        label: "{{ item.item.name | default(item.item.ip) }}"
+
+    - name: Summary - Gen2+ devices configured
+      debug:
+        msg: "{{ item.item.name }} ({{ item.item.ip }}): NTP={{ item.json.sntp.server | default('unknown') }}"
+      loop: "{{ gen2_verify.results | default([]) }}"
+      when:
+        - gen2_verify is defined
+        - gen2_verify.results is defined
+        - item.json is defined
+        - item.json.sntp is defined
+      loop_control:
+        label: "{{ item.item.name | default(item.item.ip) }}"
+
+    - name: Final summary
+      debug:
+        msg: |
+          ==========================================
+          Shelly NTP Configuration Complete
+          ==========================================
+          Total devices found: {{ shelly_devices | length }}
+          Gen1 devices: {{ shelly_devices | selectattr('gen', 'equalto', 1) | list | length }}
+          Gen2+ devices: {{ shelly_devices | selectattr('gen', 'greaterthan', 1) | list | length }}
+          NTP Server: {{ shelly_ntp_server }}
+          ==========================================

ansible/plays/configure-ssh-ca.yml

@@ -1,23 +0,0 @@
----
-# Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
-# owner certificates (principal k.renner) for root/ladmin, Claude's 1h
-# certificates (principal claude-bot) for its own claude-bot account with
-# limited sudo. The role reports the admin logins' authorized_keys; on
-# [ssh_keys_managed] they are then reduced to ssh_admin_authorized_keys.
-#
-# serial: 1 so a host whose sshd rejects the change stops the run before the
-# next host is touched. One play, so each host needs one login (one YubiKey
-# PIN from a workstation).
-- name: Configure the SSH user CA
-  hosts: ssh_ca
-  become: true
-  gather_facts: true
-  serial: 1
-  roles:
-    - ssh-ca
-  tasks:
-    - name: Reduce authorized_keys to the break-glass key
-      ansible.builtin.include_role:
-        name: ssh-ca
-        tasks_from: authorized_keys
-      when: inventory_hostname in groups['ssh_keys_managed']

ansible/plays/configure-ssh-keys.yml

@@ -0,0 +1,48 @@
+---
+- name: Deploy authorized SSH keys to all hosts
+  hosts: all:!localhost
+  become: true
+  gather_facts: false
+  # Authorized keys deployed to root and ladmin on every managed host.
+  #
+  # - krenner@fortress (human): unrestricted.
+  # - woodpecker-ci@homelab (CI): restricted with from= to the cluster's pod
+  #   network (ci_ssh_from_cidrs in group_vars/all/vars.yml). The in-cluster
+  #   Woodpecker step pods SSH to fleet hosts presenting their POD IP as source
+  #   (Cilium native-routing-cidr covers the LAN, so no SNAT; PodCIDRs are
+  #   BGP-advertised). This is a network-of-origin restriction, NOT host-pinned,
+  #   so it survives node DHCP churn and pod rescheduling. See vars.yml for the
+  #   full derivation.
+  # - ansible-runner@ci-new: removed — dead key, no such host exists.
+  vars:
+    authorized_keys:
+      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
+      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
+        options: 'from="{{ ci_ssh_from_cidrs }}"'
+  tasks:
+    - name: Check SSH connectivity
+      ansible.builtin.wait_for_connection:
+        timeout: 5
+      ignore_errors: true
+      register: ssh_check
+
+    - name: Skip unreachable hosts
+      ansible.builtin.meta: end_host
+      when: ssh_check is failed
+
+    - name: Deploy authorized keys to root
+      ansible.builtin.authorized_key:
+        user: root
+        key: "{{ item.key }}"
+        key_options: "{{ item.options | default(omit) }}"
+        state: present
+      loop: "{{ authorized_keys }}"
+
+    - name: Deploy authorized keys to ladmin
+      ansible.builtin.authorized_key:
+        user: ladmin
+        key: "{{ item.key }}"
+        key_options: "{{ item.options | default(omit) }}"
+        state: present
+      loop: "{{ authorized_keys }}"
+      ignore_errors: true

ansible/plays/configure-ssh-mesh-only.yml

@@ -1,15 +0,0 @@
----
-# sshd on the hosts in [ssh_mesh_only] accepts connections only on the NetBird
-# interface; nothing listens on their public addresses. Run from a workstation
-# in the mesh. A switchover reverts by itself unless the play logs in again
-# through the new listener.
-#
-# serial: 1 so a host that does not come back stops the run before the next
-# host is touched.
-- name: Restrict sshd to the mesh interface
-  hosts: ssh_mesh_only
-  become: true
-  gather_facts: true
-  serial: 1
-  roles:
-    - ssh-mesh-only

ansible/plays/configure-vm-time-sync.yml

@@ -29,7 +29,6 @@
     - name: Check if systemd-timesyncd is installed
       command: systemctl list-unit-files systemd-timesyncd.service
       register: timesyncd_installed
-      check_mode: false
       failed_when: false
       changed_when: false
 
@@ -57,7 +56,6 @@
     - name: Verify time sync status
       command: timedatectl show --property=NTPSynchronized
       register: timesync_status
-      check_mode: false
       changed_when: false
       failed_when: false
 

ansible/plays/install-claude-workstation.yml

@@ -0,0 +1,20 @@
+---
+# Off-site Debian VM running Claude Code — reattachable interactive sessions
+# plus an unattended notes-distill timer. Host creation (Synology VMM), the
+# Netbird dashboard route and secret material are manual prerequisites; see
+# roles/claude-workstation/README.md and docs/claude-workstation.md.
+#
+# CI reaches this off-site host over the site-to-site tunnel (home -> off-site
+# works), same as it reaches every other inventory host.
+
+- name: Deploy the Claude Code workstation
+  hosts: claude_workstation
+  become: true
+  # qemu-guest-agent is applied
+  # to every inventory host by the umbrella plays/common.yml, same as for the
+  # netbird-proxy host — run that against this host too (see RUNBOOK Step 5).
+  roles:
+    - common
+    - role: netbird-client       # mesh enrolment (needs netbird_setup_key in vault)
+      when: netbird_setup_key is defined
+    - claude-workstation

ansible/plays/install-edge-host.yml

@@ -9,6 +9,7 @@
 #
 #   plays/install-zwave-gateway.yml   [zwave_gateway]   ZWA-2 radio
 #   plays/install-heating-can.yml     [heating_can]     Innomaker USB2CAN
+#   plays/install-power-meter.yml     [power_meter]     FTDI meter head
 #
 # Splitting the roles without splitting the plays would have kept the coupling:
 # one play applying every role to one group is the monolith again, a layer up.

ansible/plays/install-netbird-client.yml

@@ -6,25 +6,26 @@
 #   - host_vars/<hostname>/vars.yml - per-host key (overrides group)
 #   - group_vars/all/vault.yml - encrypted keys
 #
-# Applied by site.yml.
+# Usage:
+#   ansible-playbook -i inventory/ plays/setup-netbird-client.yml
+#
+# Limit to specific hosts:
+#   ansible-playbook -i inventory/ plays/setup-netbird-client.yml --limit caddy_servers
 
 - name: Install NetBird client on hosts with setup key defined
   hosts: all
   become: true
-  # One host at a time: from a workstation every new SSH session asks for the
-  # YubiKey PIN and a touch, and parallel hosts would prompt at once.
-  serial: 1
-  # Facts only after the skip: gathering them first opens an SSH session (a
-  # YubiKey touch from a workstation) to every host, including skipped ones.
-  gather_facts: false
+  gather_facts: true
 
   pre_tasks:
+    - name: Check if netbird_setup_key is defined
+      ansible.builtin.debug:
+        msg: "NetBird setup key is defined for {{ inventory_hostname }}"
+      when: netbird_setup_key is defined
+
     - name: Skip hosts without netbird_setup_key
       ansible.builtin.meta: end_host
       when: netbird_setup_key is not defined
 
-    - name: Gather facts
-      ansible.builtin.setup:
-
   roles:
     - netbird-client

ansible/plays/install-netbird-proxy.yml

@@ -1,17 +1,18 @@
 ---
-# Deploy the NetBird reverse-proxy HA cluster members (the public edge).
-# Applied by site.yml.
+# Deploy a NetBird reverse-proxy HA cluster member (a second proxy host).
+# Usage: ansible-playbook -i inventory/ plays/install-netbird-proxy.yml
 #
 # Prerequisites:
 # - Proxy VPS provisioned (proxy-1 is Arch Linux), root SSH reachable, and
 #   listed under [netbird_proxy] in inventory/pve_hosts.yml. Docker is
 #   installed by the (Arch-aware) docker-install role.
 # - DNS A/AAAA for the host's own name (e.g. proxy-1.renner.uno) -> its IP.
-# - Uses the existing vault_netbird_proxy_token (shared by every member) — no
-#   new token to generate; a shared token + same NB_PROXY_DOMAIN forms the
-#   cluster.
-# - AFTER deploy + verify: add this host's A/AAAA to proxy.renner.uno so
-#   traffic round-robins across the cluster.
+# - Uses the existing vault_netbird_proxy_token (shared with VPS-1) — no new
+#   token to generate; a shared token + same NB_PROXY_DOMAIN forms the cluster.
+# - AFTER deploy + verify: add this host's IP to the proxy domain so traffic
+#   round-robins across the cluster. Repoint proxy.renner.uno away from its
+#   CNAME to netbird.renner.uno (management host) to A/AAAA records listing
+#   BOTH proxy IPs (VPS-1 + this host).
 
 - name: Deploy NetBird reverse-proxy cluster member
   hosts: netbird_proxy

ansible/plays/install-netbird-server.yml

@@ -1,6 +1,6 @@
 ---
 # Deploy NetBird Server to VPS
-# Applied by site.yml.
+# Usage: ansible-playbook -i inventory/ plays/install-netbird-server.yml
 #
 # Prerequisites:
 # - VPS provisioned with Ubuntu 22.04/24.04
@@ -18,3 +18,4 @@
     - common
     - docker-install
     - netbird-server
+    - netbird-proxy-watchdog

ansible/plays/install-node-exporter.yml

@@ -0,0 +1,22 @@
+---
+# Install Node Exporter on all managed hosts
+# Usage: ansible-playbook -i inventory/ plays/install-node-exporter.yml
+
+- name: Install Node Exporter
+  hosts: all
+  become: yes
+  gather_facts: yes
+
+  roles:
+    - node-exporter
+
+  post_tasks:
+    - name: Verify node-exporter is running
+      uri:
+        url: "http://127.0.0.1:9100/metrics"
+        status_code: 200
+      register: exporter_health
+      retries: 3
+      delay: 5
+      until: exporter_health.status == 200
+      ignore_errors: yes

ansible/plays/install-ntp-ha.yml

@@ -102,45 +102,32 @@
       ansible.builtin.pause:
         seconds: 5
 
-    - name: Verify chronyd answers
-      ansible.builtin.command:
-        argv:
-          - chronyc
-          - -n
-          - tracking
-      check_mode: false
+    - name: Verify chrony is serving NTP
+      ansible.builtin.shell: chronyc clients | head -10
+      register: chrony_clients
       changed_when: false
 
-    - name: Verify chronyd listens on udp/123
-      ansible.builtin.command:
-        argv:
-          - ss
-          - -Hlunp
-          - sport = :123
+    - name: Check NTP port is listening
+      ansible.builtin.shell: ss -lun | grep ':123 '
       register: ntp_port
-      check_mode: false
+      failed_when: false
       changed_when: false
-      failed_when: "'chronyd' not in ntp_port.stdout"
+
+    - name: Display NTP port status
+      ansible.builtin.debug:
+        msg: "NTP port 123: {{ 'LISTENING' if ntp_port.rc == 0 else 'NOT LISTENING' }}"
 
     - name: Check if this server has the VIP
-      ansible.builtin.command:
-        argv:
-          - ip
-          - -4
-          - addr
-          - show
-          - to
-          - "{{ ntp_vip }}"
+      ansible.builtin.shell: ip addr show | grep {{ ntp_vip }}
       register: vip_check
-      check_mode: false
-      failed_when: vip_check.rc != 0
+      failed_when: false
       changed_when: false
       when: ntp_keepalived_enabled | default(false)
 
     - name: Display VIP status
       ansible.builtin.debug:
         msg: >
-          {% if vip_check.stdout | default('') | length > 0 %}
+          {% if vip_check.rc == 0 %}
           This server ({{ inventory_hostname }}) is MASTER and holds the VIP {{ ntp_vip }}
           {% else %}
           This server ({{ inventory_hostname }}) is BACKUP
@@ -172,8 +159,7 @@
           - to
           - "{{ ntp_vip6 }}"
       register: vip6_check
-      check_mode: false
-      failed_when: vip6_check.rc != 0
+      failed_when: false
       changed_when: false
       when: ntp_keepalived_enabled | default(false)
 
@@ -198,28 +184,18 @@
           (keepalived not enabled on this host).
       when: not (ntp_keepalived_enabled | default(false))
 
-    # chronyc talks to the command port, which is loopback-only; this is a real
-    # NTP query. Directives on the command line mean no config file and no
-    # pidfile, and -Q opens no server or command port, so it runs alongside
-    # the live daemon.
-    - name: Query NTP through the VIP (only on master)
-      ansible.builtin.command:
-        argv:
-          - chronyd
-          - -Q
-          - -t
-          - "10"
-          - "server {{ ntp_vip }} iburst maxsamples 1"
+    - name: Test NTP via VIP (only on master)
+      ansible.builtin.shell: |
+        chronyc -h {{ ntp_vip }} tracking 2>&1 || echo "Cannot connect to VIP (may need to wait for failover)"
       register: ntp_vip_test
-      check_mode: false
       when:
         - ntp_keepalived_enabled | default(false)
-        - vip_check.stdout | default('') | length > 0
+        - vip_check.rc == 0
       changed_when: false
 
     - name: Display VIP NTP test result
       ansible.builtin.debug:
-        msg: "{{ ntp_vip_test.stderr_lines + ntp_vip_test.stdout_lines }}"
+        msg: "{{ ntp_vip_test.stdout_lines | default(['Not master, skipping VIP test']) }}"
       when:
         - ntp_keepalived_enabled | default(false)
-        - vip_check.stdout | default('') | length > 0
+        - vip_check.rc == 0

ansible/plays/install-power-meter.yml

@@ -0,0 +1,14 @@
+---
+# Power-meter reader (sml2mqtt) for hosts with the FTDI serial meter head
+# attached.
+#
+# Targets [power_meter]: membership means the meter head is physically present.
+#
+# Requires the base play (plays/install-edge-host.yml) to have run: it needs
+# docker and publishes to the local MQTT bridge.
+- name: Install power-meter reader
+  hosts: power_meter
+  become: true
+
+  roles:
+    - power-meter-sml

ansible/plays/install-uptime-kuma.yml

@@ -1,18 +0,0 @@
----
-# Deploy Uptime Kuma to the NetBird VPS
-# Applied by site.yml.
-#
-# Prerequisites:
-# - netbird-server play applied (docker, Traefik on netbird_netbird)
-# - OpenBao homelab/infrastructure/uptime-kuma with keys username, password
-# - OpenBao homelab/infrastructure/uptime-kuma/smtp with keys server, port, username, token
-# - homelab/infrastructure/uptime-kuma-metrics (key api_key) is written by the role
-# - an OpenBao token with policy uptime-kuma-vps: `bao login -method=oidc`
-#   (role owner) or VAULT_TOKEN
-# - DNS for kuma.renner.uno pointing at the VPS
-
-- name: Deploy Uptime Kuma
-  hosts: netbird_server
-  become: true
-  roles:
-    - uptime-kuma

ansible/plays/rotate-netbird-secrets.yml

@@ -43,10 +43,10 @@
 #
 # WHAT IS NOT, and must not be added:
 #
-#   vault_netbird_proxy_token    LOOKS symmetric — it is in proxy.env on the
-#                                proxies, an Ansible template — but the
-#                                management server does not validate it from
-#                                any file. It is registered
+#   vault_netbird_proxy_token    LOOKS symmetric — it is in proxy.env on both
+#                                the server and the proxies, all Ansible
+#                                templates — but the management server does not
+#                                validate it from any of them. It is registered
 #                                in the management DATABASE. Rotating it changes
 #                                only what the proxies present, so every proxy
 #                                is locked out: "proxy auth failed: too many
@@ -248,6 +248,3 @@
           - "Commit inventory/group_vars/all/netbird-rotatable.yml — it is encrypted."
           - "NOT rotated: datastore_key (needs dump/restore), auth_client_secret (Authentik side)."
           - "OpenBao is unaffected by design — its NetBird credentials are separate ones."
-
-- name: Close the SSH connections
-  ansible.builtin.import_playbook: close-connections.yml

ansible/plays/update-dist-upgrade.yml

@@ -43,6 +43,7 @@
             - ubuntu_upgrade.rc != 0 
             - "'No new release found' not in ubuntu_upgrade.stdout"
             - "'No new release found' not in ubuntu_upgrade.stderr"
+          ignore_errors: true
 
         - name: Display Upgrade Result
           ansible.builtin.debug:
@@ -110,6 +111,3 @@
       when: 
         - (ansible_distribution == 'Ubuntu' and reboot_required.stat.exists) or
           (ansible_distribution == 'Debian')
-
-- name: Close the SSH connections
-  ansible.builtin.import_playbook: close-connections.yml

ansible/plays/update-os.yml

@@ -1,33 +0,0 @@
----
-# Package upgrade + reboot for netbird_proxy, edge_host and netbird_server,
-# one host at a time.
-# Usage (from ansible/): ansible-playbook plays/update-os.yml
-#
-# Plays run in order; a failed host stops the whole run (max_fail_percentage: 0).
-
-- name: Update NetBird proxies
-  hosts: netbird_proxy
-  serial: 1
-  max_fail_percentage: 0
-  become: true
-  roles:
-    - os-update
-
-- name: Update edge hosts
-  hosts: edge_host
-  serial: 1
-  max_fail_percentage: 0
-  become: true
-  roles:
-    - os-update
-
-- name: Update NetBird server
-  hosts: netbird_server
-  serial: 1
-  max_fail_percentage: 0
-  become: true
-  roles:
-    - os-update
-
-- name: Close the SSH connections
-  ansible.builtin.import_playbook: close-connections.yml

ansible/roles/chrony-exporter/tasks/main.yml

@@ -18,7 +18,6 @@
 - name: Get installed version
   command: "{{ chrony_exporter_binary }} --version"
   register: installed_version
-  check_mode: false
   when: chrony_exporter_installed.stat.exists
   changed_when: false
   failed_when: false

ansible/roles/claude-workstation/README.md

@@ -0,0 +1,57 @@
+# claude-workstation
+
+An always-on, **off-site** headless Claude Code host (Debian 12 VM under Synology
+VMM on nas-2). It runs (a) reattachable interactive `tmux` sessions and (b) an
+unattended notes-distill timer. It deliberately holds the **least** privilege
+that still does the job.
+
+## What lives here vs. what's manual
+
+This role configures everything **inside** the VM. Three things are **manual**
+(no IaC path) and precede/accompany it — see [`RUNBOOK.md`](RUNBOOK.md) for the
+full step-by-step:
+
+1. **VM creation** in Synology VMM (`ladmin` has no sudo; the DSM API is
+   root-only), with a cloud-init drive that installs the CI SSH key.
+2. **Netbird dashboard route** advertising the k8s API to the mesh, and the
+   **site-to-site return route** on the off-site gateway (backup path).
+3. **Secret material**: `claude setup-token` (subscription OAuth), the claude-bot
+   creds, and the read-only SA kubeconfig — encrypted into the group vault.
+
+## Credential isolation
+
+Two Unix users with separate credential stores so an unattended job cannot read
+the interactive session's secrets:
+
+| User | Holds | Runs |
+|------|-------|------|
+| `claude` | OAuth token, claude-bot creds, **read-only** SA kubeconfig | interactive tmux |
+| `claude-job` | its **own** OAuth token, claude-bot creds (no kubeconfig) | notes-distill timer |
+
+**No talosconfig, no SOPS age key** ever land here (Fable review #1). The
+kubeconfig is a read-only `view` ServiceAccount (`kubernetes/apps/claude-workstation/`).
+
+## Egress lockdown (staged)
+
+`claude_ws_egress_enforce` ships **`false`** so a fresh host is never bricked.
+The deny-default nftables ruleset, the DNS-resolver, and its refresh timer are
+installed regardless. Flip to `true` (in host/group vars) only after confirming
+`claude_ws_egress_allow_fqdns` / `claude_ws_egress_allow_cidrs*` cover real
+traffic — watch the `claude-egress-dropped` counter first.
+
+## Vault keys (encrypt into `inventory/group_vars/claude_workstation/vault.yml`)
+
+```
+netbird_setup_key                  # from the Netbird dashboard
+claude_ws_oauth_token_interactive  # `claude setup-token` (interactive user)
+claude_ws_oauth_token_job          # `claude setup-token` (job user, separate)
+claude_ws_gitea_token              # claude-bot PAT
+claude_ws_gitea_git_credentials    # https://claude-bot:<pat>@gitea.renner.uno
+claude_ws_kubeconfig               # read-only SA kubeconfig (literal YAML)
+```
+
+## Kill switch
+
+Everything unattended hangs off systemd timers; `systemctl disable --now
+claude-notes-distill.timer` (and the egress refresh) stops it. Every credential
+is independently revocable (SA token, claude-bot PAT, OAuth token, Netbird peer).

ansible/roles/claude-workstation/RUNBOOK.md

@@ -0,0 +1,222 @@
+# Runbook — off-site Claude Code workstation (nas-2)
+
+Operator guide for the manual steps around the `claude-workstation` role. The
+role automates everything **inside** the VM; this covers what it can't:
+VM creation, networking, secret material, egress cutover, rotation and DR.
+
+## Architecture (scope)
+
+An always-on **Debian 13 VM under Synology VMM on the off-site nas-2**
+(DS1522+, 10.3.40.0/24, reached over the site-to-site VPN, ~55 ms). It runs:
+
+- **Reattachable interactive sessions** (`tmux` over SSH as user `claude`).
+- **Unattended notes-distill timer** (as user `claude-job`).
+- **A thin is-home-up signal** (the existing external Gatus on the Netbird VPS
+  already covers this — don't duplicate it).
+
+**Out of scope (runs in-cluster instead):** scheduled repo-automation and
+cluster-reports go in as Woodpecker cron / CronJobs with an in-cluster
+ServiceAccount — *not* on this VM, so the VM holds no cluster-write creds.
+
+Credentials on the VM are minimal: a **read-only** `view` SA kubeconfig,
+claude-bot (non-admin) gitea creds, and a Claude subscription OAuth token.
+**No talosconfig, no SOPS age key.**
+
+---
+
+## Step 1 — Create + install the VM (Synology VMM)
+
+`ladmin` has no sudo and the DSM API is root-only, so VMM is driven through the
+DSM **web UI**. Two hard facts dictate the method (all confirmed 2026-07-16, as
+built): **VMM has no cloud-init**, and its ISO-install wizard only creates blank
+disks. So the OS is installed from a **remastered Debian netinst ISO that
+auto-boots a preseed** — no console interaction, no boot-menu race.
+
+VMM gotchas that shape this (see the wiki build-record):
+- VMM refuses to change/unmount a VM's ISO while it is **running** ("Failed to
+  replace ISO files"). Do all ISO/boot-device edits with the VM **off**.
+- A **guest reboot does not re-read** VMM's boot-device/ISO config — only a full
+  VMM power-cycle applies it. So an auto-booting install ISO **loops** (reboot →
+  CD → reinstall, wiping the disk). Break it by preseeding the installer to
+  **power off** at the end: `d-i debian-installer/exit/poweroff boolean true`.
+- Attach the NIC to the **untagged `Default VM Network`** — the VLAN-40
+  `40_Server` network is isolated (no DHCP, no network). The LAN has DHCP.
+
+Procedure (build the ISO on a host with `xorriso`):
+1. Download the current Debian **netinst** (`.../debian-cd/current/amd64/iso-cd/`).
+2. Remaster it — add `/preseed.cfg` and replace `/isolinux/isolinux.cfg` with a
+   minimal one (`default preseed`, `timeout 1`, one label:
+   `kernel /install.amd/vmlinuz` /
+   `append auto=true priority=critical preseed/file=/cdrom/preseed.cfg vga=788 initrd=/install.amd/initrd.gz --- quiet`):
+   `xorriso -indev in.iso -outdev out.iso -boot_image any replay -map preseed.cfg /preseed.cfg -map isolinux.cfg /isolinux/isolinux.cfg`.
+   The preseed uses **DHCP** (`netcfg/dhcp_timeout 120` — the virtual-switch port
+   is slow to link), disk **`/dev/sda`** (VirtIO SCSI), **no user** + a locked
+   random root hash (key-only via `PermitRootLogin prohibit-password`), packages
+   `openssh-server sudo qemu-guest-agent python3`, and a `late_command` that
+   injects the SSH keys (`krenner@fortress`, `woodpecker-ci@homelab`, your
+   workstation key) into `/root/.ssh/authorized_keys`, plus the `poweroff` line.
+3. Upload to nas-2 via `ssh 'cat > /volume1/Install/<iso>' < <iso>` — **scp /
+   sftp / rsync are blocked on DSM**. Then VMM → Image → Add → From Synology NAS
+   → import (VMM **copies** it; re-import if you change the source file).
+4. Create VM: 2 vCPU, 4 GB, 50 GB on `/volume1`, **Default VM Network**, Legacy
+   BIOS, **Autostart: Yes**; attach the ISO, **Boot from DVD/CD-ROM**, power on.
+5. It installs unattended and **powers off**. Then (VM off) set **ISO =
+   Unmounted**, **Boot from = Virtual Disk**, power on → boots the installed
+   Debian. `qemu-guest-agent` reports the DHCP lease in VMM's IP column.
+6. **Pin a DSM update window** (Control Panel → Update & Restore) so an
+   auto-reboot doesn't kill tmux/jobs.
+
+> As built 2026-07-16: Debian 13 (trixie), got DHCP `10.3.40.78`, all three keys
+> injected, packages present. Freeing RAM required powering off the pre-existing
+> `pbs1` PBS VM (kept its 3.4 TB backup disk).
+
+> **At-rest encryption (Fable #6).** Off-site physical custody is weaker than
+> home. For true at-rest protection of the VM's secrets *with* unattended
+> autostart, use **network-bound decryption** (clevis + a Tang server at home):
+> a stolen disk off-site can't unlock because it can't reach Tang. Full-disk
+> LUKS with a manual passphrase would break autostart. Until Tang is stood up,
+> secrets rely on 0600 perms + VM access control — documented gap, not a fix.
+
+---
+
+## Step 2 — Networking (two paths)
+
+**Netbird mesh (done via CI):** mint a setup key in the dashboard
+(`https://netbird.renner.uno`), vault it as `netbird_setup_key` under
+`group_vars/claude_workstation/`, and run `.woodpecker/deploy-netbird-client.yml`
+(`install-netbird-client.yml --limit claude_workstation`). As built 2026-07-16
+the VM enrolled as `claude-ws-190-112.nb.renner.uno` / `100.96.190.112` (+ IPv6).
+
+> **Mesh reachability caveat.** The mesh IP (`100.96.x`) is only reachable from
+> **other Netbird peers**. The home workstation and the Woodpecker CI agent are
+> **not** peers — they reach the VM only via the site-to-site tunnel at its DHCP
+> lease. So a *stable* mesh-based `ansible_host` needs CI (or a home host) on the
+> mesh first, or a Netbird routing peer bridging tunnel↔mesh. Until then the
+> inventory targets the tunnel DHCP IP.
+
+> **DNS trap.** `claude-ws.obg.renner.uno` resolves to the VM's lease **only at
+> the off-site DNS** (`@10.3.40.1`). The home resolver catch-alls `*.renner.uno`
+> to the public proxy (`162.55.50.97`, the Netbird VPS) — so from home the FQDN
+> lands on `root@netbird`, **not** the VM. Never use it as `ansible_host`.
+
+**Optional — mesh route to the k8s API (redundant read path):** to reach the API
+over the mesh too, add a Netbird dashboard **network route** advertising the
+apiserver's ClusterIP to the `netbird-ingress` peer group, and add an egress
+rule to the `netbird-ingress-egress` CiliumNetworkPolicy
+(`kubernetes/infrastructure/network-policies/ingress-isolation.yaml`) permitting
+`app: netbird-ingress` pods to reach `kube-apiserver:443`. Deferred from this PR
+because it edits a security-critical CNP that can't be validated offline — do it
+as a follow-up when you want the redundant API path.
+
+---
+
+## Step 3 — Read-only SA kubeconfig
+
+After the `kubernetes/apps/claude-workstation/` PR merges and Flux reconciles:
+
+```sh
+NS=apps
+TOKEN=$(kubectl -n $NS get secret claude-workstation-token -o jsonpath='{.data.token}' | base64 -d)
+kubectl -n $NS get secret claude-workstation-token -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/ca.crt
+# server = the cluster's stable API endpoint (kubernetes/talos/patches/cluster.yaml).
+# Use the name, not an address: it is what the API server certificate carries as a
+# SAN, so an IP here fails TLS verification even when it is reachable. It must
+# resolve from the workstation over the site-to-site route.
+kubectl config set-cluster homelab --server=https://k8s.renner.uno:6443 \
+  --certificate-authority=/tmp/ca.crt --embed-certs=true --kubeconfig=/tmp/claude-ws.kubeconfig
+kubectl config set-credentials claude-workstation --token="$TOKEN" --kubeconfig=/tmp/claude-ws.kubeconfig
+kubectl config set-context homelab --cluster=homelab --user=claude-workstation --kubeconfig=/tmp/claude-ws.kubeconfig
+kubectl config use-context homelab --kubeconfig=/tmp/claude-ws.kubeconfig
+# verify read-only:
+KUBECONFIG=/tmp/claude-ws.kubeconfig kubectl get nodes        # works
+KUBECONFIG=/tmp/claude-ws.kubeconfig kubectl get secrets -A   # Forbidden (view excludes secrets)
+```
+
+The resulting file becomes the `claude_ws_kubeconfig` vault value.
+
+---
+
+## Step 4 — Secret material → vault
+
+```sh
+# Two independent OAuth tokens so job jobs can't impersonate the interactive user:
+claude setup-token   # -> claude_ws_oauth_token_interactive
+claude setup-token   # -> claude_ws_oauth_token_job  (run again / separate session)
+```
+
+Encrypt into `ansible/inventory/group_vars/claude_workstation/vault.yml`
+(ansible-vault; same VAULT_PASS the CI uses):
+
+```yaml
+netbird_setup_key: "..."
+claude_ws_oauth_token_interactive: "..."
+claude_ws_oauth_token_job: "..."
+claude_ws_gitea_token: "..."                # claude-bot PAT
+claude_ws_gitea_git_credentials: "https://claude-bot:<PAT>@gitea.renner.uno"
+claude_ws_kubeconfig: |
+  apiVersion: v1
+  ...                                        # the file from Step 3
+```
+
+> **Never run `claude` with `--bare`** here or on the VM — bare mode silently
+> ignores `CLAUDE_CODE_OAUTH_TOKEN` and the jobs will fail auth.
+
+---
+
+## Step 5 — Deploy
+
+Merge the PR, then trigger the pipeline (Woodpecker → Run → `PIPELINE ==
+"deploy-claude-workstation"`), or push a change under the role's paths. CI SSHes
+to `10.3.40.60` over the tunnel and applies `common`, `netbird-client`, and this
+role. Also run the umbrella `plays/common.yml` against the new host once, so it
+picks up `qemu-guest-agent` like every other inventory host.
+
+---
+
+## Step 6 — Egress cutover (staged)
+
+The nftables allowlist ships **disabled**. To enforce:
+
+1. Leave it off for a few days; watch what the box actually talks to.
+2. Confirm `claude_ws_egress_allow_fqdns` + `claude_ws_egress_allow_cidrs*`
+   (home Prometheus/Loki/API) cover it.
+3. Set `claude_ws_egress_enforce: true` in host/group vars, redeploy.
+4. Watch the `claude-egress-dropped` nft counter (`nft list ruleset`) and Loki
+   for anything legitimate being dropped; widen the allowlist as needed.
+
+---
+
+## Step 7 — Notes (Syncthing pairing)
+
+The role installs Syncthing for `claude-job` and starts
+`syncthing@claude-job`. Pair it once (manual): from the VM's GUI (SSH tunnel to
+`127.0.0.1:8384`) add your existing vault's device ID and accept the shared
+`Notes` folder pointed at `{{ claude_ws_notes_dir }}`. The distill timer runs
+daily at 04:30 and only ever **creates** new atomic notes.
+
+---
+
+## Rotation
+
+- **OAuth token** (expires/can be revoked): re-run `claude setup-token` on your
+  laptop, update the two vault keys, redeploy. Headless auth failures surface in
+  Loki (the wrapper logs `result=failure`) — don't let them go quiet.
+- **claude-bot PAT**: rotate in Gitea, update `claude_ws_gitea_*`, redeploy.
+- **SA token**: `kubectl -n apps delete secret claude-workstation-token` (Flux
+  recreates it), re-assemble the kubeconfig (Step 3), update vault, redeploy.
+
+## Kill switch
+
+```sh
+systemctl disable --now claude-notes-distill.timer claude-egress-refresh.timer
+```
+
+Every credential is independently revocable: delete the SA, revoke the PAT,
+revoke the OAuth token, or remove the Netbird peer — no shared blast radius.
+
+## Disaster recovery (cattle, not pet)
+
+The VM holds no unique state — the only real state is the Notes replica, which
+re-syncs from Syncthing. To rebuild: redo **Step 1** (VM + cloud-init), then
+**Step 5** (pipeline). No VMM snapshots needed (they'd bloat the 86%-full
+`/volume1`).

ansible/roles/claude-workstation/defaults/main.yml

@@ -0,0 +1,68 @@
+---
+# claude-workstation role defaults
+#
+# An always-on headless host that runs Claude Code for (a) reattachable
+# interactive sessions and (b) unattended notes/wiki timers. It deliberately
+# holds the LEAST privilege that still lets it do its job: a read-only k8s
+# ServiceAccount kubeconfig, claude-bot (non-admin) gitea creds, and a Claude
+# subscription OAuth token — NO talosconfig and NO SOPS age key. See README.
+
+# --- Users -----------------------------------------------------------------
+# Two Unix users with separate credential stores so an unattended (timer) job
+# cannot read the interactive session's OAuth token / kube / gitea creds.
+claude_ws_interactive_user: claude          # human tmux sessions
+claude_ws_job_user: claude-job              # unattended systemd-timer jobs
+
+claude_ws_base_dir: /opt/claude-workstation
+
+# --- Toolchain -------------------------------------------------------------
+claude_ws_node_major: 22                     # matches the ci-claude image
+claude_ws_claude_code_pkg: "@anthropic-ai/claude-code"
+claude_ws_base_packages:
+  - git
+  - tmux
+  - jq
+  - curl
+  - ca-certificates
+  - ripgrep
+  - unzip
+  - nftables
+  - python3
+  - python3-venv
+
+# --- Egress allowlist ------------------------------------------------------
+# Deny-default OUTPUT is the target end-state, but it can brick a fresh host if
+# an allowed destination is missed, so it ships DISABLED and is tightened after
+# observing real egress (staged rollout). Flip claude_ws_egress_enforce to true
+# once the allowlist below is confirmed complete. See templates/nftables.conf.j2.
+claude_ws_egress_enforce: false
+# Static destinations the host must always reach. CDN-backed hosts (Anthropic,
+# npm, apt) are matched by name at apply time via the fqdn set; home services
+# are matched by IP. Tune in host_vars.
+claude_ws_egress_allow_fqdns:
+  - api.anthropic.com
+  - statsig.anthropic.com
+  - gitea.renner.uno
+  - fsn1.your-objectstorage.com          # Hetzner Object Storage (SOPS bundle)
+  - deb.debian.org
+  - security.debian.org
+  - deb.nodesource.com
+  - pkgs.netbird.io
+  - registry.npmjs.org
+claude_ws_egress_allow_cidrs: []          # home Prometheus/Loki/API (v4), per-host
+claude_ws_egress_allow_cidrs6: []         # same (v6) — IPv6-first, prefer these
+
+# --- Notes / wiki pipeline -------------------------------------------------
+# The Obsidian vault syncs onto this host via Syncthing (matches the user's
+# existing multi-device setup). Unattended pipeline jobs must CREATE new atomic
+# files rather than rewrite existing ones, to keep Syncthing conflicts rare.
+claude_ws_notes_enabled: true
+claude_ws_notes_dir: "/home/{{ claude_ws_job_user }}/Notes"
+claude_ws_syncthing_gui_bind: "127.0.0.1:8384"
+
+# --- Secrets (populated from the ansible-vault'd group_vars — see README) ---
+# claude_ws_oauth_token_interactive : CLAUDE_CODE_OAUTH_TOKEN for the human user
+# claude_ws_oauth_token_job         : CLAUDE_CODE_OAUTH_TOKEN for timer jobs
+# claude_ws_gitea_token             : claude-bot PAT
+# claude_ws_gitea_git_credentials   : https://claude-bot:<pat>@gitea.renner.uno line
+# claude_ws_kubeconfig              : read-only SA kubeconfig (base64 or literal)

ansible/roles/claude-workstation/files/claude-notes-distill-prompt.md

@@ -0,0 +1,22 @@
+You are running unattended as a scheduled job on the claude-workstation host.
+Your job is a conservative distillation pass over the Obsidian wiki vault.
+
+Scope and rules (follow exactly — you have only Read, Write, Glob, Grep):
+
+1. Look at `~/Notes/raw/` (ignore `~/Notes/raw/_processed/` and `~/Notes/_archive/`
+   entirely) for source material.
+2. For each raw source, check whether `~/Notes/wiki/` already contains a note
+   grounded in it (Grep the wiki for the source filename / its `sources:` slug).
+   If a note already exists, SKIP that source — do not rewrite or duplicate it.
+3. For raw sources with NO corresponding wiki note yet, create new atomic
+   concept notes in `~/Notes/wiki/`: one concept per file, self-contained,
+   tagged, cross-linked with `[[wikilinks]]`, every claim grounded in a
+   `sources:` entry. Never invent facts not present in the source.
+4. CREATE new files only. Never rewrite, move, delete, or truncate an existing
+   file in `~/Notes/` — appending a missing cross-link to a note you created
+   THIS run is fine; touching pre-existing notes is not.
+5. If there is nothing new to distill, do nothing and stop. Do not manufacture
+   work.
+
+Preserve disagreement: if sources conflict, surface the conflict in the note
+rather than silently picking a winner. Keep each note short and single-purpose.

ansible/roles/claude-workstation/files/tmux.conf

@@ -0,0 +1,8 @@
+# claude-workstation shared tmux config — reattachable long-running sessions.
+set -g history-limit 100000
+set -g mouse on
+set -g base-index 1
+setw -g pane-base-index 1
+set -g status-right "#H  %Y-%m-%d %H:%M"
+# Keep sessions alive across client disconnects (the whole point of this host).
+set -g destroy-unattached off

ansible/roles/claude-workstation/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Reload nftables
+  ansible.builtin.systemd:
+    name: nftables.service
+    state: reloaded
+  when: claude_ws_egress_enforce | bool

ansible/roles/claude-workstation/tasks/firewall.yml

@@ -0,0 +1,42 @@
+---
+# Deny-default egress allowlist. Ships DISABLED (claude_ws_egress_enforce:
+# false) so a fresh host is never bricked by a missed destination; the ruleset,
+# the FQDN-resolver, and its refresh timer are installed regardless, and the
+# operator flips the toggle to true once egress is confirmed complete against
+# real traffic (staged rollout). CDN-backed hosts are matched via nft named
+# sets that the refresh unit repopulates from DNS; static home services are
+# matched by claude_ws_egress_allow_cidrs.
+
+- name: Deploy the egress-refresh resolver
+  ansible.builtin.template:
+    src: claude-egress-refresh.sh.j2
+    dest: "{{ claude_ws_base_dir }}/claude-egress-refresh.sh"
+    mode: "0755"
+
+- name: Deploy the egress-refresh service + timer
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop:
+    - claude-egress-refresh.service
+    - claude-egress-refresh.timer
+  notify: Reload systemd
+
+- name: Deploy the nftables ruleset
+  ansible.builtin.template:
+    src: nftables.conf.j2
+    dest: /etc/nftables.conf
+    mode: "0644"
+    validate: "nft -c -f %s"
+  notify: Reload nftables
+
+- name: Enforce or relax egress lockdown
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: "{{ claude_ws_egress_enforce | bool }}"
+    state: "{{ 'started' if (claude_ws_egress_enforce | bool) else 'stopped' }}"
+    daemon_reload: true
+  loop:
+    - nftables.service
+    - claude-egress-refresh.timer

ansible/roles/claude-workstation/tasks/main.yml

@@ -0,0 +1,183 @@
+---
+# claude-workstation: an always-on headless Claude Code host.
+# Debian 12 VM on the off-site nas-2 (Synology VMM). See README for the manual
+# bootstrap (VM creation, Netbird route, secret material) that precedes this.
+#
+# common.yml already applies `common` and `qemu-guest-agent`
+# to every inventory host, so this role owns only the
+# Claude-specific surface: toolchain, the two credential-isolated users, the
+# read-only creds, notes sync, the unattended timers, and egress lockdown.
+
+- name: Assert we are on Debian (role is apt-only)
+  ansible.builtin.assert:
+    that:
+      - ansible_facts['os_family'] == 'Debian'
+    fail_msg: "claude-workstation targets Debian; got {{ ansible_facts['os_family'] }}."
+
+# --- Base packages ---------------------------------------------------------
+- name: Install base packages
+  ansible.builtin.apt:
+    name: "{{ claude_ws_base_packages }}"
+    state: present
+    update_cache: true
+
+# --- Node.js 22 + Claude Code (matches the ci-claude image toolchain) -------
+- name: Add NodeSource signing key
+  ansible.builtin.get_url:
+    url: https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key
+    dest: /usr/share/keyrings/nodesource.gpg
+    mode: "0644"
+    force: false
+
+- name: Add NodeSource apt repository
+  ansible.builtin.apt_repository:
+    repo: "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_{{ claude_ws_node_major }}.x nodistro main"
+    state: present
+    filename: nodesource
+
+- name: Install Node.js
+  ansible.builtin.apt:
+    name: nodejs
+    state: present
+    update_cache: true
+
+# NodeSource installs Node under /usr, so global npm modules land in
+# /usr/lib/node_modules and `claude` is on PATH for every user/timer. `creates`
+# keeps it idempotent (re-run `npm install -g` manually to bump the version).
+- name: Install Claude Code globally
+  ansible.builtin.command:
+    cmd: "npm install -g {{ claude_ws_claude_code_pkg }}"
+    creates: /usr/lib/node_modules/@anthropic-ai/claude-code
+
+# --- Credential-isolated users --------------------------------------------
+- name: Create Claude users
+  ansible.builtin.user:
+    name: "{{ item }}"
+    shell: /bin/bash
+    create_home: true
+  loop:
+    - "{{ claude_ws_interactive_user }}"
+    - "{{ claude_ws_job_user }}"
+
+- name: Create the role base directory (scripts, prompts)
+  ansible.builtin.file:
+    path: "{{ claude_ws_base_dir }}"
+    state: directory
+    mode: "0755"
+
+# Let the interactive user's tmux server survive SSH logout / reboot.
+- name: Enable lingering for the interactive user
+  ansible.builtin.command: "loginctl enable-linger {{ claude_ws_interactive_user }}"
+  args:
+    creates: "/var/lib/systemd/linger/{{ claude_ws_interactive_user }}"
+
+- name: Deploy shared tmux config
+  ansible.builtin.copy:
+    src: tmux.conf
+    dest: "/home/{{ item }}/.tmux.conf"
+    owner: "{{ item }}"
+    group: "{{ item }}"
+    mode: "0644"
+  loop:
+    - "{{ claude_ws_interactive_user }}"
+    - "{{ claude_ws_job_user }}"
+
+# --- Secrets: each user gets only what its role needs ----------------------
+# The interactive user gets gitea + OAuth + read-only kubeconfig. The job user
+# gets its OWN OAuth token + gitea (to open PRs) but NO kubeconfig.
+- name: Create ~/.config/claude-bot for the interactive user
+  ansible.builtin.file:
+    path: "/home/{{ item }}/.config/claude-bot"
+    state: directory
+    owner: "{{ item }}"
+    group: "{{ item }}"
+    mode: "0700"
+  loop:
+    - "{{ claude_ws_interactive_user }}"
+    - "{{ claude_ws_job_user }}"
+
+- name: Deploy claude-bot gitea token
+  ansible.builtin.copy:
+    content: "{{ claude_ws_gitea_token }}\n"
+    dest: "/home/{{ item }}/.config/claude-bot/gitea-token"
+    owner: "{{ item }}"
+    group: "{{ item }}"
+    mode: "0600"
+  loop:
+    - "{{ claude_ws_interactive_user }}"
+    - "{{ claude_ws_job_user }}"
+  no_log: true
+
+- name: Deploy claude-bot git-credentials
+  ansible.builtin.copy:
+    content: "{{ claude_ws_gitea_git_credentials }}\n"
+    dest: "/home/{{ item }}/.config/claude-bot/git-credentials"
+    owner: "{{ item }}"
+    group: "{{ item }}"
+    mode: "0600"
+  loop:
+    - "{{ claude_ws_interactive_user }}"
+    - "{{ claude_ws_job_user }}"
+  no_log: true
+
+# CLAUDE_CODE_OAUTH_TOKEN is delivered as an EnvironmentFile so both interactive
+# shells and timer units can source it without it landing in shell history.
+# NEVER run claude with --bare here: bare mode silently ignores this token.
+- name: Deploy per-user Claude OAuth environment file
+  ansible.builtin.copy:
+    content: "CLAUDE_CODE_OAUTH_TOKEN={{ item.token }}\n"
+    dest: "/home/{{ item.user }}/.config/claude-code.env"
+    owner: "{{ item.user }}"
+    group: "{{ item.user }}"
+    mode: "0600"
+  loop:
+    - { user: "{{ claude_ws_interactive_user }}", token: "{{ claude_ws_oauth_token_interactive }}" }
+    - { user: "{{ claude_ws_job_user }}", token: "{{ claude_ws_oauth_token_job }}" }
+  loop_control:
+    label: "{{ item.user }}"
+  no_log: true
+
+- name: Create ~/.kube for the interactive user
+  ansible.builtin.file:
+    path: "/home/{{ claude_ws_interactive_user }}/.kube"
+    state: directory
+    owner: "{{ claude_ws_interactive_user }}"
+    group: "{{ claude_ws_interactive_user }}"
+    mode: "0700"
+
+- name: Deploy read-only SA kubeconfig (interactive user only)
+  ansible.builtin.copy:
+    content: "{{ claude_ws_kubeconfig }}"
+    dest: "/home/{{ claude_ws_interactive_user }}/.kube/config"
+    owner: "{{ claude_ws_interactive_user }}"
+    group: "{{ claude_ws_interactive_user }}"
+    mode: "0600"
+  no_log: true
+
+# This VM's only git identity IS claude-bot (unlike the shared human
+# workstation, where a global credential.helper is forbidden), so a per-user
+# ~/.gitconfig pointing at each user's own claude-bot store is appropriate.
+- name: Configure git identity + credential store for the Claude users
+  ansible.builtin.copy:
+    dest: "/home/{{ item }}/.gitconfig"
+    owner: "{{ item }}"
+    group: "{{ item }}"
+    mode: "0644"
+    content: |
+      [user]
+      	name = Claude Bot
+      	email = claude-bot@renner.uno
+      [credential]
+      	helper = store --file=/home/{{ item }}/.config/claude-bot/git-credentials
+  loop:
+    - "{{ claude_ws_interactive_user }}"
+    - "{{ claude_ws_job_user }}"
+
+# --- Notes / wiki pipeline (Syncthing + timers) ----------------------------
+- name: Notes pipeline
+  ansible.builtin.include_tasks: notes.yml
+  when: claude_ws_notes_enabled | bool
+
+# --- Egress lockdown -------------------------------------------------------
+- name: Egress firewall
+  ansible.builtin.include_tasks: firewall.yml

ansible/roles/claude-workstation/tasks/notes.yml

@@ -0,0 +1,48 @@
+---
+# Notes/wiki pipeline: the Obsidian vault syncs onto this host via Syncthing
+# (matching the user's existing multi-device setup), and a hardened, bounded
+# systemd timer runs an unattended `claude -p` distill pass over it as the
+# credential-isolated job user. Device pairing is a one-time manual step (see
+# README) — this only installs Syncthing and the timer.
+
+- name: Install Syncthing
+  ansible.builtin.apt:
+    name: syncthing
+    state: present
+    update_cache: true
+
+- name: Enable Syncthing for the job user
+  ansible.builtin.systemd:
+    name: "syncthing@{{ claude_ws_job_user }}.service"
+    enabled: true
+    state: started
+    daemon_reload: true
+
+- name: Deploy the notes-distill prompt
+  ansible.builtin.copy:
+    src: claude-notes-distill-prompt.md
+    dest: "{{ claude_ws_base_dir }}/claude-notes-distill-prompt.md"
+    mode: "0644"
+
+- name: Deploy the notes-distill wrapper
+  ansible.builtin.template:
+    src: claude-notes-distill.sh.j2
+    dest: "{{ claude_ws_base_dir }}/claude-notes-distill.sh"
+    mode: "0755"
+
+- name: Deploy the notes-distill service + timer
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop:
+    - claude-notes-distill.service
+    - claude-notes-distill.timer
+  notify: Reload systemd
+
+- name: Enable the notes-distill timer
+  ansible.builtin.systemd:
+    name: claude-notes-distill.timer
+    enabled: true
+    state: started
+    daemon_reload: true

ansible/roles/claude-workstation/templates/claude-egress-refresh.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+# {{ ansible_managed }}
+Description=Refresh claude-workstation egress allowlist from DNS
+After=nftables.service network-online.target
+Wants=network-online.target
+Requires=nftables.service
+
+[Service]
+Type=oneshot
+ExecStart={{ claude_ws_base_dir }}/claude-egress-refresh.sh

ansible/roles/claude-workstation/templates/claude-egress-refresh.sh.j2

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# {{ ansible_managed }}
+# Resolve the egress FQDN allowlist to current IPs and load them into the
+# nftables @allow4/@allow6 sets. CDN IPs rotate, so this runs on a timer.
+# No-op unless the claude_egress table is loaded (i.e. enforcement is on).
+set -euo pipefail
+
+TABLE="inet claude_egress"
+FQDNS=(
+{% for fqdn in claude_ws_egress_allow_fqdns %}
+    "{{ fqdn }}"
+{% endfor %}
+)
+
+nft list table $TABLE >/dev/null 2>&1 || { echo "claude_egress table not loaded; enforcement off"; exit 0; }
+
+v4=(); v6=()
+for host in "${FQDNS[@]}"; do
+    while read -r ip; do [ -n "$ip" ] && v4+=("$ip"); done < <(getent ahostsv4 "$host" | awk '{print $1}' | sort -u)
+    while read -r ip; do [ -n "$ip" ] && v6+=("$ip"); done < <(getent ahostsv6 "$host" | awk '{print $1}' | sort -u)
+done
+
+nft flush set $TABLE allow4
+nft flush set $TABLE allow6
+[ ${#v4[@]} -gt 0 ] && nft add element $TABLE allow4 "{ $(IFS=,; echo "${v4[*]}") }"
+[ ${#v6[@]} -gt 0 ] && nft add element $TABLE allow6 "{ $(IFS=,; echo "${v6[*]}") }"
+echo "claude-egress-refresh: ${#v4[@]} v4 + ${#v6[@]} v6 addresses loaded"

ansible/roles/claude-workstation/templates/claude-egress-refresh.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+# {{ ansible_managed }}
+Description=Periodically refresh the claude-workstation egress allowlist
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=15min
+
+[Install]
+WantedBy=timers.target

ansible/roles/claude-workstation/templates/claude-notes-distill.service.j2

@@ -0,0 +1,32 @@
+[Unit]
+# {{ ansible_managed }}
+Description=Claude Code unattended notes-distill pass
+After=network-online.target syncthing@{{ claude_ws_job_user }}.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User={{ claude_ws_job_user }}
+Group={{ claude_ws_job_user }}
+EnvironmentFile=/home/{{ claude_ws_job_user }}/.config/claude-code.env
+ExecStart={{ claude_ws_base_dir }}/claude-notes-distill.sh
+# Hard wall-clock kill — the runaway/kill switch for this unattended agent.
+TimeoutStartSec=1800
+
+# --- Hardening: an unattended LLM job on internet-influenced input ----------
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ReadWritePaths=/home/{{ claude_ws_job_user }}
+# Credential isolation: cannot read the interactive session's tokens/kubeconfig.
+InaccessiblePaths=/home/{{ claude_ws_interactive_user }}
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictSUIDSGID=true
+RestrictNamespaces=true
+LockPersonality=true
+MemoryDenyWriteExecute=false
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/claude-workstation/templates/claude-notes-distill.sh.j2

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# {{ ansible_managed }}
+# Unattended notes-distill pass. Prompt via stdin (never shell-substituted),
+# tool-allowlisted, no --bare (bare silently drops CLAUDE_CODE_OAUTH_TOKEN).
+# Bounded by the unit's TimeoutStartSec, not a CLI flag (--max-turns is absent
+# in the installed Claude Code). Emits a terminal state line to the journal on
+# EVERY outcome so a Loki alert can catch silent failures (dead-man switch).
+set -euo pipefail
+
+export HOME="/home/{{ claude_ws_job_user }}"
+NOTES="{{ claude_ws_notes_dir }}"
+PROMPT="{{ claude_ws_base_dir }}/claude-notes-distill-prompt.md"
+
+log() { logger -t claude-notes-distill -- "$*"; echo "claude-notes-distill: $*"; }
+
+if [ ! -d "$NOTES/raw" ]; then
+  log "result=skipped reason=no-vault path=$NOTES"
+  exit 0
+fi
+
+log "result=start"
+if claude -p \
+     --add-dir "$NOTES" \
+     --allowedTools "Read,Write,Glob,Grep" \
+     --model sonnet \
+     < "$PROMPT"; then
+  log "result=success"
+else
+  rc=$?
+  log "result=failure rc=$rc"
+  exit "$rc"
+fi

ansible/roles/claude-workstation/templates/claude-notes-distill.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+# {{ ansible_managed }}
+Description=Run the Claude Code notes-distill pass daily
+
+[Timer]
+OnCalendar=*-*-* 04:30:00
+Persistent=true
+RandomizedDelaySec=900
+
+[Install]
+WantedBy=timers.target

ansible/roles/claude-workstation/templates/nftables.conf.j2

@@ -0,0 +1,55 @@
+#!/usr/sbin/nft -f
+# {{ ansible_managed }}
+# Deny-default egress for the claude-workstation. CDN-backed destinations are
+# matched via the @allow4/@allow6 named sets, repopulated from DNS by
+# claude-egress-refresh. Static home services are matched by the @static* sets.
+# Only loaded when claude_ws_egress_enforce is true.
+flush ruleset
+
+table inet claude_egress {
+    set allow4 { type ipv4_addr; flags interval; }
+    set allow6 { type ipv6_addr; flags interval; }
+
+    set static4 {
+        type ipv4_addr
+        flags interval
+{% if (claude_ws_egress_allow_cidrs | default([])) | length > 0 %}
+        elements = { {{ claude_ws_egress_allow_cidrs | join(', ') }} }
+{% endif %}
+    }
+    set static6 {
+        type ipv6_addr
+        flags interval
+{% if (claude_ws_egress_allow_cidrs6 | default([])) | length > 0 %}
+        elements = { {{ claude_ws_egress_allow_cidrs6 | join(', ') }} }
+{% endif %}
+    }
+
+    chain output {
+        type filter hook output priority 0; policy drop;
+
+        oif "lo" accept
+        ct state established,related accept
+
+        # DNS — required to resolve the allowlist itself.
+        udp dport 53 accept
+        tcp dport 53 accept
+
+        # Diagnostics + DHCP lease renewal on the bridged interface.
+        ip protocol icmp accept
+        ip6 nexthdr icmpv6 accept
+        udp dport { 67, 68 } accept
+
+        # NetBird / WireGuard control + data plane (initial handshakes).
+        udp dport { 3478, 51820 } accept
+
+        # Allowed destinations.
+        ip  daddr @allow4  accept
+        ip6 daddr @allow6  accept
+        ip  daddr @static4 accept
+        ip6 daddr @static6 accept
+
+        # Everything else is dropped and counted for inspection.
+        counter comment "claude-egress-dropped"
+    }
+}

ansible/roles/common/tasks/main.yml

@@ -51,8 +51,10 @@
 # -------------------------------------------------------
 # 4. SSH Configuration
 # -------------------------------------------------------
-# SSH access (CA trust, admin authorized_keys) is managed by the ssh-ca role
-# via plays/configure-ssh-ca.yml.
+# Authorized keys (human + woodpecker-ci, the latter from=-restricted) are
+# managed by the dedicated ansible/plays/configure-ssh-keys.yml play. The old
+# "Add Ansible Runner SSH Key" task deployed ansible-runner@ci-new, a dead key
+# for a host that no longer exists — removed (issue #184).
 
 # -------------------------------------------------------
 # 5. Disable NTP on VMs/LXC (use host time)

ansible/roles/crowdsec-edge-bouncer/defaults/main.yml

@@ -3,7 +3,7 @@
 #
 # Enforces the decisions the in-cluster CrowdSec already detects, at the point
 # where public traffic actually enters: these hosts terminate TLS for every
-# public renner.uno name. A drop here never reaches the mesh, the relay or the
+# *.renner.uno name. A drop here never reaches the mesh, the relay or the
 # cluster.
 #
 # Detection stays in the cluster. The reverse-proxy produces no parseable

ansible/roles/crowdsec-edge-bouncer/tasks/main.yml

@@ -29,11 +29,9 @@
 # upstream ships no container image for this bouncer, which is expected —
 # it manipulates host netfilter.
 #
-# The controller has v4, so it does the fetching. The checksum is verified
-# there, before anything is copied to a host. Also in check mode: the fetch
-# only writes on the controller, and the copy below needs the file.
+# The CI runner has v4, so it does the fetching. The checksum is verified
+# there, before anything is copied to a host.
 - name: Fetch the pinned bouncer release on the controller
-  check_mode: false
   ansible.builtin.get_url:
     url: >-
       https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{

ansible/roles/heating-can/tasks/main.yml

@@ -87,7 +87,6 @@
   ansible.builtin.command:
     argv: [pgrep, -af, "edge-host/(open3e|e3oncan)"]
   register: heating_can_survivors
-  check_mode: false
   changed_when: false
   failed_when: false
 

ansible/roles/keepalived-ntp/tasks/main.yml

@@ -65,7 +65,6 @@
       - /etc/network/interfaces
       - /etc/network/interfaces.d/
   register: keepalived_ntp_static6_ifupdown
-  check_mode: false
   changed_when: false
   failed_when: false
   when:
@@ -124,7 +123,6 @@
       - "{{ keepalived_ntp_vip6.split('/')[0] }}\\b"
       - /etc/systemd/network/
   register: keepalived_ntp_static6_networkd
-  check_mode: false
   changed_when: false
   failed_when: false
   when:
@@ -142,8 +140,9 @@
       Handing the address to keepalived while networkd also assigns it means
       it would NOT be withdrawn when check_ntp fails -- the host would keep
       serving time on an address VRRP believes it has released.
-      Re-apply the edge-vlan role first (site.yml runs it before this play),
-      whose template no longer carries the [Address] block.
+      Re-apply the edge-vlan role first, whose template no longer carries the
+      [Address] block:
+      ansible-playbook -i inventory/pve_hosts.yml plays/install-edge-host.yml
     success_msg: >-
       No static networkd assignment of {{ keepalived_ntp_vip6 }}; safe for
       keepalived to own it.

ansible/roles/mqtt-bridge/defaults/main.yml

@@ -10,14 +10,3 @@
 edge_mqtt_cluster_host: mqtt.renner.uno
 edge_mqtt_cluster_tls_port: 8883
 edge_mqtt_bridge_cafile: /etc/ssl/certs/ca-certificates.crt
-
-# --- Bridge credential from mqtt-account-controller (REN-1307) ------------
-# The cluster account the bridge logs in as (register entry in
-# kubernetes/apps/home-assistant/mqtt-accounts.yaml). The controller pushes
-# its password to edge-mqtt-cred and rotates it when the bridge reports
-# "not connected"; there is no copy in ansible-vault.
-edge_mqtt_bridge_account: edge-1
-# HTTPS port of edge-mqtt-cred (esp32 device API, mTLS). Only a client
-# certificate from the MQTT CA with this CN is accepted.
-edge_mqtt_cred_port: 8443
-edge_mqtt_cred_allowed_cn: mqtt-account-controller

ansible/roles/mqtt-bridge/files/edge-mqtt-cred.py

@@ -1,141 +0,0 @@
-#!/usr/bin/env python3
-# Managed by ansible (roles/mqtt-bridge). Receives the cluster bridge password
-# from mqtt-account-controller (REN-1307) so no plaintext copy lives in
-# ansible-vault and no play run is needed to rotate it.
-#
-# Speaks the controller's esp32 device API over HTTPS:
-#   GET /api/status   -> {"mqtt": {"connected": bool}}
-#   PUT /api/settings <- {"mqtt": {"user": "...", "pass": "..."}}
-# A client certificate from the MQTT CA with CN mqtt-account-controller is
-# required; nothing else may call it. The server key is generated on this
-# host and never leaves it (the controller pins it on first contact).
-#
-# PUT writes the bridge file (template + credentials), removes the legacy
-# vault-password bridges and restarts the local mosquitto. The password is
-# kept only in that 0640 root:mosquitto file.
-import grp
-import json
-import os
-import socket
-import ssl
-import subprocess
-import sys
-import tempfile
-import threading
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-
-CONF = json.load(open(sys.argv[1]))
-WRITE = threading.Lock()
-
-
-def connected():
-    try:
-        out = subprocess.run(
-            ["mosquitto_sub", "-h", "127.0.0.1", "-p", str(CONF["local_port"]),
-             "-t", CONF["state_topic"], "-C", "1", "-W", "3"],
-            capture_output=True, text=True, timeout=10).stdout
-    except (OSError, subprocess.TimeoutExpired):
-        return False
-    return out.strip() == "1"
-
-
-def write_bridge(user, password):
-    with open(CONF["template"]) as f:
-        body = f.read()
-    body += f"remote_username {user}\nremote_password {password}\n"
-    d = os.path.dirname(CONF["bridge_file"])
-    fd, tmp = tempfile.mkstemp(dir=d, prefix=".edge-mqtt-cred.")
-    with os.fdopen(fd, "w") as f:
-        f.write(body)
-    os.chown(tmp, 0, grp.getgrnam("mosquitto").gr_gid)
-    os.chmod(tmp, 0o640)
-    os.replace(tmp, CONF["bridge_file"])
-    for legacy in CONF["legacy_files"]:
-        if os.path.exists(legacy):
-            os.remove(legacy)
-    subprocess.run(["systemctl", "restart", "mosquitto"], check=True, timeout=60)
-
-
-def valid(s):
-    # Controller passwords are random URL-safe tokens; anything with
-    # whitespace or a control character would break the config line.
-    return isinstance(s, str) and 0 < len(s) <= 256 and s.isprintable() and not any(c.isspace() for c in s)
-
-
-class Handler(BaseHTTPRequestHandler):
-    def authorized(self):
-        cert = self.connection.getpeercert() or {}
-        cns = [v for rdn in cert.get("subject", ()) for k, v in rdn if k == "commonName"]
-        if cns != [CONF["allowed_cn"]]:
-            self.send_error(403)
-            return False
-        return True
-
-    def reply(self, code, obj=None):
-        data = b"" if obj is None else json.dumps(obj).encode()
-        self.send_response(code)
-        if data:
-            self.send_header("Content-Type", "application/json")
-        self.send_header("Content-Length", str(len(data)))
-        self.end_headers()
-        self.wfile.write(data)
-
-    def do_GET(self):
-        if not self.authorized():
-            return
-        if self.path != "/api/status":
-            return self.send_error(404)
-        self.reply(200, {"mqtt": {"connected": connected()}})
-
-    def do_PUT(self):
-        if not self.authorized():
-            return
-        if self.path != "/api/settings":
-            return self.send_error(404)
-        try:
-            n = int(self.headers.get("Content-Length", "0"))
-            if not 0 < n <= 4096:
-                raise ValueError("body size")
-            m = json.loads(self.rfile.read(n))["mqtt"]
-            user, password = m["user"], m["pass"]
-        except (ValueError, KeyError, TypeError):
-            return self.send_error(400)
-        if user != CONF["user"] or not valid(password):
-            return self.send_error(400)
-        with WRITE:
-            write_bridge(user, password)
-        self.log_message("bridge credentials for %s replaced, mosquitto restarted", user)
-        self.reply(204)
-
-    def log_message(self, fmt, *args):
-        sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args))
-
-
-class Server(ThreadingHTTPServer):
-    address_family = socket.AF_INET6
-
-    def server_bind(self):
-        self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
-        super().server_bind()
-
-    def get_request(self):
-        # The TLS handshake runs in the request thread, with a timeout, so a
-        # stalled peer cannot block the accept loop.
-        sock, addr = self.socket.accept()
-        sock.settimeout(10)
-        return self.ctx.wrap_socket(sock, server_side=True, do_handshake_on_connect=False), addr
-
-
-def main():
-    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-    ctx.minimum_version = ssl.TLSVersion.TLSv1_2
-    ctx.load_cert_chain(CONF["cert"], CONF["key"])
-    ctx.load_verify_locations(CONF["ca"])
-    ctx.verify_mode = ssl.CERT_REQUIRED
-    srv = Server(("::", CONF["port"]), Handler)
-    srv.ctx = ctx
-    srv.serve_forever()
-
-
-if __name__ == "__main__":
-    main()

ansible/roles/mqtt-bridge/files/mqtt-ca.crt

@@ -1,10 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIBdTCCARygAwIBAgIUN8ne4k4Wif9O83GDyKiBP0Ma7EIwCgYIKoZIzj0EAwIw
-GTEXMBUGA1UEAxMOUmVubmVyIE1RVFQgQ0EwHhcNMjYxMDA2MTUzMDE1WhcNMzYx
-MDAzMTUzMDE1WjAZMRcwFQYDVQQDEw5SZW5uZXIgTVFUVCBDQTBZMBMGByqGSM49
-AgEGCCqGSM49AwEHA0IABOZhZF2JDoyvVg2oIXSbJAuAWDcoG22RxPA4oMKRMA/b
-XzIY566D2k6cVnWqbil9s/q4O2mj+Epo7plBw8NwuOOjQjBAMA4GA1UdDwEB/wQE
-AwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQcqSDDiLZzhrvqq9f16Y08
-CO7QfzAKBggqhkjOPQQDAgNHADBEAiBFkxd9pXnqIcjReEcOTvCBZ3HxKu2jNQGL
-/le7gunTGgIgZapWAcu4awa+wmZcR/z2luOjD2GhIT15um14Z+sstJk=
------END CERTIFICATE-----

ansible/roles/mqtt-bridge/handlers/main.yml

@@ -5,9 +5,3 @@
   ansible.builtin.service:
     name: mosquitto
     state: restarted
-
-- name: Restart edge-mqtt-cred
-  ansible.builtin.systemd:
-    name: edge-mqtt-cred
-    state: restarted
-    daemon_reload: true

ansible/roles/mqtt-bridge/tasks/main.yml

@@ -11,39 +11,13 @@
     name: mosquitto
     state: present
 
-- name: Create the bridge directory
-  ansible.builtin.file:
-    path: /etc/mosquitto/bridge.d
-    state: directory
-    owner: root
-    group: mosquitto
-    mode: "0750"
-
-- name: Check for the controller-managed bridge file
-  ansible.builtin.stat:
-    path: "/etc/mosquitto/bridge.d/{{ edge_mqtt_bridge_account }}.conf"
-  register: edge_mqtt_bridge_managed
-
-# Until mqtt-account-controller has pushed the first password, keep the
-# vault-password bridges so the cluster never loses edge-1's data. Once the
-# managed file exists, never render them again (edge-mqtt-cred removed them).
-- name: Deploy legacy vault-password bridges
-  ansible.builtin.template:
-    src: edge-mqtt-bridge-legacy.conf.j2
-    dest: /etc/mosquitto/bridge.d/legacy.conf
-    owner: root
-    group: mosquitto
-    mode: "0640"   # carries the legacy cluster bridge credentials
-  when: not edge_mqtt_bridge_managed.stat.exists
-  notify: Restart mosquitto
-
-- name: Deploy local mosquitto config
+- name: Deploy local mosquitto bridge config
   ansible.builtin.template:
     src: edge-mqtt-bridge.conf.j2
     dest: /etc/mosquitto/mosquitto.conf
     owner: root
     group: mosquitto
-    mode: "0640"
+    mode: "0640"   # carries the cluster bridge credentials
   notify: Restart mosquitto
 
 - name: Enable local mosquitto bridge
@@ -51,59 +25,3 @@
     name: mosquitto
     enabled: true
     state: started
-
-# --- edge-mqtt-cred (REN-1307): receives the bridge password from
-# mqtt-account-controller. edge-1 holds only the public MQTT CA and its own
-# server key, generated here and never copied off the host.
-- name: Create edge-mqtt-cred config directory
-  ansible.builtin.file:
-    path: /etc/edge-mqtt-cred
-    state: directory
-    owner: root
-    group: root
-    mode: "0700"
-
-- name: Generate edge-mqtt-cred server key and certificate
-  ansible.builtin.command:
-    cmd: >-
-      openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1
-      -nodes -days 36500 -subj "/CN={{ inventory_hostname }}"
-      -keyout /etc/edge-mqtt-cred/tls.key -out /etc/edge-mqtt-cred/tls.crt
-    creates: /etc/edge-mqtt-cred/tls.key
-  notify: Restart edge-mqtt-cred
-
-- name: Deploy edge-mqtt-cred files
-  ansible.builtin.copy:
-    src: "{{ item.src }}"
-    dest: "{{ item.dest }}"
-    owner: root
-    group: root
-    mode: "{{ item.mode }}"
-  loop:
-    - {src: edge-mqtt-cred.py, dest: /usr/local/sbin/edge-mqtt-cred, mode: "0755"}
-    # Public CA of the cluster's MQTT client certificates
-    # (kubernetes/infrastructure/post-crds/mqtt-ca.yaml), valid until 2036-10-03.
-    - {src: mqtt-ca.crt, dest: /etc/edge-mqtt-cred/mqtt-ca.crt, mode: "0644"}
-  notify: Restart edge-mqtt-cred
-
-- name: Deploy edge-mqtt-cred config and bridge template
-  ansible.builtin.template:
-    src: "{{ item.src }}"
-    dest: "{{ item.dest }}"
-    owner: root
-    group: root
-    mode: "{{ item.mode }}"
-  loop:
-    - {src: edge-mqtt-cred.json.j2, dest: /etc/edge-mqtt-cred/config.json, mode: "0600"}
-    # Read only on the controller's next push: a change here reaches the
-    # running bridge when the password is next set.
-    - {src: edge-1-bridge.conf.in.j2, dest: /etc/edge-mqtt-cred/bridge.conf.in, mode: "0600"}
-    - {src: edge-mqtt-cred.service.j2, dest: /etc/systemd/system/edge-mqtt-cred.service, mode: "0644"}
-  notify: Restart edge-mqtt-cred
-
-- name: Enable edge-mqtt-cred
-  ansible.builtin.systemd:
-    name: edge-mqtt-cred
-    enabled: true
-    state: started
-    daemon_reload: true

ansible/roles/mqtt-bridge/templates/edge-1-bridge.conf.in.j2

@@ -1,21 +0,0 @@
-# Managed by roles/mqtt-bridge (REN-1307). edge-mqtt-cred appends
-# remote_username/remote_password to this and writes the result to
-# /etc/mosquitto/bridge.d/{{ edge_mqtt_bridge_account }}.conf. One bridge for
-# open3e, as the register account `{{ edge_mqtt_bridge_account }}`
-# (ACL readwrite open3e/#). sml2mqtt is retired (SML reader, 2026-09-19).
-connection {{ edge_mqtt_bridge_account }}
-address {{ edge_mqtt_cluster_host }}:{{ edge_mqtt_cluster_tls_port }}
-bridge_cafile {{ edge_mqtt_bridge_cafile }}
-bridge_insecure false
-remote_clientid edge1-bridge
-cleansession false
-try_private true
-restart_timeout 5 30
-queue_qos0_messages true
-# The bridge state goes to the LOCAL broker only (retained 1/0); edge-mqtt-cred
-# reads it to answer the controller's "connected?".
-notifications true
-notifications_local_only true
-notification_topic $SYS/broker/connection/{{ edge_mqtt_bridge_account }}/state
-topic open3e/# out 0
-topic open3e/cmnd in 0

ansible/roles/mqtt-bridge/templates/edge-mqtt-bridge-legacy.conf.j2

@@ -1,21 +0,0 @@
-# Managed by roles/mqtt-bridge. Legacy vault-password bridges, rendered only
-# while /etc/mosquitto/bridge.d/{{ edge_mqtt_bridge_account }}.conf does not
-# exist; edge-mqtt-cred deletes this file on the controller's first push
-# (REN-1307). Remove with the vault keys once edge-1 is on the register account.
-# --- Bridge 1: open3e datapoints + command topic, as the cluster `open3e`
-# user (its ACL is `readwrite open3e/#`, which covers publishing datapoints
-# and subscribing to open3e/cmnd). e3oncan also publishes under open3e/#.
-connection edge1-open3e
-address {{ edge_mqtt_cluster_host }}:{{ edge_mqtt_cluster_tls_port }}
-bridge_cafile {{ edge_mqtt_bridge_cafile }}
-bridge_insecure false
-remote_username {{ edge_open3e_mqtt_user }}
-remote_password {{ edge_open3e_mqtt_password }}
-remote_clientid edge1-bridge-open3e
-cleansession false
-try_private true
-notifications false
-restart_timeout 5 30
-queue_qos0_messages true
-topic open3e/# out 0
-topic open3e/cmnd in 0

ansible/roles/mqtt-bridge/templates/edge-mqtt-bridge.conf.j2

@@ -22,9 +22,36 @@ log_type notice
 listener 1883 127.0.0.1
 allow_anonymous true
 
-# --- Bridges to the cluster broker (REN-1307). bridge.d holds either
-# {{ edge_mqtt_bridge_account }}.conf, written by edge-mqtt-cred with the password
-# mqtt-account-controller pushed, or, until that first push, legacy.conf with
-# the two vault-password bridges. edge-mqtt-cred deletes legacy.conf when it
-# writes the new file.
-include_dir /etc/mosquitto/bridge.d
+# --- Bridge 1: open3e datapoints + command topic, as the cluster `open3e`
+# user (its ACL is `readwrite open3e/#`, which covers publishing datapoints
+# and subscribing to open3e/cmnd). e3oncan also publishes under open3e/#.
+connection edge1-open3e
+address {{ edge_mqtt_cluster_host }}:{{ edge_mqtt_cluster_tls_port }}
+bridge_cafile {{ edge_mqtt_bridge_cafile }}
+bridge_insecure false
+remote_username {{ edge_open3e_mqtt_user }}
+remote_password {{ edge_open3e_mqtt_password }}
+remote_clientid edge1-bridge-open3e
+cleansession false
+try_private true
+notifications false
+restart_timeout 5 30
+queue_qos0_messages true
+topic open3e/# out 0
+topic open3e/cmnd in 0
+
+# --- Bridge 2: power-meter topics, as the cluster `sml2mqtt` user (ACL
+# `write sml2mqtt/#`). Publish-only, so no `in` topics.
+connection edge1-sml2mqtt
+address {{ edge_mqtt_cluster_host }}:{{ edge_mqtt_cluster_tls_port }}
+bridge_cafile {{ edge_mqtt_bridge_cafile }}
+bridge_insecure false
+remote_username sml2mqtt
+remote_password {{ edge_sml2mqtt_mqtt_password }}
+remote_clientid edge1-bridge-sml2mqtt
+cleansession false
+try_private true
+notifications false
+restart_timeout 5 30
+queue_qos0_messages true
+topic sml2mqtt/# out 0

ansible/roles/mqtt-bridge/templates/edge-mqtt-cred.json.j2

@@ -1,13 +0,0 @@
-{{ {
-  "port": edge_mqtt_cred_port,
-  "allowed_cn": edge_mqtt_cred_allowed_cn,
-  "user": edge_mqtt_bridge_account,
-  "cert": "/etc/edge-mqtt-cred/tls.crt",
-  "key": "/etc/edge-mqtt-cred/tls.key",
-  "ca": "/etc/edge-mqtt-cred/mqtt-ca.crt",
-  "template": "/etc/edge-mqtt-cred/bridge.conf.in",
-  "bridge_file": "/etc/mosquitto/bridge.d/" ~ edge_mqtt_bridge_account ~ ".conf",
-  "legacy_files": ["/etc/mosquitto/bridge.d/legacy.conf"],
-  "local_port": edge_mqtt_port,
-  "state_topic": "$SYS/broker/connection/" ~ edge_mqtt_bridge_account ~ "/state",
-} | to_nice_json }}

ansible/roles/mqtt-bridge/templates/edge-mqtt-cred.service.j2

@@ -1,19 +0,0 @@
-# Managed by roles/mqtt-bridge (REN-1307). See /usr/local/sbin/edge-mqtt-cred.
-[Unit]
-Description=Receives the MQTT bridge credential from mqtt-account-controller
-After=network-online.target mosquitto.service
-Wants=network-online.target
-
-[Service]
-ExecStart=/usr/bin/python3 /usr/local/sbin/edge-mqtt-cred /etc/edge-mqtt-cred/config.json
-Restart=always
-RestartSec=5
-# Root only to write the bridge file and restart mosquitto.
-ProtectSystem=strict
-ReadWritePaths=/etc/mosquitto/bridge.d
-ProtectHome=true
-PrivateTmp=true
-NoNewPrivileges=true
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/netbird-client/defaults/main.yml

@@ -6,13 +6,3 @@ netbird_management_url: "https://netbird.renner.uno"
 
 # Setup key - should be defined in group_vars or host_vars
 # netbird_setup_key: ""
-
-# Release installed on hosts without a NetBird package repository (Arch).
-# Debian/Ubuntu follow the apt repository instead.
-netbird_client_version: "0.75.0"
-netbird_release_url: "https://github.com/netbirdio/netbird/releases/download/v{{ netbird_client_version }}"
-# On the controller; the archive is copied from there to the hosts.
-netbird_download_dir: "{{ lookup('ansible.builtin.env', 'HOME') }}/.cache/ansible/netbird"
-
-# Leave the host's DNS configuration alone (hosts with a pinned resolv.conf).
-netbird_disable_dns: false

ansible/roles/netbird-client/tasks/main.yml

@@ -2,79 +2,47 @@
 # NetBird Client Installation Role
 # Installs and configures NetBird client on hosts with netbird_setup_key defined
 
-- name: Install NetBird from the NetBird apt repository
-  when: ansible_facts['os_family'] == 'Debian'
-  block:
-    - name: Install required packages
-      ansible.builtin.apt:
-        name:
-          - curl
-          - ca-certificates
-          - gnupg
-        state: present
-        update_cache: true
+- name: Install required packages
+  ansible.builtin.apt:
+    name:
+      - curl
+      - ca-certificates
+      - gnupg
+    state: present
+    update_cache: true
 
-    - name: Add NetBird repository key
-      ansible.builtin.shell:
-        cmd: |
-          curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
-        creates: /usr/share/keyrings/netbird-archive-keyring.gpg
+- name: Add NetBird repository key
+  ansible.builtin.shell:
+    cmd: |
+      curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
+    creates: /usr/share/keyrings/netbird-archive-keyring.gpg
 
-    - name: Add NetBird repository
-      ansible.builtin.apt_repository:
-        repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
-        state: present
-        filename: netbird
+- name: Add NetBird repository
+  ansible.builtin.apt_repository:
+    repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
+    state: present
+    filename: netbird
 
-    - name: Install NetBird client
-      ansible.builtin.apt:
-        name: netbird
-        state: present
-        update_cache: true
+- name: Install NetBird client
+  ansible.builtin.apt:
+    name: netbird
+    state: present
+    update_cache: true
 
-# Arch has no netbird package in its repositories; the release binary is
-# verified against the release's checksums file. It is downloaded on the
-# controller and copied over: github.com has no IPv6 address, and proxy-1 is
-# IPv6-only. The controller steps also run in check mode: they only write on
-# the controller, and the install step needs the archive.
-- name: Install NetBird from the release tarball
-  when: ansible_facts['os_family'] == 'Archlinux'
-  block:
-    - name: Create the controller download directory
-      ansible.builtin.file:
-        path: "{{ netbird_download_dir }}"
-        state: directory
-        mode: "0755"
-      delegate_to: localhost
-      become: false
-      run_once: true
-      check_mode: false
+- name: Check if NetBird is already connected
+  ansible.builtin.command: netbird status
+  register: netbird_status
+  changed_when: false
+  failed_when: false
 
-    - name: Download the NetBird release on the controller
-      check_mode: false
-      ansible.builtin.get_url:
-        url: "{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
-        dest: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
-        checksum: "sha256:{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_checksums.txt"
-        mode: "0644"
-      delegate_to: localhost
-      become: false
-      run_once: true
-
-    - name: Install the NetBird binary
-      ansible.builtin.unarchive:
-        src: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
-        dest: /usr/local/bin
-        include:
-          - netbird
-        owner: root
-        group: root
-        mode: "0755"
-
-    - name: Install the NetBird service
-      ansible.builtin.command: netbird service install
-      args:
-        creates: /etc/systemd/system/netbird.service
+- name: Connect NetBird to management server
+  ansible.builtin.command: >
+    netbird up
+    --management-url {{ netbird_management_url }}
+    --setup-key {{ netbird_setup_key }}
+  when: "'Connected' not in netbird_status.stdout"
+  register: netbird_connect
+  changed_when: netbird_connect.rc == 0
 
 - name: Enable NetBird service
   ansible.builtin.service:
@@ -82,29 +50,9 @@
     state: started
     enabled: true
 
-- name: Check if NetBird is already connected
-  ansible.builtin.command: netbird status
-  register: netbird_status
-  check_mode: false
-  changed_when: false
-  failed_when: false
-
-# "Disconnected" contains "Connected", so match the management line.
-- name: Connect NetBird to management server
-  ansible.builtin.command: >
-    netbird up
-    --management-url {{ netbird_management_url }}
-    --setup-key {{ netbird_setup_key }}
-    {{ '--disable-dns' if netbird_disable_dns else '' }}
-  when: "'Management: Connected' not in netbird_status.stdout"
-  register: netbird_connect
-  changed_when: netbird_connect.rc == 0
-  no_log: true
-
 - name: Get NetBird status
   ansible.builtin.command: netbird status
   register: netbird_final_status
-  check_mode: false
   changed_when: false
 
 - name: Display NetBird status

ansible/roles/netbird-proxy-watchdog/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-# Self-healing watchdog for the NetBird reverse proxy on the proxy hosts.
+# Self-healing watchdog for the NetBird reverse proxy (VPS-1 and proxy-1).
 #
 # Why it exists: the proxy's embedded NetBird client pins each proxied
 # domain's route to a specific routing-peer identity. The netbird-ingress
@@ -20,7 +20,9 @@ netbird_watchdog_probe_hosts:
   - home.renner.uno
   - immich.renner.uno
 
-# Address the probe connects to. The proxy binds :443 on the host directly.
+# Address the probe connects to. 127.0.0.1 reaches the local proxy on both
+# members: VPS-1 routes it via Traefik's HostSNI(*) TCP passthrough, proxy-1
+# binds :443 directly.
 netbird_watchdog_probe_address: 127.0.0.1
 
 netbird_watchdog_container: netbird-proxy

ansible/roles/netbird-proxy-watchdog/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 # Deploys the reverse-proxy watchdog (see defaults/main.yml for why).
 
-- name: Ensure curl is installed (probe dependency)
+- name: Ensure curl is installed (probe dependency; proxy-1 is Arch, VPS-1 Debian)
   ansible.builtin.package:
     name: curl
     state: present

ansible/roles/netbird-proxy/defaults/main.yml

@@ -1,8 +1,8 @@
 ---
 # NetBird reverse-proxy HA cluster member (proxy-only host, e.g. proxy-1).
-# Every member uses the same NB_PROXY_DOMAIN + token. NetBird load-balances/
-# fails the cluster over via DNS round-robin on the proxy domain (see
-# plays/install-netbird-proxy.yml).
+# Joins the same proxy cluster as the VPS-1 proxy: identical NB_PROXY_DOMAIN
+# + the same token. NetBird load-balances/fails the cluster over via DNS
+# round-robin on the proxy domain (see plays/install-netbird-proxy.yml).
 
 # MUST match the netbird-server role's netbird_version — the reverse-proxy
 # release line has to track the management server it enrolls against.
@@ -11,13 +11,11 @@ netbird_version: "0.73.2"
 # Management server this proxy enrolls against (public API, TLS).
 netbird_server_host: netbird.renner.uno
 
-# Proxy domain — MUST be identical across every cluster member, and must NOT
-# be a subdomain of the management domain (netbird.renner.uno) per NetBird
-# docs (TLS/routing conflicts).
+# Proxy domain — MUST be identical across every cluster member.
 netbird_proxy_domain: "proxy.renner.uno"
 
-# A shared NB_PROXY_TOKEN across members with the same NB_PROXY_DOMAIN is a
-# valid HA cluster, so one token serves every member.
+# Reuse VPS-1's proxy token — a shared NB_PROXY_TOKEN across members with the
+# same NB_PROXY_DOMAIN is a valid HA cluster, so no separate token is needed.
 netbird_proxy_token: "{{ vault_netbird_proxy_token }}"
 
 # Relay auth secret — MUST be identical across every relay instance and the

ansible/roles/netbird-proxy/templates/docker-compose.yml.j2

@@ -1,7 +1,8 @@
 # NetBird reverse-proxy — HA cluster member on a proxy-only host.
-# Just the reverse-proxy container, bound directly to :443. No Traefik: the
-# proxy owns the port and terminates TLS + answers its own tls-alpn-01 ACME
-# challenges directly.
+# Just the reverse-proxy container, bound directly to :443. No Traefik:
+# unlike VPS-1 (where the proxy shares :443 with management/signal/relay via
+# SNI routing), this host runs nothing else, so the proxy owns the port and
+# terminates TLS + answers its own tls-alpn-01 ACME challenges directly.
 services:
   proxy:
     image: netbirdio/reverse-proxy:{{ netbird_version }}

ansible/roles/netbird-server/defaults/main.yml

@@ -23,3 +23,9 @@ netbird_datastore_key: "{{ vault_netbird_datastore_key }}"
 
 # Relay secret for WebSocket relay authentication
 netbird_relay_secret: "{{ vault_netbird_relay_secret }}"
+
+# Reverse proxy configuration
+# IMPORTANT: Proxy domain must NOT be a subdomain of the management domain
+# (netbird.renner.uno) per NetBird docs - causes TLS/routing conflicts
+netbird_proxy_domain: "proxy.renner.uno"
+netbird_proxy_token: "{{ vault_netbird_proxy_token }}"

ansible/roles/netbird-server/handlers/main.yml

@@ -16,3 +16,7 @@
 - name: Restart netbird coturn
   ansible.builtin.shell: |
     cd /opt/netbird && docker compose up -d --force-recreate coturn
+
+- name: Restart netbird proxy
+  ansible.builtin.shell: |
+    cd /opt/netbird && docker compose up -d --force-recreate proxy

ansible/roles/netbird-server/tasks/main.yml

@@ -36,6 +36,7 @@
     - /opt/netbird/management
     - /opt/netbird/signal
     - /opt/netbird/letsencrypt
+    - /opt/netbird/proxy/certs
 
 - name: Deploy docker-compose.yml
   ansible.builtin.template:
@@ -58,119 +59,45 @@
     mode: "0644"
   notify: Restart netbird coturn
 
-# The public edge is the proxy hosts (plays/install-netbird-proxy.yml); this
-# host runs no reverse proxy. The tasks below remove the one it used to run,
-# its watchdogs first so they cannot restart it mid-removal, and can go once
-# the role has run against every host that ever carried it.
-- name: Check for reverse-proxy watchdog units
-  ansible.builtin.stat:
-    path: "/etc/systemd/system/{{ item }}"
-  loop:
-    - netbird-proxy-watchdog.timer
-    - netbird-proxy-flood-watchdog.service
-  register: _proxy_watchdog_units
-
-- name: Stop and disable reverse-proxy watchdogs
-  ansible.builtin.systemd:
-    name: "{{ item.item }}"
-    state: stopped
-    enabled: false
-  loop: "{{ _proxy_watchdog_units.results }}"
-  loop_control:
-    label: "{{ item.item }}"
-  when: item.stat.exists
-
-- name: Remove reverse-proxy watchdog units and scripts
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: absent
-  loop:
-    - /etc/systemd/system/netbird-proxy-watchdog.timer
-    - /etc/systemd/system/netbird-proxy-watchdog.service
-    - /etc/systemd/system/netbird-proxy-flood-watchdog.service
-    - /usr/local/bin/netbird-proxy-watchdog
-    - /usr/local/bin/netbird-proxy-flood-watchdog
-    - /run/netbird-proxy-watchdog
-  register: _proxy_watchdog_files
-
-- name: Reload systemd after removing the watchdog units
-  ansible.builtin.systemd:
-    daemon_reload: true
-  when: _proxy_watchdog_files.changed
+- name: Deploy proxy.env
+  ansible.builtin.template:
+    src: proxy.env.j2
+    dest: /opt/netbird/proxy.env
+    mode: "0644"
+  notify: Restart netbird proxy
 
 - name: Pull NetBird container images
   ansible.builtin.shell: |
     cd /opt/netbird && docker compose pull
   changed_when: false
 
-# --remove-orphans removes containers of services no longer in the compose
-# file (the reverse proxy).
 - name: Start NetBird containers
   ansible.builtin.shell: |
-    cd /opt/netbird && docker compose up -d --remove-orphans
+    cd /opt/netbird && docker compose up -d
   register: docker_compose_result
-  changed_when: >-
-    'Started' in docker_compose_result.stdout or
-    'Creating' in docker_compose_result.stdout or
-    'Removed' in docker_compose_result.stdout or
-    'Removed' in docker_compose_result.stderr
+  changed_when: "'Started' in docker_compose_result.stdout or 'Creating' in docker_compose_result.stdout"
 
-- name: Remove reverse-proxy volumes
-  ansible.builtin.shell: |
-    set -eu
-    for v in netbird_netbird_proxy_certs netbird_netbird_proxy_data netbird_netbird_proxy_state; do
-      if [ -n "$(docker volume ls -q --filter "name=^${v}$")" ]; then
-        docker volume rm "$v"
-        echo "removed $v"
-      fi
-    done
-  register: _proxy_volumes
-  changed_when: "'removed' in _proxy_volumes.stdout"
-
-- name: Remove reverse-proxy config and state
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: absent
-  loop:
-    - /opt/netbird/proxy.env
-    - /opt/netbird/proxy
-    - /opt/netbird/proxy-state
-
-# The management store has no cluster-side backup; the cluster's backup
-# key is gone with the managed authorized_keys (ssh-ca role).
+# The management store has no cluster-side backup. Pulling one put a root
+# key on this host that the cluster held, and this server's own state is
+# its own business to protect rather than the cluster's. Dropping the two
+# tasks that placed them would only stop ansible managing the key and the
+# script — the pair below revokes them instead, and can go once the role
+# has run against every host that ever carried them.
 - name: Remove the backup dump script
   ansible.builtin.file:
     path: /usr/local/bin/netbird-backup-dump
     state: absent
 
+- name: Revoke the cluster backup key
+  ansible.posix.authorized_key:
+    user: root
+    key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeW/L9bHMTSa5/Qb0ClppCf4Z0hL39JwzI6tBmKd2FG netbird-backup@homelab-cluster"
+    state: absent
+
 - name: Wait for management container to be ready
   ansible.builtin.pause:
     seconds: 10
 
-# The host's mesh client must not manage DNS: containers here resolve through
-# the host stub, and the renner.uno nameserver group is unreachable from it.
-- name: Check for the host NetBird client
-  ansible.builtin.stat:
-    path: /usr/bin/netbird
-  register: _netbird_client
-
-- name: Read the host NetBird client config
-  ansible.builtin.command: netbird debug config
-  register: _netbird_client_config
-  check_mode: false
-  changed_when: false
-  when: _netbird_client.stat.exists
-
-# disable-dns is only applied on a fresh `up`, not to a connected client.
-- name: Reconnect the host NetBird client with DNS management disabled
-  ansible.builtin.shell: |
-    set -eu
-    netbird down
-    netbird up --disable-dns
-  when:
-    - _netbird_client.stat.exists
-    - not (_netbird_client_config.stdout | from_json).disableDns
-
 - name: Display deployment status
   ansible.builtin.debug:
     msg: |

ansible/roles/netbird-server/templates/docker-compose.yml.j2

@@ -61,10 +61,6 @@ services:
       - AUTH_AUDIENCE={{ netbird_auth_audience }}
       - AUTH_CLIENT_ID={{ netbird_auth_client_id }}
       - AUTH_AUTHORITY={{ netbird_auth_issuer }}
-      # The CSP allows AUTH_AUTHORITY only as a path (/application/o/netbird/);
-      # Authentik's token/userinfo endpoints sit outside it, so the browser
-      # would block the code exchange and every token refresh.
-      - NETBIRD_CSP={{ netbird_auth_issuer | urlsplit('scheme') }}://{{ netbird_auth_issuer | urlsplit('netloc') }}
       - USE_AUTH0=false
       - AUTH_SUPPORTED_SCOPES=openid profile email
       - NETBIRD_TOKEN_SOURCE=idToken
@@ -191,9 +187,10 @@ services:
       - "traefik.http.routers.netbird-management.service=netbird-management"
       - "traefik.http.services.netbird-management.loadbalancer.server.port=33073"
       - "traefik.http.services.netbird-management.loadbalancer.server.scheme=h2c"
-      # gRPC reverse-proxy service — the proxy hosts (proxy-1, proxy-2)
-      # connect here via NB_PROXY_MANAGEMENT_ADDRESS=https://host:443 and get
-      # a 404 without it. Same backend (h2c 33073).
+      # gRPC reverse-proxy service — remote reverse-proxy instances (e.g.
+      # proxy-1) connect here via NB_PROXY_MANAGEMENT_ADDRESS=https://host:443.
+      # The local proxy uses management:33073 directly, so it never needed
+      # this route; remote ones get a 404 without it. Same backend (h2c 33073).
       - "traefik.http.routers.netbird-proxyservice.rule=Host(`{{ netbird_server_host }}`) && PathPrefix(`/management.ProxyService/`)"
       - "traefik.http.routers.netbird-proxyservice.entrypoints=websecure"
       - "traefik.http.routers.netbird-proxyservice.tls=true"
@@ -224,6 +221,37 @@ services:
         max-size: "500m"
         max-file: "2"
 
+  # Reverse Proxy Service (per docs.netbird.io/selfhosted/migration/enable-reverse-proxy)
+  proxy:
+    image: netbirdio/reverse-proxy:{{ netbird_version }}
+    container_name: netbird-proxy
+    restart: unless-stopped
+    networks:
+      - netbird
+    depends_on:
+      - management
+    env_file:
+      - ./proxy.env
+    volumes:
+      - netbird_proxy_certs:/certs
+      - /opt/netbird/proxy-state:/var/lib/netbird
+    labels:
+      - "traefik.enable=true"
+      - "traefik.tcp.routers.proxy-passthrough.entrypoints=websecure"
+      - "traefik.tcp.routers.proxy-passthrough.rule=HostSNI(`*`)"
+      - "traefik.tcp.routers.proxy-passthrough.tls.passthrough=true"
+      - "traefik.tcp.routers.proxy-passthrough.service=proxy-tls"
+      - "traefik.tcp.routers.proxy-passthrough.priority=1"
+      - "traefik.tcp.services.proxy-tls.loadbalancer.server.port=8443"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "500m"
+        max-file: "2"
+
 networks:
   netbird:
     driver: bridge
+
+volumes:
+  netbird_proxy_certs:

ansible/roles/netbird-server/templates/proxy.env.j2

@@ -0,0 +1,8 @@
+NB_PROXY_DOMAIN={{ netbird_proxy_domain }}
+NB_PROXY_TOKEN={{ netbird_proxy_token }}
+NB_PROXY_MANAGEMENT_ADDRESS=http://management:33073
+NB_PROXY_ALLOW_INSECURE=true
+NB_PROXY_ADDRESS=:8443
+NB_PROXY_ACME_CERTIFICATES=true
+NB_PROXY_ACME_CHALLENGE_TYPE=tls-alpn-01
+NB_PROXY_CERTIFICATE_DIRECTORY=/certs

ansible/roles/node-exporter/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+# Node Exporter defaults
+
+node_exporter_version: "1.8.2"
+node_exporter_port: 9100
+node_exporter_user: node-exporter
+node_exporter_group: node-exporter
+
+# Installation paths
+node_exporter_install_dir: /opt/node-exporter
+node_exporter_binary: "{{ node_exporter_install_dir }}/node_exporter"
+
+# Collectors to enable (defaults are usually sufficient)
+node_exporter_enabled_collectors:
+  - systemd
+  - processes
+  - timex
+
+# Collectors to disable
+node_exporter_disabled_collectors: []

ansible/roles/node-exporter/files/chrony-metrics.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# Generate Prometheus metrics from chronyc tracking output
+# Runs as a cron job and outputs to textfile collector directory
+
+OUTPUT_FILE="${1:-/var/lib/node-exporter/chrony.prom}"
+
+# Get chrony tracking info
+tracking=$(chronyc tracking 2>/dev/null)
+
+if [ -z "$tracking" ]; then
+    echo "chrony_up 0" > "$OUTPUT_FILE.tmp"
+    mv "$OUTPUT_FILE.tmp" "$OUTPUT_FILE"
+    exit 0
+fi
+
+# Parse values
+stratum=$(echo "$tracking" | grep "^Stratum" | awk '{print $3}')
+system_time=$(echo "$tracking" | grep "^System time" | awk '{print $4}')
+root_delay=$(echo "$tracking" | grep "^Root delay" | awk '{print $4}')
+root_dispersion=$(echo "$tracking" | grep "^Root dispersion" | awk '{print $4}')
+last_offset=$(echo "$tracking" | grep "^Last offset" | awk '{print $4}')
+rms_offset=$(echo "$tracking" | grep "^RMS offset" | awk '{print $4}')
+frequency=$(echo "$tracking" | grep "^Frequency" | awk '{print $3}')
+ref_id=$(echo "$tracking" | grep "^Reference ID" | awk '{print $4}' | tr -d '()')
+
+# Handle negative offsets (remove leading + or keep -)
+system_time=$(echo "$system_time" | sed 's/^+//')
+last_offset=$(echo "$last_offset" | sed 's/^+//')
+rms_offset=$(echo "$rms_offset" | sed 's/^+//')
+
+# Output Prometheus metrics
+cat > "$OUTPUT_FILE.tmp" << EOF
+# HELP chrony_stratum Current stratum of the NTP server
+# TYPE chrony_stratum gauge
+chrony_stratum ${stratum:-0}
+# HELP chrony_system_time_offset_seconds System time offset from NTP time
+# TYPE chrony_system_time_offset_seconds gauge
+chrony_system_time_offset_seconds ${system_time:-0}
+# HELP chrony_root_delay_seconds Total round-trip delay to the primary reference clock
+# TYPE chrony_root_delay_seconds gauge
+chrony_root_delay_seconds ${root_delay:-0}
+# HELP chrony_root_dispersion_seconds Total dispersion to the primary reference clock
+# TYPE chrony_root_dispersion_seconds gauge
+chrony_root_dispersion_seconds ${root_dispersion:-0}
+# HELP chrony_last_offset_seconds Estimated offset of the last update
+# TYPE chrony_last_offset_seconds gauge
+chrony_last_offset_seconds ${last_offset:-0}
+# HELP chrony_rms_offset_seconds RMS offset
+# TYPE chrony_rms_offset_seconds gauge
+chrony_rms_offset_seconds ${rms_offset:-0}
+# HELP chrony_frequency_ppm Frequency offset in parts per million
+# TYPE chrony_frequency_ppm gauge
+chrony_frequency_ppm ${frequency:-0}
+# HELP chrony_up Whether chrony is running and synchronized
+# TYPE chrony_up gauge
+chrony_up 1
+# HELP chrony_reference_info Reference clock information
+# TYPE chrony_reference_info gauge
+chrony_reference_info{reference="${ref_id:-unknown}"} 1
+EOF
+
+mv "$OUTPUT_FILE.tmp" "$OUTPUT_FILE"

ansible/roles/node-exporter/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Reload systemd
+  systemd:
+    daemon_reload: yes
+
+- name: Restart node-exporter
+  systemd:
+    name: node-exporter
+    state: restarted

ansible/roles/node-exporter/tasks/main.yml

@@ -0,0 +1,88 @@
+---
+# Install and configure Node Exporter
+
+- name: Create node-exporter system group
+  group:
+    name: "{{ node_exporter_group }}"
+    system: yes
+    state: present
+
+- name: Create node-exporter system user
+  user:
+    name: "{{ node_exporter_user }}"
+    group: "{{ node_exporter_group }}"
+    system: yes
+    shell: /usr/sbin/nologin
+    home: "{{ node_exporter_install_dir }}"
+    createhome: no
+
+- name: Create installation directory
+  file:
+    path: "{{ node_exporter_install_dir }}"
+    state: directory
+    owner: "{{ node_exporter_user }}"
+    group: "{{ node_exporter_group }}"
+    mode: '0755'
+
+- name: Check if node_exporter is already installed
+  stat:
+    path: "{{ node_exporter_binary }}"
+  register: node_exporter_installed
+
+- name: Get installed version
+  command: "{{ node_exporter_binary }} --version"
+  register: installed_version
+  when: node_exporter_installed.stat.exists
+  changed_when: false
+  failed_when: false
+
+- name: Download node_exporter
+  get_url:
+    url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz"
+    dest: /tmp/node_exporter.tar.gz
+    mode: '0644'
+  when: not node_exporter_installed.stat.exists or (installed_version.stdout is defined and node_exporter_version not in installed_version.stdout)
+
+- name: Extract node_exporter
+  unarchive:
+    src: /tmp/node_exporter.tar.gz
+    dest: /tmp
+    remote_src: yes
+  when: not node_exporter_installed.stat.exists or (installed_version.stdout is defined and node_exporter_version not in installed_version.stdout)
+
+- name: Install node_exporter binary
+  copy:
+    src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter"
+    dest: "{{ node_exporter_binary }}"
+    owner: "{{ node_exporter_user }}"
+    group: "{{ node_exporter_group }}"
+    mode: '0755'
+    remote_src: yes
+  when: not node_exporter_installed.stat.exists or (installed_version.stdout is defined and node_exporter_version not in installed_version.stdout)
+  notify: Restart node-exporter
+
+- name: Create systemd service
+  template:
+    src: node_exporter.service.j2
+    dest: /etc/systemd/system/node-exporter.service
+    owner: root
+    group: root
+    mode: '0644'
+  notify:
+    - Reload systemd
+    - Restart node-exporter
+
+- name: Enable and start node-exporter service
+  systemd:
+    name: node-exporter
+    enabled: yes
+    state: started
+    daemon_reload: yes
+
+- name: Clean up downloaded files
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /tmp/node_exporter.tar.gz
+    - "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64"

ansible/roles/node-exporter/templates/node_exporter.service.j2

@@ -0,0 +1,38 @@
+[Unit]
+Description=Prometheus Node Exporter
+Documentation=https://github.com/prometheus/node_exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ node_exporter_user }}
+Group={{ node_exporter_group }}
+ExecStart={{ node_exporter_binary }} \
+  --web.listen-address=:{{ node_exporter_port }} \
+{% for collector in node_exporter_enabled_collectors %}
+  --collector.{{ collector }} \
+{% endfor %}
+{% for collector in node_exporter_disabled_collectors %}
+  --no-collector.{{ collector }} \
+{% endfor %}
+{% if node_exporter_textfile_dir is defined %}
+  --collector.textfile \
+  --collector.textfile.directory={{ node_exporter_textfile_dir }} \
+{% endif %}
+  --collector.filesystem.mount-points-exclude='^/(sys|proc|dev|host|etc)($$|/)'
+Restart=on-failure
+RestartSec=10s
+
+# Security hardening
+NoNewPrivileges=yes
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+ReadOnlyPaths=/
+{% if node_exporter_textfile_dir is defined %}
+ReadWritePaths={{ node_exporter_textfile_dir }}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/os-update/tasks/main.yml

@@ -1,142 +0,0 @@
----
-- name: Fail on unsupported distribution
-  ansible.builtin.fail:
-    msg: >-
-      os-update supports Debian and Archlinux only;
-      this host reports os_family={{ ansible_facts['os_family'] }}
-  when: ansible_facts['os_family'] not in ['Debian', 'Archlinux']
-
-# --- Baseline: what is healthy before the upgrade must be healthy after ---
-- name: Record failed units before upgrade
-  ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager
-  register: os_update_failed_before
-  changed_when: false
-  check_mode: false
-
-- name: Check for docker
-  ansible.builtin.stat:
-    path: /usr/bin/docker
-  register: os_update_docker
-
-- name: Record running containers before upgrade
-  ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}"
-  register: os_update_containers_before
-  changed_when: false
-  check_mode: false
-  when: os_update_docker.stat.exists
-
-- name: Assert at least 1 GiB free on /
-  ansible.builtin.assert:
-    that: >-
-      (ansible_facts['mounts'] | selectattr('mount', 'equalto', '/') | first)['size_available']
-      >= 1073741824
-    fail_msg: "Less than 1 GiB free on / -- free space before upgrading"
-
-# --- Arch Linux ---
-- name: Upgrade keyring first (Arch)
-  community.general.pacman:
-    name: archlinux-keyring
-    state: latest
-    update_cache: true
-  when: ansible_facts['os_family'] == 'Archlinux'
-
-- name: Upgrade all packages (Arch)
-  community.general.pacman:
-    upgrade: true
-  register: os_update_pacman
-  when: ansible_facts['os_family'] == 'Archlinux'
-
-# --- Debian/Ubuntu ---
-- name: Upgrade all packages (Debian)
-  ansible.builtin.apt:
-    update_cache: true
-    upgrade: full
-    autoremove: true
-  environment:
-    DEBIAN_FRONTEND: noninteractive
-  when: ansible_facts['os_family'] == 'Debian'
-
-- name: Check reboot-required flag (Debian)
-  ansible.builtin.stat:
-    path: /var/run/reboot-required
-  register: os_update_reboot_flag
-  when: ansible_facts['os_family'] == 'Debian'
-
-# --- Reboot ---
-- name: Read running kernel before reboot (Arch)
-  ansible.builtin.command: uname -r
-  register: os_update_kernel_before
-  changed_when: false
-  check_mode: false
-  when: ansible_facts['os_family'] == 'Archlinux'
-
-# Missing modules dir: a newer kernel is installed but not booted, including
-# from an earlier interrupted run where pacman now reports no change.
-- name: Check running kernel is still installed (Arch)
-  ansible.builtin.stat:
-    path: "/usr/lib/modules/{{ os_update_kernel_before.stdout }}/vmlinuz"
-  register: os_update_kernel_present
-  when: ansible_facts['os_family'] == 'Archlinux'
-
-- name: Reboot
-  ansible.builtin.reboot:
-    msg: "Rebooting after package upgrade"
-    reboot_timeout: 900
-    post_reboot_delay: 30
-  when: >-
-    (ansible_facts['os_family'] == 'Archlinux' and
-      (os_update_pacman is changed or not os_update_kernel_present.stat.exists)) or
-    (ansible_facts['os_family'] == 'Debian' and os_update_reboot_flag.stat.exists)
-
-# --- Verify ---
-- name: Read running kernel
-  ansible.builtin.command: uname -r
-  register: os_update_kernel
-  changed_when: false
-  check_mode: false
-
-# Arch keeps only the installed kernel's modules; the directory existing means
-# the host booted the kernel pacman installed.
-- name: Assert running kernel is the installed one (Arch)
-  ansible.builtin.stat:
-    path: "/usr/lib/modules/{{ os_update_kernel.stdout }}/vmlinuz"
-  register: os_update_kernel_installed
-  failed_when: not os_update_kernel_installed.stat.exists
-  when: ansible_facts['os_family'] == 'Archlinux'
-
-- name: Assert no reboot pending (Debian)
-  ansible.builtin.stat:
-    path: /var/run/reboot-required
-  register: os_update_reboot_flag_after
-  failed_when: os_update_reboot_flag_after.stat.exists
-  when: ansible_facts['os_family'] == 'Debian'
-
-- name: Wait for previously running containers
-  ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}"
-  register: os_update_containers_after
-  changed_when: false
-  check_mode: false
-  until: >-
-    os_update_containers_before.stdout_lines
-    | difference(os_update_containers_after.stdout_lines)
-    | length == 0
-  retries: 20
-  delay: 15
-  when: os_update_docker.stat.exists
-
-- name: Wait for no new failed units
-  ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager
-  register: os_update_failed_after
-  changed_when: false
-  check_mode: false
-  until: >-
-    os_update_failed_after.stdout_lines
-    | map('split') | map('first')
-    | difference(os_update_failed_before.stdout_lines | map('split') | map('first'))
-    | length == 0
-  retries: 10
-  delay: 15
-
-- name: Report kernel
-  ansible.builtin.debug:
-    msg: "{{ inventory_hostname }} runs {{ os_update_kernel.stdout }}"

ansible/roles/power-meter-sml/defaults/main.yml

@@ -0,0 +1,23 @@
+---
+# Power-meter reader: sml2mqtt against the FTDI serial meter head.
+
+# Service root. Kept per-role so the role is usable on its own.
+edge_base_dir: /opt/edge-host
+
+# Own compose project directory. Unchanged from the path used under the
+# previous shared project.
+sml_base_dir: "{{ edge_base_dir }}/sml2mqtt"
+
+# The compose file that used to hold both this service and zwave-js. Removed
+# once, so the fixed container_name is released for the new projects.
+edge_legacy_compose: "{{ edge_base_dir }}/docker-compose.yml"
+
+# Stable by-id symlink -- see the zwave-gateway role.
+edge_stromzaehler_device: /dev/serial/by-id/usb-FTDI_FT230X_Basic_UART_DK0E1GP5-if00-port0
+
+# Hand-pinned on purpose: critical meter bridge, deliberately excluded from
+# image automation.
+edge_sml2mqtt_image: spacemanspiff2007/sml2mqtt:3.6.0
+
+# Secret from inventory/group_vars/edge_host/vault.yml:
+#   edge_sml2mqtt_mqtt_password

ansible/roles/power-meter-sml/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart sml2mqtt
+  ansible.builtin.command:
+    cmd: docker compose up -d --force-recreate
+    chdir: "{{ sml_base_dir }}"
+  changed_when: true

ansible/roles/power-meter-sml/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+# Power-meter reader: sml2mqtt against the FTDI serial meter head.
+#
+# Its own compose project. It shared a compose file with the Z-Wave gateway,
+# which meant a change to either recreated both -- two unrelated services bound
+# to different USB devices. Data paths are unchanged.
+
+- name: Create sml2mqtt directory
+  ansible.builtin.file:
+    path: "{{ sml_base_dir }}"
+    state: directory
+    mode: "0755"
+
+# sml2mqtt reads only a config file and has no env override for the password,
+# so the rendered config carries it -- 0600, root-owned, same trust boundary as
+# the k8s init-container approach it replaced.
+- name: Deploy sml2mqtt config
+  ansible.builtin.template:
+    src: sml2mqtt-config.yml.j2
+    dest: "{{ sml_base_dir }}/config.yml"
+    mode: "0600"
+  notify: Restart sml2mqtt
+
+- name: Deploy sml2mqtt compose project
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ sml_base_dir }}/docker-compose.yml"
+    mode: "0644"
+  notify: Restart sml2mqtt
+
+- name: Pull sml2mqtt image
+  ansible.builtin.command:
+    cmd: docker compose pull
+    chdir: "{{ sml_base_dir }}"
+  changed_when: false
+
+# Tolerates a missing USB device, as for the Z-Wave gateway.
+- name: Start sml2mqtt
+  ansible.builtin.command:
+    cmd: docker compose up -d
+    chdir: "{{ sml_base_dir }}"
+  register: sml_compose_up
+  failed_when: >-
+    sml_compose_up.rc != 0
+    and 'error gathering device information' not in sml_compose_up.stderr
+  changed_when: >-
+    'Started' in sml_compose_up.stderr or 'Created' in sml_compose_up.stderr
+    or 'Started' in sml_compose_up.stdout or 'Creating' in sml_compose_up.stdout
+
+- name: Warn if sml2mqtt is waiting for its USB device
+  ansible.builtin.debug:
+    msg: >-
+      sml2mqtt could not start -- the FTDI meter head is not present:
+      {{ sml_compose_up.stderr_lines | select('search', 'device') | list }}
+  when: sml_compose_up.rc != 0