fix: [alert] KubeJobFailed (gitea) (#1214 )

2026-10-09 01:42:49 +00:00
124 changed files with 2010 additions and 4289 deletions
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -408,28 +408,3 @@ steps:
          --dockerfile Containerfile \
          --destination gitea.renner.uno/gitea_admin/bird:$TAG \
          --destination gitea.renner.uno/gitea_admin/bird:latest
-
-  # Orchestrator for the monthly backup-verify CronJob (velero ns).
-  - name: build-backup-verify
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
-    directory: ci/images/backup-verify
-    depends_on: []
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/backup-verify/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-backup-verify:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-backup-verify:latest
@@ -35,24 +35,18 @@ when:
 steps:
  - name: model-selection-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh

  - name: promtool-rules-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-promtool-rules.sh

  - name: yaml-citations-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-yaml-citations.sh
-
-  - name: backup-verify-tests
-    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-backup-verify:2026-10-10
-    commands:
-      - bash kubernetes/infrastructure/velero/backup-verify/tests/run.sh
@@ -26,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,41 +1,301 @@
-# PR review bot — per-repo STUB. All logic lives centrally in
-# k.renner/homelab: ci/agents/pr-review/run.sh (plus prompt.md,
-# settings.json and scripts/ci/). This file is deployed byte-identical into every
-# reviewed repo (k.renner/homelab, shelly-manager, authentik-samba-bridge,
-# homeassistant) and holds NO logic, so the copies cannot drift apart: change the
-# review in homelab and every repo picks it up on its next PR event. Only edit this
-# stub if the trigger, the image or the secrets change — then change all copies in
-# one go.
+# PR review bot — event-driven. Woodpecker fires this on `pull_request` (which
+# covers BOTH "PR opened" and "new commits pushed to an open PR"), so a review
+# is produced the moment a PR appears or is updated — no polling, no cron.
 #
-# Triggers on pull_request: PR opened AND new commits pushed to an open PR.
-# Approves only when the review found nothing red/yellow (see run.sh); humans merge.
+# This file is deployed UNCHANGED into every repo the fleet wants reviewed
+# (k.renner/homelab, shelly-manager, authentik-samba-bridge, homeassistant).
+# Woodpecker events are per-repo, so each repo triggers a review of ITS OWN PR
+# only; the triggering PR is identified from the CI event (CI_REPO +
+# CI_COMMIT_PULL_REQUEST) instead of the old hourly cross-repo discovery loop.
+# Because it runs on the PR event, Woodpecker's built-in clone hands the step
+# THIS repo already checked out at the PR head — so, unlike the old cron model,
+# there is no manual clone of the target repo (the source of an earlier token
+# leak); the agents read the checkout in the workspace directly.
 #
-# The central code is cloned from public homelab@master on every run, never taken
-# from the PR checkout, so a PR cannot alter the rubric or helper scripts it is
-# reviewed with.
+# Prompts: the two agent prompts live once, in k.renner/homelab under
+# ci/agents/pr-review/. The homelab checkout already has them; the other repos
+# don't, so the step falls back to a shallow clone of public homelab@master.
+# One source of truth for the review rubric across all repos.
 #
-# Image: ci-claude:latest — the refresh pipeline keeps it on current Claude Code,
-# and a pinned tag would be the one thing in this file that diverges per repo.
+# Model selection: no model id — and no model family — is baked into this file.
+# scripts/ci/fetch-models.sh pulls the LIVE catalog (Anthropic /v1/models, via
+# CLAUDE_CODE_OAUTH_TOKEN), and a triage agent reads the PR diff + stats and
+# picks whichever currently-available model best fits the review, judged on the
+# catalog's own capability metadata. A model released tomorrow is eligible the
+# day it appears. The shell validates the pick against the catalog snapshot taken
+# BEFORE the triage runs; if the fetch fails, the pick is rejected, or the review
+# is complex/security-sensitive (where the injectable pick is deliberately
+# ignored), it leaves the model unset and the review runs on the CLI's default
+# model. A wrong-way miss only over/under-spends one review.
 #
-# Secrets (both must be granted the pull_request event):
-#   claude_code_oauth_token -> CLAUDE_CODE_OAUTH_TOKEN
-#   review_bot_token        -> REVIEW_TOKEN (review-bot PAT, write:repository)
+# Posture: posted as a Gitea PR review, event COMMENT normally or
+# REQUEST_CHANGES on a serious defect. It NEVER approves — a bot must not
+# satisfy branch protection. Humans still merge.
 #
-# (Note: keep literal dollar-brace out of THIS file, even in comments — Woodpecker
-# runs its variable substitution over the whole YAML text.)
+# Dedup (guards pipeline retries/restarts, not cron re-runs anymore): the PR is
+# skipped when review-bot already left a review carrying the marker
+# "<!-- pr-review:sha=<head_sha> -->" for the PR's CURRENT head commit. A pushed
+# commit changes the head sha and fires a fresh pull_request event -> fresh
+# review.
+#
+# Tokens:
+#   CLAUDE_CODE_OAUTH_TOKEN (claude_code_oauth_token) — Claude subscription
+#     OAuth, how the `claude` CLI authenticates (same as the other agents).
+#   REVIEW_TOKEN (review_bot_token) — a dedicated `review-bot` Gitea account's
+#     PAT, scope write:repository. Reads the PR + diff and POSTs the review, all
+#     over the Gitea API (ephemeral curl Authorization headers — the token is
+#     never written to disk; the repo is NOT cloned). review-bot must be a
+#     collaborator (read) on every reviewed repo.
+#   Both secrets MUST be granted the `pull_request` event, or Woodpecker will
+#     withhold them from this pipeline. Because homelab is PUBLIC, it is also set
+#     to require approval for forked-PR pipelines, so an untrusted fork cannot
+#     rewrite this file to exfiltrate the tokens.
+#
+# Triggering: pull_request only — automatic on PR open + push-to-open-PR. To
+# re-review a PR, push a commit (the new head sha clears the dedup marker below
+# and earns a fresh review). There is deliberately no on-demand trigger for a
+# specific PR: Woodpecker exposes injected variables to YAML substitution and
+# `when` filters, NOT to a step's shell environment, so a PR=<n> re-review would
+# silently no-op (the step would find no PR number and exit green).
+# (Note: keep literal dollar-brace out of THIS file entirely, even in comments —
+# Woodpecker runs its variable substitution over the whole YAML text, comments
+# included, and an unparseable one errors the pipeline before any step runs.)
 when:
  - event: pull_request

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:latest
-    pull: true
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
      REVIEW_TOKEN:
        from_secret: review_bot_token
    commands:
-      - rm -rf /tmp/pr-review-central
-      - git clone --quiet --depth 1 https://gitea.renner.uno/k.renner/homelab /tmp/pr-review-central
-      - sh /tmp/pr-review-central/ci/agents/pr-review/run.sh
+      - |
+        # ---- Resolve the single triggering PR --------------------------------
+        # Woodpecker hands us the repo (CI_REPO, canonical owner/repo) and the PR
+        # number (CI_COMMIT_PULL_REQUEST, a pull_request-event built-in). Build
+        # /tmp/pr-review/worklist.jsonl with 0 or 1 entries; an empty file makes
+        # the next block a no-op (the agent never starts).
+        set -eu
+        API="https://gitea.renner.uno/api/v1"
+        AUTH="Authorization: token $$REVIEW_TOKEN"
+        SLUG="$$CI_REPO"
+        NUM="$${CI_COMMIT_PULL_REQUEST:-}"
+        REVIEW_BOT_LOGIN="review-bot"
+        mkdir -p /tmp/pr-review
+        : > /tmp/pr-review/worklist.jsonl
+
+        if [ -z "$$NUM" ]; then
+          echo "No PR number in CI env (event=$${CI_PIPELINE_EVENT:-?}) — nothing to review."
+          exit 0
+        fi
+        echo "Triggered for $$SLUG#$$NUM (event $${CI_PIPELINE_EVENT:-?})."
+
+        if ! curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM" > /tmp/pr-review/pr.json; then
+          echo "Cannot fetch $$SLUG#$$NUM (review-bot not a collaborator / PR gone?) — skipping."
+          exit 0
+        fi
+        STATE=$$(jq -r '.state'        /tmp/pr-review/pr.json)
+        DRAFT=$$(jq -r '.draft // false' /tmp/pr-review/pr.json)
+        TITLE=$$(jq -r '.title'        /tmp/pr-review/pr.json)
+        HEAD=$$(jq -r '.head.sha'      /tmp/pr-review/pr.json)
+        SHORT=$$(printf '%s' "$$HEAD" | cut -c1-8)
+
+        # Skip closed/merged, drafts, and WIP-titled PRs — not ready for review.
+        case "$$STATE" in open) ;; *) echo "#$$NUM is $$STATE — skip."; exit 0;; esac
+        case "$$DRAFT" in true) echo "#$$NUM draft — skip."; exit 0;; esac
+        case "$$TITLE" in [Ww][Ii][Pp]:*|[Ww][Ii][Pp]\ *) echo "#$$NUM WIP — skip."; exit 0;; esac
+
+        # Dedup: already reviewed at THIS head sha? (guards re-runs / restarts)
+        REVIEWED=$$(curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM/reviews?limit=50" \
+          | jq -r --arg u "$$REVIEW_BOT_LOGIN" --arg m "pr-review:sha=$$HEAD" \
+              '[.[] | select(.user.login == $$u) | select(.body | contains($$m))] | length' \
+          || echo 0)
+        if [ "$${REVIEWED:-0}" -gt 0 ]; then
+          echo "#$$NUM already reviewed at $$SHORT — skip."
+          exit 0
+        fi
+
+        jq -c --arg slug "$$SLUG" \
+          '{slug: $$slug, number: .number, head: .head.sha, base: .base.sha,
+            title: .title, author: .user.login}' \
+          /tmp/pr-review/pr.json >> /tmp/pr-review/worklist.jsonl
+        echo "#$$NUM enqueued (head $$SHORT)."
+      - |
+        # ---- Clone, pick model, review, post ---------------------------------
+        set -eu
+        [ -s /tmp/pr-review/worklist.jsonl ] || { echo "Nothing to review."; exit 0; }
+        API="https://gitea.renner.uno/api/v1"
+        AUTH="Authorization: token $$REVIEW_TOKEN"
+        WS="$$PWD"                     # workspace = this repo, checked out at the PR head by Woodpecker
+        MAX_DIFF_LINES=20000
+
+        # Agent prompts: use the local homelab checkout if present, else clone
+        # public homelab@master for them (one source of truth for the rubric).
+        PROMPTS="$$WS/ci/agents/pr-review"
+        HOMELAB="$$WS"                 # repo root holding ci/ AND scripts/ci/
+        # NB: for a homelab PR, HOMELAB is the PR head checkout, so the
+        # fetch-models.sh run below executes branch-supplied shell. It is wrapped
+        # in `env -u REVIEW_TOKEN`, so the only credential it can see is
+        # CLAUDE_CODE_OAUTH_TOKEN. This is not a new exposure regardless: on
+        # pull_request Woodpecker already takes .woodpecker/*.yml itself from the
+        # PR branch, so a branch can run arbitrary commands in this step anyway.
+        if [ ! -f "$$PROMPTS/prompt.md" ]; then
+          rm -rf /tmp/pr-review/homelab
+          git clone --quiet --depth 1 https://gitea.renner.uno/k.renner/homelab \
+            /tmp/pr-review/homelab
+          PROMPTS=/tmp/pr-review/homelab/ci/agents/pr-review
+          HOMELAB=/tmp/pr-review/homelab
+        fi
+        echo "Using prompts from $$PROMPTS"
+
+        gitea_post() {                 # method url datafile ; echoes+checks HTTP
+          code=$$(curl -s -o /tmp/pr-review/resp.json -w '%{http_code}' \
+            -X "$$1" -H "$$AUTH" -H "Content-Type: application/json" \
+            --data @"$$3" "$$2")
+          echo "$$1 $$2 -> HTTP $$code"
+          if [ "$$code" -lt 200 ] || [ "$$code" -ge 300 ]; then
+            echo "Gitea API error body:"; cat /tmp/pr-review/resp.json; echo; return 1
+          fi
+        }
+
+        while IFS= read -r ITEM; do
+          SLUG=$$(printf '%s' "$$ITEM"  | jq -r '.slug')
+          NUM=$$(printf '%s' "$$ITEM"   | jq -r '.number')
+          HEAD=$$(printf '%s' "$$ITEM"  | jq -r '.head')
+          TITLE=$$(printf '%s' "$$ITEM" | jq -r '.title')
+          SHORT=$$(printf '%s' "$$HEAD" | cut -c1-8)
+          echo "=== Reviewing $$SLUG#$$NUM ($$SHORT) ==="
+          D="/tmp/pr-review/$$NUM"
+          rm -rf "$$D"; mkdir -p "$$D"
+
+          # PR metadata + unified diff (authenticated; private repos need it).
+          curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM" > "$$D/pr.json"
+          curl -sf -H "$$AUTH" "$$API/repos/$$SLUG/pulls/$$NUM.diff" > "$$D/diff.full.patch" \
+            || { echo "Could not fetch diff for $$SLUG#$$NUM — skipping."; continue; }
+          FILES=$$(grep -c '^diff --git ' "$$D/diff.full.patch" || true)
+          ADDED=$$(grep -c '^+[^+]' "$$D/diff.full.patch" || true)
+          DELETED=$$(grep -c '^-[^-]' "$$D/diff.full.patch" || true)
+          TOTAL=$$(wc -l < "$$D/diff.full.patch")
+          if [ "$$TOTAL" -gt "$$MAX_DIFF_LINES" ]; then
+            head -n "$$MAX_DIFF_LINES" "$$D/diff.full.patch" > "$$D/diff.patch"
+            printf '\n... [diff truncated at %s of %s lines] ...\n' \
+              "$$MAX_DIFF_LINES" "$$TOTAL" >> "$$D/diff.patch"
+          else
+            cp "$$D/diff.full.patch" "$$D/diff.patch"
+          fi
+          printf 'repo: %s\nPR: #%s\nfiles_changed: %s\nlines_added: %s\nlines_deleted: %s\ndiff_total_lines: %s\n' \
+            "$$SLUG" "$$NUM" "$$FILES" "$$ADDED" "$$DELETED" "$$TOTAL" > "$$D/stats.txt"
+
+          # Cheaper inputs for the TRIAGE agent (which now runs on a capable
+          # default model): the COMPLETE changed-file list, plus a diff capped far
+          # smaller than the review's. Triage only gauges size/shape/sensitivity,
+          # so it never needs the full (up to 20k-line) review diff — this keeps
+          # "triage is cheap" true now that its model is no longer pinned to Haiku.
+          grep '^diff --git ' "$$D/diff.full.patch" | sed 's|^diff --git a/||; s| b/.*$$||' \
+            | sort -u > "$$D/files.txt"
+          head -n 4000 "$$D/diff.patch" > "$$D/diff.triage.patch"
+
+          # No manual clone of the target repo: this runs on a pull_request
+          # event, so Woodpecker's built-in clone already checked THIS repo out
+          # at the PR head in $$WS — the agents (which cd into $$WS) read it
+          # directly for context. The old cron model had to clone each target
+          # repo by hand (its workspace was homelab@master), and that clone
+          # embedded REVIEW_TOKEN in the URL, persisting a write-scoped PAT in
+          # work/target/.git/config — readable by the review agent whose report
+          # is posted publicly. Per-repo pull_request triggering makes the manual
+          # clone redundant AND keeps the token off disk (it now lives only in
+          # the ephemeral curl Authorization headers used for the API calls).
+
+          # Inputs the agents read live at ./.pr-review (symlink into $$D).
+          rm -f "$$WS/.pr-review"; ln -s "$$D" "$$WS/.pr-review"
+
+          # --- Triage: an agent picks the model from the LIVE catalog, so no
+          #     model id and no model family is baked into this file.
+          #     fetch-models.sh asks the Anthropic /v1/models list (via the same
+          #     CLAUDE_CODE_OAUTH_TOKEN) and we hand the triage the result at
+          #     ./.pr-review/models.txt. If the fetch fails the file is empty and
+          #     MODEL stays empty below, so the review runs on the CLI default.
+          if MODELS_TABLE=$$(env -u REVIEW_TOKEN sh "$$HOMELAB/scripts/ci/fetch-models.sh" --table); then
+            printf '%s\n' "$$MODELS_TABLE" > "$$D/models.txt"
+            echo "Live model catalog ($$(wc -l < "$$D/models.txt") models) written for triage."
+          else
+            : > "$$D/models.txt"
+            echo "Model catalog fetch failed — triage/review will use the CLI default model."
+          fi
+          # env -u REVIEW_TOKEN: the agents never call Gitea (the shell does, via
+          # $$AUTH), so the write:repository PAT must not sit in the agent's env
+          # where a prompt-injected diff could read /proc/self/environ. Defence in
+          # depth alongside the settings.json Read-sandbox. Triage itself runs on
+          # the CLI default model (no --model) — we don't name a family even to
+          # bootstrap the pick.
+          cat "$$PROMPTS/triage-prompt.md" | env -u REVIEW_TOKEN claude \
+            --permission-mode acceptEdits \
+            --allowedTools "Read,Glob,Grep,Write" \
+            --settings "$$PROMPTS/settings.json" \
+            -p || echo "(triage agent exited non-zero)"
+          # Use the triage's pick only if it is a currently-available id;
+          # otherwise (hallucinated id, or empty catalog) leave MODEL empty so
+          # the review runs on the CLI default — a run never dies on selection.
+          MODEL=$$(jq -r '.model // empty' "$$D/triage.json" 2>/dev/null || true)
+          CPLX=$$(jq -r '.complexity // "unknown"' "$$D/triage.json" 2>/dev/null || echo unknown)
+          # triage.json is UNTRUSTED (the triage reads the attacker-controlled
+          # diff and its Write is not path-scoped). valid-model.sh is the single,
+          # unit-tested gate (scripts/ci/test-model-selection.sh): it echoes the id
+          # only if it is a safe [A-Za-z0-9._-] token present in the catalog, else
+          # nothing — so a value like "claude-x\n--dangerously-skip-permissions"
+          # can never reach the unquoted --model expansion below. Empty -> default.
+          # CRITICAL: validate against $$MODELS_TABLE — the catalog captured in a
+          # shell variable BEFORE the triage ran — NOT the on-disk ./.pr-review/
+          # models.txt, which the triage's unscoped Write could overwrite to smuggle
+          # an attacker-chosen token past the membership check (the triage reads the
+          # file, but only $$MODELS_TABLE is trusted). This mirrors pick-model.sh,
+          # which validates its pick against its in-memory table for the same reason.
+          # env -u REVIEW_TOKEN: like fetch-models.sh above, this runs a script from
+          # $$HOMELAB (branch-supplied for a homelab PR) — it must not see the PAT.
+          # `|| true`: keep the "a run never dies on selection" invariant even if
+          # the script is absent (e.g. a PR that moves scripts/ci/).
+          MODEL=$$(printf '%s\n' "$$MODELS_TABLE" | env -u REVIEW_TOKEN sh "$$HOMELAB/scripts/ci/valid-model.sh" "$$MODEL" || true)
+          [ -n "$$MODEL" ] || echo "Triage model rejected or not in live catalog — using the CLI default model."
+          # Defence in depth: for a complex / security-sensitive review, do NOT
+          # trust the triage's pick (it read the attacker-controlled diff and could
+          # be injected into naming the weakest catalog entry to get a shallow
+          # review). Fall back to the CLI default — the strongest configured model.
+          if [ "$$CPLX" = complex ] && [ -n "$$MODEL" ]; then
+            echo "complex review — ignoring triage pick '$$MODEL', using the CLI default (strongest) model."
+            MODEL=
+          fi
+          if [ -n "$$MODEL" ]; then echo "Triage: complexity=$$CPLX -> model=$$MODEL"; else echo "Triage: complexity=$$CPLX -> CLI default model"; fi
+
+          # --- Review with the chosen model ---
+          ( cd "$$WS" && cat "$$PROMPTS/prompt.md" | env -u REVIEW_TOKEN claude $${MODEL:+--model $$MODEL} \
+              --permission-mode acceptEdits \
+              --allowedTools "Read,Glob,Grep,Write" \
+              --settings "$$PROMPTS/settings.json" \
+              -p ) || echo "(review agent exited non-zero)"
+
+          if [ ! -s "$$D/report.md" ]; then
+            echo "Agent wrote no report for $$SLUG#$$NUM — skipping post."
+            continue
+          fi
+          EVENT=$$(jq -r '.event // "COMMENT"' "$$D/verdict.json" 2>/dev/null || echo COMMENT)
+          case "$$EVENT" in REQUEST_CHANGES|COMMENT) ;; *) EVENT=COMMENT;; esac   # never APPROVE
+
+          # Assemble the review body: header + report + marker footer.
+          {
+            printf '🤖 **Automated review** — model `%s` (complexity: %s), commit `%s`.\n\n' \
+              "$${MODEL:-CLI default}" "$$CPLX" "$${HEAD}"
+            printf '> Advisory review by the `review-bot` pipeline. It never approves; a human merges. Push new commits to get a fresh review.\n\n---\n\n'
+            cat "$$D/report.md"
+            printf '\n\n<!-- pr-review:sha=%s -->\n' "$$HEAD"
+          } > "$$D/body.md"
+          jq -n --rawfile b "$$D/body.md" --arg e "$$EVENT" --arg c "$$HEAD" \
+            '{body: $$b, event: $$e, commit_id: $$c}' > "$$D/review.json"
+          gitea_post POST "$$API/repos/$$SLUG/pulls/$$NUM/reviews" "$$D/review.json" \
+            && echo "Posted $$EVENT review on $$SLUG#$$NUM." \
+            || echo "FAILED to post review on $$SLUG#$$NUM (continuing)."
+          rm -f "$$WS/.pr-review"
+        done < /tmp/pr-review/worklist.jsonl
+        echo "PR review run complete."
@@ -70,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -17,7 +17,7 @@ when:

 steps:
  - name: checkout-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -54,7 +54,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: distill
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -89,14 +89,14 @@ steps:
          -p

  - name: audit
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    commands:
      # Findings are report lines in output/, not a pipeline failure.
      - python3 -B notes/bin/wiki_audit.py --vault "$$PWD/notes"

  - name: consolidate
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -145,7 +145,7 @@ steps:
        exit $$rc

  - name: push-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.296
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -54,35 +54,16 @@ Look for, in priority order:

 Ground every finding in a specific location: `path:line` (or a hunk) from the
 diff or the checkout. **Do not invent** bugs, file paths, or behavior you have
-not confirmed.
-
-**Verify every claim yourself before you write it down. Never hand the
-verification to the author.** "I could not find X", "X may not exist", "this is
-unverified" is not a finding — it is unfinished work. For each suspicion:
-
-1. Check the checkout: callers, the rest of the function, config elsewhere,
-   existing uses of the same construct, the repo's `CLAUDE.md`/`README`.
-   Shallow read-only clones of the other fleet repos that are publicly
-   readable (at least `homelab`) are under `./.pr-review/context/<name>/` —
-   look there when the change depends on how a sibling repo does something.
-   A repo missing there simply was not readable; do not treat that as a finding.
-2. If it depends on an external API, syntax, device behavior or product
-   version (a Home Assistant trigger/action, a Shelly generation, a Kubernetes
-   field, a library call), read the authoritative documentation or source with
-   WebFetch/WebSearch (official docs, the project's own repo or README). Your
-   training data is NOT evidence: this repo targets current releases that may
-   postdate it, and "I have not seen this syntax" says nothing about whether
-   it exists.
-3. Only after both: state what you found, with the source (`path:line` or URL).
-
-If after a real attempt you still cannot settle it, list it under
-"Unverified" with what you checked and why that was not enough. An unverified
-suspicion is never Blocking.
+not confirmed by reading the code. The whole repo is on disk in your working
+directory, so verify: if a judgement depends on a caller, the rest of a
+function, or config elsewhere, open it. If after checking you still cannot
+confirm a suspicion, say so explicitly and give the one check that would settle
+it — do not state a hypothesis as fact. A concern you could have checked by
+reading the checkout but did not is a weakness in your review.

 Distinguish severity honestly. A blocking issue is a concrete defect — a bug
-that will misbehave, a security hole, or a breaking change — that you
-**verified** and can point at. Style preferences, optional improvements and
-anything unverified are not blocking.
+that will misbehave, a security hole, or a breaking change — that you can point
+at. Style preferences and optional improvements are not blocking.

 ## Output

@@ -103,11 +84,6 @@ Structure it as:
 why it is wrong. Omit this whole section if there are none — and say so in the
 Summary.>

-## ⚪ Unverified
-
-<Suspicions you could not settle after checking the checkout and the docs, each
-with what you checked. Omit if none.>
-
 ## 🟡 Suggestions

 <Non-blocking improvements worth considering, each anchored to a location.>
@@ -134,21 +110,17 @@ even if the PR added one; refer to it by location and flag it as blocking.

 ```json
 {
-  "event": "APPROVE | COMMENT | REQUEST_CHANGES",
+  "event": "COMMENT | REQUEST_CHANGES",
  "headline": "<short one-line summary of the outcome>"
 }
 ```

 - Use **`REQUEST_CHANGES`** only when your report lists at least one real
  **Blocking** issue.
- Use **`APPROVE`** when the report has no 🔴 Blocking, no 🟡 Suggestions and no
-  ⚪ Unverified section — i.e. you checked the change and found nothing to
-  raise. Nits (🟢) alone do not prevent it.
- Use **`COMMENT`** otherwise.
- The pipeline sets the final review event from the sections present in your
-  report, so keep the section headings exactly as above and omit a section
-  entirely when it is empty. Do not add a 🟡/🔴/⚪ heading unless you have a
-  finding under it.
+- Use **`COMMENT`** otherwise (including when you have only suggestions/nits, or
+  the PR looks good).
+- You cannot approve — there is no APPROVE option, by design. A clean PR gets a
+  `COMMENT` review saying so.

 If `pr.json` or `diff.patch` is missing or empty, write a `report.md` stating
 you had no diff to review and a `verdict.json` with `event: "COMMENT"`, then
@@ -1,191 +0,0 @@
-#!/bin/sh
-# PR review bot — the ONE implementation, run by every reviewed repo's
-# .woodpecker/pr-review.yml stub (k.renner/homelab, shelly-manager,
-# authentik-samba-bridge, homeassistant). Edit it here only; the stubs hold no
-# logic. Design notes (event-driven trigger, dedup marker, tokens, model
-# selection, posture) are in the stub and in prompt.md.
-#
-# ---- Resolve the single triggering PR --------------------------------
-# Woodpecker hands us the repo (CI_REPO, canonical owner/repo) and the PR
-# number (CI_COMMIT_PULL_REQUEST, a pull_request-event built-in). Build
-# /tmp/pr-review/worklist.jsonl with 0 or 1 entries; an empty file makes
-# the next block a no-op (the agent never starts).
-set -eu
-API="https://gitea.renner.uno/api/v1"
-AUTH="Authorization: token $REVIEW_TOKEN"
-SLUG="$CI_REPO"
-NUM="${CI_COMMIT_PULL_REQUEST:-}"
-REVIEW_BOT_LOGIN="review-bot"
-mkdir -p /tmp/pr-review
-: > /tmp/pr-review/worklist.jsonl
-
-if [ -z "$NUM" ]; then
-  echo "No PR number in CI env (event=${CI_PIPELINE_EVENT:-?}) — nothing to review."
-  exit 0
-fi
-echo "Triggered for $SLUG#$NUM (event ${CI_PIPELINE_EVENT:-?})."
-
-if ! curl -sf -H "$AUTH" "$API/repos/$SLUG/pulls/$NUM" > /tmp/pr-review/pr.json; then
-  echo "Cannot fetch $SLUG#$NUM (review-bot not a collaborator / PR gone?) — skipping."
-  exit 0
-fi
-STATE=$(jq -r '.state'        /tmp/pr-review/pr.json)
-DRAFT=$(jq -r '.draft // false' /tmp/pr-review/pr.json)
-TITLE=$(jq -r '.title'        /tmp/pr-review/pr.json)
-HEAD=$(jq -r '.head.sha'      /tmp/pr-review/pr.json)
-SHORT=$(printf '%s' "$HEAD" | cut -c1-8)
-
-# Skip closed/merged, drafts, and WIP-titled PRs — not ready for review.
-case "$STATE" in open) ;; *) echo "#$NUM is $STATE — skip."; exit 0;; esac
-case "$DRAFT" in true) echo "#$NUM draft — skip."; exit 0;; esac
-case "$TITLE" in [Ww][Ii][Pp]:*|[Ww][Ii][Pp]\ *) echo "#$NUM WIP — skip."; exit 0;; esac
-
-# Dedup: already reviewed at THIS head sha? (guards re-runs / restarts)
-REVIEWED=$(curl -sf -H "$AUTH" "$API/repos/$SLUG/pulls/$NUM/reviews?limit=50" \
-  | jq -r --arg u "$REVIEW_BOT_LOGIN" --arg m "pr-review:sha=$HEAD" \
-      '[.[] | select(.user.login == $u) | select(.body | contains($m))] | length' \
-  || echo 0)
-if [ "${REVIEWED:-0}" -gt 0 ]; then
-  echo "#$NUM already reviewed at $SHORT — skip."
-  exit 0
-fi
-
-jq -c --arg slug "$SLUG" \
-  '{slug: $slug, number: .number, head: .head.sha, base: .base.sha,
-    title: .title, author: .user.login}' \
-  /tmp/pr-review/pr.json >> /tmp/pr-review/worklist.jsonl
-echo "#$NUM enqueued (head $SHORT)."
-# ---- Clone, pick model, review, post ---------------------------------
-set -eu
-[ -s /tmp/pr-review/worklist.jsonl ] || { echo "Nothing to review."; exit 0; }
-API="https://gitea.renner.uno/api/v1"
-AUTH="Authorization: token $REVIEW_TOKEN"
-WS="$PWD"                     # workspace = this repo, checked out at the PR head by Woodpecker
-MAX_DIFF_LINES=20000
-
-# This script, the prompts and scripts/ci/ all come from the homelab@master
-# clone the per-repo stub made (see .woodpecker/pr-review.yml) — never from the
-# PR checkout, so a PR cannot change the rubric or the helper scripts it is
-# reviewed with.
-PROMPTS="$(cd "$(dirname "$0")" && pwd)"
-HOMELAB="$(cd "$PROMPTS/../../.." && pwd)"   # repo root holding ci/ AND scripts/ci/
-echo "Using prompts from $PROMPTS"
-
-gitea_post() {                 # method url datafile ; echoes+checks HTTP
-  code=$(curl -s -o /tmp/pr-review/resp.json -w '%{http_code}' \
-    -X "$1" -H "$AUTH" -H "Content-Type: application/json" \
-    --data @"$3" "$2")
-  echo "$1 $2 -> HTTP $code"
-  if [ "$code" -lt 200 ] || [ "$code" -ge 300 ]; then
-    echo "Gitea API error body:"; cat /tmp/pr-review/resp.json; echo; return 1
-  fi
-}
-
-while IFS= read -r ITEM; do
-  SLUG=$(printf '%s' "$ITEM"  | jq -r '.slug')
-  NUM=$(printf '%s' "$ITEM"   | jq -r '.number')
-  HEAD=$(printf '%s' "$ITEM"  | jq -r '.head')
-  TITLE=$(printf '%s' "$ITEM" | jq -r '.title')
-  SHORT=$(printf '%s' "$HEAD" | cut -c1-8)
-  echo "=== Reviewing $SLUG#$NUM ($SHORT) ==="
-  D="/tmp/pr-review/$NUM"
-  rm -rf "$D"; mkdir -p "$D"
-
-  # PR metadata + unified diff (authenticated; private repos need it).
-  curl -sf -H "$AUTH" "$API/repos/$SLUG/pulls/$NUM" > "$D/pr.json"
-  curl -sf -H "$AUTH" "$API/repos/$SLUG/pulls/$NUM.diff" > "$D/diff.full.patch" \
-    || { echo "Could not fetch diff for $SLUG#$NUM — skipping."; continue; }
-  FILES=$(grep -c '^diff --git ' "$D/diff.full.patch" || true)
-  ADDED=$(grep -c '^+[^+]' "$D/diff.full.patch" || true)
-  DELETED=$(grep -c '^-[^-]' "$D/diff.full.patch" || true)
-  TOTAL=$(wc -l < "$D/diff.full.patch")
-  if [ "$TOTAL" -gt "$MAX_DIFF_LINES" ]; then
-    head -n "$MAX_DIFF_LINES" "$D/diff.full.patch" > "$D/diff.patch"
-    printf '\n... [diff truncated at %s of %s lines] ...\n' \
-      "$MAX_DIFF_LINES" "$TOTAL" >> "$D/diff.patch"
-  else
-    cp "$D/diff.full.patch" "$D/diff.patch"
-  fi
-  printf 'repo: %s\nPR: #%s\nfiles_changed: %s\nlines_added: %s\nlines_deleted: %s\ndiff_total_lines: %s\n' \
-    "$SLUG" "$NUM" "$FILES" "$ADDED" "$DELETED" "$TOTAL" > "$D/stats.txt"
-
-  # No manual clone of the target repo: this runs on a pull_request
-  # event, so Woodpecker's built-in clone already checked THIS repo out
-  # at the PR head in $WS — the agents (which cd into $WS) read it
-  # directly for context. The old cron model had to clone each target
-  # repo by hand (its workspace was homelab@master), and that clone
-  # embedded REVIEW_TOKEN in the URL, persisting a write-scoped PAT in
-  # work/target/.git/config — readable by the review agent whose report
-  # is posted publicly. Per-repo pull_request triggering makes the manual
-  # clone redundant AND keeps the token off disk (it now lives only in
-  # the ephemeral curl Authorization headers used for the API calls).
-
-  # Inputs the agents read live at ./.pr-review (symlink into $D).
-  rm -f "$WS/.pr-review"; ln -s "$D" "$WS/.pr-review"
-
-  # Cross-repo context for the review agent: shallow, UNAUTHENTICATED clones of
-  # the other fleet repos at ./.pr-review/context/<name> (a change here often
-  # depends on how a sibling repo does it). No credential is involved, so only
-  # publicly readable repos come through; a repo that needs auth fails fast
-  # (GIT_TERMINAL_PROMPT=0) and is skipped, never fatal.
-  CTX="$D/context"; mkdir -p "$CTX"
-  for REPO in k.renner/homelab k.renner/homeassistant konairius/shelly-manager k.renner/authentik-samba-bridge; do
-    [ "$REPO" = "$SLUG" ] && continue
-    GIT_TERMINAL_PROMPT=0 git clone --quiet --depth 1 "https://gitea.renner.uno/$REPO" "$CTX/${REPO#*/}" 2>/dev/null \
-      || echo "No public context clone of $REPO (skipped)."
-  done
-
-  # --- Model: always the most capable CURRENT one, from the live catalog. ---
-  # No model id, no family name and no complexity triage is baked in here: a
-  # triage that downgrades "trivial" PRs to a small model is exactly what made
-  # reviews shallow, and reviews are rare enough that the strongest model is the
-  # right default. pick-model.sh reads the live catalog (Anthropic /v1/models via
-  # CLAUDE_CODE_OAUTH_TOKEN), prefers the newest of comparable models, validates
-  # its pick against that catalog and echoes nothing on any failure — then the
-  # review runs on the CLI's own default model. A model released tomorrow is
-  # eligible the day the API lists it. REVIEW_TOKEN is stripped: the picker never
-  # needs the write-scoped PAT.
-  MODEL=$(env -u REVIEW_TOKEN sh "$HOMELAB/scripts/ci/pick-model.sh" \
-    "careful, high-stakes code review of a pull request: verify claims against the code and external documentation, find real defects, avoid false positives; always pick the most capable and newest model available, regardless of PR size" || true)
-  if [ -n "$MODEL" ]; then echo "Review model: $MODEL"; else echo "Review model: CLI default (catalog/picker unavailable)"; fi
-
-  # --- Review with the chosen model ---
-  ( cd "$WS" && cat "$PROMPTS/prompt.md" | env -u REVIEW_TOKEN claude ${MODEL:+--model $MODEL} \
-      --permission-mode acceptEdits \
-      --allowedTools "Read,Glob,Grep,Write,WebFetch,WebSearch" \
-      --settings "$PROMPTS/settings.json" \
-      -p ) || echo "(review agent exited non-zero)"
-
-  if [ ! -s "$D/report.md" ]; then
-    echo "Agent wrote no report for $SLUG#$NUM — skipping post."
-    continue
-  fi
-  # Review event, derived from the report's own severity sections (not trusted
-  # from verdict.json): any red (Blocking) -> REQUEST_CHANGES; no red, yellow
-  # (Suggestions) or Unverified section on a non-empty diff -> APPROVE; else
-  # COMMENT. An empty diff or a report that names no section never approves.
-  if grep -q '^## 🔴' "$D/report.md"; then
-    EVENT=REQUEST_CHANGES
-  elif [ "$TOTAL" -gt 0 ] && ! grep -q -e '^## 🟡' -e '^## ⚪' "$D/report.md" \
-       && grep -q '^## Recommendation' "$D/report.md"; then
-    EVENT=APPROVE
-  else
-    EVENT=COMMENT
-  fi
-
-  # Assemble the review body: header + report + marker footer.
-  {
-    printf '🤖 **Automated review** — model `%s`, commit `%s`.\n\n' \
-      "${MODEL:-CLI default}" "${HEAD}"
-    printf '> Automated review by the `review-bot` pipeline: it approves only when it found nothing red or yellow, and a human still merges. Push new commits to get a fresh review.\n\n---\n\n'
-    cat "$D/report.md"
-    printf '\n\n<!-- pr-review:sha=%s -->\n' "$HEAD"
-  } > "$D/body.md"
-  jq -n --rawfile b "$D/body.md" --arg e "$EVENT" --arg c "$HEAD" \
-    '{body: $b, event: $e, commit_id: $c}' > "$D/review.json"
-  gitea_post POST "$API/repos/$SLUG/pulls/$NUM/reviews" "$D/review.json" \
-    && echo "Posted $EVENT review on $SLUG#$NUM." \
-    || echo "FAILED to post review on $SLUG#$NUM (continuing)."
-  rm -f "$WS/.pr-review"
-done < /tmp/pr-review/worklist.jsonl
-echo "PR review run complete."
@@ -1,19 +0,0 @@
-FROM hetznercloud/cli:v1.69.0@sha256:febb8898949b31a2249384aa9728f2254dfb752ff2f69be709178c4a3285b05f AS hcloud
-
-FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
-
-RUN apk add --no-cache \
-      bash \
-      ca-certificates \
-      curl \
-      jq \
-      kubectl \
-      openssh-client \
-      openssh-keygen \
-      tar
-
-COPY --from=hcloud /ko-app/hcloud /usr/local/bin/hcloud
-
-RUN adduser -D -u 65532 -h /work verify
-
-RUN hcloud version && kubectl version --client && jq --version && ssh -V && ssh-keygen -t ed25519 -N '' -q -f /tmp/k && rm -f /tmp/k /tmp/k.pub
@@ -1,10 +1,7 @@
 #!/usr/bin/env python3
 """netbird-service-sync: reconcile NetBird reverse-proxy services from Ingresses.

-Source of truth: every Ingress carrying one of the public labels.
-dns.renner.uno/public also takes the host out of the LAN DNS (external-dns-unifi
-skips it); netbird.renner.uno/public publishes on the proxy only and leaves the
-LAN record in place.
+Source of truth: every Ingress carrying the public label (dns.renner.uno/public).
 For each host on such an Ingress the NetBird management gets one reverse-proxy
 service in TLS-passthrough mode on :443, targeting Traefik's ClusterIP through a
 subnet resource that covers the cluster's service CIDR. SNI and Host reach
@@ -45,7 +42,7 @@ NETWORK_NAME = os.environ.get("NETWORK_NAME", "k8s public ingress")
 ROUTER_GROUP = os.environ.get("ROUTER_GROUP_NAME", "k8s-ingress-router")
 SOURCE_GROUP = os.environ.get("SOURCE_GROUP_NAME", "All")
 TRAEFIK_NS, TRAEFIK_SVC = os.environ.get("TRAEFIK_SERVICE", "ingress/traefik").split("/", 1)
-PUBLIC_LABELS = os.environ.get("PUBLIC_LABELS", "dns.renner.uno/public,netbird.renner.uno/public").split(",")
+PUBLIC_LABEL = os.environ.get("PUBLIC_LABEL", "dns.renner.uno/public")
 LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "443"))
 INTERVAL = int(os.environ.get("INTERVAL_SECONDS", "60"))
 HTTP_TIMEOUT = 20
@@ -89,12 +86,10 @@ def k8s(path):


 def desired_hosts():
-    items = []
-    for label in PUBLIC_LABELS:
-        sel = urllib.parse.quote(f"{label}=true")
-        items += k8s(f"/apis/networking.k8s.io/v1/ingresses?labelSelector={sel}").get("items", [])
+    sel = urllib.parse.quote(f"{PUBLIC_LABEL}=true")
+    ingresses = k8s(f"/apis/networking.k8s.io/v1/ingresses?labelSelector={sel}")
    hosts = {}
-    for ing in items:
+    for ing in ingresses.get("items", []):
        ref = f'{ing["metadata"]["namespace"]}/{ing["metadata"]["name"]}'
        for rule in ing.get("spec", {}).get("rules", []):
            host = rule.get("host")
@@ -1,443 +0,0 @@
-# Gitea MCP Server Implementation Plan
-
-> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
-
-**Goal:** Deploy the official Gitea MCP server in-cluster at `https://gitea-mcp.renner.uno/mcp`, published on the NetBird proxy, authenticating every request with the caller's own Gitea PAT and rejecting tokenless requests.
-
-**Architecture:** One stateless Deployment in a new `gitea-mcp` namespace, Flux-managed through its own Kustomization. Traefik Ingress with the proxy-only public label; netbird-service-sync publishes it. gitea-mcp runs in `-oauth` mode purely to get the 401-on-missing-token gate; it talks to Gitea via the in-cluster Service.
-
-**Tech Stack:** Kubernetes manifests, Flux (Kustomization, image automation), Cilium CiliumNetworkPolicy, Traefik Ingress, cert-manager.
-
-**Spec:** `docs/superpowers/specs/2026-10-11-gitea-mcp-design.md`
-
-## Global Constraints
-
- Image `docker.gitea.com/gitea-mcp-server`, initial tag `1.8.1`, ImagePolicy semver `>=1.8.1`, marker `# {"$imagepolicy": "flux-system:gitea-mcp"}`.
- Args exactly: `-t http -p 8080 -oauth -oauth-public-url https://gitea-mcp.renner.uno -H http://gitea.gitea.svc.cluster.local:3000`. No `-T`/`GITEA_ACCESS_TOKEN`, no `-d`.
- Host `gitea-mcp.renner.uno`; Ingress label `netbird.renner.uno/public: "true"` (not `dns.renner.uno/public`); cert issuer `letsencrypt-cluster`; TLS secret `gitea-mcp-tls`.
- Pod label `app.kubernetes.io/name: gitea-mcp`; namespace `gitea-mcp`; Service `gitea-mcp:8080`.
- No `world` egress from `gitea-mcp`.
- Manifest comments: one terse factual line at most; rationale goes in commit/PR text.
- Commits as claude-bot: `git -c user.name='Claude Bot' -c user.email='claude-bot@renner.uno' commit …`; each git command its own Bash call.
-
-## Review Focus
-
-1. Tokenless `POST /mcp` from the internet → must be 401, never anonymous Gitea data (Task 4 step 4).
-2. Request with an invalid/revoked PAT → 401 `invalid_token` (Task 4 step 5).
-3. Kubelet probes with `Host: <podIP>` → `/healthz` must stay 200 despite the oauth Host check (Task 3 readiness, Task 4 step 2).
-4. LAN client resolving `gitea-mcp.renner.uno` → must reach Traefik VIP directly, not the proxy (Task 4 step 3).
-5. Gitea refusing gitea-mcp's connection (missing CNP half) → MCP calls fail; authenticated `list_my_repos` must succeed (Task 4 step 6).
-
---
-
-### Task 1: Namespace and network policies
-
-**Files:**
- Modify: `kubernetes/infrastructure/namespaces/namespaces.yaml` (append after the `gitea` Namespace)
- Create: `kubernetes/infrastructure/network-policies/gitea-mcp-ns.yaml`
- Modify: `kubernetes/infrastructure/network-policies/kustomization.yaml` (add entry after `gitea-ns.yaml`)
- Modify: `kubernetes/apps/gitea/networkpolicy.yaml` (append CNP)
-
-**Interfaces:**
- Produces: namespace `gitea-mcp`; ingress to pods labelled `app.kubernetes.io/name: gitea-mcp` on 8080 from Traefik; egress from `gitea-mcp` to `gitea` pod :3000; Gitea-side ingress allowance from namespace `gitea-mcp` on :3000.
-
- [ ] **Step 1: Add the Namespace** — insert after the `name: gitea` Namespace document in `namespaces.yaml`:
-
-```yaml
---
-# Gitea MCP server. Declared here so it exists before infrastructure-crds applies its CNPs.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: gitea-mcp
-```
-
- [ ] **Step 2: Create `gitea-mcp-ns.yaml`**
-
-```yaml
---
-# Baseline policy for the `gitea-mcp` namespace.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: default-deny
-  namespace: gitea-mcp
-spec:
-  endpointSelector: {}
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea-mcp
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": monitoring
-    - fromEntities:
-        - host
-        - remote-node
-    - fromEndpoints:
-        - matchExpressions:
-            - key: io.kubernetes.pod.namespace
-              operator: Exists
-      icmps:
-        - fields:
-            - { type: 0,   family: IPv4 }
-            - { type: 3,   family: IPv4 }
-            - { type: 8,   family: IPv4 }
-            - { type: 11,  family: IPv4 }
-            - { type: 12,  family: IPv4 }
-            - { type: 1,   family: IPv6 }
-            - { type: 2,   family: IPv6 }
-            - { type: 3,   family: IPv6 }
-            - { type: 4,   family: IPv6 }
-            - { type: 128, family: IPv6 }
-            - { type: 129, family: IPv6 }
-            - { type: 133, family: IPv6 }
-            - { type: 134, family: IPv6 }
-            - { type: 135, family: IPv6 }
-            - { type: 136, family: IPv6 }
-            - { type: 143, family: IPv6 }
-  egress:
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea-mcp
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": kube-system
-            k8s-app: kube-dns
-      toPorts:
-        - ports:
-            - port: "53"
-              protocol: UDP
-            - port: "53"
-              protocol: TCP
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea
-            app.kubernetes.io/name: gitea
-      toPorts:
-        - ports:
-            - port: "3000"
-              protocol: TCP
---
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: gitea-mcp-allow-traefik
-  namespace: gitea-mcp
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: gitea-mcp
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ingress
-            app.kubernetes.io/name: traefik
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
-```
-
- [ ] **Step 3: Register it** — in `network-policies/kustomization.yaml` add `  - gitea-mcp-ns.yaml` directly after `  - gitea-ns.yaml`.
-
- [ ] **Step 4: Gitea-side allowance** — append to `kubernetes/apps/gitea/networkpolicy.yaml`:
-
-```yaml
---
-# gitea-mcp reaches the in-cluster Service; egress half is in network-policies/gitea-mcp-ns.yaml.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: gitea-allow-gitea-mcp
-  namespace: gitea
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: gitea
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea-mcp
-            app.kubernetes.io/name: gitea-mcp
-      toPorts:
-        - ports:
-            - port: "3000"
-              protocol: TCP
-```
-
- [ ] **Step 5: Validate builds**
-
-Run each separately:
-`kubectl kustomize kubernetes/infrastructure/namespaces` → output contains `name: gitea-mcp`
-`kubectl kustomize kubernetes/infrastructure/network-policies` → output contains `gitea-mcp-allow-traefik`
-`kubectl kustomize kubernetes/apps/gitea` → output contains `gitea-allow-gitea-mcp`
-
- [ ] **Step 6: Server-side dry-run of the Gitea-side CNP** (namespace already exists):
-`kubectl apply --dry-run=server -f kubernetes/apps/gitea/networkpolicy.yaml`
-Expected: every CNP `configured (server dry run)` or `unchanged`, `gitea-allow-gitea-mcp created (server dry run)`.
-
- [ ] **Step 7: Commit**
-
-`git add kubernetes/infrastructure/namespaces/namespaces.yaml kubernetes/infrastructure/network-policies/gitea-mcp-ns.yaml kubernetes/infrastructure/network-policies/kustomization.yaml kubernetes/apps/gitea/networkpolicy.yaml`
-`git -c user.name='Claude Bot' -c user.email='claude-bot@renner.uno' commit -m "feat(gitea-mcp): namespace and network policies" -m "Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>"`
-
-### Task 2: Image automation
-
-**Files:**
- Create: `kubernetes/infrastructure/image-automation/gitea-mcp.yaml`
- Modify: `kubernetes/infrastructure/image-automation/kustomization.yaml` (insert `  - gitea-mcp.yaml` alphabetically after `  - gatus.yaml`)
-
-**Interfaces:**
- Produces: ImagePolicy `flux-system:gitea-mcp`, referenced by the marker in Task 3.
-
- [ ] **Step 1: Create `gitea-mcp.yaml`**
-
-```yaml
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImageRepository
-metadata:
-  name: gitea-mcp
-  namespace: flux-system
-spec:
-  image: docker.gitea.com/gitea-mcp-server
-  interval: 1h
---
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImagePolicy
-metadata:
-  name: gitea-mcp
-  namespace: flux-system
-spec:
-  imageRepositoryRef:
-    name: gitea-mcp
-  policy:
-    semver:
-      range: ">=1.8.1"
-```
-
- [ ] **Step 2: Register it** in `image-automation/kustomization.yaml` after `  - gatus.yaml`.
-
- [ ] **Step 3: Validate**
-`kubectl kustomize kubernetes/infrastructure/image-automation` → contains `docker.gitea.com/gitea-mcp-server`
-`kubectl apply --dry-run=server -f kubernetes/infrastructure/image-automation/gitea-mcp.yaml` → both objects `created (server dry run)`.
-
- [ ] **Step 4: Commit**
-
-`git add kubernetes/infrastructure/image-automation/gitea-mcp.yaml kubernetes/infrastructure/image-automation/kustomization.yaml`
-`git -c user.name='Claude Bot' -c user.email='claude-bot@renner.uno' commit -m "feat(gitea-mcp): track docker.gitea.com/gitea-mcp-server via image automation" -m "Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>"`
-
-### Task 3: App manifests and Flux Kustomization
-
-**Files:**
- Create: `kubernetes/apps/gitea-mcp/deployment.yaml`, `service.yaml`, `ingress.yaml`, `kustomization.yaml`
- Create: `kubernetes/clusters/homelab/gitea-mcp.yaml`
-
-**Interfaces:**
- Consumes: namespace + CNPs (Task 1), ImagePolicy `flux-system:gitea-mcp` (Task 2).
- Produces: `https://gitea-mcp.renner.uno/mcp` and `/healthz`.
-
- [ ] **Step 1: `deployment.yaml`**
-
-```yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gitea-mcp
-  namespace: gitea-mcp
-  labels:
-    app.kubernetes.io/name: gitea-mcp
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gitea-mcp
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: gitea-mcp
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: gitea-mcp
-          image: docker.gitea.com/gitea-mcp-server:1.8.1 # {"$imagepolicy": "flux-system:gitea-mcp"}
-          args:
-            - -t
-            - http
-            - -p
-            - "8080"
-            # Only for the 401 on tokenless requests; clients send their own Gitea PAT.
-            - -oauth
-            - -oauth-public-url
-            - https://gitea-mcp.renner.uno
-            - -H
-            - http://gitea.gitea.svc.cluster.local:3000
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          ports:
-            - containerPort: 8080
-              name: http
-          volumeMounts:
-            - name: tmp
-              mountPath: /tmp
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: http
-            initialDelaySeconds: 5
-            periodSeconds: 30
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: http
-            initialDelaySeconds: 2
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 10m
-              memory: 32Mi
-            limits:
-              memory: 128Mi
-      volumes:
-        - name: tmp
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
-```
-
- [ ] **Step 2: `service.yaml`**
-
-```yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: gitea-mcp
-  namespace: gitea-mcp
-  labels:
-    app.kubernetes.io/name: gitea-mcp
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: gitea-mcp
-  ports:
-    - name: http
-      port: 8080
-      targetPort: http
-```
-
- [ ] **Step 3: `ingress.yaml`**
-
-```yaml
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: gitea-mcp
-  namespace: gitea-mcp
-  labels:
-    app.kubernetes.io/name: gitea-mcp
-    # Published on the NetBird proxy; LAN DNS stays on the Traefik VIP.
-    netbird.renner.uno/public: "true"
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-cluster
-spec:
-  ingressClassName: traefik
-  rules:
-    - host: gitea-mcp.renner.uno
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: gitea-mcp
-                port:
-                  number: 8080
-  tls:
-    - secretName: gitea-mcp-tls
-      hosts:
-        - gitea-mcp.renner.uno
-```
-
- [ ] **Step 4: `kustomization.yaml`**
-
-```yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - deployment.yaml
-  - service.yaml
-  - ingress.yaml
-```
-
- [ ] **Step 5: `kubernetes/clusters/homelab/gitea-mcp.yaml`**
-
-```yaml
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: gitea-mcp
-  namespace: flux-system
-spec:
-  interval: 10m
-  retryInterval: 2m
-  timeout: 5m
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./kubernetes/apps/gitea-mcp
-  prune: true
-  wait: true
-  dependsOn:
-    - name: infrastructure
-    - name: infrastructure-crds
-```
-
- [ ] **Step 6: Validate**
-`kubectl kustomize kubernetes/apps/gitea-mcp` → renders Deployment, Service, Ingress, all in namespace `gitea-mcp`.
-`kubectl apply --dry-run=server -f kubernetes/clusters/homelab/gitea-mcp.yaml` → `created (server dry run)`.
-Confirm `kubernetes/clusters/homelab/` is picked up without a listing file: `ls kubernetes/clusters/homelab/flux-system` and read its `kustomization.yaml`; if it lists files explicitly, add `gitea-mcp.yaml` where `gitea.yaml` is listed.
-
- [ ] **Step 7: Commit**
-
-`git add kubernetes/apps/gitea-mcp kubernetes/clusters/homelab/gitea-mcp.yaml`
-`git -c user.name='Claude Bot' -c user.email='claude-bot@renner.uno' commit -m "feat(gitea-mcp): deploy the Gitea MCP server at gitea-mcp.renner.uno" -m "Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>"`
-
-### Task 4: PR, rollout and live verification
-
- [ ] **Step 1: Push and open the PR**
-
-`git -c credential.helper='store --file=/home/krenner/.config/claude-bot/creds' push -u origin feat/gitea-mcp`
-`tea pr create --login claude-bot --repo k.renner/homelab --base master --head feat/gitea-mcp --title "feat(gitea-mcp): in-cluster Gitea MCP server on the NetBird proxy" --description "<summary of spec + verification plan; ends with 🤖 Generated with [Claude Code](https://claude.com/claude-code)>"`
-Wait for `ci-manifests-test` green (`woodpecker` CLI). Owner merges; never self-merge.
-
- [ ] **Step 2: After merge — rollout**
-`flux get kustomization gitea-mcp` → Ready True. `kubectl -n gitea-mcp get pods` → 1/1 Running, 0 restarts.
-`curl -sS -o /dev/stdout -w "\nHTTP %{http_code}\n" https://gitea-mcp.renner.uno/healthz` → `ok`, HTTP 200.
-
- [ ] **Step 3: DNS paths**
-`getent ahosts gitea-mcp.renner.uno` from the LAN → Traefik VIP (fd99::86 / 10.2.99.86).
-Public answer: query Cloudflare's authoritative NS directly (UDM intercepts `@1.1.1.1`) → NetBird proxy address. NetBird management lists a reverse-proxy service for `gitea-mcp.renner.uno` (netbird-service-sync logs via `logcli`).
-
- [ ] **Step 4: Tokenless request is rejected**
-
-```
-curl -sS -i https://gitea-mcp.renner.uno/mcp -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' --data '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"curl","version":"0"}}}'
-```
-Expected: `HTTP/2 401` with `WWW-Authenticate: Bearer resource_metadata=…`.
-
- [ ] **Step 5: Invalid token is rejected** — same request plus `-H 'Authorization: Bearer invalid'` → 401 with `error="invalid_token"`.
-
- [ ] **Step 6: Authenticated calls work** — same `initialize` request with `--oauth2-bearer "$(cat /home/krenner/.config/claude-bot/gitea-token)"` → 200 with `serverInfo`. Then `tools/call` `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"list_my_repos","arguments":{}}}` with the same token → non-empty repo list; `list_my_repos` resolves the caller from the token, so a result proves the PAT was used. If the server demands `Mcp-Protocol-Version`/`Mcp-Method` headers, take the required values from the error and resend.
-
- [ ] **Step 7: Image policy** — `flux get image policy gitea-mcp` → latest image `docker.gitea.com/gitea-mcp-server:1.8.1` or newer.
-
- [ ] **Step 8: Wiki** — route through `wiki-ingest` → `wiki-distill`: netbird-service-sync label mechanism (`dns.renner.uno/public` vs `netbird.renner.uno/public`), and gitea-mcp deployment facts (oauth-mode-as-auth-gate, PAT passthrough, `/healthz` outside the Host check).
@@ -1,180 +0,0 @@
-# Backup verification on an ephemeral Hetzner Cloud VM — design
-
-Date: 2026-10-10
-Status: draft, awaiting review
-
-## Goal
-
-Replace the monthly Velero `restore-test` with a check that proves every PVC
-backup is readable and complete, **without writing restored data to the
-homelab Ceph OSDs**. The work runs on a throwaway Hetzner Cloud VM next to the
-backup bucket; the cluster only orchestrates.
-
-Full disaster-recovery rehearsals (Velero restores of manifests into a fresh
-cluster) are a later, separate project. This spec only covers data
-verifiability.
-
-## Why
-
- `restore-test` restores every schedule into the production Ceph pools. On
-  2026-10-01 the immich restore took the fullest OSD from 74% to 95% and Ceph
-  went `OSD_FULL` (REN-451). The test is a large share of the OSD write budget.
- Its file-count parity compares the restored PVC against the *live* volume.
-  That needs a source-side snapshot probe per StorageClass (it broke for
-  `ceph-filesystem-bulk`, see PR #1224) and volatile-path exclusions.
- The restore only covers `persistentvolumeclaims`/`persistentvolumes`, so a
-  data-level check loses no coverage that the current test has.
-
-## Decisions
-
-| Topic | Decision |
-|---|---|
-| What is verified | Data only (option A): `kopia snapshot verify --verify-files-percent=100` on the latest Kopia snapshot of every DataUpload of the latest `Completed` backup per schedule, plus a size check against Velero's own `DataUpload.status.progress.totalBytes`. No restore, no Kubernetes on the VM. |
-| Where it runs | Ephemeral Hetzner Cloud VM in `fsn1` (same location as bucket `renner-uno-k8s-velero`). Bucket traffic stays inside Hetzner's eu-central zone. |
-| Orchestrator | In-cluster CronJob `backup-verify` (namespace `velero`), monthly `0 8 1 * *` UTC. It reads Kubernetes, drives the VM via the Hetzner API and SSH, and writes nothing but logs. |
-| Alerting | Exit 1 on any `FAIL` → existing `KubeJobFailed`. Results in the Job log (Loki). |
-| Coverage | All DataUploads, including CNPG volumes (read-only, no cost). Schedules discovered by label `app.kubernetes.io/part-of=homelab-backup`. |
-| SSH source restriction | None. Key-only auth with a per-run key, pinned host key, password auth off, VM lives ~1 h. |
-| Kopia version | CLI pinned to **0.16.0**, the version Velero 1.18.2 vendors (`project-velero/kopia` fork of 0.16.0). sha256 pinned per arch. |
-
-## Architecture
-
-```
-CronJob backup-verify (velero ns)            Hetzner project homelab-backup-verify
-  image ci-backup-verify                       ┌──────────────────────────────┐
-  ├─ 1 sweep stale resources by label ───────▶ │ servers / ssh-keys / firewalls│
-  ├─ 2 build plan from Backups+DataUploads     │   label managed-by=backup-verify
-  ├─ 3 create ssh-key, firewall, server ─────▶ │ VM cx23 (fallback cax11),     │
-  ├─ 4 ssh: run vm-verify.sh, creds on stdin ▶ │ debian-12, fsn1               │
-  │      ◀─ OK/FAIL lines ─────────────────────│  kopia 0.16.0 --readonly ─────┼─▶ S3 renner-uno-k8s-velero
-  ├─ 5 summary, exit code                      └──────────────────────────────┘     kopia/<ns>/
-  └─ 6 trap: delete server, key, firewall
-```
-
-### 1. Sweep
-
-Delete every server, SSH key and firewall in the project labelled
-`managed-by=backup-verify`. `concurrencyPolicy: Forbid` guarantees no other run
-is live, so everything carrying the label is residue.
-
-### 2. Plan
-
-For each Schedule labelled `app.kubernetes.io/part-of=homelab-backup`:
-
- newest Backup with `status.phase == Completed` for that schedule
-  (`PartiallyFailed` → the schedule is reported `FAIL` without verifying);
- its DataUploads (`velero.io/backup-name` label): `spec.sourceNamespace`,
-  `spec.sourcePVC`, `status.snapshotID`, `status.progress.totalBytes`,
-  `status.phase` (anything but `Completed` → `FAIL`);
- a schedule with no DataUploads (e.g. `authentik`) is `SKIP` (nothing to verify).
-
-`ONLY_NAMESPACES` (comma list, optional) limits the plan for test runs.
-An empty plan is `FAIL`.
-
-### 3. VM
-
- Per-run ed25519 client key → `hcloud ssh-key create`.
- Per-run ed25519 **host** key, injected via cloud-init `ssh_keys`; the
-  orchestrator writes the matching `known_hosts` entry and connects with
-  `StrictHostKeyChecking=yes`.
- `hcloud firewall create` + inbound rule `tcp/22` from `0.0.0.0/0, ::/0`.
- `hcloud server create --type cx23 --image debian-12 --location fsn1`
-  with labels; if `cx23` is not orderable in `fsn1`, retry once with `cax11`
-  (ARM). The chosen arch selects the Kopia download.
- cloud-init: disable password auth, mount a tmpfs at `/run/verify`,
-  `shutdown -h +240` as a backstop (a halted server is still billed; the next
-  run's sweep deletes it).
- No credentials in user-data.
-
-### 4. Verify (on the VM, `vm-verify.sh`)
-
- Download Kopia 0.16.0 release tarball for the arch, check sha256, install.
- Credentials arrive on SSH stdin (S3 access/secret key, repository password)
-  and live only in `/run/verify` (tmpfs).
- Per namespace in the plan:
-  - `kopia repository connect s3 --readonly --bucket renner-uno-k8s-velero
-    --prefix kopia/<ns>/ --endpoint fsn1.your-objectstorage.com --region fsn1`,
-    config and cache under `/run/verify`, cache hard limits set small.
-  - `kopia snapshot list --all --json` once; per planned `snapshotID`, find the
-    manifest by `id` and take its root object ID and `TotalFileSize`.
-  - `kopia snapshot verify --directory-id <root> --verify-files-percent=100`.
-  - Emit one line per PVC: `OK|FAIL <ns>/<pvc> snapshot=<id> bytes=<kopia>/<velero> <reason>`.
-  - `kopia repository disconnect`.
- Size check: Kopia `TotalFileSize` must equal DataUpload `totalBytes` (both
-  come from the same upload; Velero reports `manifest.Stats.TotalFileSize`).
-  `incrementalBytes` is not used (observed negative values).
-
-Unverified details the first test run must confirm before the check is
-trusted: the JSON field names in `snapshot list --json`, that `--directory-id`
-verifies exactly that snapshot, and the exit code of `verify` on error.
-
-### 5. Summary
-
-PASS/FAIL/SKIP counts, the list of failures and skips, wall time, server type
-used. Exit 1 on any `FAIL`.
-
-### 6. Teardown
-
-A `trap` on EXIT and TERM deletes the server, SSH key and firewall by label.
-`activeDeadlineSeconds` 4 h, `terminationGracePeriodSeconds` 120 s (enough for
-the hcloud deletes).
-
-## Security
-
- **Hetzner Cloud token**: dedicated project `homelab-backup-verify`
-  (Read & Write; tokens cannot be scoped finer, so the project is the
-  boundary) with a budget alert. OpenBao `homelab/infrastructure/hcloud-backup-verify`,
-  key `token`, synced by an ExternalSecret into `velero`.
- **S3 + repository password**: existing Secrets `cloud-credentials-hetzner`
-  and `velero-repo-credentials` mounted into the pod. Hetzner S3 keys are
-  project-wide and not read-only; mitigated by `--readonly`, tmpfs-only
-  storage on the VM, and the VM's ~1 h lifetime.
- **RBAC**: ServiceAccount `backup-verify`: `get/list` on `schedules`,
-  `backups`, `datauploads` (`velero.io`). No access to Secrets via the API.
- **Network**: CiliumNetworkPolicy for the pod: egress to the kube-apiserver,
-  DNS, `api.hetzner.cloud:443` (FQDN), and `tcp/22` to world (VM address is
-  not known in advance).
-
-## Image
-
-`ci/images/backup-verify/Containerfile`: Alpine with `bash`, `kubectl`, `jq`,
-`openssh-client`; `hcloud` copied from `hetznercloud/cli:v1.69.0`. Built by
-`.woodpecker/build-ci-images.yml`, published as
-`gitea.renner.uno/gitea_admin/ci-backup-verify`, with ImageRepository,
-ImagePolicy and setter marker in the first PR.
-
-Scripts `backup-verify.sh` and `vm-verify.sh` in
-`kubernetes/infrastructure/velero/`, packaged into a ConfigMap by
-`configMapGenerator` like `restore-test.sh`.
-
-## Testing
-
-1. `ONLY_NAMESPACES=apps` manual run (`kubectl create job --from=cronjob/backup-verify`):
-   confirms the unverified Kopia details above.
-2. Negative test: a forged `snapshotID` must end `FAIL` with exit 1.
-3. Abort test: delete the Job mid-run; afterwards no labelled resource may
-   remain in the Hetzner project.
-4. Full run: record wall time and cost.
-
-## Rollout
-
- **PR 1**: image + image automation, scripts, CronJob `backup-verify` with
-  `suspend: true`, RBAC, ExternalSecret, CiliumNetworkPolicy. `restore-test`
-  unchanged. Tests 1–4 run manually after merge.
- **PR 2** (after tests pass, before 2026-11-01): unsuspend `backup-verify`;
-  remove `restore-test` (CronJob, RBAC, script, its rule in
-  `network-policies/monitoring-ns.yaml`); rewrite the restore-test section of
-  `kubernetes/RECOVERY.md`. The failed `restore-test-29847360` Job goes with
-  the CronJob, clearing the open `KubeJobFailed` (velero) alert.
- Close PR #1224 as superseded.
- The `*-snapshot-readonly` StorageClasses stay (the node-agent uses them for
-  backups).
- After the first green full run: ingest the run facts into the wiki.
-
-## Out of scope
-
- Disaster-recovery rehearsal (manifests, Velero restore into a fresh
-  cluster) — separate project.
- CNPG logical restore checks (`RECOVERY.md` "CNPG restore-test").
- Whether the Velero repository password is still the upstream default
-  `static-passw0rd` — noted for a separate review.
@@ -1,127 +0,0 @@
-# Gitea MCP server — design
-
-Date: 2026-10-11
-Status: draft, awaiting review
-
-## Goal
-
-Run the official Gitea MCP server (`gitea.com/gitea/gitea-mcp`) in the cluster
-and publish it on the NetBird proxy, so MCP clients anywhere (Claude Code on
-the workstation, Paperclip agents, other machines) can use Gitea through MCP
-with their own Gitea personal access token.
-
-## Decisions
-
-| Topic | Decision |
-|---|---|
-| Hosting | In-cluster HTTP service, own namespace `gitea-mcp`, one stateless replica. |
-| Exposure | Public via NetBird proxy, host `gitea-mcp.renner.uno`, label `netbird.renner.uno/public` (proxy-only; the LAN record on the Traefik VIP stays, so LAN clients don't hairpin). |
-| Client auth | Bearer PAT per request (`Authorization: Bearer <pat>` or `token <pat>`). The server holds no token. Every call runs with exactly the caller's Gitea permissions. |
-| Anonymous access | Rejected with 401. Achieved by running with `-oauth`: in that mode a missing token is refused, a present token is passed through unchanged and checked with one `GET /api/v1/user`. No OAuth app or client secret is needed server-side. |
-| Gitea backend | `http://gitea.gitea.svc.cluster.local:3000` (in-cluster Service, no hairpin via the ingress LB). |
-| Image | `docker.gitea.com/gitea-mcp-server`, starting at `1.8.1`, tracked by Flux image automation (semver). |
-| Read-only mode | Off; write tools exposed. Write access is bounded by each caller's token. |
-
-## Why the `-oauth` flag with PATs
-
-Gitea answers anonymous API calls: `GET /api/v1/repos/search` without
-credentials returns `k.renner/homelab`, `konairius/Wiki` and
-`gitea_admin/_cargo-index`. In plain HTTP mode (no `-oauth`, no server token)
-gitea-mcp forwards tokenless requests as an anonymous Gitea client, so the MCP
-endpoint would serve those repos to anyone. Upstream has no "require token"
-flag outside OAuth mode.
-
-With `-oauth -oauth-public-url https://gitea-mcp.renner.uno` (upstream
-`pkg/oauth/oauth.go`):
-
- `/mcp` without a token → 401 with an RFC 9728 `WWW-Authenticate` challenge.
- `/mcp` with a token → `TokenRejected` calls Gitea `GET /user` with it; only a
-  401 there rejects. The token is then used as-is for the API calls, so a plain
-  PAT works. There is no JWT/introspection/audience check.
- The server only publishes `/.well-known/oauth-protected-resource` metadata;
-  config is the public URL and the Gitea URL, nothing else.
- `/mcp` additionally checks `Host` against the public URL's host (or
-  loopback) and returns 421 otherwise. `/healthz` is outside both checks.
-
-Costs: one extra `/user` call per MCP request. The metadata document advertises
-the in-cluster Gitea URL as authorization server; this only matters for
-OAuth-flow clients (claude.ai connectors), which are out of scope.
-
-## Architecture
-
-```
-client ──HTTPS, Authorization: Bearer <pat>──▶ gitea-mcp.renner.uno
-  ├─ internet: NetBird proxy (TLS passthrough, PROXY v2) ─▶ Traefik
-  └─ LAN:      Traefik VIP (LAN DNS record)                ─▶ Traefik
-Traefik (TLS termination, letsencrypt-cluster cert) ──HTTP :8080──▶ gitea-mcp pod
-gitea-mcp ──HTTP :3000, same token──▶ gitea.gitea.svc (Gitea API)
-```
-
-Traefik forwards `Host` and the `Mcp-*` headers unchanged by default, which the
-Host check and the MCP transport need. `/mcp` is stateless POST only, so no
-sticky sessions or long-lived streams.
-
-## Components
-
-All paths under `kubernetes/`.
-
- `infrastructure/namespaces/namespaces.yaml` — Namespace `gitea-mcp`.
- `clusters/homelab/gitea-mcp.yaml` — Flux Kustomization, path
-  `./kubernetes/apps/gitea-mcp`, `prune: true`, `wait: true`, dependsOn
-  `infrastructure`, `infrastructure-crds`.
- `apps/gitea-mcp/`
-  - `deployment.yaml` — 1 replica; label `app.kubernetes.io/name: gitea-mcp`;
-    args `-t http -p 8080 -oauth -oauth-public-url https://gitea-mcp.renner.uno
-    -H http://gitea.gitea.svc.cluster.local:3000`; no `GITEA_ACCESS_TOKEN`, no
-    `-d` (debug may log tokens via the SDK); pod/container securityContext as in
-    gatus (non-root 65532, drop ALL, no privilege escalation, RuntimeDefault
-    seccomp) plus `readOnlyRootFilesystem: true` with a disk-backed `emptyDir`
-    (`sizeLimit: 2Gi`) at `/tmp` for the app's own rotated log file; liveness/readiness `httpGet /healthz` on `http`;
-    requests `cpu: 10m, memory: 32Mi`, limit `memory: 128Mi`; image line
-    carries the `$imagepolicy` marker.
-  - `service.yaml` — ClusterIP `gitea-mcp`, port 8080 → `http`.
-  - `ingress.yaml` — host `gitea-mcp.renner.uno`, `ingressClassName: traefik`,
-    `cert-manager.io/cluster-issuer: letsencrypt-cluster`, TLS secret
-    `gitea-mcp-tls`, label `netbird.renner.uno/public: "true"`.
-  - `kustomization.yaml`.
- `infrastructure/network-policies/gitea-mcp-ns.yaml` (+ entry in that
-  directory's `kustomization.yaml`) — default-deny CNP following
-  `gitea-ns.yaml`: ingress from same namespace, `monitoring`, `host`/
-  `remote-node`, ICMP; ingress from Traefik (`ingress` ns,
-  `app.kubernetes.io/name: traefik`) on 8080; egress to kube-dns and to
-  `gitea` ns `app.kubernetes.io/name: gitea` on 3000. No `world` egress.
- `apps/gitea/networkpolicy.yaml` — new CNP `gitea-allow-gitea-mcp`, modelled on
-  `gitea-allow-flux-rotator`: ingress to Gitea on 3000 from the `gitea-mcp`
-  namespace.
- `infrastructure/image-automation/gitea-mcp.yaml` (+ kustomization entry) —
-  ImageRepository `docker.gitea.com/gitea-mcp-server`, ImagePolicy semver
-  `>=1.8.1`.
-
-## Error handling
-
- Invalid/expired token → Gitea `/user` 401 → gitea-mcp returns 401
-  `invalid_token`.
- Gitea unreachable → upstream's token check fails open (transport errors are
-  not treated as rejection), but the subsequent API call fails, so the client
-  gets a tool error, never data.
- Pod down → Traefik 503; Flux `wait` surfaces a failed rollout as
-  `FluxResourceNotReady`.
-
-## Verification (after Flux reconciles)
-
-1. `gitea-mcp` Kustomization Ready; pod Ready.
-2. `https://gitea-mcp.renner.uno/healthz` → 200, from the LAN and through the
-   NetBird proxy (public resolution).
-3. `POST /mcp` `initialize` without a token → 401.
-4. `POST /mcp` with the `claude-bot` PAT: `initialize`, then `tools/call`
-   `list_my_repos` → returns repos.
-5. NetBird management lists a reverse-proxy service for
-   `gitea-mcp.renner.uno`.
-6. ImagePolicy `gitea-mcp` resolves `1.8.1` (or newer).
-
-## Out of scope
-
- Registering the server in Claude Code / other clients.
- OAuth browser flow for claude.ai connectors (would need a Gitea OAuth2 app
-  and `-H https://gitea.renner.uno`).
- Read-only mode, tool filtering (`-O`/`-S`).
@@ -165,33 +165,26 @@ permanent recovery.

 ---

-## Velero backup verification (backup-verify)
+## Velero restore-test: 80% OSD capacity gate

-Monthly (1st, 08:00 UTC) CronJob `velero/backup-verify`. It creates a throwaway
-Hetzner Cloud VM in fsn1 (project `homelab-backup-verify`) and runs
-`kopia snapshot verify --verify-files-percent=100` read-only against the latest
-snapshot of every DataUpload of each schedule's newest finished backup. Snapshot
-sizes are checked against `DataUpload.status.progress.totalBytes`. Nothing is
-restored into the cluster. A full run takes about an hour (2026-10-10: 39 PVCs,
-~410 GB, 61 min, cx23).
-
- Results: `kubectl -n velero logs job/<backup-verify-...>`; `FAIL` lines name the
-  backup, PVC, snapshot and cause.
- A failed run fires `KubeJobFailed` (velero).
- Manual run: `kubectl -n velero create job bv-manual --from=cronjob/backup-verify`;
-  limit it with env `ONLY_NAMESPACES=apps,gitea`.
- Every run first deletes everything labelled `managed-by=backup-verify` in the
-  Hetzner project, so a leftover VM is cleaned up by the next run. Do not start a
-  manual run while another one is running: its sweep deletes the other run's VM.
- This checks data readability, not a full restore. Disaster-recovery rehearsal is separate.
+The monthly Velero restore-test restores into the production Ceph pools. Before
+each schedule's restore it projects the restore's raw size (DataUpload
+`totalBytes` x the pool's replication/EC factor, spread over all OSDs) onto
+current OSD usage from Prometheus. If the fullest OSD would go above **80%**
+(`OSD_MAX_PCT` in `restore-test-cronjob.yaml`), that schedule is not restored:
+the Job log shows it as **SKIP** with the projected %, and the summary lists it
+under "Skipped (backup NOT restore-tested)". A SKIP does not fail the Job, so
+read the summary each month. If the projection cannot be made (Prometheus
+down, unsized DataUpload), the schedule is not restored and counts as FAIL.
+Background: REN-441, REN-442.

 ---

 ## CNPG restore-test (verify backups are restorable — non-destructive)

-The monthly Velero backup verification (`kubernetes/infrastructure/velero/`)
-checks the Velero PVC backups but **does not cover CNPG** (barman-cloud is a
-separate code path). Run this drill periodically to prove the CNPG backups are actually
+The monthly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
+PVC restores but **explicitly does not touch CNPG** (barman-cloud is a separate
+code path). Run this drill periodically to prove the CNPG backups are actually
 restorable, **without touching the live clusters**. It restores into a throwaway
 cluster in the *same* namespace (so it can reference the existing namespaced
 `ObjectStore` and creds) and declares the barman plugin with
@@ -2189,23 +2189,6 @@ data:
          negate: false
          failure_result: false

-      # OpenProject, decommissioned. Application first: Application.provider
-      # is on_delete=PROTECT. Its policy binding cascades with it.
-      - model: authentik_core.application
-        identifiers:
-          slug: openproject
-        state: absent
-
-      - model: authentik_providers_oauth2.oauth2provider
-        identifiers:
-          name: Provider for OpenProject
-        state: absent
-
-      - model: authentik_core.group
-        identifiers:
-          name: OpenProject Users
-        state: absent
-
      # Superseded by the single passkey-webauthn-validation stage above.
      # Their flowstagebindings are FK-CASCADEd away with them; the
      # identification stage's webauthn_stage FK is SET_NULL and is
@@ -1,75 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gitea-mcp
-  namespace: gitea-mcp
-  labels:
-    app.kubernetes.io/name: gitea-mcp
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gitea-mcp
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: gitea-mcp
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: gitea-mcp
-          image: docker.gitea.com/gitea-mcp-server:1.8.1 # {"$imagepolicy": "flux-system:gitea-mcp"}
-          # The image has no ENTRYPOINT; args alone would replace its CMD.
-          command:
-            - /app/gitea-mcp
-          args:
-            - -t
-            - http
-            - -p
-            - "8080"
-            # Only for the 401 on tokenless requests; clients send their own Gitea PAT.
-            - -oauth
-            - -oauth-public-url
-            - https://gitea-mcp.renner.uno
-            - -H
-            - http://gitea.gitea.svc.cluster.local:3000
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          ports:
-            - containerPort: 8080
-              name: http
-          volumeMounts:
-            - name: tmp
-              mountPath: /tmp
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: http
-            initialDelaySeconds: 5
-            periodSeconds: 30
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: http
-            initialDelaySeconds: 2
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 10m
-              memory: 32Mi
-            limits:
-              memory: 128Mi
-      volumes:
-        # Holds the app's own rotated log file (100MB x 11 max); stdout carries the same lines.
-        - name: tmp
-          emptyDir:
-            sizeLimit: 2Gi
@@ -1,28 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: gitea-mcp
-  namespace: gitea-mcp
-  labels:
-    app.kubernetes.io/name: gitea-mcp
-    # Published on the NetBird proxy; LAN DNS stays on the Traefik VIP.
-    netbird.renner.uno/public: "true"
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-cluster
-spec:
-  ingressClassName: traefik
-  rules:
-    - host: gitea-mcp.renner.uno
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: gitea-mcp
-                port:
-                  number: 8080
-  tls:
-    - secretName: gitea-mcp-tls
-      hosts:
-        - gitea-mcp.renner.uno
@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - deployment.yaml
-  - service.yaml
-  - ingress.yaml
@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gitea-mcp
-  namespace: gitea-mcp
-  labels:
-    app.kubernetes.io/name: gitea-mcp
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: gitea-mcp
-  ports:
-    - name: http
-      port: 8080
-      targetPort: http
@@ -5,9 +5,6 @@ metadata:
  namespace: gitea
  labels:
    app.kubernetes.io/name: gitea
-    # Published on the NetBird proxy; LAN DNS stays on the Traefik VIP so
-    # SSH (gitea-ssh) and node image pulls stay local.
-    netbird.renner.uno/public: "true"
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-cluster
    # Long-running git fetch/push and container-registry blob uploads:
@@ -72,23 +72,3 @@ spec:
        - ports:
            - port: "6443"
              protocol: TCP
---
-# gitea-mcp reaches the in-cluster Service; egress half is in network-policies/gitea-mcp-ns.yaml.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: gitea-allow-gitea-mcp
-  namespace: gitea
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: gitea
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea-mcp
-            app.kubernetes.io/name: gitea-mcp
-      toPorts:
-        - ports:
-            - port: "3000"
-              protocol: TCP
@@ -8,6 +8,8 @@ import os
 import re
 import ssl
 import sys
+import time
+import urllib.error
 import urllib.parse
 import urllib.request
 from datetime import datetime
@@ -21,6 +23,7 @@ GROUP_SECONDS = int(os.environ["GROUP_SECONDS"])
 MAX_DELETES = int(os.environ["MAX_DELETES"])
 EXCLUDE = re.compile(os.environ["EXCLUDE_PACKAGES"])
 DRY_RUN = os.environ.get("DRY_RUN", "false") == "true"
+WAIT_SECONDS = int(os.environ.get("WAIT_SECONDS", "1200"))

 # A package is a CI build package if at least one of its tags looks like this.
 BUILD_TAG = re.compile(r"^(build\.[0-9]+|main-[0-9a-f]{7,40})$")
@@ -42,6 +45,22 @@ def gitea(method, path):
        return json.loads(body) if body else None


+def wait_for_gitea():
+    """Gitea may be restarting at 22:30 (upgrade, crashloop; REN-1367). With
+    backoffLimit 0 a single refused connection fails the run, so poll
+    /api/healthz for up to WAIT_SECONDS (inside activeDeadlineSeconds) first."""
+    deadline = time.monotonic() + WAIT_SECONDS
+    while True:
+        try:
+            with urllib.request.urlopen(GITEA + "/api/healthz", timeout=10):
+                return
+        except (urllib.error.URLError, OSError) as e:
+            if time.monotonic() >= deadline:
+                raise
+            log(f"gitea not ready ({e}); retrying in 30s")
+            time.sleep(30)
+
+
 def ts(s):
    # Gitea returns RFC 3339, e.g. 2026-10-07T18:07:32Z or +02:00.
    return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
@@ -116,6 +135,7 @@ def plan(versions, protected):


 def main():
+    wait_for_gitea()
    in_use, unsafe = running_images()
    log(f"running tags from {REGISTRY_HOST}: {sum(len(t) for t in in_use.values())}")
    work = []
@@ -37,7 +37,7 @@ spec:
          # both handled as explicit steps in MIGRATION-gitea-woodpecker.md. No
          # imagepolicy marker: a minor bump migrates the live SQLite database and
          # needs a snapshot first (volumesnapshot-pre-upgrade.yaml). 1.21.4 ->
-          # 1.27.3 in REN-1367; the
+          # 1.27.3 in REN-1367 (OpenProject needs the 1.22+ webhook fields); the
          # next step, 28.x, changes the outgoing-request (webhook) host lists.
          image: gitea/gitea:1.27.3-rootless@sha256:1c17ecaead42eb3b5391553d8708103a4beb0e86edf5b9ebc1eb269c318845f2
          env:
@@ -115,9 +115,6 @@ spec:
            # Nothing here authenticates via query tokens (CI uses webhooks/OAuth).
            - name: GITEA__security__DISABLE_QUERY_AUTH_TOKEN
              value: "true"
-            # Webhook targets Gitea may call; every configured webhook host.
-            - name: GITEA__security__ALLOWED_HOST_LIST
-              value: "ci.renner.uno,paperclip-api.renner.uno,wiki.ai.svc.cluster.local"
          ports:
            - name: http
              containerPort: 3000
@@ -95,7 +95,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.10.1 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.10.0 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -123,3 +123,23 @@ spec:
      interval: 30s
  podTargetLabels:
    - role
+---
+# op-pg (OpenProject, REN-1363). Added with the cluster, like ha-pg, so the
+# CNPG backup/WAL rules see it from the first moment.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cnpg-op-pg
+  namespace: openproject
+  labels:
+    app.kubernetes.io/name: cloudnative-pg
+    release: kube-prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      cnpg.io/cluster: op-pg
+  podMetricsEndpoints:
+    - port: metrics
+      interval: 30s
+  podTargetLabels:
+    - role
@@ -71,6 +71,11 @@ spec:
        # health, not reachability — the root path returns 404 by design, since
        # the service only answers on calendar paths.
        - https://ical-proxy.renner.uno/health
+        # OpenProject (REN-1363). Ungated (the app has its own login), so the
+        # public name reaches the app: /health_checks/default answers 200 only
+        # when the web process and op-pg are up. App health and the public
+        # edge path in one target.
+        - https://openproject.renner.uno/health_checks/default
        # APP-HEALTH probes for the gated monitoring UIs, from the correct
        # in-cluster vantage (monitoring-vantage-point-rule): the gate is
        # only on the ingress, so the ClusterIP /-/healthy endpoints are
@@ -15,6 +15,9 @@ spec:
          # Per-check rather than ceph_health_status, so one permanently-active
          # check cannot mask every other Ceph warning.
          #
+          # AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a
+          # kernel supporting aes256k over the CSI mount path (#527).
+          #
          # POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes
          # `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked
          # maintenance and not just data risk — do not exclude it for a pool
@@ -31,14 +34,20 @@ spec:
          # both episodes in the 30d to 2026-08-23 ran ~26h and cleared
          # untouched, the first ~24h after the last stall.
          #
-          # Not excludable: it is also the earliest signal of a failing drive.
-          expr: ceph_health_detail == 1
+          # Both began inside the Velero restore-test sweep — a ~5h sustained
+          # restore that outruns the SLC cache on these QLC NVMe. The sweep
+          # runs monthly, 08:00-14:00Z on the 1st.
+          #
+          # Expected for the hardware, but not excludable:
+          # this is also the earliest signal of a genuinely failing drive, and
+          # all three are at 80-87% endurance used (see cephcluster.yaml).
+          expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
          for: 6h
          labels:
            severity: warning
          annotations:
            summary: "Ceph health check {{ $labels.name }} active for 6h"
-            description: "Ceph health check {{ $labels.name }} has been active for six hours. BLUESTORE_SLOW_OP_ALERT stays for 24h after the last stall even when latency is back to normal; find the OSD in the detail output and check that drive's SMART data (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the monthly Velero restore-test window (08:00-14:00Z on the 1st), which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."

        - alert: CephPoolLostRedundancy
          # Names the pool, which POOL_NO_REDUNDANCY does not.
@@ -174,18 +174,18 @@ spec:
            forward_to = [loki.process.udm_cef.receiver]
          }

-          // CEF parser for UDM SIEM payload. The listener also receives the
-          // UDM's plain syslog (log_all_contents), which does not match and
-          // passes through unlabelled. CEF extension keys differ per event
-          // type, so the regex anchors only on the header
-          // (vendor|product|product_version|sig_id|name|sev), UNIFIcategory,
-          // UNIFIhost and the trailing msg. The full raw line stays the log
-          // content.
+          // CEF parser for UDM SIEM payload. One regex with named groups
+          // for header (vendor|product|product_version|sig_id|name|sev) and
+          // extension (UNIFIhost / UNIFIdeviceName / -Model / -Ip / -Mac /
+          // -Version / msg). Only deviceName may contain spaces — use a
+          // non-greedy match anchored to the literal next-key boundary.
+          // Keep the full raw line as the log content so unparsed senders
+          // (or future CEF format drift) still land queryably.
          loki.process "udm_cef" {
            forward_to = [loki.write.default.receiver]

            stage.regex {
-              expression = "CEF:\\d+\\|(?P<vendor>[^|]*)\\|(?P<product>[^|]*)\\|(?P<product_version>[^|]*)\\|(?P<signature_id>[^|]*)\\|(?P<event_name>[^|]*)\\|(?P<cef_severity>[^|]*)\\|UNIFIcategory=(?P<category>.+?) UNIFIhost=(?P<unifi_host>.+?) \\S+=.* msg=(?P<msg>.+)$"
+              expression = "CEF:\\d+\\|(?P<vendor>[^|]*)\\|(?P<product>[^|]*)\\|(?P<product_version>[^|]*)\\|(?P<signature_id>[^|]*)\\|(?P<event_name>[^|]*)\\|(?P<cef_severity>[^|]*)\\|UNIFIhost=(?P<unifi_host>\\S+) UNIFIdeviceName=(?P<device_name>.+?) UNIFIdeviceModel=(?P<device_model>\\S+) UNIFIdeviceIp=(?P<device_ip>\\S+) UNIFIdeviceMac=(?P<device_mac>\\S+) UNIFIdeviceVersion=(?P<device_version>\\S+) msg=(?P<msg>.+)$"
            }

            stage.labels {
@@ -194,7 +194,8 @@ spec:
                product      = "product",
                signature_id = "signature_id",
                event_name   = "event_name",
-                category     = "category",
+                device_model = "device_model",
+                host         = "device_name",
              }
            }
          }
@@ -55,7 +55,7 @@ spec:
        # stdlib only; Flux keeps the tag in lockstep via the shared image
        # policy) — no extra dependency.
        - name: reconcile-publish-ip
-          image: ghcr.io/music-assistant/server:2.10.6 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.5 # {"$imagepolicy": "flux-system:music-assistant"}
          command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
          volumeMounts:
            - name: data
@@ -65,7 +65,7 @@ spec:
              readOnly: true
      containers:
        - name: music-assistant
-          image: ghcr.io/music-assistant/server:2.10.6 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.5 # {"$imagepolicy": "flux-system:music-assistant"}
          env:
            - name: LOG_LEVEL
              value: info
@@ -0,0 +1,71 @@
+# openproject
+
+OpenProject Community Edition, the owner's project board (REN-1346,
+REN-1363). <https://openproject.renner.uno>, public through the NetBird
+proxy like immich and matrix; OpenProject's own login guards it.
+
+## Layout
+
+- `openproject` Deployment, one pod: init `seeder` (schema load or
+  migrations, then seed), then `web` (puma, :8080) and
+  `worker` (GoodJob). One pod because both need the RWO attachments PVC.
+- `op-pg` CNPG cluster (PostgreSQL 17, 2 instances), WAL archive and daily
+  base backup to Hetzner `s3://renner-uno-k8s-cnpg/op-pg`, 30 days.
+- `openproject-assets` PVC (attachments), Velero `openproject-daily` at
+  10:15 local, 30 days.
+- `openproject-secrets` is generated once in-cluster by ESO:
+  `SECRET_KEY_BASE` and `OPENPROJECT_SEED_ADMIN_USER_PASSWORD` (login
+  `admin`). The password only seeds the first start; a change in the UI is
+  not reverted.
+
+## Image updates
+
+The image is pinned by tag and digest (`openproject/openproject:<x.y.z>-slim`)
+and Renovate opens the bump. The `seeder` init container migrates the schema
+before the new web container starts. A major bump (17 -> 18) gets its own
+card and a check of the upstream upgrade notes.
+
+## Monitoring
+
+- Blackbox `https://openproject.renner.uno/health_checks/default`: public
+  edge, Traefik, web and database in one target (catch-all
+  `BlackboxProbeFailed`).
+- `cnpg-op-pg` PodMonitor: the CNPG backup and WAL-archive rules cover op-pg.
+- Velero schedule rules cover `openproject-daily`.
+
+## Restore
+
+Database (physical restore of the latest base backup plus WAL):
+
+1. Check a backup exists: `kubectl -n openproject get backups.postgresql.cnpg.io`
+   and the bucket path `op-pg/`.
+2. Follow `kubernetes/RECOVERY.md` (cold rebuild) with this stanza in
+   `cluster.yaml` in place of `bootstrap.initdb`:
+
+   ```yaml
+   bootstrap:
+     recovery:
+       source: op-pg-store
+       database: openproject
+       owner: openproject
+   externalClusters:
+     - name: op-pg-store
+       plugin:
+         name: barman-cloud.cloudnative-pg.io
+         parameters:
+           barmanObjectName: op-pg-store
+           serverName: op-pg
+   ```
+
+   Scale the `openproject` Deployment to 0 first, so nothing writes to an
+   empty database while the cluster rebuilds. Revert to `initdb` afterwards.
+3. A non-destructive test restore into a side cluster: "CNPG restore-test"
+   in `kubernetes/RECOVERY.md`.
+
+Attachments: scale the Deployment to 0, delete the `openproject-assets` PVC,
+then `velero restore create --from-backup <openproject-daily-...>
+--include-namespaces openproject`. Restore database and attachments from the
+same day, so attachment rows and files match.
+
+`openproject-secrets`: if lost, ESO generates a new one. Sessions end; the
+admin password in the database stays what it was.
@@ -0,0 +1,42 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: op-pg
+  namespace: openproject
+spec:
+  instances: 2
+  smartShutdownTimeout: 10
+  stopDelay: 60
+  # PG 17, not the repo-standard 16: OpenProject 17 ships its all-in-one
+  # image and packages on 17 and its system requirements point there. Bare
+  # major tag like the other clusters (minor bumps flow, majors stay an
+  # explicit decision).
+  imageName: ghcr.io/cloudnative-pg/postgresql:17
+  storage:
+    storageClass: ceph-block
+    size: 5Gi
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: op-pg-store
+  bootstrap:
+    initdb:
+      database: openproject
+      owner: openproject
+  postgresql:
+    parameters:
+      shared_buffers: "128MB"
+      max_connections: "100"
+      # Same write-reduction settings as the other clusters; see
+      # apps/trmnl/cluster.yaml for the reasoning and the RPO trade.
+      wal_compression: "zstd"
+      archive_timeout: "30min"
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      memory: 512Mi
@@ -1,15 +1,15 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
-  name: hcloud-backup-verify
-  namespace: velero
+  name: hetzner-s3-creds
+  namespace: openproject
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: openbao
  target:
-    name: hcloud-backup-verify
+    name: hetzner-s3-creds
  dataFrom:
    - extract:
-        key: infrastructure/hcloud-backup-verify
+        key: infrastructure/hetzner-s3
@@ -0,0 +1,31 @@
+# Public, same edge path as immich and matrix: the dns.renner.uno/public
+# label makes netbird-service-sync publish the host through the NetBird proxy
+# (Cloudflare wildcard -> proxy.renner.uno -> TLS passthrough to Traefik) and
+# external-dns-unifi skip it; LAN clients follow the public answer and the UDM
+# hairpins back in. OpenProject's own login guards it.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: openproject
+  namespace: openproject
+  labels:
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - openproject.renner.uno
+      secretName: openproject-tls
+  rules:
+    - host: openproject.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: openproject
+                port:
+                  number: 80
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secrets.yaml
+  - pvcs.yaml
+  - openproject.yaml
+  - ingress.yaml
+  # op-pg CNPG Postgres + offsite Barman backup (PostgreSQL 17, the version
+  # OpenProject 17 targets — see cluster.yaml).
+  - cluster.yaml
+  - objectstore.yaml
+  - scheduled-backup.yaml
+  - hetzner-creds-externalsecret.yaml
@@ -0,0 +1,30 @@
+---
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: op-pg-store
+  namespace: openproject
+spec:
+  configuration:
+    # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG
+    # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud
+    # (plugin #646), so at-rest encryption is unavailable; transit is TLS.
+    destinationPath: s3://renner-uno-k8s-cnpg/op-pg
+    endpointURL: https://fsn1.your-objectstorage.com
+    s3Credentials:
+      accessKeyId:
+        name: hetzner-s3-creds
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: hetzner-s3-creds
+        key: ACCESS_SECRET_KEY
+      # Region is required (REGION=fsn1) — without it boto3's HeadBucket
+      # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s.
+      region:
+        name: hetzner-s3-creds
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  retentionPolicy: "30d"
@@ -0,0 +1,199 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: openproject-config
+  namespace: openproject
+data:
+  OPENPROJECT_HOST__NAME: "openproject.renner.uno"
+  # TLS ends at Traefik; this makes OpenProject build https:// links and set
+  # secure cookies.
+  OPENPROJECT_HTTPS: "true"
+  # Log in with the generated password directly instead of being forced to
+  # change it on first login (see secrets.yaml).
+  OPENPROJECT_SEED_ADMIN_USER_PASSWORD_RESET: "false"
+  # One user base and a small team: two puma workers instead of the
+  # per-CPU default keep the pod inside its memory limit.
+  OPENPROJECT_WEB_WORKERS: "2"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: openproject
+  namespace: openproject
+  labels:
+    app.kubernetes.io/name: openproject
+  annotations:
+    # Env vars are read once at container start; roll the pod when the
+    # ConfigMap or the generated Secret changes.
+    configmap.reloader.stakater.com/reload: openproject-config
+    secret.reloader.stakater.com/reload: openproject-secrets
+spec:
+  replicas: 1
+  # Web and worker share the RWO attachments PVC, and the seeder init
+  # container migrates the schema; two pods would race on both.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: openproject
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: openproject
+    spec:
+      enableServiceLinks: false
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        # `app` in the official image.
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # Loads the schema on an empty database, else runs pending
+        # migrations, then seeds (idempotent; creates `admin` on first run).
+        # Runs on every pod start, so an image bump migrates before the new
+        # web container serves. Restarted by the kubelet until op-pg accepts
+        # connections.
+        - name: seeder
+          image: openproject/openproject:17.9.1-slim@sha256:2beeecac788d4f423c46a95ffbd04e6479491938b9924b610e9f47d81a38303c
+          args: ["/app/docker/prod/seeder"]
+          envFrom:
+            - configMapRef:
+                name: openproject-config
+            - secretRef:
+                name: openproject-secrets
+          env:
+            # CNPG generates this Secret; `uri` carries the rotated password
+            # and points at op-pg-rw.
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: op-pg-app
+                  key: uri
+          volumeMounts:
+            - name: assets
+              mountPath: /var/openproject/assets
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              memory: 2Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+      containers:
+        - name: web
+          image: openproject/openproject:17.9.1-slim@sha256:2beeecac788d4f423c46a95ffbd04e6479491938b9924b610e9f47d81a38303c
+          args: ["/app/docker/prod/web"]
+          ports:
+            - name: http
+              containerPort: 8080
+          envFrom:
+            - configMapRef:
+                name: openproject-config
+            - secretRef:
+                name: openproject-secrets
+          env:
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: op-pg-app
+                  key: uri
+          volumeMounts:
+            - name: assets
+              mountPath: /var/openproject/assets
+          # /health_checks/default checks the database and the web process.
+          # The Host header is required: the request goes to the pod IP,
+          # which OpenProject's host check would answer with 404.
+          startupProbe:
+            httpGet:
+              path: /health_checks/default
+              port: http
+              httpHeaders:
+                - name: Host
+                  value: localhost
+            periodSeconds: 10
+            # ~5 min for puma to boot and preload the app.
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              path: /health_checks/default
+              port: http
+              httpHeaders:
+                - name: Host
+                  value: localhost
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health_checks/default
+              port: http
+              httpHeaders:
+                - name: Host
+                  value: localhost
+            periodSeconds: 10
+            timeoutSeconds: 5
+          resources:
+            requests:
+              cpu: 250m
+              memory: 1Gi
+            limits:
+              memory: 2Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+        # Background jobs (GoodJob): mails, notifications, exports, and the
+        # cron-like jobs. Needs the attachments volume for exports/imports.
+        - name: worker
+          image: openproject/openproject:17.9.1-slim@sha256:2beeecac788d4f423c46a95ffbd04e6479491938b9924b610e9f47d81a38303c
+          args: ["/app/docker/prod/worker"]
+          envFrom:
+            - configMapRef:
+                name: openproject-config
+            - secretRef:
+                name: openproject-secrets
+          env:
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: op-pg-app
+                  key: uri
+          volumeMounts:
+            - name: assets
+              mountPath: /var/openproject/assets
+          resources:
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              memory: 1536Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+      volumes:
+        - name: assets
+          persistentVolumeClaim:
+            claimName: openproject-assets
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openproject
+  namespace: openproject
+  labels:
+    app.kubernetes.io/name: openproject
+spec:
+  selector:
+    app.kubernetes.io/name: openproject
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
@@ -0,0 +1,19 @@
+# Attachments (OPENPROJECT_ATTACHMENTS__STORAGE__PATH defaults to
+# $APP_DATA_PATH/files = /var/openproject/assets/files). Web and worker share
+# one pod, so RWO is enough and the Deployment uses `Recreate`. Backed up by
+# the Velero schedule openproject-daily (selected by these labels).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: openproject-assets
+  namespace: openproject
+  labels:
+    app.kubernetes.io/name: openproject
+    app.kubernetes.io/component: assets
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ceph-block
+  resources:
+    requests:
+      storage: 10Gi
@@ -0,0 +1,17 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: op-pg-daily
+  namespace: openproject
+spec:
+  # 03:00 local, same slot as the other CNPG clusters. `immediate` takes the
+  # first base backup as soon as the cluster is up. The attachments PVC is
+  # covered by the Velero schedule openproject-daily.
+  schedule: "0 0 3 * * *"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: op-pg
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
@@ -0,0 +1,42 @@
+# Generated once in-cluster by ESO; nothing here is seeded by hand.
+#
+#   SECRET_KEY_BASE                       Rails session/cookie key.
+#   OPENPROJECT_SEED_ADMIN_USER_PASSWORD  Password of the `admin` account the
+#                                         seeder creates on the first start.
+#                                         Only read at seed time: changing the
+#                                         password in the UI later is not
+#                                         reverted, and this value goes stale.
+#
+# Deleting the Secret regenerates both; that logs everyone out and does NOT
+# change an existing admin password.
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: openproject-secrets
+  namespace: openproject
+spec:
+  length: 64
+  digits: 16
+  symbols: 0
+  allowRepeat: true
+  noUpper: false
+  secretKeys:
+    - SECRET_KEY_BASE
+    - OPENPROJECT_SEED_ADMIN_USER_PASSWORD
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: openproject-secrets
+  namespace: openproject
+spec:
+  # Generate once; Periodic would rotate the key every refresh.
+  refreshPolicy: CreatedOnce
+  target:
+    name: openproject-secrets
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: openproject-secrets
@@ -50,9 +50,11 @@ data:
  PAPERLESS_TIKA_ENABLED: "1"
  PAPERLESS_TIKA_ENDPOINT: "http://tika:9998"
  PAPERLESS_TIKA_GOTENBERG_ENDPOINT: "http://gotenberg:3000"
-  # AI suggestions against Mistral AI's hosted API (OpenAI-compatible). Document
-  # text is sent to Mistral. The key is PAPERLESS_AI_LLM_API_KEY, from the
-  # paperless-secrets Secret (OpenBao apps/paperless, field minstral-token).
+  # AI suggestions against the cluster's own LLM (kubernetes/apps/llama-server).
+  # Nothing leaves the cluster: both endpoints are ClusterIP Services, so
+  # document text is never sent to a third party and there is no API key to
+  # rotate — llama-server has no authentication, and the ai namespace's network
+  # policy is what restricts who may reach it.
  #
  # CAVEAT, and it is a real one: paperless reads these from an
  # ApplicationConfiguration row in the database and falls back to the
@@ -61,14 +63,20 @@ data:
  # here — and leave those fields untouched in the UI.
  PAPERLESS_AI_ENABLED: "true"
  PAPERLESS_AI_LLM_BACKEND: "openai-like"
-  PAPERLESS_AI_LLM_ENDPOINT: "https://api.mistral.ai/v1"
-  PAPERLESS_AI_LLM_MODEL: "mistral-small-2603"
-  # The embedding endpoint is the in-cluster llama-embed, a ClusterIP address,
-  # and paperless blocks non-public endpoints unless this is set. Default is
-  # already true; explicit because the embedding integration silently stops
-  # working if it ever flips.
+  PAPERLESS_AI_LLM_ENDPOINT: "http://llama-server.ai.svc.cluster.local:8080/v1"
+  # Must match --alias in kubernetes/apps/llama-server/deployment.yaml. The
+  # server ignores the field on the wire, but paperless sends it and the value
+  # is what appears in its logs.
+  PAPERLESS_AI_LLM_MODEL: "qwen3.6-35b-a3b"
+  # ClusterIP addresses are private, and paperless blocks non-public endpoints
+  # unless this is set. Default is already true; explicit because the whole
+  # integration silently stops working if it ever flips.
  PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS: "true"
+  # Under llama-server's own 16384, leaving room for the response.
  PAPERLESS_AI_LLM_CONTEXT_SIZE: "12288"
+  # The 120s default is too tight at ~11 tok/s: a classification is roughly
+  # 2-4k prompt tokens plus a second round trip for localisation.
+  PAPERLESS_AI_LLM_REQUEST_TIMEOUT: "300"
  # Matched against Django's settings.LANGUAGES, so the region is required —
  # "de" alone does not resolve. Without it the model returns German tags but
  # English document types, which was reproducible against this endpoint.
@@ -78,9 +86,6 @@ data:
  # primarily English archives — the better fit for a German corpus.
  PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
  PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
-  # The embedding key falls back to PAPERLESS_AI_LLM_API_KEY when unset, which
-  # would send the Mistral token to llama-embed. llama-embed has no auth.
-  PAPERLESS_AI_LLM_EMBEDDING_API_KEY: "unused"
  # Must match --alias in kubernetes/apps/llama-server/embed.yaml. Changing the
  # embedding model invalidates every stored vector, so a swap means the LLM
  # index has to be rebuilt.
@@ -195,11 +200,6 @@ spec:
                secretKeyRef:
                  name: paperless-secrets
                  key: secret-key
-            - name: PAPERLESS_AI_LLM_API_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-secrets
-                  key: minstral-token
            - name: PAPERLESS_DBUSER
              valueFrom:
                secretKeyRef:
@@ -1,18 +0,0 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: gitea-mcp
-  namespace: flux-system
-spec:
-  interval: 10m
-  retryInterval: 2m
-  timeout: 5m
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./kubernetes/apps/gitea-mcp
-  prune: true
-  wait: true
-  dependsOn:
-    - name: infrastructure
-    - name: infrastructure-crds
@@ -0,0 +1,28 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: openproject
+  namespace: flux-system
+spec:
+  # OpenProject Community (REN-1363): web + worker, the op-pg CNPG Cluster +
+  # ObjectStore + ScheduledBackup and the attachments PVC. Own KS so a
+  # failure here never holds up the shared apps KS.
+  interval: 10m
+  retryInterval: 1m
+  timeout: 30m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/apps/openproject
+  prune: true
+  wait: true
+  dependsOn:
+    - name: infrastructure
+    - name: infrastructure-crds
+    - name: infrastructure-ceph-cluster
+    - name: infrastructure-cnpg-barman
+  healthCheckExprs:
+    - apiVersion: postgresql.cnpg.io/v1
+      kind: Cluster
+      failed: "has(status.phase) && status.phase.startsWith('Failed')"
+      current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"
@@ -4,9 +4,7 @@ metadata:
  name: alertmanager
  namespace: flux-system
 spec:
-  # The notification controller POSTs to spec.address verbatim (it does
-  # not append a path), so the address must be the full Alertmanager v2
-  # alerts endpoint. The bare base URL is answered with 405 Method Not
-  # Allowed and every Flux notification is dropped.
+  # The notification controller appends `/api/v2/alerts/` itself, so
+  # spec.address is the bare base URL with trailing slash.
  type: alertmanager
-  address: http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/api/v2/alerts/
+  address: http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/
@@ -1,23 +0,0 @@
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImageRepository
-metadata:
-  name: backup-verify
-  namespace: flux-system
-spec:
-  image: gitea.renner.uno/gitea_admin/ci-backup-verify
-  interval: 30m
---
-# build-ci-images.yml tags each build YYYY-MM-DD; newest date wins.
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImagePolicy
-metadata:
-  name: backup-verify
-  namespace: flux-system
-spec:
-  imageRepositoryRef:
-    name: backup-verify
-  filterTags:
-    pattern: '^\d{4}-\d{2}-\d{2}$'
-  policy:
-    alphabetical:
-      order: asc
@@ -1,20 +0,0 @@
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImageRepository
-metadata:
-  name: gitea-mcp
-  namespace: flux-system
-spec:
-  image: docker.gitea.com/gitea-mcp-server
-  interval: 1h
---
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImagePolicy
-metadata:
-  name: gitea-mcp
-  namespace: flux-system
-spec:
-  imageRepositoryRef:
-    name: gitea-mcp
-  policy:
-    semver:
-      range: ">=1.8.1"
@@ -3,11 +3,9 @@ kind: Kustomization
 resources:
  - update-automation.yaml
  - audiobookshelf.yaml
-  - backup-verify.yaml
  - ceph.yaml
  - cnpg-barman-plugin.yaml
  - gatus.yaml
-  - gitea-mcp.yaml
  - gotenberg.yaml
  - home-assistant.yaml
  - immich.yaml
@@ -152,6 +152,12 @@ kind: Namespace
 metadata:
  name: matrix
 ---
+# OpenProject Community, the owner's project board (REN-1363).
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openproject
+---
 # Terminus (TRMNL BYOS server) driving the entrance-way e-ink panel.
 apiVersion: v1
 kind: Namespace
@@ -198,12 +204,6 @@ kind: Namespace
 metadata:
  name: gitea
 ---
-# Gitea MCP server. Declared here so it exists before infrastructure-crds applies its CNPs.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: gitea-mcp
---
 # netbird-service-sync, moving out of `openbao` (issue #876). Declared here like
 # `woodpecker` above, so it exists before infrastructure-crds applies its
 # CiliumNetworkPolicies. No pod-security labels: the cluster default applies.
@@ -1,6 +1,5 @@
 # Reconciles NetBird reverse-proxy services from Ingresses that carry
-# dns.renner.uno/public or netbird.renner.uno/public (proxy only, LAN DNS
-# untouched): one TLS-passthrough service per host on :443,
+# dns.renner.uno/public: one TLS-passthrough service per host on :443,
 # targeting Traefik's ClusterIP through a subnet resource over the service
 # CIDR (ci/images/netbird-service-sync/controller.py).
 #
@@ -81,7 +80,7 @@ spec:
          # image directory (branch and master alike) and the registry keeps
          # only the newest manifest per tag, so the digest must come from
          # the MASTER build of the code change.
-          image: gitea.renner.uno/gitea_admin/netbird-service-sync:2026-10-11@sha256:2e8c5e45cffd515cc93e04dc9fd47403a7910368fc7516e94c78d8f53dcd5272
+          image: gitea.renner.uno/gitea_admin/netbird-service-sync:2026-09-23@sha256:64f6057c508640e1dbc50457dc26e9b2de058a111fd46ca096021d7444b5bbff
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
@@ -25,9 +25,9 @@ resources:
  - woodpecker-ci-ns.yaml
  - woodpecker-ns.yaml
  - gitea-ns.yaml
-  - gitea-mcp-ns.yaml
  - netbird-service-sync-ns.yaml
  - trmnl-ns.yaml
+  - openproject-ns.yaml
  - matrix-ns.yaml
  - wordpress-ns.yaml
  - ai-ns.yaml
@@ -50,6 +50,17 @@ spec:
        - ports:
            - port: "9090"
              protocol: TCP
+    # velero restore-test Job: its capacity gate reads OSD usage from
+    # Prometheus :9090 (restore-test.sh, preflight_capacity). Scoped to the
+    # Job's pod label; the velero namespace has no egress policy.
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": velero
+            app.kubernetes.io/name: restore-test
+      toPorts:
+        - ports:
+            - port: "9090"
+              protocol: TCP
    # Kubelet probes + control-plane (PrometheusRule/ServiceMonitor admission
    # webhooks via kube-prometheus-stack-operator are called by kube-apiserver).
    # Also covers external alloy-syslog traffic — ETP=Cluster on the LB SNATs
@@ -1,22 +1,38 @@
---
-# Baseline policy for the `gitea-mcp` namespace.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
-  name: default-deny
-  namespace: gitea-mcp
+  name: default-deny-ingress
+  namespace: openproject
 spec:
  endpointSelector: {}
  ingress:
+    # Intra-namespace — openproject → op-pg.
    - fromEndpoints:
        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea-mcp
+            "k8s:io.kubernetes.pod.namespace": openproject
+    # openproject.renner.uno via Traefik.
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ingress
+            app.kubernetes.io/name: traefik
+    # Prometheus scrape.
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": monitoring
+    # Velero backups via PVC mount.
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": velero
+    # CNPG operator (in `database` ns) reconciles op-pg — needs to reach
+    # the postgres pods for health, failover, role checks.
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": database
    - fromEntities:
        - host
        - remote-node
+    # ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See
+    # home-assistant-ns.yaml for the rationale. INGRESS ONLY.
    - fromEndpoints:
        - matchExpressions:
            - key: io.kubernetes.pod.namespace
@@ -39,10 +55,29 @@ spec:
            - { type: 135, family: IPv6 }
            - { type: 136, family: IPv6 }
            - { type: 143, family: IPv6 }
+
+---
+# Egress for the OpenProject pod only, so the op-pg CNPG pods keep
+# unrestricted egress for the barman-cloud WAL archive to Hetzner (same trade
+# as trmnl-ns.yaml).
+#
+# OpenProject needs nothing outside the namespace today: no SMTP, no OIDC, no
+# file-storage integration. Adding one of those needs a rule here, or it
+# fails closed.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: openproject-egress
+  namespace: openproject
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: openproject
  egress:
+    # Intra-namespace — op-pg-rw.
    - toEndpoints:
        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea-mcp
+            "k8s:io.kubernetes.pod.namespace": openproject
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": kube-system
@@ -53,30 +88,3 @@ spec:
              protocol: UDP
            - port: "53"
              protocol: TCP
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": gitea
-            app.kubernetes.io/name: gitea
-      toPorts:
-        - ports:
-            - port: "3000"
-              protocol: TCP
---
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: gitea-mcp-allow-traefik
-  namespace: gitea-mcp
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: gitea-mcp
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ingress
-            app.kubernetes.io/name: traefik
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
@@ -6,7 +6,7 @@ metadata:
 spec:
  endpointSelector: {}
  ingress:
-    # Intra-namespace — velero server ↔ node-agent.
+    # Intra-namespace — velero server ↔ node-agent ↔ restore-test job.
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": velero
@@ -42,44 +42,3 @@ spec:
            - { type: 135, family: IPv6 }
            - { type: 136, family: IPv6 }
            - { type: 143, family: IPv6 }
---
-# backup-verify CronJob: kube-apiserver, DNS, Hetzner Cloud API, SSH to its VM.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: backup-verify-egress
-  namespace: velero
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: backup-verify
-  egress:
-    - toEntities:
-        - kube-apiserver
-        - host
-        - remote-node
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": kube-system
-            k8s-app: kube-dns
-      toPorts:
-        - ports:
-            - port: "53"
-              protocol: UDP
-            - port: "53"
-              protocol: TCP
-          rules:
-            dns:
-              - matchPattern: "*"
-    - toFQDNs:
-        - matchName: "api.hetzner.cloud"
-      toPorts:
-        - ports:
-            - port: "443"
-              protocol: TCP
-    - toEntities:
-        - world
-      toPorts:
-        - ports:
-            - port: "22"
-              protocol: TCP
@@ -25,7 +25,6 @@ resources:
  - netbird-ingress-externalsecret.yaml
  - velero-externalsecret.yaml
  - velero-hetzner-creds-externalsecret.yaml
-  - hcloud-backup-verify-externalsecret.yaml
  # traefik.io CRDs, same reasoning — installed by the Traefik HelmRelease in
  # infrastructure. Traefik is deliberately absent from that Kustomization's
  # healthChecks, so this KS may dry-run before the chart lands; it retries on
@@ -29,6 +29,7 @@ spec:
        - ingress
        - matrix # synapse-secrets, mx-pg barman creds (apps/matrix)
        - monitoring
+        - openproject # op-pg barman creds (apps/openproject)
        - paperclip
        - paperless
        - renovate # renovate-bot token (apps/renovate/externalsecret.yaml)
@@ -368,3 +368,30 @@ spec:
    labelSelector:
      matchLabels:
        app.kubernetes.io/name: synapse
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: openproject-daily
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: homelab-backup
+spec:
+  # Daily at 10:15 local (next free stagger slot after matrix-daily at
+  # 09:45). openproject-assets only: the attachments. Selected by the label
+  # this repo sets, like wordpress-daily; the op-pg database is covered by
+  # Barman.
+  schedule: "15 10 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    includedNamespaces:
+      - openproject
+    includedResources:
+      - persistentvolumeclaims
+    storageLocation: default
+    ttl: 720h0m0s
+    snapshotMoveData: true
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/name: openproject
+        app.kubernetes.io/component: assets
@@ -1,74 +0,0 @@
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: backup-verify
-  namespace: velero
-spec:
-  schedule: "0 8 1 * *"
-  timeZone: Etc/UTC
-  suspend: false
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 2
-  failedJobsHistoryLimit: 3
-  startingDeadlineSeconds: 3600
-  jobTemplate:
-    spec:
-      backoffLimit: 0
-      activeDeadlineSeconds: 14400
-      ttlSecondsAfterFinished: 3456000
-      template:
-        metadata:
-          labels:
-            app.kubernetes.io/name: backup-verify
-        spec:
-          serviceAccountName: backup-verify
-          restartPolicy: Never
-          terminationGracePeriodSeconds: 120
-          securityContext:
-            runAsNonRoot: true
-            runAsUser: 65532
-            runAsGroup: 65532
-            seccompProfile:
-              type: RuntimeDefault
-          containers:
-            - name: backup-verify
-              image: gitea.renner.uno/gitea_admin/ci-backup-verify:2026-10-10 # {"$imagepolicy": "flux-system:backup-verify"}
-              command: ["bash", "/script/backup-verify.sh"]
-              env:
-                - name: HOME
-                  value: /work
-              securityContext:
-                allowPrivilegeEscalation: false
-                readOnlyRootFilesystem: true
-                capabilities:
-                  drop: ["ALL"]
-              volumeMounts:
-                - { name: script, mountPath: /script, readOnly: true }
-                - { name: work, mountPath: /work }
-                - { name: hcloud, mountPath: /secrets/hcloud, readOnly: true }
-                - { name: s3, mountPath: /secrets/s3, readOnly: true }
-                - { name: repo, mountPath: /secrets/repo, readOnly: true }
-              resources:
-                requests:
-                  cpu: 20m
-                  memory: 64Mi
-                limits:
-                  memory: 384Mi
-          volumes:
-            - name: script
-              configMap:
-                name: backup-verify-scripts
-                defaultMode: 0755
-            - name: work
-              emptyDir:
-                medium: Memory
-                sizeLimit: 128Mi
-            - name: hcloud
-              secret:
-                secretName: hcloud-backup-verify
-            - name: s3
-              secret:
-                secretName: cloud-credentials-hetzner
-            - name: repo
-              secret:
-                secretName: velero-repo-credentials
@@ -1,29 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: backup-verify
-  namespace: velero
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: backup-verify
-  namespace: velero
-rules:
-  - apiGroups: ["velero.io"]
-    resources: ["schedules", "backups", "datauploads"]
-    verbs: ["get", "list"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: backup-verify
-  namespace: velero
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: backup-verify
-subjects:
-  - kind: ServiceAccount
-    name: backup-verify
-    namespace: velero
@@ -1,180 +0,0 @@
-#!/usr/bin/env bash
-# Monthly backup verification: plan from Velero, verify on a throwaway
-# Hetzner Cloud VM, summarize, always delete the VM.
-set -euo pipefail
-
-SCRIPT_DIR=${SCRIPT_DIR:-/script}
-WORK=${WORK:-/work}
-LABEL=managed-by=backup-verify
-LOCATION=${LOCATION:-fsn1}
-SERVER_IMAGE=${SERVER_IMAGE:-debian-12}
-SERVER_TYPES=${SERVER_TYPES:-cx23 cax11}
-ONLY_NAMESPACES=${ONLY_NAMESPACES:-}
-SSH_WAIT_S=${SSH_WAIT_S:-600}
-HCLOUD_TOKEN_FILE=${HCLOUD_TOKEN_FILE:-/secrets/hcloud/token}
-S3_CREDENTIALS_FILE=${S3_CREDENTIALS_FILE:-/secrets/s3/cloud}
-REPO_PASSWORD_FILE=${REPO_PASSWORD_FILE:-/secrets/repo/repository-password}
-RUN_ID=${RUN_ID:-$(date +%s)}
-NAME="backup-verify-$RUN_ID"
-SSH_OPTS=()
-VM_ADDR=""
-SERVER_TYPE=none
-START_S=$(date +%s)
-
-log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
-
-delete_kind() {
-  local id
-  for id in $(hcloud "$1" list -l "$LABEL" -o noheader -o columns=id); do
-    log "deleting $1 $id"
-    hcloud "$1" delete "$id" >&2 || log "WARN: could not delete $1 $id"
-  done
-}
-
-sweep() {
-  local kind
-  for kind in server firewall ssh-key; do delete_kind "$kind"; done
-}
-
-build_plan() {
-  local only
-  only=$(jq -cn --arg s "$ONLY_NAMESPACES" '$s | split(",") | map(select(. != ""))')
-  kubectl -n velero get schedules.velero.io -l app.kubernetes.io/part-of=homelab-backup -o json > "$WORK/schedules.json"
-  kubectl -n velero get backups.velero.io -o json > "$WORK/backups.json"
-  kubectl -n velero get datauploads.velero.io -o json > "$WORK/datauploads.json"
-  jq -n --slurpfile s "$WORK/schedules.json" --slurpfile b "$WORK/backups.json" --slurpfile d "$WORK/datauploads.json" \
-     '{schedules: $s[0].items, backups: $b[0].items, datauploads: $d[0].items}' \
-    | jq --argjson only "$only" -f "$SCRIPT_DIR/plan.jq" \
-    | jq --arg f "${FORGE_SNAPSHOT_ID:-}" \
-         'if $f == "" then . else map(if .status == "VERIFY" then .snapshotID = $f else . end) end' \
-    > "$WORK/plan.json"
-  if [ -n "${FORGE_SNAPSHOT_ID:-}" ]; then log "NEGATIVE TEST: every snapshotID replaced by $FORGE_SNAPSHOT_ID"; fi
-  log "plan: $(jq -r 'group_by(.status) | map("\(.[0].status)=\(length)") | join(" ")' "$WORK/plan.json")"
-}
-
-# cloud-config as JSON (a YAML subset), so jq does the escaping.
-render_user_data() {
-  printf '#cloud-config\n'
-  jq -n --rawfile priv "$WORK/host_key" --arg pub "$(cat "$WORK/host_key.pub")" '{
-    ssh_pwauth: false,
-    ssh_deletekeys: true,
-    ssh_genkeytypes: [],
-    ssh_keys: {ed25519_private: $priv, ed25519_public: $pub},
-    packages: ["jq", "curl", "ca-certificates"],
-    runcmd: [
-      "mkdir -p /run/verify && mount -t tmpfs -o size=1500m,mode=0700 tmpfs /run/verify",
-      ["shutdown", "-h", "+240"]
-    ]
-  }'
-}
-
-create_vm() {
-  local type
-  ssh-keygen -q -t ed25519 -N '' -C "$NAME" -f "$WORK/client_key"
-  ssh-keygen -q -t ed25519 -N '' -C "$NAME-host" -f "$WORK/host_key"
-  render_user_data > "$WORK/user-data.yaml"
-  hcloud ssh-key create --name "$NAME" --public-key-from-file "$WORK/client_key.pub" --label "$LABEL" >&2
-  hcloud firewall create --name "$NAME" --label "$LABEL" >&2
-  hcloud firewall add-rule "$NAME" --direction in --protocol tcp --port 22 --source-ips 0.0.0.0/0 --source-ips ::/0 >&2
-  for type in $SERVER_TYPES; do
-    if hcloud server create --name "$NAME" --type "$type" --image "$SERVER_IMAGE" --location "$LOCATION" \
-         --ssh-key "$NAME" --firewall "$NAME" --label "$LABEL" \
-         --user-data-from-file "$WORK/user-data.yaml" -o json > "$WORK/server.json"; then
-      log "server $NAME created (type $type)"
-      SERVER_TYPE=$type
-      return 0
-    fi
-    log "server type $type not available"
-    # A create that timed out may still have left the server behind under $NAME.
-    delete_kind server
-  done
-  return 1
-}
-
-wait_ssh() {
-  local v4 v6 hostkey addr deadline rc
-  v6=$(hcloud server ip -6 "$NAME")
-  v4=$(hcloud server ip "$NAME")
-  hostkey=$(cut -d' ' -f1,2 "$WORK/host_key.pub")
-  printf '%s %s\n%s %s\n' "$v6" "$hostkey" "$v4" "$hostkey" > "$WORK/known_hosts"
-  SSH_OPTS=(-i "$WORK/client_key" -o "UserKnownHostsFile=$WORK/known_hosts" -o StrictHostKeyChecking=yes
-            -o ConnectTimeout=10 -o BatchMode=yes -o ServerAliveInterval=30)
-  deadline=$(( $(date +%s) + SSH_WAIT_S ))
-  while [ "$(date +%s)" -lt "$deadline" ]; do
-    for addr in "$v6" "$v4"; do
-      # Background + wait so a TERM is handled while cloud-init is still running.
-      ssh "${SSH_OPTS[@]}" "root@$addr" 'cloud-init status --wait > /run/cloud-init-wait.log' < /dev/null &
-      if wait $!; then
-        VM_ADDR=$addr
-        log "ssh ready via $addr"
-        return 0
-      else
-        rc=$?
-      fi
-      # 255 is an ssh connection error; anything else is cloud-init itself failing.
-      if [ "$rc" -ne 255 ]; then
-        log "cloud-init failed on $addr (exit $rc):"
-        ssh "${SSH_OPTS[@]}" "root@$addr" 'cloud-init status --long' < /dev/null >&2 || true
-        return 1
-      fi
-    done
-    sleep 10
-  done
-  log "ssh not reachable within ${SSH_WAIT_S}s"
-  return 1
-}
-
-secrets() {
-  awk -F' *= *' '/^aws_access_key_id/ {print "AWS_ACCESS_KEY_ID=" $2} /^aws_secret_access_key/ {print "AWS_SECRET_ACCESS_KEY=" $2}' "$S3_CREDENTIALS_FILE"
-  printf 'KOPIA_PASSWORD=%s\n' "$(cat "$REPO_PASSWORD_FILE")"
-}
-
-push() { ssh "${SSH_OPTS[@]}" "root@$VM_ADDR" "cat > /run/verify/$2" < "$1"; }
-
-run_vm() {
-  jq '[.[] | select(.status == "VERIFY")]' "$WORK/plan.json" > "$WORK/vm-plan.json"
-  if ! { ssh "${SSH_OPTS[@]}" "root@$VM_ADDR" 'mkdir -p /run/verify' < /dev/null \
-         && push "$WORK/vm-plan.json" plan.json \
-         && push "$SCRIPT_DIR/check.jq" check.jq \
-         && push "$SCRIPT_DIR/vm-verify.sh" vm-verify.sh; }; then
-    log "could not stage files on the VM; every VERIFY entry will report no result"
-    return 0
-  fi
-  # Background + wait: bash runs the TERM trap during `wait`, not during a foreground child.
-  ( secrets | ssh "${SSH_OPTS[@]}" "root@$VM_ADDR" 'bash /run/verify/vm-verify.sh /run/verify/plan.json' \
-      | tee "$WORK/vm.out" >&2 ) &
-  wait $! || log "vm-verify exited non-zero"
-}
-
-summarize() {
-  { grep $'^RESULT\t' "$WORK/vm.out" || true; } \
-    | jq -R 'split("\t") | {status: .[1], namespace: .[2], pvc: .[3], snapshotID: .[4], reason: .[5]}' \
-    | jq -s . > "$WORK/results.json"
-  jq -n --slurpfile p "$WORK/plan.json" --slurpfile r "$WORK/results.json" '{plan: $p[0], results: $r[0]}' \
-    | jq -f "$SCRIPT_DIR/summary.jq" > "$WORK/summary.json"
-  jq -r '.entries[] | "\(.status)\t\(.schedule // "-")\t\(.backup // "-")\t\(.namespace // "-")/\(.pvc // "-")\tsnapshot=\(.snapshotID // "-")\t\(.reason // "")"' "$WORK/summary.json"
-  jq -r '"PASS=\(.pass) FAIL=\(.fail) SKIP=\(.skip)"' "$WORK/summary.json"
-  printf 'elapsed=%ss server_type=%s\n' "$(( $(date +%s) - START_S ))" "$SERVER_TYPE"
-  [ "$(jq .fail "$WORK/summary.json")" -eq 0 ]
-}
-
-main() {
-  mkdir -p "$WORK"
-  : > "$WORK/vm.out"
-  HCLOUD_TOKEN=$(cat "$HCLOUD_TOKEN_FILE")
-  export HCLOUD_TOKEN
-  trap sweep EXIT
-  trap 'log "TERM received, tearing down"; exit 143' TERM INT
-  sweep
-  build_plan
-  if [ "$(jq '[.[] | select(.status == "VERIFY")] | length' "$WORK/plan.json")" -gt 0 ]; then
-    if create_vm && wait_ssh; then
-      run_vm
-    else
-      log "VM setup failed; every VERIFY entry will report no result"
-    fi
-  fi
-  summarize
-}
-
-if [ "${BASH_SOURCE[0]}" = "$0" ]; then main "$@"; fi
@@ -1,21 +0,0 @@
-# kopia `snapshot list --json` -> verdict per $plan entry (one namespace).
-(map({key: .id, value: .}) | from_entries) as $m
-| [ $plan[] as $e
-    | $m[$e.snapshotID] as $s
-    | if $s == null then
-        $e + {status: "FAIL", reason: "snapshot not in repository"}
-      else
-        ($s.stats.totalSize // $s.rootEntry.summ.size) as $kb
-        | $e + {rootID: ($s.rootEntry.obj // ""), kopiaBytes: $kb}
-        | if ($s.incomplete // "") != "" then
-            . + {status: "FAIL", reason: "snapshot incomplete: \($s.incomplete)"}
-          elif ($s.rootEntry.summ.numFailed // 0) > 0 then
-            . + {status: "FAIL", reason: "\($s.rootEntry.summ.numFailed) entries failed at backup time"}
-          elif .rootID == "" then
-            . + {status: "FAIL", reason: "snapshot has no root object"}
-          elif $kb != $e.totalBytes then
-            . + {status: "FAIL", reason: "size mismatch kopia=\($kb) velero=\($e.totalBytes)"}
-          else
-            . + {status: "VERIFY"}
-          end
-      end ]
@@ -1,37 +0,0 @@
-# Velero objects -> plan entries. Input: {schedules, backups, datauploads}
-# (kubectl .items). $only: namespaces to keep; [] keeps all.
-def finished: . as $p | ["Completed", "PartiallyFailed", "Failed", "FailedValidation"] | index($p) != null;
-
-.backups as $backups
-| .datauploads as $dus
-| [ .schedules | sort_by(.metadata.name)[]
-    | .metadata.name as $s
-    | ([ $backups[]
-         | select(.metadata.labels["velero.io/schedule-name"] == $s)
-         | select((.status.phase // "") | finished) ]
-       | sort_by(.metadata.creationTimestamp) | last) as $b
-    | if $b == null then
-        {schedule: $s, status: "FAIL", reason: "no finished backup"}
-      elif $b.status.phase != "Completed" then
-        {schedule: $s, backup: $b.metadata.name, status: "FAIL",
-         reason: "latest finished backup phase=\($b.status.phase)"}
-      else
-        [ $dus[] | select(.metadata.labels["velero.io/backup-name"] == $b.metadata.name) ] as $mine
-        | if ($mine | length) == 0 then
-            {schedule: $s, backup: $b.metadata.name, status: "SKIP", reason: "no DataUploads"}
-          else
-            $mine | sort_by(.spec.sourceNamespace, .spec.sourcePVC)[]
-            | {schedule: $s, backup: $b.metadata.name,
-               namespace: .spec.sourceNamespace, pvc: .spec.sourcePVC,
-               snapshotID: (.status.snapshotID // ""),
-               totalBytes: (.status.progress.totalBytes // 0)}
-              + if .status.phase != "Completed" then
-                  {status: "FAIL", reason: "DataUpload phase=\(.status.phase)"}
-                elif (.status.snapshotID // "") == "" then
-                  {status: "FAIL", reason: "DataUpload has no snapshotID"}
-                else
-                  {status: "VERIFY"}
-                end
-          end
-      end ]
-| if ($only | length) > 0 then map(select(.namespace as $n | $only | index($n) != null)) else . end
@@ -1,15 +0,0 @@
-# {plan, results} -> {entries, pass, fail, skip}.
-(.results | map({key: "\(.namespace)/\(.pvc)/\(.snapshotID)", value: .}) | from_entries) as $r
-| [ .plan[]
-    | if .status == "VERIFY" then
-        $r["\(.namespace)/\(.pvc)/\(.snapshotID)"] as $x
-        | if $x == null then . + {status: "FAIL", reason: "no result from VM"}
-          else . + {status: $x.status, reason: $x.reason} end
-      else . end ] as $e
-| {entries: $e,
-   pass: ([$e[] | select(.status == "OK")] | length),
-   fail: ([$e[] | select(.status == "FAIL")] | length),
-   skip: ([$e[] | select(.status == "SKIP")] | length)}
-| if (.pass + .fail) == 0 then
-    .fail += 1 | .entries += [{status: "FAIL", reason: "nothing verified"}]
-  else . end
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "rootID": "k1234", "kopiaBytes": 5005752, "status": "FAIL", "reason": "2 entries failed at backup time"}]
@@ -1 +0,0 @@
-[{"id": "57d5999303c1bf59e9f8b984d7e71f83", "stats": {"totalSize": 5005752}, "rootEntry": {"obj": "k1234", "summ": {"size": 5005752, "numFailed": 2}}}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "VERIFY"}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "rootID": "k1234", "kopiaBytes": 5005752, "status": "FAIL", "reason": "snapshot incomplete: checkpoint"}]
@@ -1 +0,0 @@
-[{"id": "57d5999303c1bf59e9f8b984d7e71f83", "incomplete": "checkpoint", "stats": {"totalSize": 5005752}, "rootEntry": {"obj": "k1234", "summ": {"size": 5005752}}}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "VERIFY"}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "FAIL", "reason": "snapshot not in repository"}]
@@ -1 +0,0 @@
-[{"id": "ffff", "rootEntry": {"obj": "kold", "summ": {"size": 1}}}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "VERIFY"}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "rootID": "k1234", "kopiaBytes": 5005752, "status": "VERIFY"}]
@@ -1,4 +0,0 @@
-[
-  {"id": "ffff", "rootEntry": {"obj": "kold", "summ": {"size": 1, "numFailed": 0}}},
-  {"id": "57d5999303c1bf59e9f8b984d7e71f83", "stats": {"totalSize": 5005752}, "rootEntry": {"obj": "k1234", "summ": {"size": 5005752, "numFailed": 0}}}
-]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "VERIFY"}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "rootID": "k1234", "kopiaBytes": 5000000, "status": "FAIL", "reason": "size mismatch kopia=5000000 velero=5005752"}]
@@ -1 +0,0 @@
-[{"id": "57d5999303c1bf59e9f8b984d7e71f83", "stats": {"totalSize": 5000000}, "rootEntry": {"obj": "k1234", "summ": {"size": 5000000, "numFailed": 0}}}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "VERIFY"}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "rootID": "k1234", "kopiaBytes": 5005752, "status": "VERIFY"}]
@@ -1 +0,0 @@
-[{"id": "57d5999303c1bf59e9f8b984d7e71f83", "rootEntry": {"obj": "k1234", "summ": {"size": 5005752, "numFailed": 0}}}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "VERIFY"}]
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "apps-daily-2", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "s1", "totalBytes": 5, "status": "VERIFY"}]
@@ -1,11 +0,0 @@
-{
-  "schedules": [{"metadata": {"name": "apps-daily"}}, {"metadata": {"name": "gitea-daily"}}],
-  "backups": [
-    {"metadata": {"name": "apps-daily-2", "creationTimestamp": "2026-10-10T07:15:23Z", "labels": {"velero.io/schedule-name": "apps-daily"}}, "status": {"phase": "Completed"}},
-    {"metadata": {"name": "g-1", "creationTimestamp": "2026-10-10T07:45:00Z", "labels": {"velero.io/schedule-name": "gitea-daily"}}, "status": {"phase": "Completed"}}
-  ],
-  "datauploads": [
-    {"metadata": {"labels": {"velero.io/backup-name": "apps-daily-2"}}, "spec": {"sourceNamespace": "apps", "sourcePVC": "gatus-data"}, "status": {"phase": "Completed", "snapshotID": "s1", "progress": {"totalBytes": 5}}},
-    {"metadata": {"labels": {"velero.io/backup-name": "g-1"}}, "spec": {"sourceNamespace": "gitea", "sourcePVC": "data-gitea-0"}, "status": {"phase": "Completed", "snapshotID": "s2", "progress": {"totalBytes": 6}}}
-  ]
-}
@@ -1,4 +0,0 @@
-[
-  {"schedule": "gitea-daily", "backup": "g-1", "namespace": "gitea", "pvc": "data-gitea-0", "snapshotID": "", "totalBytes": 0, "status": "FAIL", "reason": "DataUpload phase=Failed"},
-  {"schedule": "gitea-daily", "backup": "g-1", "namespace": "gitea", "pvc": "data-gitea-1", "snapshotID": "", "totalBytes": 3, "status": "FAIL", "reason": "DataUpload has no snapshotID"}
-]
@@ -1,8 +0,0 @@
-{
-  "schedules": [{"metadata": {"name": "gitea-daily"}}],
-  "backups": [{"metadata": {"name": "g-1", "creationTimestamp": "2026-10-10T07:45:00Z", "labels": {"velero.io/schedule-name": "gitea-daily"}}, "status": {"phase": "Completed"}}],
-  "datauploads": [
-    {"metadata": {"labels": {"velero.io/backup-name": "g-1"}}, "spec": {"sourceNamespace": "gitea", "sourcePVC": "data-gitea-0"}, "status": {"phase": "Failed", "progress": {}}},
-    {"metadata": {"labels": {"velero.io/backup-name": "g-1"}}, "spec": {"sourceNamespace": "gitea", "sourcePVC": "data-gitea-1"}, "status": {"phase": "Completed", "progress": {"totalBytes": 3}}}
-  ]
-}
@@ -1 +0,0 @@
-[]
@@ -1 +0,0 @@
-{"schedules": [], "backups": [], "datauploads": []}
@@ -1 +0,0 @@
-[{"schedule": "apps-daily", "backup": "apps-daily-2", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "status": "VERIFY"}]
@@ -1,11 +0,0 @@
-{
-  "schedules": [{"metadata": {"name": "apps-daily"}}],
-  "backups": [
-    {"metadata": {"name": "apps-daily-1", "creationTimestamp": "2026-10-09T07:15:23Z", "labels": {"velero.io/schedule-name": "apps-daily"}}, "status": {"phase": "Completed"}},
-    {"metadata": {"name": "apps-daily-2", "creationTimestamp": "2026-10-10T07:15:23Z", "labels": {"velero.io/schedule-name": "apps-daily"}}, "status": {"phase": "Completed"}}
-  ],
-  "datauploads": [
-    {"metadata": {"labels": {"velero.io/backup-name": "apps-daily-1"}}, "spec": {"sourceNamespace": "apps", "sourcePVC": "gatus-data"}, "status": {"phase": "Completed", "snapshotID": "old", "progress": {"totalBytes": 1}}},
-    {"metadata": {"labels": {"velero.io/backup-name": "apps-daily-2"}}, "spec": {"sourceNamespace": "apps", "sourcePVC": "gatus-data"}, "status": {"phase": "Completed", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "progress": {"totalBytes": 5005752}}}
-  ]
-}
@@ -1 +0,0 @@
-[{"schedule": "wordpress-daily", "backup": "wp-1", "namespace": "wordpress", "pvc": "wordpress-content", "snapshotID": "aaa", "totalBytes": 10, "status": "VERIFY"}]
@@ -1,10 +0,0 @@
-{
-  "schedules": [{"metadata": {"name": "wordpress-daily"}}],
-  "backups": [
-    {"metadata": {"name": "wp-1", "creationTimestamp": "2026-10-09T08:45:00Z", "labels": {"velero.io/schedule-name": "wordpress-daily"}}, "status": {"phase": "Completed"}},
-    {"metadata": {"name": "wp-2", "creationTimestamp": "2026-10-10T08:45:00Z", "labels": {"velero.io/schedule-name": "wordpress-daily"}}, "status": {"phase": "InProgress"}}
-  ],
-  "datauploads": [
-    {"metadata": {"labels": {"velero.io/backup-name": "wp-1"}}, "spec": {"sourceNamespace": "wordpress", "sourcePVC": "wordpress-content"}, "status": {"phase": "Completed", "snapshotID": "aaa", "progress": {"totalBytes": 10}}}
-  ]
-}
@@ -1 +0,0 @@
-[{"schedule": "new-daily", "status": "FAIL", "reason": "no finished backup"}]
@@ -1 +0,0 @@
-{"schedules": [{"metadata": {"name": "new-daily"}}], "backups": [], "datauploads": []}
@@ -1 +0,0 @@
-[{"schedule": "authentik-daily", "backup": "ak-1", "status": "SKIP", "reason": "no DataUploads"}]
@@ -1,5 +0,0 @@
-{
-  "schedules": [{"metadata": {"name": "authentik-daily"}}],
-  "backups": [{"metadata": {"name": "ak-1", "creationTimestamp": "2026-10-10T04:15:00Z", "labels": {"velero.io/schedule-name": "authentik-daily"}}, "status": {"phase": "Completed"}}],
-  "datauploads": []
-}
@@ -1 +0,0 @@
-[{"schedule": "immich-daily", "backup": "im-1", "status": "FAIL", "reason": "latest finished backup phase=PartiallyFailed"}]
@@ -1,5 +0,0 @@
-{
-  "schedules": [{"metadata": {"name": "immich-daily"}}],
-  "backups": [{"metadata": {"name": "im-1", "creationTimestamp": "2026-10-10T05:15:00Z", "labels": {"velero.io/schedule-name": "immich-daily"}}, "status": {"phase": "PartiallyFailed"}}],
-  "datauploads": []
-}
@@ -1,7 +0,0 @@
-{
-  "entries": [
-    {"schedule": "apps-daily", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "s1", "status": "OK", "reason": "verified 5 bytes"},
-    {"schedule": "authentik-daily", "status": "SKIP", "reason": "no DataUploads"}
-  ],
-  "pass": 1, "fail": 0, "skip": 1
-}
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`[{"schedule": "apps-daily", "backup": "b", "namespace": "apps", "pvc": "gatus-data", "snapshotID": "57d5999303c1bf59e9f8b984d7e71f83", "totalBytes": 5005752, "rootID": "k1234", "kopiaBytes": 5005752, "status": "FAIL", "reason": "2 entries failed at backup time"}]`
				`@@ -1 +0,0 @@`
				`[{"id": "57d5999303c1bf59e9f8b984d7e71f83", "stats": {"totalSize": 5005752}, "rootEntry": {"obj": "k1234", "summ": {"size": 5005752, "numFailed": 2}}}]`
				`@@ -1 +0,0 @@`
				`[{"id": "ffff", "rootEntry": {"obj": "kold", "summ": {"size": 1}}}]`
				`@@ -1 +0,0 @@`
				`{"schedules": [], "backups": [], "datauploads": []}`
				`@@ -1 +0,0 @@`
				`[{"schedule": "wordpress-daily", "backup": "wp-1", "namespace": "wordpress", "pvc": "wordpress-content", "snapshotID": "aaa", "totalBytes": 10, "status": "VERIFY"}]`
				`@@ -1 +0,0 @@`
				`[{"schedule": "new-daily", "status": "FAIL", "reason": "no finished backup"}]`
				`@@ -1 +0,0 @@`
				`{"schedules": [{"metadata": {"name": "new-daily"}}], "backups": [], "datauploads": []}`
				`@@ -1 +0,0 @@`
				`[{"schedule": "authentik-daily", "backup": "ak-1", "status": "SKIP", "reason": "no DataUploads"}]`
				`@@ -1 +0,0 @@`
				`[{"schedule": "immich-daily", "backup": "im-1", "status": "FAIL", "reason": "latest finished backup phase=PartiallyFailed"}]`