fix: [alert] CephOSDNearfullApproaching (cluster) (#1089 )

Merge pull request 'monitoring: stop wear alerts on EPHEMERAL NVMe drives (REN-419)' (#1088 ) from atlas/ren-419-ephemeral-wear-alert into master
monitoring: feed disk-wear case A at 5m so the 6h for completes (REN-438)
2026-10-01 09:44:40 +00:00 · 2026-10-01 09:19:18 +00:00 · 2026-10-01 09:07:58 +00:00 · 2026-10-01 09:03:17 +00:00 · 2026-10-01 05:40:32 +00:00 · 2026-09-30 23:19:33 +00:00
239 changed files with 13984 additions and 1528 deletions
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -0,0 +1,66 @@
+# Paperclip agent runtime images (REN-284 / REN-286): the stock
+# agent-runtime-claude plus a tool set, one image per sandbox environment.
+# Built with kaniko like build-ci-images.yml and pushed with the same
+# ci_images_token (package read+write only, see the header there).
+#
+# Own workflow, not a step in build-ci-images.yml. A change to that file
+# rebuilds all of its images at once and fills the 24-CPU woodpecker-ci-quota;
+# on the REN-286 merge (pipeline 11306) this step then got no pod
+# ("exceeded quota") and the go image was never built. Here it runs only when
+# its own sources change, and it can be re-run alone with a deployment to
+# target `build-agent-runtime`, without rebuilding the other images.
+#
+# The name looks odd on purpose. The kubernetes sandbox plugin keeps the image
+# name and tag `agent-runtime-claude:v1` and swaps only the registry prefix
+# (`imageRegistry`), replacing host + org. So each variant needs its own
+# prefix with the same name under it:
+#   imageRegistry gitea.renner.uno/konairius/agent-runtime-go
+#   -> gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1
+# `v1` is what the plugin pulls and moves on every build; the date tag is the
+# rollback point. konairius/ because the ci_images_token can push there and
+# it is the target of the gitea_admin/* migration (#330).
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/agent-runtime-*/**"
+      - ".woodpecker/build-agent-runtime.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-agent-runtime"'
+
+steps:
+  - name: build-agent-runtime
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images
+    # Context is ci/images (not the variant dir) so both Containerfiles can
+    # COPY the shared agent-runtime-common/ files.
+    # One pod, both images one after the other (kaniko --cleanup between
+    # builds), at 1 CPU so it still fits if it overlaps another pipeline.
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: "1"
+            memory: 4Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        for VARIANT in go ops; do
+          /kaniko/executor \
+            --cleanup \
+            --context . \
+            --dockerfile "agent-runtime-$VARIANT/Containerfile" \
+            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:$TAG" \
+            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:v1" \
+            || exit 1
+        done
@@ -26,8 +26,14 @@
 # 38-reference change tracked separately in #330 rather than bundled here.
 # Note this pipeline runs no LLM agent, so the exfil path that motivated the
 # other cutovers does not apply to it.
+#
+# Push builds run on master only. Without the branch filter a feature branch
+# that touches this file started all steps at once: it pushed unreviewed
+# images and used the whole woodpecker-ci-quota, so the PR checks could not
+# get a pod (REN-286).
 when:
  - event: push
+    branch: master
    path:
      - "ci/images/**"
      - "secret-rotator/**"
@@ -350,17 +356,17 @@ steps:
          --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
          --destination gitea.renner.uno/gitea_admin/heating-can:latest

-  # netbird-proxy-services: the controller that reconciles NetBird reverse-proxy
+  # netbird-service-sync: the controller that reconciles NetBird reverse-proxy
  # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
  # Runtime artifact like heating-can; same registry so the audit tracks it.
-  - name: build-netbird-proxy-services
+  - name: build-netbird-service-sync
    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
-    directory: ci/images/netbird-proxy-services
+    directory: ci/images/netbird-service-sync
    depends_on: []
    when:
      - event: deployment
      - event: push
-        path: ["ci/images/netbird-proxy-services/**", ".woodpecker/build-ci-images.yml"]
+        path: ["ci/images/netbird-service-sync/**", ".woodpecker/build-ci-images.yml"]
    environment:
      GITEA_USER: gitea_admin
      GITEA_TOKEN:
@@ -374,39 +380,8 @@ steps:
        /kaniko/executor \
          --context . \
          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
-          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
-
-  # wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea and
-  # queried by ~/Notes/bin/wiki_rank.py. Runtime artifact like netbird-proxy-services.
-  - name: build-wiki-index
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
-    directory: ci/images/wiki-index
-    depends_on: []
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/wiki-index/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      # Also a per-commit tag: several builds a day (branch pushes and the
-      # merge) all rewrite the dated tag, and the deployment pin raced them
-      # on 2026-09-20. The commit tag is immutable and needs no digest lookup.
-      - SHORT=$(printf '%s' "$CI_COMMIT_SHA" | cut -c1-8)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/wiki-index:$TAG \
-          --destination gitea.renner.uno/gitea_admin/wiki-index:$TAG-$SHORT \
-          --destination gitea.renner.uno/gitea_admin/wiki-index:latest
+          --destination gitea.renner.uno/gitea_admin/netbird-service-sync:$TAG \
+          --destination gitea.renner.uno/gitea_admin/netbird-service-sync:latest

  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
  # artifact like heating-can; same registry so the audit tracks it.
@@ -0,0 +1,69 @@
+# Paperclip server image (REN-401): upstream paperclipai/paperclip at a pinned
+# master commit plus the patches in ci/images/paperclip/. Needed while the k8s
+# sandbox fix (upstream PR #13733) is not merged; see the README there for how
+# to go back to upstream nightly.
+#
+# Own workflow for the reason given in build-agent-runtime.yml: a step in
+# build-ci-images.yml runs on every change to that file and competes for the
+# woodpecker-ci-quota. Re-run alone with a deployment to target
+# `build-paperclip`.
+#
+# Tag `build.<unix-ts>` is what the Flux ImagePolicy `paperclip` sorts on
+# (image-automation/paperclip.yaml); image automation then rolls it out.
+# konairius/ and ci_images_token as for the agent-runtime images.
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/paperclip/**"
+      - ".woodpecker/build-paperclip.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-paperclip"'
+
+steps:
+  - name: prepare-source
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh ci/images/paperclip/prepare.sh build/paperclip-src
+
+  - name: build-paperclip
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: build/paperclip-src
+    depends_on: [prepare-source]
+    # Rust runner + UI + server build in one pod. The server build alone sets
+    # a 4 GiB Node heap, so the memory limit is the LimitRange max (8Gi).
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: "1"
+            memory: 2Gi
+          limits:
+            cpu: "4"
+            memory: 8Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - . ../../ci/images/paperclip/upstream.env
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=build.$(date +%s)
+      # Same target and build args as upstream's docker.yml for nightly.
+      # --compressed-caching=false: pipeline 11717 was killed at the 8Gi limit
+      # (exit 137) in the server build; kaniko otherwise holds compressed
+      # layer tarballs in memory next to the 4 GiB Node heap.
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Dockerfile.kaniko \
+          --target production \
+          --skip-unused-stages \
+          --snapshot-mode=redo \
+          --compressed-caching=false \
+          --build-arg PAPERCLIP_BUILD_COMMIT=$PAPERCLIP_COMMIT \
+          --build-arg PAPERCLIP_BUILD_VERSION=$PAPERCLIP_VERSION \
+          --build-arg CLI_TOOLS_CACHE_EPOCH=$(date -u +%G-W%V) \
+          --destination gitea.renner.uno/konairius/paperclip:$TAG
@@ -14,8 +14,8 @@
 #                 and  ci-claude:latest.  The Containerfile makes this layer
 #                 byte-reproducible, so rebuilding an unchanged version dedups
 #                 in the registry instead of orphaning a fresh ~260 MB copy.
-#   3. bump-pr  — (only when the version actually changed) rewrite the 5
-#                 consumer pins to  ci-claude:<version>  and open a PR. A human
+#   3. bump-pr  — (only when any pin is off the latest version) rewrite every
+#                 consumer pin to  ci-claude:<version>  and open a PR. A human
 #                 reviews and merges it; nothing auto-merges.
 #
 # TAG SCHEME — deliberately version tags (ci-claude:<claude-version>), NOT the dated
@@ -107,31 +107,22 @@ steps:
          [0-9]*.[0-9]*.[0-9]*) : ;;
          *) echo "npm returned an implausible version '$$NEW' — aborting."; exit 1 ;;
        esac
-        # The tag every consumer currently references. They are kept identical,
-        # so this must resolve to EXACTLY ONE non-empty tag; anything else
-        # (no match, or a partial/disagreeing set) aborts rather than risk a
-        # corrupting sed later. Counting with `grep -c .` (non-empty lines)
-        # instead of `wc -l` is deliberate: an empty result yields 0 here but
-        # 1 under wc -l, which would slip past the guard.
-        CUR=$$(grep -hoE 'ci-claude:[0-9A-Za-z._-]+' $$CONSUMERS | sed 's/^ci-claude://' | sort -u)
-        NTAGS=$$(printf '%s' "$$CUR" | grep -c . || true)
-        if [ -z "$$CUR" ] || [ "$$NTAGS" -ne 1 ]; then
-          echo "Expected exactly one ci-claude tag across the consumers, got [$$CUR]."
-          echo "Refusing to guess — fix the pins by hand."; exit 1
-        fi
-        echo "latest=$$NEW  current=$$CUR  event=$$CI_PIPELINE_EVENT"
+        # Distinct tags the consumers currently reference. Divergent pins are
+        # normal (a pipeline bumped on its own); the bump rewrites all of them.
+        CUR=$$(grep -hoE 'ci-claude:[0-9][0-9A-Za-z._-]*' $$CONSUMERS | sed 's/^ci-claude://' | sort -u | tr '\n' ' ' | sed 's/ $$//')
+        [ -n "$$CUR" ] || { echo "No ci-claude pin found in the consumers — aborting."; exit 1; }
+        echo "latest=$$NEW  current=[$$CUR]  event=$$CI_PIPELINE_EVENT"
        printf '%s' "$$NEW" > "$$CI_WORKSPACE/.ci-refresh/version"
        printf '%s' "$$CUR" > "$$CI_WORKSPACE/.ci-refresh/current"
-        # Build when the version changed, or on a push (Containerfile edit must
-        # ship even if the version is unchanged). Bump the pins only when the
-        # version the consumers reference is not the latest.
+        # Build when any pin is off latest, or on a push (Containerfile edit must
+        # ship even if the version is unchanged). Bump when any pin is off latest.
        if [ "$$CUR" != "$$NEW" ] || [ "$$CI_PIPELINE_EVENT" = "push" ]; then
          touch "$$CI_WORKSPACE/.ci-refresh/do-build"; echo "will build $$NEW"
        else
          echo "already on $$NEW — nothing to build"
        fi
        if [ "$$CUR" != "$$NEW" ]; then
-          touch "$$CI_WORKSPACE/.ci-refresh/do-bump"; echo "will open bump PR $$CUR -> $$NEW"
+          touch "$$CI_WORKSPACE/.ci-refresh/do-bump"; echo "will open bump PR [$$CUR] -> $$NEW"
        else
          echo "consumers already pin $$NEW — no PR"
        fi
@@ -175,6 +166,9 @@ steps:
      - |
        [ -f "$$CI_WORKSPACE/.ci-refresh/do-bump" ] || { echo "no version change — no PR"; exit 0; }
        set -eu
+        # Secret values can carry a trailing newline; it breaks the push URL and the auth header.
+        PUSH_TOKEN=$$(printf '%s' "$$PUSH_TOKEN" | tr -d '\r\n')
+        [ -n "$$PUSH_TOKEN" ] || { echo "PUSH_TOKEN is empty — aborting"; exit 1; }
        NEW=$$(cat "$$CI_WORKSPACE/.ci-refresh/version")
        CUR=$$(cat "$$CI_WORKSPACE/.ci-refresh/current")
        # Both were validated non-empty by the decide step; re-assert before a
@@ -190,9 +184,11 @@ steps:
        # is not a maintained list.
        CONSUMERS=$$(grep -rlE 'ci-claude:[0-9]' .woodpecker)
        [ -n "$$CONSUMERS" ] || { echo "No ci-claude consumers found — refusing to continue."; exit 1; }
-        # Exact-string swap of the current tag for the new one, consumers only.
+        # NEW goes into a sed replacement: allow only a plain x.y.z version.
+        printf '%s' "$$NEW" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$$' || { echo "refusing non-x.y.z version '$$NEW'"; exit 1; }
+        # Every version-pinned tag becomes NEW, whatever it was; consumers only.
        for f in $$CONSUMERS; do
-          sed -i "s#ci-claude:$$CUR#ci-claude:$$NEW#g" "$$f"
+          sed -i "s#ci-claude:[0-9][0-9A-Za-z._-]*#ci-claude:$$NEW#g" "$$f"
        done
        # Author is claude-bot (the PAT's account); the commit subject makes
        # the automated-bump nature clear.
@@ -17,30 +17,44 @@
 # here is a static check that either passes or fails on the checkout alone, so it
 # can gate PRs without becoming a bottleneck or a flake source.
 when:
-  # Path-filtered on purpose, unlike ci-scripts-test.yml. That pipeline is one
-  # step on one already-cached image, so running it everywhere is free; this one
-  # pulls three images and builds every kustomization in the tree, which is real
+  # Path-filtered on purpose, unlike ci-scripts-test.yml. That pipeline is two
+  # steps on already-cached images, so running it everywhere is free; this one
+  # pulls four images and builds every kustomization in the tree, which is real
  # cost to put on a docs-only PR. The filter covers everything the steps actually read:
-  # kubernetes/** for the manifests and scripts, .woodpecker/** for the lint (and
-  # so an edit to this file re-runs its own checks).
+  # kubernetes/** for the manifests, scripts and promtool suites, .woodpecker/**
+  # for the lint (and so an edit to this file re-runs its own checks), and the
+  # promtool driver script by name.
+  #
+  # That last entry is NOT covered by kubernetes/**: promtool-rules.sh lives
+  # under scripts/, so without it a PR that only edits the script would never run
+  # the check it drives. It is listed as one file rather than scripts/ci/**
+  # because the rest of that directory is CI-agent helpers with nothing to do
+  # with this pipeline, and pulling four images for them is the cost this filter
+  # exists to avoid. Their own tests run in ci-scripts-test.yml on every PR.
  - event: pull_request
    path:
      - "kubernetes/**"
      - ".woodpecker/**"
+      - "scripts/ci/promtool-rules.sh"
+      - "scripts/ci/yaml-citations.sh"

-# ADDING A THIRD DISCOVERY CHECK: kustomize-build and shellcheck both hand-roll
-# the same shape — `find`, a floor guard against empty discovery, then a
-# file-fed `while read`. Two copies is deliberate: the two traps being commented
-# around (BusyBox find lacking `-printf`, and a piped loop running in a subshell
-# where `fail` cannot propagate) are different, and both are worth spelling out
-# where they bite. A third instance is the point to stop copying and factor it
-# into a shared script under `scripts/ci/` — which is already covered by
-# ci-scripts-test.yml, so it would gain unit tests in the process.
+# THE THIRD DISCOVERY CHECK WENT TO A SCRIPT. kustomize-build and shellcheck
+# both hand-roll the same shape — `find`, a floor guard against empty discovery,
+# then a file-fed `while read`. Two copies was deliberate: the two traps being
+# commented around (BusyBox find lacking `-printf`, and a piped loop running in
+# a subshell where `fail` cannot propagate) are different, and both are worth
+# spelling out where they bite. An earlier revision of this comment said a third
+# instance would be the point to stop copying and factor it into `scripts/ci/`,
+# where it would gain unit tests — and that is what `promtool` below does. Its
+# discovery, floors and guards live in scripts/ci/promtool-rules.sh and are
+# tested by scripts/ci/test-promtool-rules.sh on every PR via
+# ci-scripts-test.yml. A FOURTH inline copy should go the same way; do not add
+# one here.
 #
-# Every step carries `depends_on: []` so all four run in PARALLEL and every
+# Every step carries `depends_on: []` so all six run in PARALLEL and every
 # failure shows in one run. Woodpecker's default is sequential, which would stop
 # at the first failure and make a contributor fix-and-repush to discover the
-# next — and these four are independent, cheap, offline checks with no ordering
+# next — and these six are independent, cheap, offline checks with no ordering
 # between them. Keep it on every step: `depends_on` is per-step, so dropping it
 # from one silently re-serialises that step behind the others.
 steps:
@@ -258,3 +272,72 @@ steps:
          exit 1
        fi
        echo "ok    no digest pins under openbao/rotation/"
+
+  # Machine-check the alert rules: parse every PrometheusRule, run the committed
+  # promtool unit suites, then damage the rules on purpose and require the suites
+  # to notice. Everything it does lives in scripts/ci/promtool-rules.sh; read
+  # that file and kubernetes/apps/monitoring-rules/tests/README.md for the why.
+  #
+  # The gap it closes: the other four steps prove a PrometheusRule assembles into
+  # valid YAML and nothing more — not that an expression parses, not that a label
+  # join matches anything, not that a `for` window does what its comment claims.
+  # On REN-129 the CNPG surge rule was written `for: 2m`, which fires on six of
+  # seven measured nights and would have mailed the owner twice a night at 04:16
+  # and 04:22 local for a backlog that clears itself. Nothing here would have
+  # caught it, and the arithmetic that did catch it was hand-written from scratch
+  # twice, in two runs, and thrown away both times because there was nowhere in
+  # the repo to commit it. REN-152 retired that rule; its `for` margin is now
+  # guarded by mutations.txt:backlog-for-12m.
+  #
+  # Offline, like the rest of this pipeline: no cluster, no registry, no secrets,
+  # no network. promtool reads files and exits.
+  #
+  # IMAGE — prom/prometheus, not a registry-local ci-* image, and pinned to an
+  # exact upstream version rather than a dated tag. The dated-tag convention
+  # guarded by no-digest-pins above is about OUR images, whose digests are
+  # garbage-collected out from under a pin; this is upstream and immutable.
+  #
+  # The version must be 3.x, and that is not a preference. The cluster runs
+  # whatever kube-prometheus-stack >= 72.0.0 ships, which is Prometheus 3, and
+  # alert `for` evaluation is version-sensitive — a 2.x promtool would be
+  # checking a different evaluator than production, so the margins the suites
+  # assert would be claims about the wrong program. promtool-rules.sh asserts the
+  # major version it was handed and fails on 2.x, so a careless bump here is a
+  # red step rather than a quietly wrong check. Keep this at the latest stable
+  # 3.x, per the standing rule that any software uses the latest stable version.
+  #
+  # This image is NOT covered by image-automation (same reason as
+  # woodpecker-lint above: a Woodpecker pipeline file has no apiVersion/kind, so
+  # an $imagepolicy marker here is silently ignored — fluxcd/flux2#2826). The
+  # bump is manual.
+  #
+  # `prom/prometheus` is busybox-based and ships /bin/sh, so it can run a
+  # `commands:` block — unlike the plain woodpecker-cli tag, which cannot. Its
+  # find/grep/awk/sed are BusyBox, which is why promtool-rules.sh avoids
+  # `find -printf` and `grep -r --include`; it was verified against BusyBox
+  # 1.35.0 as well as GNU. The image also declares `USER nobody`, so the script
+  # writes its stripped mirror under mktemp and never into the workspace.
+  - name: promtool
+    depends_on: [] # see note above `steps:`
+    image: prom/prometheus:v3.15.0
+    commands:
+      - sh scripts/ci/promtool-rules.sh
+
+  # Comments cite other YAML blocks by anchor name, never by line number
+  # (REN-183; the exact syntax is in CONTRIBUTING.md, not repeated here because
+  # an example written in this comment would itself be checked). Line numbers drift on any edit
+  # above them and nothing noticed: three drift fixes were merged by hand
+  # before this step existed. This fails on a new line-number citation and on
+  # a citation whose file or anchor marker does not exist.
+  # Existence only — it never judges whether the anchored block still says
+  # what the citing comment claims, so ordinary edits cannot turn it red.
+  #
+  # It scans every YAML file in the checkout, but this pipeline only runs on
+  # the paths in `when:` above, so an edit confined to ansible/ or esphome/ is
+  # checked on the next PR that does touch kubernetes/ or .woodpecker/.
+  # Self-test: scripts/ci/test-yaml-citations.sh, in ci-scripts-test.yml.
+  - name: yaml-citations
+    depends_on: [] # see note above `steps:`
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh scripts/ci/yaml-citations.sh
@@ -8,15 +8,45 @@
 # against adversarial PR input. Running it here fails the offending PR's checks
 # before it can merge.
 #
+# scripts/ci/test-promtool-rules.sh is the same idea for the alert-rule gate.
+# What it tests is not whether promtool evaluates PromQL — ci-manifests-test.yml
+# does that with the real binary against the real rules — but whether
+# promtool-rules.sh REFUSES TO REPORT SUCCESS when it has checked nothing: zero
+# manifests discovered, zero suites discovered, a `rule_files` path that does not
+# resolve (promtool warns, loads zero rules, and passes every assertion), a
+# mutation the suites no longer catch. Those are the states a green pipeline
+# cannot tell apart from real coverage, so they need a test that asserts the
+# failure. It stubs promtool, so it needs neither the binary nor the heavy image.
+#
+# Both are unfiltered by path on purpose: this pipeline is two steps on one
+# already-cached image, so running it everywhere is free, and a guard whose test
+# only runs when someone remembers to touch the right directory is not a guard.
+#
 # Only in homelab (scripts/ci/ lives here); the file is not deployed to the
-# mirrored repos, so it does not run on their PRs. The test is offline (no token,
-# no network, no cluster) — the ci-claude image just provides sh + jq, and it is
-# already pulled by pr-review.yml on the same event, so the runner has it cached.
+# mirrored repos, so it does not run on their PRs. All tests here are offline (no
+# token, no network, no cluster) — the ci-claude image just provides sh + jq, and
+# it is already pulled by pr-review.yml on the same event, so the runner has it
+# cached.
 when:
  - event: pull_request

+# `depends_on: []` on every step so they run in parallel and one failure does not hide
+# the other. Same reasoning as the note above `steps:` in ci-manifests-test.yml.
 steps:
  - name: model-selection-tests
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    depends_on: []
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    commands:
      - sh scripts/ci/test-model-selection.sh
+
+  - name: promtool-rules-guard-tests
+    depends_on: []
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    commands:
+      - sh scripts/ci/test-promtool-rules.sh
+
+  - name: yaml-citations-guard-tests
+    depends_on: []
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    commands:
+      - sh scripts/ci/test-yaml-citations.sh
@@ -4,9 +4,6 @@ when:
    branch: master
    path:
      - "ansible/roles/netbird-server/**"
-      # install-netbird-server.yml also applies the netbird-proxy-watchdog role,
-      # so changes to it must redeploy this host too.
-      - "ansible/roles/netbird-proxy-watchdog/**"
      - "ansible/plays/install-netbird-server.yml"
      - ".woodpecker/deploy-netbird-server.yml"
  - event: deployment
@@ -26,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -70,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,12 +1,13 @@
 # Nightly wiki maintenance for the Notes vault (k.renner/notes), replacing the
-# workstation's wiki-maintenance.timer: distill the raw/ backlog into wiki/,
-# then audit wiki/. Each agent is its own `claude` run, not a subagent of an
-# orchestrator, so nothing depends on background tasks in print mode (the
-# workstation job died on their 600 s ceiling). The pipeline commits and pushes
-# as Claude Bot; the push webhook then refreshes the wiki-index service.
+# workstation's wiki-maintenance and wiki-consolidate timers: distill the raw/
+# backlog into wiki/ (wiki-distill agent), audit wiki/ with the vault's own
+# script (structural checks, no model), then consolidate (wiki-consolidate
+# agent: verifies a cohort of notes against sources, repos and the live
+# cluster and edits them). The pipeline commits and pushes as Claude Bot; the
+# push webhook then refreshes the wiki service.
 #
-# The agents and their retrieval scripts address the vault as ~/Notes, so each
-# agent step links $HOME/Notes to the checkout instead of rewriting them.
+# The agents address the vault as ~/Notes and the repos as ~/source/<repo>, so
+# each agent step links those into the workspace instead of rewriting them.
 when:
  - event: cron
    cron: wiki-maintenance
@@ -16,7 +17,7 @@ when:

 steps:
  - name: checkout-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -30,9 +31,30 @@ steps:
        git -c http.extraheader="AUTHORIZATION: token $$TOKEN" \
          clone --depth 50 --branch main https://gitea.renner.uno/k.renner/notes.git notes
        echo "raw backlog: $$(ls notes/raw/*.md | grep -v _index.md | wc -l) file(s)"
+        # Full history for the consolidate agent's `git log` checks; the
+        # workspace clone is shallow and tree-filtered.
+        mkdir -p sources
+        git clone --quiet https://gitea.renner.uno/k.renner/homelab.git sources/homelab
+        git -c http.extraheader="AUTHORIZATION: token $$TOKEN" \
+          clone --quiet https://gitea.renner.uno/k.renner/homeassistant.git sources/homeassistant
+
+  - name: fetch-diag
+    # Read-only live-cluster access for the consolidate agent, same mechanism
+    # as issue-fix.yml: ./.ci-diag/kubectl plus a 1h ci-diagnostics token.
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    environment:
+      SOPS_AGE_KEY:
+        from_secret: sops_age_key
+      S3_ACCESS_KEY_ID:
+        from_secret: hetzner_s3_access_key
+      S3_SECRET_ACCESS_KEY:
+        from_secret: hetzner_s3_secret_key
+    commands:
+      - bash scripts/ci/fetch-kubeconfig.sh
+      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: distill
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -44,25 +66,37 @@ steps:
    commands:
      - |
        set -eu
+        # An empty queue must not start the agent: its startup context alone
+        # costs more than the whole audit.
+        BACKLOG=$$(ls notes/raw/*.md | grep -v _index.md | wc -l)
+        if [ "$$BACKLOG" -eq 0 ]; then
+          echo "distill: raw/ is empty, skipping the agent"
+          exit 0
+        fi
        ln -sfn "$$PWD/notes" "$$HOME/Notes"
        mkdir -p "$$HOME/.config/wiki-index"
        printf '%s' "$$WIKI_INDEX_BASIC_AUTH" | tr -d '\r\n' > "$$HOME/.config/wiki-index/basic-auth"
        chmod 600 "$$HOME/.config/wiki-index/basic-auth"
        "$$HOME/Notes/bin/wiki_rank.py" --probe-index
-      - |
-        set -eu
        MODEL=$$(sh scripts/ci/pick-model.sh "distill raw source notes into atomic wiki notes: read sources, check the wiki for existing coverage, write or merge notes; careful writing, moderate reasoning")
        if [ -n "$$MODEL" ]; then echo "distill: selected model $$MODEL"; else echo "distill: using the CLI default model"; fi
        cd notes
        cat ../ci/agents/wiki-maintenance/distill.md | claude $${MODEL:+--model $$MODEL} \
          --agent wiki-distill \
          --permission-mode acceptEdits \
-          --allowedTools "Read,Edit,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*),Bash(mv /root/Notes/raw/:*),Bash(mkdir -p /root/Notes/:*)" \
+          --allowedTools "Read,Edit,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*),Bash(mv /root/Notes/raw/:*),Bash(mkdir -p /root/Notes/:*),Bash(~/Notes/bin/wiki_rank.py:*),Bash(~/Notes/bin/wiki_ask.py:*),Bash(mv ~/Notes/raw/:*),Bash(mkdir -p ~/Notes/:*)" \
          --settings ../ci/agents/wiki-maintenance/settings.json \
          -p

  - name: audit
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    pull: true
+    commands:
+      # Findings are report lines in output/, not a pipeline failure.
+      - python3 -B notes/bin/wiki_audit.py --vault "$$PWD/notes"
+
+  - name: consolidate
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -75,23 +109,43 @@ steps:
      - |
        set -eu
        ln -sfn "$$PWD/notes" "$$HOME/Notes"
+        ln -sfn "$$PWD/sources" "$$HOME/source"
        mkdir -p "$$HOME/.config/wiki-index"
        printf '%s' "$$WIKI_INDEX_BASIC_AUTH" | tr -d '\r\n' > "$$HOME/.config/wiki-index/basic-auth"
        chmod 600 "$$HOME/.config/wiki-index/basic-auth"
-      - |
-        set -eu
-        MODEL=$$(sh scripts/ci/pick-model.sh "structural audit of ~1700 markdown notes: frontmatter completeness, broken links, orphans; mechanical checks, little reasoning")
-        if [ -n "$$MODEL" ]; then echo "audit: selected model $$MODEL"; else echo "audit: using the CLI default model"; fi
+        if [ -f .ci-diag/kubeconfig ]; then
+          ln -sf "$$PWD/.ci-diag/kubectl" /usr/local/bin/kubectl
+          export KUBECONFIG="$$PWD/.ci-diag/kubeconfig"
+        else
+          echo "consolidate: no ci-diag kubeconfig, live cluster checks unavailable"
+        fi
        cd notes
-        cat ../ci/agents/wiki-maintenance/audit.md | claude $${MODEL:+--model $$MODEL} \
-          --agent wiki-audit \
+        # JSON output: the agent's report and the run's cost come out of one
+        # record at the end instead of a 45-minute silent log.
+        # One source for the Bash allowlist: the settings file (run 10608: the
+        # agent writes ~/Notes, the rules carried only /root/Notes).
+        BASH_RULES=$$(jq -r '[.permissions.allow[] | select(startswith("Bash("))] | join(",")' ../ci/agents/wiki-maintenance/consolidate-settings.json)
+        rc=0
+        cat ../ci/agents/wiki-maintenance/consolidate.md | claude \
+          --agent wiki-consolidate \
          --permission-mode acceptEdits \
-          --allowedTools "Read,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*)" \
-          --settings ../ci/agents/wiki-maintenance/settings.json \
-          -p
+          --allowedTools "Read,Edit,Write,Glob,Grep,Agent,$$BASH_RULES" \
+          --settings ../ci/agents/wiki-maintenance/consolidate-settings.json \
+          --output-format json \
+          -p > /tmp/consolidate.json || rc=$$?
+        # A killed or crashed claude leaves no valid JSON; report rc, not a jq error.
+        if [ -s /tmp/consolidate.json ] && jq -e . /tmp/consolidate.json > /dev/null; then
+          jq -r '.result // empty' /tmp/consolidate.json
+          jq -c '{duration_ms, num_turns, total_cost_usd, is_error}' /tmp/consolidate.json
+        else
+          echo "consolidate: no result record (exit $$rc)"; head -c 2000 /tmp/consolidate.json || true
+        fi
+        echo "consolidate: git log --oneline of the run:"
+        git log --oneline origin/main..HEAD
+        exit $$rc

  - name: push-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -102,7 +156,13 @@ steps:
        cd notes
        git config user.email "claude-bot@renner.uno"
        git config user.name "Claude Bot"
-        if git diff --quiet && git diff --cached --quiet && [ -z "$$(git ls-files --others --exclude-standard)" ]; then
+        # Only the vault content is committed; anything an agent leaves elsewhere
+        # in the checkout (run 10362: scratch scripts in the root) stays behind.
+        # The consolidate agent commits its own snapshots, so HEAD may already be
+        # ahead of origin/main with a clean tree.
+        git add -A -- wiki raw output .wiki-consolidation-state.json
+        git diff --cached --quiet || git commit -q -m "wiki maintenance $$(date -u +%F): distill + audit + consolidate (CI run $$CI_PIPELINE_NUMBER)"
+        if [ "$$(git rev-list --count origin/main..HEAD)" -eq 0 ]; then
          echo "wiki-maintenance: no changes"
          exit 0
        fi
@@ -112,13 +172,11 @@ steps:
        # leaves the PR open for a human and fails the run.
        BRANCH="wiki-maintenance/$$(date -u +%F)-$$CI_PIPELINE_NUMBER"
        git checkout -q -b "$$BRANCH"
-        git add -A
-        git commit -q -m "wiki maintenance $$(date -u +%F): distill + audit (CI run $$CI_PIPELINE_NUMBER)"
        TOKEN=$$(printf '%s' "$$WIKI_BOT_TOKEN" | tr -d '\r\n')
        git -c http.extraheader="AUTHORIZATION: token $$TOKEN" push origin "HEAD:$$BRANCH"
        API="https://gitea.renner.uno/api/v1/repos/k.renner/notes"
        jq -n --arg title "wiki maintenance $$(date -u +%F) (CI run $$CI_PIPELINE_NUMBER)" \
-              --arg body "Nightly distill + audit. Merged by the pipeline; open only if the merge failed." \
+              --arg body "Nightly distill + audit + consolidate. Merged by the pipeline; open only if the merge failed." \
              --arg head "$$BRANCH" '{title: $$title, body: $$body, head: $$head, base: "main"}' > /tmp/pr.json
        code=$$(curl -s -o /tmp/pr-resp.json -w '%{http_code}' -X POST \
          -H "Authorization: token $$TOKEN" -H "Content-Type: application/json" --data @/tmp/pr.json "$$API/pulls")
@@ -0,0 +1,7 @@
+# Contributing
+
+## Citing other YAML from a comment
+
+- Mark the block you cite with a comment line `# anchor: <name>` (kebab-case, unique in that file). If the block is inside a `|` block scalar, put the marker just above the key, never inside the scalar.
+- Cite it as `path.yaml#<name>`, with the path relative to the citing file's directory or to the repo root. Never cite `path.yaml:<line>`, because line numbers go stale on any edit above them.
+- `scripts/ci/yaml-citations.sh` enforces this in `ci-manifests-test.yml`: it fails on line-number citations and on citations whose file or anchor does not exist.
@@ -0,0 +1,70 @@
+# homelab
+
+GitOps source for the Renner homelab cluster. Flux applies `master` to
+production, so every merge is a production change. Changes go in by pull
+request only; see [CONTRIBUTING.md](CONTRIBUTING.md) for YAML conventions.
+
+## Change rules that protect Home Assistant
+
+Owner decision (REN-240, REN-243). Home Assistant runs the house, so it
+gets two rules on top of normal review.
+
+### 1. Disruptive work runs only 02:00–05:00 Europe/Berlin
+
+Planned work that can restart Home Assistant or disrupt the cluster is
+merged and applied only between **02:00 and 05:00 Europe/Berlin**. That
+covers:
+
+- node drain, node reboot, Talos upgrade
+- CNI (Cilium, Multus) and cluster-wide network policy
+- storage (Rook/Ceph, CSI, snapshot controller, CNPG operator)
+- Flux core (`kubernetes/clusters/*/flux-system/`, Flux controllers)
+- Authentik
+- OpenBao, including secret rotation that restarts consumers
+- anything in `kubernetes/apps/home-assistant/`
+
+A PR in this list states in its description that it waits for the window.
+Flux applies on merge, so "wait for the window" means **merge** in the
+window, not just apply. If unsure whether a change is disruptive, treat it
+as disruptive.
+
+Not covered: Home Assistant image bumps by Flux image automation. They stay
+fully automatic by owner decision. If one breaks Home Assistant, roll it
+back with [RUNBOOK-home-assistant-image-rollback.md](kubernetes/RUNBOOK-home-assistant-image-rollback.md).
+An incident rollback is not planned work; the owner decides its timing.
+
+### 2. HA-impact review before merge
+
+Every PR that touches `kubernetes/apps/home-assistant/`, or makes a
+cluster-wide disruptive change from the list above, needs an **HA-impact
+review** before merge: a named reviewer states in the PR what Home
+Assistant will notice (restart, lost integration, lost history, none) and
+approves.
+
+- Reviewer: Hestia, the Home Assistant maintainer.
+- Until Hestia exists: the owner does the HA-impact review.
+
+This is in addition to the normal review and merge by Argus. Argus does
+not merge such a PR without the HA-impact approval.
+
+## Agent runtime images
+
+Pod images for the Paperclip Kubernetes sandbox (REN-284). Each is the stock
+`ghcr.io/paperclipai/agent-runtime-claude` (pinned by digest) plus a tool
+set. Base user (1000, non-root), entrypoint and cmd are unchanged; the
+images run with a read-only root FS. Source in `ci/images/agent-runtime-*`,
+built by `.woodpecker/build-agent-runtime.yml` (re-run alone: deployment to
+target `build-agent-runtime`), versions bumped by Renovate. Both also
+carry `claude-agent-acp` for the ACP engine, pinned and patched like
+Paperclip ships it, SDK override included
+(`ci/images/agent-runtime-common/`); bump it by hand
+together with Paperclip. All tools are on the default `PATH`.
+
+| Image | `imageRegistry` | Adds |
+|---|---|---|
+| `gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-go` | Go toolchain, golangci-lint |
+| `gitea.renner.uno/konairius/agent-runtime-ops/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-ops` | kubectl, flux, kustomize, helm |
+
+The plugin keeps the name `agent-runtime-claude:v1` and swaps only host and
+org, hence the nested path. Every build also pushes a date tag
+(`:YYYY-MM-DD`) as the rollback point. `konairius` is public: no pull secret.
@@ -1,5 +1,5 @@
 ---
-# Deploy a NetBird reverse-proxy HA cluster member (a second proxy host).
+# Deploy the NetBird reverse-proxy HA cluster members (the public edge).
 # Usage: ansible-playbook -i inventory/ plays/install-netbird-proxy.yml
 #
 # Prerequisites:
@@ -7,12 +7,11 @@
 #   listed under [netbird_proxy] in inventory/pve_hosts.yml. Docker is
 #   installed by the (Arch-aware) docker-install role.
 # - DNS A/AAAA for the host's own name (e.g. proxy-1.renner.uno) -> its IP.
-# - Uses the existing vault_netbird_proxy_token (shared with VPS-1) — no new
-#   token to generate; a shared token + same NB_PROXY_DOMAIN forms the cluster.
-# - AFTER deploy + verify: add this host's IP to the proxy domain so traffic
-#   round-robins across the cluster. Repoint proxy.renner.uno away from its
-#   CNAME to netbird.renner.uno (management host) to A/AAAA records listing
-#   BOTH proxy IPs (VPS-1 + this host).
+# - Uses the existing vault_netbird_proxy_token (shared by every member) — no
+#   new token to generate; a shared token + same NB_PROXY_DOMAIN forms the
+#   cluster.
+# - AFTER deploy + verify: add this host's A/AAAA to proxy.renner.uno so
+#   traffic round-robins across the cluster.

 - name: Deploy NetBird reverse-proxy cluster member
  hosts: netbird_proxy
@@ -18,4 +18,3 @@
    - common
    - docker-install
    - netbird-server
-    - netbird-proxy-watchdog
@@ -43,10 +43,10 @@
 #
 # WHAT IS NOT, and must not be added:
 #
-#   vault_netbird_proxy_token    LOOKS symmetric — it is in proxy.env on both
-#                                the server and the proxies, all Ansible
-#                                templates — but the management server does not
-#                                validate it from any of them. It is registered
+#   vault_netbird_proxy_token    LOOKS symmetric — it is in proxy.env on the
+#                                proxies, an Ansible template — but the
+#                                management server does not validate it from
+#                                any file. It is registered
 #                                in the management DATABASE. Rotating it changes
 #                                only what the proxies present, so every proxy
 #                                is locked out: "proxy auth failed: too many
@@ -1,5 +1,5 @@
 ---
-# Self-healing watchdog for the NetBird reverse proxy (VPS-1 and proxy-1).
+# Self-healing watchdog for the NetBird reverse proxy on the proxy hosts.
 #
 # Why it exists: the proxy's embedded NetBird client pins each proxied
 # domain's route to a specific routing-peer identity. The netbird-ingress
@@ -20,9 +20,7 @@ netbird_watchdog_probe_hosts:
  - home.renner.uno
  - immich.renner.uno

-# Address the probe connects to. 127.0.0.1 reaches the local proxy on both
-# members: VPS-1 routes it via Traefik's HostSNI(*) TCP passthrough, proxy-1
-# binds :443 directly.
+# Address the probe connects to. The proxy binds :443 on the host directly.
 netbird_watchdog_probe_address: 127.0.0.1

 netbird_watchdog_container: netbird-proxy
@@ -1,7 +1,7 @@
 ---
 # Deploys the reverse-proxy watchdog (see defaults/main.yml for why).

- name: Ensure curl is installed (probe dependency; proxy-1 is Arch, VPS-1 Debian)
+- name: Ensure curl is installed (probe dependency)
  ansible.builtin.package:
    name: curl
    state: present
@@ -1,8 +1,8 @@
 ---
 # NetBird reverse-proxy HA cluster member (proxy-only host, e.g. proxy-1).
-# Joins the same proxy cluster as the VPS-1 proxy: identical NB_PROXY_DOMAIN
-# + the same token. NetBird load-balances/fails the cluster over via DNS
-# round-robin on the proxy domain (see plays/install-netbird-proxy.yml).
+# Every member uses the same NB_PROXY_DOMAIN + token. NetBird load-balances/
+# fails the cluster over via DNS round-robin on the proxy domain (see
+# plays/install-netbird-proxy.yml).

 # MUST match the netbird-server role's netbird_version — the reverse-proxy
 # release line has to track the management server it enrolls against.
@@ -11,11 +11,13 @@ netbird_version: "0.73.2"
 # Management server this proxy enrolls against (public API, TLS).
 netbird_server_host: netbird.renner.uno

-# Proxy domain — MUST be identical across every cluster member.
+# Proxy domain — MUST be identical across every cluster member, and must NOT
+# be a subdomain of the management domain (netbird.renner.uno) per NetBird
+# docs (TLS/routing conflicts).
 netbird_proxy_domain: "proxy.renner.uno"

-# Reuse VPS-1's proxy token — a shared NB_PROXY_TOKEN across members with the
-# same NB_PROXY_DOMAIN is a valid HA cluster, so no separate token is needed.
+# A shared NB_PROXY_TOKEN across members with the same NB_PROXY_DOMAIN is a
+# valid HA cluster, so one token serves every member.
 netbird_proxy_token: "{{ vault_netbird_proxy_token }}"

 # Relay auth secret — MUST be identical across every relay instance and the
@@ -1,8 +1,7 @@
 # NetBird reverse-proxy — HA cluster member on a proxy-only host.
-# Just the reverse-proxy container, bound directly to :443. No Traefik:
-# unlike VPS-1 (where the proxy shares :443 with management/signal/relay via
-# SNI routing), this host runs nothing else, so the proxy owns the port and
-# terminates TLS + answers its own tls-alpn-01 ACME challenges directly.
+# Just the reverse-proxy container, bound directly to :443. No Traefik: the
+# proxy owns the port and terminates TLS + answers its own tls-alpn-01 ACME
+# challenges directly.
 services:
  proxy:
    image: netbirdio/reverse-proxy:{{ netbird_version }}
@@ -23,9 +23,3 @@ netbird_datastore_key: "{{ vault_netbird_datastore_key }}"

 # Relay secret for WebSocket relay authentication
 netbird_relay_secret: "{{ vault_netbird_relay_secret }}"
-
-# Reverse proxy configuration
-# IMPORTANT: Proxy domain must NOT be a subdomain of the management domain
-# (netbird.renner.uno) per NetBird docs - causes TLS/routing conflicts
-netbird_proxy_domain: "proxy.renner.uno"
-netbird_proxy_token: "{{ vault_netbird_proxy_token }}"
@@ -16,7 +16,3 @@
 - name: Restart netbird coturn
  ansible.builtin.shell: |
    cd /opt/netbird && docker compose up -d --force-recreate coturn
-
- name: Restart netbird proxy
-  ansible.builtin.shell: |
-    cd /opt/netbird && docker compose up -d --force-recreate proxy
@@ -36,7 +36,6 @@
    - /opt/netbird/management
    - /opt/netbird/signal
    - /opt/netbird/letsencrypt
-    - /opt/netbird/proxy/certs

 - name: Deploy docker-compose.yml
  ansible.builtin.template:
@@ -59,23 +58,83 @@
    mode: "0644"
  notify: Restart netbird coturn

- name: Deploy proxy.env
-  ansible.builtin.template:
-    src: proxy.env.j2
-    dest: /opt/netbird/proxy.env
-    mode: "0644"
-  notify: Restart netbird proxy
+# The public edge is the proxy hosts (plays/install-netbird-proxy.yml); this
+# host runs no reverse proxy. The tasks below remove the one it used to run,
+# its watchdogs first so they cannot restart it mid-removal, and can go once
+# the role has run against every host that ever carried it.
+- name: Check for reverse-proxy watchdog units
+  ansible.builtin.stat:
+    path: "/etc/systemd/system/{{ item }}"
+  loop:
+    - netbird-proxy-watchdog.timer
+    - netbird-proxy-flood-watchdog.service
+  register: _proxy_watchdog_units
+
+- name: Stop and disable reverse-proxy watchdogs
+  ansible.builtin.systemd:
+    name: "{{ item.item }}"
+    state: stopped
+    enabled: false
+  loop: "{{ _proxy_watchdog_units.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: item.stat.exists
+
+- name: Remove reverse-proxy watchdog units and scripts
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/systemd/system/netbird-proxy-watchdog.timer
+    - /etc/systemd/system/netbird-proxy-watchdog.service
+    - /etc/systemd/system/netbird-proxy-flood-watchdog.service
+    - /usr/local/bin/netbird-proxy-watchdog
+    - /usr/local/bin/netbird-proxy-flood-watchdog
+    - /run/netbird-proxy-watchdog
+  register: _proxy_watchdog_files
+
+- name: Reload systemd after removing the watchdog units
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: _proxy_watchdog_files.changed

 - name: Pull NetBird container images
  ansible.builtin.shell: |
    cd /opt/netbird && docker compose pull
  changed_when: false

+# --remove-orphans removes containers of services no longer in the compose
+# file (the reverse proxy).
 - name: Start NetBird containers
  ansible.builtin.shell: |
-    cd /opt/netbird && docker compose up -d
+    cd /opt/netbird && docker compose up -d --remove-orphans
  register: docker_compose_result
-  changed_when: "'Started' in docker_compose_result.stdout or 'Creating' in docker_compose_result.stdout"
+  changed_when: >-
+    'Started' in docker_compose_result.stdout or
+    'Creating' in docker_compose_result.stdout or
+    'Removed' in docker_compose_result.stdout or
+    'Removed' in docker_compose_result.stderr
+
+- name: Remove reverse-proxy volumes
+  ansible.builtin.shell: |
+    set -eu
+    for v in netbird_netbird_proxy_certs netbird_netbird_proxy_data netbird_netbird_proxy_state; do
+      if [ -n "$(docker volume ls -q --filter "name=^${v}$")" ]; then
+        docker volume rm "$v"
+        echo "removed $v"
+      fi
+    done
+  register: _proxy_volumes
+  changed_when: "'removed' in _proxy_volumes.stdout"
+
+- name: Remove reverse-proxy config and state
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /opt/netbird/proxy.env
+    - /opt/netbird/proxy
+    - /opt/netbird/proxy-state

 # The management store has no cluster-side backup. Pulling one put a root
 # key on this host that the cluster held, and this server's own state is
@@ -187,10 +187,9 @@ services:
      - "traefik.http.routers.netbird-management.service=netbird-management"
      - "traefik.http.services.netbird-management.loadbalancer.server.port=33073"
      - "traefik.http.services.netbird-management.loadbalancer.server.scheme=h2c"
-      # gRPC reverse-proxy service — remote reverse-proxy instances (e.g.
-      # proxy-1) connect here via NB_PROXY_MANAGEMENT_ADDRESS=https://host:443.
-      # The local proxy uses management:33073 directly, so it never needed
-      # this route; remote ones get a 404 without it. Same backend (h2c 33073).
+      # gRPC reverse-proxy service — the proxy hosts (proxy-1, proxy-2)
+      # connect here via NB_PROXY_MANAGEMENT_ADDRESS=https://host:443 and get
+      # a 404 without it. Same backend (h2c 33073).
      - "traefik.http.routers.netbird-proxyservice.rule=Host(`{{ netbird_server_host }}`) && PathPrefix(`/management.ProxyService/`)"
      - "traefik.http.routers.netbird-proxyservice.entrypoints=websecure"
      - "traefik.http.routers.netbird-proxyservice.tls=true"
@@ -221,37 +220,6 @@ services:
        max-size: "500m"
        max-file: "2"

-  # Reverse Proxy Service (per docs.netbird.io/selfhosted/migration/enable-reverse-proxy)
-  proxy:
-    image: netbirdio/reverse-proxy:{{ netbird_version }}
-    container_name: netbird-proxy
-    restart: unless-stopped
-    networks:
-      - netbird
-    depends_on:
-      - management
-    env_file:
-      - ./proxy.env
-    volumes:
-      - netbird_proxy_certs:/certs
-      - /opt/netbird/proxy-state:/var/lib/netbird
-    labels:
-      - "traefik.enable=true"
-      - "traefik.tcp.routers.proxy-passthrough.entrypoints=websecure"
-      - "traefik.tcp.routers.proxy-passthrough.rule=HostSNI(`*`)"
-      - "traefik.tcp.routers.proxy-passthrough.tls.passthrough=true"
-      - "traefik.tcp.routers.proxy-passthrough.service=proxy-tls"
-      - "traefik.tcp.routers.proxy-passthrough.priority=1"
-      - "traefik.tcp.services.proxy-tls.loadbalancer.server.port=8443"
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "500m"
-        max-file: "2"
-
 networks:
  netbird:
    driver: bridge
-
-volumes:
-  netbird_proxy_certs:
@@ -1,8 +0,0 @@
-NB_PROXY_DOMAIN={{ netbird_proxy_domain }}
-NB_PROXY_TOKEN={{ netbird_proxy_token }}
-NB_PROXY_MANAGEMENT_ADDRESS=http://management:33073
-NB_PROXY_ALLOW_INSECURE=true
-NB_PROXY_ADDRESS=:8443
-NB_PROXY_ACME_CERTIFICATES=true
-NB_PROXY_ACME_CHALLENGE_TYPE=tls-alpn-01
-NB_PROXY_CERTIFICATE_DIRECTORY=/certs
@@ -1,5 +0,0 @@
-You are running as the scheduled wiki-maintenance job in CI. The Notes vault is checked out at ~/Notes (the repository k.renner/notes).
-
-Run the standard audit pass over ~/Notes/wiki/ and write the dated report to ~/Notes/output/. Do not modify any wiki note. Do not commit or push; the pipeline does that.
-
-End with two lines: notes scanned, and issues found by category.
@@ -0,0 +1,120 @@
+{
+  "//": "Permission scope for the wiki-consolidate agent in CI: read the vault and the",
+  "//2": "source repos cloned under /root/source, read the live cluster through the",
+  "//3": "ci-diagnostics kubeconfig, edit wiki/, output/ and the rotation state only.",
+  "//4": "Deny rules win; git and kubectl are read-only subcommands, no network tools.",
+  "permissions": {
+    "allow": [
+      "Read(//root/Notes/**)",
+      "Read(//root/source/**)",
+      "Edit(//root/Notes/wiki/**)",
+      "Edit(//root/Notes/output/**)",
+      "Edit(//root/Notes/.wiki-consolidation-state.json)",
+      "Bash(/root/Notes/bin/wiki-snapshot.sh:*)",
+      "Bash(/root/Notes/bin/wiki_rank.py:*)",
+      "Bash(/root/Notes/bin/wiki_ask.py:*)",
+      "Bash(~/Notes/bin/wiki-snapshot.sh:*)",
+      "Bash(~/Notes/bin/wiki_rank.py:*)",
+      "Bash(~/Notes/bin/wiki_ask.py:*)",
+      "Bash(~/Notes/bin/wiki-delete.sh:*)",
+      "Bash(git -C /root/Notes log:*)",
+      "Bash(git -C /root/Notes show:*)",
+      "Bash(git -C /root/Notes diff:*)",
+      "Bash(git -C /root/Notes status:*)",
+      "Bash(git -C /root/Notes rev-parse:*)",
+      "Bash(git -C /root/Notes ls-files:*)",
+      "Bash(git -C ~/Notes log:*)",
+      "Bash(git -C ~/Notes show:*)",
+      "Bash(git -C ~/Notes diff:*)",
+      "Bash(git -C ~/Notes status:*)",
+      "Bash(git -C ~/Notes rev-parse:*)",
+      "Bash(git -C ~/Notes ls-files:*)",
+      "Bash(kubectl get:*)",
+      "Bash(kubectl describe:*)",
+      "Bash(kubectl version:*)",
+      "Bash(kubectl api-resources:*)",
+      "Bash(kubectl explain:*)",
+      "Bash(git log:*)",
+      "Bash(git show:*)",
+      "Bash(git status:*)",
+      "Bash(git diff:*)",
+      "Bash(git rev-parse:*)",
+      "Bash(git ls-files:*)",
+      "Bash(git -C /root/source/homelab log:*)",
+      "Bash(git -C /root/source/homelab show:*)",
+      "Bash(git -C /root/source/homelab diff:*)",
+      "Bash(git -C /root/source/homelab status:*)",
+      "Bash(git -C /root/source/homelab rev-parse:*)",
+      "Bash(git -C /root/source/homelab ls-files:*)",
+      "Bash(git -C /root/source/homeassistant log:*)",
+      "Bash(git -C /root/source/homeassistant show:*)",
+      "Bash(git -C /root/source/homeassistant diff:*)",
+      "Bash(git -C /root/source/homeassistant status:*)",
+      "Bash(git -C /root/source/homeassistant rev-parse:*)",
+      "Bash(git -C /root/source/homeassistant ls-files:*)",
+      "Bash(git -C ~/source/homelab log:*)",
+      "Bash(git -C ~/source/homelab show:*)",
+      "Bash(git -C ~/source/homelab diff:*)",
+      "Bash(git -C ~/source/homelab status:*)",
+      "Bash(git -C ~/source/homelab rev-parse:*)",
+      "Bash(git -C ~/source/homelab ls-files:*)",
+      "Bash(git -C ~/source/homeassistant log:*)",
+      "Bash(git -C ~/source/homeassistant show:*)",
+      "Bash(git -C ~/source/homeassistant diff:*)",
+      "Bash(git -C ~/source/homeassistant status:*)",
+      "Bash(git -C ~/source/homeassistant rev-parse:*)",
+      "Bash(git -C ~/source/homeassistant ls-files:*)",
+      "Bash(/root/Notes/bin/wiki-delete.sh:*)",
+      "Bash(ls:*)",
+      "Bash(date:*)",
+      "Bash(wc:*)",
+      "Bash(stat:*)",
+      "Bash(head:*)",
+      "Bash(tail:*)"
+    ],
+    "deny": [
+      "Read(//proc/**)",
+      "Read(//etc/**)",
+      "Read(//root/.config/**)",
+      "Read(//root/.kube/**)",
+      "Read(//woodpecker/**/.ci-diag/**)",
+      "Read(//root/Notes/.git/**)",
+      "Read(//root/source/*/.git/**)",
+      "Read(//root/Notes/_archive/**)",
+      "Edit(//root/Notes/raw/**)",
+      "Edit(//root/Notes/_archive/**)",
+      "Edit(//root/source/**)",
+      "Bash(kubectl apply:*)",
+      "Bash(kubectl delete:*)",
+      "Bash(kubectl patch:*)",
+      "Bash(kubectl edit:*)",
+      "Bash(kubectl scale:*)",
+      "Bash(kubectl rollout:*)",
+      "Bash(kubectl exec:*)",
+      "Bash(kubectl cp:*)",
+      "Bash(kubectl drain:*)",
+      "Bash(kubectl cordon:*)",
+      "Bash(kubectl label:*)",
+      "Bash(kubectl annotate:*)",
+      "Bash(git push:*)",
+      "Bash(git reset:*)",
+      "Bash(git checkout:*)",
+      "Bash(git clean:*)",
+      "Bash(git rebase:*)",
+      "Bash(git config:*)",
+      "Bash(git -C /root/Notes config:*)",
+      "Bash(git -C ~/Notes config:*)",
+      "Bash(git -C /root/source/homelab config:*)",
+      "Bash(git -C /root/source/homeassistant config:*)",
+      "Bash(git -C ~/source/homelab config:*)",
+      "Bash(git -C ~/source/homeassistant config:*)",
+      "Bash(rm:*)",
+      "Bash(rm -rf:*)",
+      "Bash(curl:*)",
+      "Bash(wget:*)",
+      "Bash(ssh:*)",
+      "Bash(tea:*)",
+      "Bash(woodpecker:*)"
+    ]
+  }
+}
@@ -0,0 +1,5 @@
+You are running as the scheduled wiki-maintenance job in CI. The Notes vault is checked out at ~/Notes (the repository k.renner/notes); the homelab and homeassistant repositories are checked out under ~/source/. `kubectl` reads the live cluster with a read-only identity.
+
+Run the wiki-consolidate agent now for today's scheduled consolidation pass. Follow its instructions exactly: snapshot the vault to git first, abort if that fails, select today's rotating cohort of 20 from the state file, verify claims against both their sources and the live systems, act on the verdicts, write the dated report to ~/Notes/output/, update the state file, and snapshot again at the end. Do not push; the pipeline pushes the run's commits.
+
+Report the counts and the coverage progress when you are done.
@@ -1,5 +1,5 @@
 {
-  "//": "Permission scope for the wiki-maintenance agents (wiki-distill, wiki-audit) in CI.",
+  "//": "Permission scope for the wiki-maintenance agent (wiki-distill) in CI.",
  "//2": "The step's environment holds a Gitea push token, a TypeSafe key and the wiki-index",
  "//3": "client credential, so the Read tool is kept away from /proc (env dumps), /root outside",
  "//4": "the vault symlink target, /etc and the checkout's .git/. The vault itself is the work.",
@@ -15,6 +15,10 @@
      "Bash(/root/Notes/bin/wiki_ask.py:*)",
      "Bash(mv /root/Notes/raw/:*)",
      "Bash(mkdir -p /root/Notes/:*)",
+      "Bash(~/Notes/bin/wiki_rank.py:*)",
+      "Bash(~/Notes/bin/wiki_ask.py:*)",
+      "Bash(mv ~/Notes/raw/:*)",
+      "Bash(mkdir -p ~/Notes/:*)",
      "Bash(ls:*)",
      "Bash(date:*)",
      "Bash(wc:*)",
@@ -0,0 +1,66 @@
+diff --git a/dist/acp-agent.js b/dist/acp-agent.js
+index 0214a509eb5e7b777a69e4e34637941c275d27b1..403be02df66e25eee29e7d5bef14627011e02b30 100644
+--- a/dist/acp-agent.js
+++ b/dist/acp-agent.js
+@@ -2922,9 +2922,19 @@ export class ClaudeAcpAgent {
+                                             amount: message.total_cost_usd,
+                                             currency: "USD",
+                                         },
+-                                        ...(message.origin && {
+-                                            _meta: { "_claude/origin": message.origin },
+-                                        }),
+                                        _meta: {
+                                            ...(message.origin && { "_claude/origin": message.origin }),
+                                            // ACP's core Usage shape exposes context occupancy and
+                                            // cost, but not the billable input/output split. Keep the
+                                            // SDK's bounded aggregate in extension metadata so ACPX can
+                                            // normalize it without persisting a raw provider message.
+                                            usage: {
+                                                input_tokens: message.usage.input_tokens,
+                                                output_tokens: message.usage.output_tokens,
+                                                cache_read_input_tokens: message.usage.cache_read_input_tokens,
+                                                cache_creation_input_tokens: message.usage.cache_creation_input_tokens,
+                                            },
+                                        },
+                                     },
+                                 });
+                             }
+@@ -5311,6 +5321,10 @@ export class ClaudeAcpAgent {
+             settingSources: ["user", "project", "local"],
+             ...(thinking !== undefined && { thinking }),
+             ...userProvidedOptions,
+            // Paperclip Runner owns the complete session context. Its isolated
+            // user root contains only assigned skills; project/local settings
+            // would reintroduce host prompts, plugins, skills, and MCP servers.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && { settingSources: ["user"] }),
+             ...(settings && { settings }),
+             env,
+             // Override certain fields that must be controlled by ACP
+@@ -5318,7 +5332,9 @@ export class ClaudeAcpAgent {
+             includePartialMessages: true,
+             forwardSubagentText,
+             mcpServers: {
+-                ...(userProvidedOptions?.mcpServers || {}),
+                ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1"
+                    ? {}
+                    : (userProvidedOptions?.mcpServers || {})),
+                 ...mcpServers,
+                 ...(fileChangeAuditSupport
+                     ? { [FILE_CHANGE_AUDIT_SERVER_NAME]: fileChangeAuditSupport.mcpServer }
+@@ -5329,6 +5345,16 @@ export class ClaudeAcpAgent {
+             allowDangerouslySkipPermissions: ALLOW_BYPASS,
+             permissionMode: initialPermissionMode,
+             canUseTool: this.canUseTool(sessionId),
+            // The runner pins this actual MCP endpoint before provider launch.
+            // Exempt only task delivery/control tools at the SDK dispatch boundary,
+            // not arbitrary provider permission metadata or MCP name prefixes.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && {
+                allowedTools: mcpServers.paperclip?.type === "http"
+                    && mcpServers.paperclip.url === process.env.PAPERCLIP_ACPX_TASK_TOOL_BRIDGE_URL
+                    ? ["paperclip_finish", "paperclip_block", "read_current_wake_comments", "request_human_input"]
+                        .map((tool) => `mcp__paperclip__${tool}`)
+                    : [],
+            }),
+             // Forward MCP elicitation requests onto ACP elicitation. Only attached
+             // when the client advertised support, so non-supporting clients keep the
+             // SDK's default (auto-decline) behavior. (AskUserQuestion is handled in
@@ -0,0 +1,9 @@
+{
+  "private": true,
+  "dependencies": {
+    "@agentclientprotocol/claude-agent-acp": "0.73.0"
+  },
+  "overrides": {
+    "@anthropic-ai/claude-agent-sdk": "0.3.280"
+  }
+}
@@ -0,0 +1,60 @@
+# Paperclip agent runtime + Go toolchain (REN-284 / REN-286). Runs as the pod
+# image of the `k8s-go` sandbox environment.
+#
+# Every tool comes from its upstream image, pinned by tag AND digest, so the
+# dockerfile manager in Renovate bumps version and digest together. A curl +
+# sha256 pin (as in ci/images/k8s) would break on every Renovate bump.
+#
+# Both binaries are static; they run on the Ubuntu 22.04 base unchanged.
+FROM golang:1.27.1-bookworm@sha256:69a7b9788769bec032d238959b61854e9ae87f57be9029ec04e9885fabf99195 AS go
+FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS golangci-lint
+
+# upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
+# digest current but cannot order commit tags; a newer base is a manual bump.
+FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518742e33ce2f@sha256:33ad285f9476b8637d6d940c08d577797030125675bdf4f497a8ed1f79261c8b
+
+COPY --from=go /usr/local/go /usr/local/go
+COPY --from=golangci-lint /usr/bin/golangci-lint /usr/local/bin/golangci-lint
+# go and gofmt also in /usr/local/bin: a process started with the default PATH
+# (not the image ENV) still finds them.
+USER root
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/go \
+    && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
+USER 1000:1000
+
+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000
+
+# The sandbox pod has a read-only root FS. Writable mounts: $HOME
+# (/home/paperclip, 1Gi), $HOME/.cache (1Gi), /tmp (2Gi), /workspace.
+# - GOPATH (module cache) goes to /tmp, the biggest one.
+# - GOCACHE and the golangci-lint cache keep their default under $HOME/.cache.
+# - GOTOOLCHAIN=local: a go.mod asking for a newer Go fails clearly instead of
+#   downloading a toolchain through the egress allow-list.
+ENV PATH=/usr/local/go/bin:$PATH \
+    GOPATH=/tmp/go \
+    GOTOOLCHAIN=local
+
+# Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
+# The PATH check uses the base image's default PATH, not the ENV above.
+RUN go version && golangci-lint version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in go gofmt golangci-lint claude-agent-acp; do command -v "$t"; done'
@@ -0,0 +1,51 @@
+# Paperclip agent runtime + cluster tools (REN-284 / REN-286). Runs as the pod
+# image of the `k8s-ops` sandbox environment.
+#
+# Every tool comes from its upstream image, pinned by tag AND digest, so the
+# dockerfile manager in Renovate bumps version and digest together. A curl +
+# sha256 pin (as in ci/images/k8s) would break on every Renovate bump.
+#
+# kubectl and flux match the ci-k8s image. All four binaries are static; they
+# run on the Ubuntu 22.04 base unchanged.
+FROM registry.k8s.io/kubectl:v1.36.5@sha256:0667e3f3e6ef435d7cd4c16684244703ebd67314c17511c2a9f87ea44f6cdf6b AS kubectl
+FROM ghcr.io/fluxcd/flux-cli:v2.9.5@sha256:704d5529535570a495249b643159c93a9aa88ffca07813c974a54b7ba6edbd86 AS flux
+FROM registry.k8s.io/kustomize/kustomize:v5.8.1@sha256:899fcd3bc898160e62bcaf82932b0cb29ba38d16272353db2e7acbba82129429 AS kustomize
+FROM alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2 AS helm
+
+# upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
+# digest current but cannot order commit tags; a newer base is a manual bump.
+FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518742e33ce2f@sha256:33ad285f9476b8637d6d940c08d577797030125675bdf4f497a8ed1f79261c8b
+
+COPY --from=kubectl /bin/kubectl /usr/local/bin/kubectl
+COPY --from=flux /usr/local/bin/flux /usr/local/bin/flux
+COPY --from=kustomize /app/kustomize /usr/local/bin/kustomize
+COPY --from=helm /usr/bin/helm /usr/local/bin/helm
+
+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000
+
+# Read-only root FS: kubectl (~/.kube/cache) and helm (~/.cache, ~/.config)
+# write only under $HOME, which the sandbox pod mounts writable. No ENV needed.
+
+# Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
+RUN kubectl version --client && flux version --client && kustomize version && helm version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in kubectl flux kustomize helm claude-agent-acp; do command -v "$t"; done'
@@ -1,4 +1,4 @@
-FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2
+FROM node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1

 # Default to the latest Claude Code. Pinning a fixed version is the wrong
 # default here: old releases eventually stop working against the API, so a
@@ -1,6 +1,6 @@
 FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d

-ARG TALOS_VERSION=v1.13.9
+ARG TALOS_VERSION=v1.13.10
 ARG VELERO_VERSION=v1.16.0
 ARG KUBECTL_VERSION=v1.36.3
 ARG FLUX_VERSION=2.9.5
@@ -18,7 +18,7 @@ RUN apk add --no-cache \

 RUN curl -sfL -o /usr/local/bin/talosctl \
      "https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
- && echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e  /usr/local/bin/talosctl" | sha256sum -c - \
+ && echo "28e9ccd9aebd690fa03d8cf2582221d87ea38b2a1704f411d75a03589c58f2aa  /usr/local/bin/talosctl" | sha256sum -c - \
 && chmod +x /usr/local/bin/talosctl

 RUN curl -sfL -o /usr/local/bin/kubectl \
@@ -1,4 +1,4 @@
-# netbird-proxy-services: reconciles NetBird reverse-proxy services (public
+# netbird-service-sync: reconciles NetBird reverse-proxy services (public
 # domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
 # public label. Standard library only, so the image is the interpreter plus one
 # file; runs unprivileged as its own user.
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""netbird-proxy-services: reconcile NetBird reverse-proxy services from Ingresses.
+"""netbird-service-sync: reconcile NetBird reverse-proxy services from Ingresses.

 Source of truth: every Ingress carrying the public label (dns.renner.uno/public).
 For each host on such an Ingress the NetBird management gets one reverse-proxy
@@ -30,11 +30,11 @@ import urllib.error
 import urllib.parse
 import urllib.request

-LOG = logging.getLogger("netbird-proxy-services")
+LOG = logging.getLogger("netbird-service-sync")

 NB = os.environ.get("NB_MANAGEMENT_URL", "https://netbird.renner.uno").rstrip("/")
 BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
-BAO_ROLE = os.environ.get("BAO_ROLE", "netbird-proxy-services")
+BAO_ROLE = os.environ.get("BAO_ROLE", "netbird-service-sync")
 BAO_PAT_PATH = os.environ.get("BAO_PAT_PATH", "homelab/data/infrastructure/secret-rotators-bootstrap/netbird")
 BAO_PAT_KEY = os.environ.get("BAO_PAT_KEY", "pat")
 SERVICE_CIDR = os.environ.get("SERVICE_CIDR", "10.96.0.0/12")
@@ -156,7 +156,7 @@ def ensure_network(nb):
        if n["name"] == NETWORK_NAME:
            return n["id"]
    n = nb.post("networks", {"name": NETWORK_NAME,
-                             "description": "managed by netbird-proxy-services; public ingress via TLS passthrough"})
+                             "description": "managed by netbird-service-sync; public ingress via TLS passthrough"})
    LOG.info("created network %s (%s)", NETWORK_NAME, n["id"])
    return n["id"]

@@ -166,7 +166,7 @@ def ensure_resource(nb, net):
        if r.get("address") == SERVICE_CIDR:
            return r["id"]
    r = nb.post(f"networks/{net}/resources", {"name": "cluster service CIDR", "address": SERVICE_CIDR,
-                                              "description": "managed by netbird-proxy-services",
+                                              "description": "managed by netbird-service-sync",
                                              "enabled": True, "groups": []})
    LOG.info("created subnet resource %s (%s)", SERVICE_CIDR, r["id"])
    return r["id"]
@@ -0,0 +1,55 @@
+# Paperclip server image (REN-401)
+
+Upstream `paperclipai/paperclip` at the commit in `upstream.env`, plus every
+`*.patch` in this directory. Built by `.woodpecker/build-paperclip.yml`,
+pushed as `gitea.renner.uno/konairius/paperclip:build.<unix-ts>`, rolled out
+by Flux image automation (`kubernetes/infrastructure/image-automation/paperclip.yaml`).
+
+## Why
+
+k8s sandbox runs fail: the run path never sets the lease `remoteCwd`, so file
+sync into the pod fails (paperclipai/paperclip#13864). The fix is
+paperclipai/paperclip#13733, open and not merged.
+`pr-13733-lease-remote-cwd.patch` is that PR's diff (head `5f320bf6`).
+
+#13733 covers only the run path. Adapter environment tests (binding an agent
+to a k8s environment) sync managed config before any workspace is realized,
+so the lease still has no `remoteCwd` and the Claude test fails with
+`claude_managed_config_dir_failed` (REN-409). The fix is
+paperclipai/paperclip#14769, also open: the plugin sets `remoteCwd:
+/workspace` when it acquires or resumes a lease.
+`pr-14769-probe-remote-cwd.patch` is that PR's diff (head `10dc7371`).
+
+## Files
+
+- `upstream.env` — pinned commit and the version string stamped into the
+  image. Bump both together.
+- `prepare.sh` — fetches the commit, applies the patches, writes
+  `Dockerfile.kaniko` (upstream's `COPY --parents` line split into plain
+  `COPY` lines; kaniko does not support `--parents`), and adds the build of
+  the Kubernetes sandbox provider plugin to the `build` stage (REN-405):
+  upstream's `production` target ships only its source, so the plugin fails
+  with "no longer exposes a Paperclip manifest". The build fails if
+  `dist/manifest.js` is missing.
+- `*.patch` — applied in name order with `git apply`.
+
+## Moving to a newer upstream commit
+
+Set `PAPERCLIP_COMMIT` to the new master commit and `PAPERCLIP_VERSION` to
+`git describe --tags --match 'v*' --long <commit>`, then run
+`sh prepare.sh /tmp/src`: it fails if a patch no longer applies.
+
+## Exit: when #13733 and #14769 are merged upstream
+
+Renovate cannot watch an upstream PR, so this is a manual step. Once
+upstream nightly contains both fixes:
+
+1. `kubernetes/infrastructure/image-automation/paperclip.yaml`: back to
+   `ghcr.io/paperclipai/paperclip` with `filterTags.pattern: '^nightly$'`
+   (the version before REN-401 in git history).
+2. `renovate.json`: swap `gitea.renner.uno/konairius/paperclip` back to
+   `ghcr.io/paperclipai/paperclip` in the image-automation exclusion list.
+3. Delete this directory and `.woodpecker/build-paperclip.yml`.
+
+Image automation then rewrites `kubernetes/apps/paperclip/paperclip.yaml`
+to the nightly digest by itself.
@@ -0,0 +1,93 @@
+diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts
+index f3d1fc659..812cae357 100644
+--- a/server/src/__tests__/environment-run-orchestrator.test.ts
+++ b/server/src/__tests__/environment-run-orchestrator.test.ts
+@@ -430,6 +430,51 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
+     expect(result.persistedExecutionWorkspace).toEqual(updatedEw);
+   });
+ 
+  it("persists the realized cwd as remoteCwd when realization returns no workspaceRealization payload", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({ environment: makeEnvironment("sandbox") }),
+    );
+
+    expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(
+      "lease-1",
+      expect.objectContaining({ remoteCwd: "/workspace" }),
+    );
+    // Nothing to record about the realization shape, so the execution workspace
+    // row is left untouched.
+    expect(mockUpdateExecutionWorkspace).not.toHaveBeenCalled();
+  });
+
+  it("does not rewrite lease metadata when the realized cwd already matches the lease", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({
+        environment: makeEnvironment("sandbox"),
+        lease: makeLease({ metadata: { remoteCwd: "/workspace" } }),
+      }),
+    );
+
+    expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled();
+  });
+
+   it("runs a remote provision command after workspace realization when configured", async () => {
+     mockBuildWorkspaceRealizationRequest.mockReturnValue({
+       version: 1,
+diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts
+index 97cca667e..e76f64913 100644
+--- a/server/src/services/environment-run-orchestrator.ts
+++ b/server/src/services/environment-run-orchestrator.ts
+@@ -485,16 +485,29 @@ export function environmentRunOrchestrator(
+     }
+ 
+     // Step 3: Persist realization metadata on lease and execution workspace
+-    if (Object.keys(workspaceRealization).length > 0) {
+    const hasWorkspaceRealization = Object.keys(workspaceRealization).length > 0;
+    // A driver that realizes the workspace reports the directory it prepared
+    // through the realization `cwd`. Persist it on the lease as `remoteCwd`, so
+    // every later step that receives only the lease — native file sync, a
+    // resumed lease, a second run against the same lease — confines itself to
+    // the same root. Without it a driver whose realization returns no
+    // `workspaceRealization` payload leaves the lease without a root at all,
+    // and the sync step fails even though realization succeeded.
+    const currentRemoteCwd =
+      typeof lease.metadata?.remoteCwd === "string" ? lease.metadata.remoteCwd.trim() : "";
+    const realizedRemoteCwd =
+      realizedWorkspaceCwd && realizedWorkspaceCwd !== currentRemoteCwd ? realizedWorkspaceCwd : null;
+    if (hasWorkspaceRealization || realizedRemoteCwd) {
+       const nextLeaseMetadata = {
+         ...(lease.metadata ?? {}),
+-        workspaceRealization,
+        ...(hasWorkspaceRealization ? { workspaceRealization } : {}),
+        ...(realizedRemoteCwd ? { remoteCwd: realizedRemoteCwd } : {}),
+       };
+       const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, nextLeaseMetadata);
+       if (updatedLease) {
+         lease = updatedLease;
+       }
+-      if (persistedExecutionWorkspace) {
+      if (hasWorkspaceRealization && persistedExecutionWorkspace) {
+         const updatedEw = await executionWorkspacesSvc.update(persistedExecutionWorkspace.id, {
+           metadata: {
+             ...(persistedExecutionWorkspace.metadata ?? {}),
@@ -0,0 +1,86 @@
+diff --git a/packages/plugins/sandbox-providers/kubernetes/README.md b/packages/plugins/sandbox-providers/kubernetes/README.md
+index a51c6ecee60e..72618cb11296 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/README.md
+++ b/packages/plugins/sandbox-providers/kubernetes/README.md
+@@ -140,6 +140,8 @@ Every agent pod is:
+ 
+ Plus per-namespace `pod-security.kubernetes.io/enforce: restricted` and a deny-all NetworkPolicy baseline with explicit egress allow-list (DNS, paperclip-server, configured FQDNs/CIDRs).
+ 
+The lease starts with `/workspace` as the native file-sync confinement root. Adapter environment tests can upload managed config before the server realizes a workspace. When resuming a lease, the provider keeps a previously recorded `remoteCwd` so a custom realized workspace is not replaced with the default.
+
+ The per-run Secret carrying the bootstrap token and adapter API keys has `ownerReferences` pointing at the owning Job, so a single `kubectl delete job …` cascades cleanly to the Pod and Secret.
+ 
+ ## Optional Kata-FC microVM isolation
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+index 8ae47de8ecac..98016385da05 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+@@ -454,6 +454,9 @@ const plugin = definePlugin({
+       secretName,
+       phase: "Pending",
+       backend: config.backend,
+      // Adapter environment probes sync managed assets before workspace
+      // realization. Give native sync the pod's mounted confinement root now.
+      remoteCwd: "/workspace",
+       scopedNetworkPolicyName,
+       scopedNetworkEgress,
+       // Native file sync streams over a pod exec; only the sandbox-cr backend
+@@ -529,6 +532,10 @@ const plugin = definePlugin({
+       secretName,
+       phase: check.phase,
+       backend: leaseBackend,
+      remoteCwd:
+        typeof params.leaseMetadata?.remoteCwd === "string" && params.leaseMetadata.remoteCwd.trim()
+          ? params.leaseMetadata.remoteCwd.trim()
+          : "/workspace",
+       scopedNetworkPolicyName:
+         typeof params.leaseMetadata?.scopedNetworkPolicyName === "string"
+           ? params.leaseMetadata.scopedNetworkPolicyName
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/types.ts b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+index 5626daa93846..ef3d0d889a4c 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+@@ -90,6 +90,8 @@ export interface KubernetesLeaseMetadata {
+   phase: "Pending" | "Running" | "Succeeded" | "Failed";
+   /** Which backend provisioned this lease. */
+   backend: "sandbox-cr" | "job";
+  /** Native sync confinement root; present before workspace realization for probes. */
+  remoteCwd: string;
+   scopedNetworkPolicyName: string | null;
+   scopedNetworkEgress: {
+     allowFqdns: string[];
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+index 7ad0ca4807e5..4c23410d4d15 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+@@ -80,6 +80,7 @@ describe("onEnvironmentResumeLease", () => {
+         secretName: "pc-abc-env",
+         phase: "Running",
+         backend: "sandbox-cr",
+        remoteCwd: "/workspace",
+         resumedLease: true,
+         // sandbox-cr has a pod-exec channel, so native file sync stays enabled.
+         nativeFileSyncUnsupported: false,
+@@ -87,6 +88,22 @@ describe("onEnvironmentResumeLease", () => {
+     );
+   });
+ 
+  it("keeps a custom realized cwd when resuming a sandbox lease", async () => {
+    h.clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockResolvedValue(readySandboxCr("pc-abc-pod")) },
+      core: { readNamespacedPod: vi.fn().mockResolvedValue({ status: { phase: "Running" } }) },
+    };
+    const lease = await plugin.definition.onEnvironmentResumeLease!({
+      driverKey: "kubernetes",
+      companyId: "acme",
+      environmentId: "env-1",
+      config: CONFIG,
+      providerLeaseId: "pc-abc",
+      leaseMetadata: leaseMetadata({ remoteCwd: "/workspace/custom" }),
+    });
+    expect(lease.metadata?.remoteCwd).toBe("/workspace/custom");
+  });
+
+   it("flags a resumed job-backend lease as native-sync-unsupported so the server keeps the base64 fallback", async () => {
+     h.clients = {
+       batch: {
@@ -0,0 +1,59 @@
+#!/bin/sh
+# Fetch upstream Paperclip at the pinned commit, apply our patches and make
+# the Dockerfile buildable with kaniko. Usage: prepare.sh <target-dir>
+set -eu
+HERE=$(cd "$(dirname "$0")" && pwd)
+. "$HERE/upstream.env"
+OUT=$1
+
+rm -rf "$OUT"
+git init -q "$OUT"
+cd "$OUT"
+git fetch -q --depth 1 https://github.com/paperclipai/paperclip.git "$PAPERCLIP_COMMIT"
+git -c advice.detachedHead=false checkout -q FETCH_HEAD
+test "$(git rev-parse HEAD)" = "$PAPERCLIP_COMMIT"
+
+for p in "$HERE"/*.patch; do
+  echo "applying $(basename "$p")"
+  git apply "$p"
+done
+
+# kaniko does not know `COPY --parents` (BuildKit only). Upstream uses it for
+# one line: every sandbox provider's package.json. Replace it with one plain
+# COPY per provider directory, which copies the same files to the same place.
+grep -c -- '^COPY --parents packages/plugins/sandbox-providers/\./\*/package.json packages/plugins/sandbox-providers/$' Dockerfile | grep -qx 1
+awk '
+  /^COPY --parents packages\/plugins\/sandbox-providers\// {
+    while (("ls -d packages/plugins/sandbox-providers/*/package.json" | getline f) > 0) {
+      d = f; sub(/package.json$/, "", d)
+      print "COPY " f " " d
+    }
+    next
+  }
+  { print }
+' Dockerfile > Dockerfile.kaniko
+if grep -q -- '--parents' Dockerfile.kaniko; then
+  echo "Dockerfile.kaniko still uses COPY --parents" >&2
+  exit 1
+fi
+
+# Build the Kubernetes sandbox provider into the image (REN-405). Upstream's
+# `production` target ships only its source; the plugin loader needs
+# dist/manifest.js. Same commands as upstream's `cloud-plugins` stage, run in
+# the `build` stage right after the server build, so production's
+# `COPY --from=build /app /app` carries dist/ and node_modules along. No extra
+# stage: kaniko would snapshot the whole build stage again for it.
+export ANCHOR='RUN test -f server/dist/index.js || (echo "ERROR: server build output missing" && exit 1)'
+grep -c -F -x -- "$ANCHOR" Dockerfile.kaniko | grep -qx 1
+K=packages/plugins/sandbox-providers/kubernetes
+awk -v k="$K" '
+  { print }
+  $0 == ENVIRON["ANCHOR"] {
+    print "RUN pnpm -C " k " install --ignore-workspace --no-lockfile \\"
+    print "  && pnpm -C " k " build \\"
+    print "  && test -f " k "/dist/manifest.js"
+  }
+' Dockerfile.kaniko > Dockerfile.kaniko.new
+mv Dockerfile.kaniko.new Dockerfile.kaniko
+grep -q -- "test -f $K/dist/manifest.js" Dockerfile.kaniko
+rm -rf .git
@@ -0,0 +1,7 @@
+# Upstream paperclipai/paperclip commit this image is built from (REN-401).
+# a027f76 is the master commit the nightly image ran when this was pinned, so
+# the only change against that image is the patch next to this file.
+# PAPERCLIP_VERSION is `git describe --tags --match 'v*' --long` of that
+# commit, the value upstream stamps into its own images. Bump both together.
+PAPERCLIP_COMMIT=a027f76a726e8a556674eded807c2de0e55f5cc0
+PAPERCLIP_VERSION=v2026.916.0-219-ga027f76a7
@@ -14,7 +14,7 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # kubernetes/talos/patches/cluster.yaml. The phase reads the version from that
 # file at run time and passes it to the provisioner; this pin only decides which
 # client talks to it.
-ARG TALOS_VERSION=v1.13.9
+ARG TALOS_VERSION=v1.13.10
 ARG KUBECTL_VERSION=v1.36.2
 ARG FLUX_VERSION=2.9.2
 ARG KOPIA_VERSION=0.23.1
@@ -49,7 +49,7 @@ RUN apk add --no-cache \

 RUN curl -sfL -o /usr/local/bin/talosctl \
      "https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
- && echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e  /usr/local/bin/talosctl" | sha256sum -c - \
+ && echo "28e9ccd9aebd690fa03d8cf2582221d87ea38b2a1704f411d75a03589c58f2aa  /usr/local/bin/talosctl" | sha256sum -c - \
 && chmod +x /usr/local/bin/talosctl

 RUN curl -sfL -o /usr/local/bin/kubectl \
@@ -1,10 +0,0 @@
-# wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea,
-# queried by ~/Notes/bin/wiki_rank.py. Standard library only; runs unprivileged.
-FROM python:3.13-slim
-
-RUN useradd --system --no-create-home --shell /usr/sbin/nologin wikiindex
-COPY wiki_index.py /opt/wiki_index.py
-
-ENV PYTHONUNBUFFERED=1
-USER wikiindex
-ENTRYPOINT ["python3", "/opt/wiki_index.py"]
@@ -1,178 +0,0 @@
-#!/usr/bin/env python3
-"""Unit tests for wiki_index. Run: python3 -m unittest test_wiki_index"""
-
-import hashlib
-import hmac
-import json
-import tempfile
-import unittest
-from pathlib import Path
-
-import wiki_index as wi
-
-NOTE = """---
-title: 'UDM intercepts DNS'
-type: wiki
-tags: [dns, unifi]
-aliases: [udm dns interception, dig 1.1.1.1 wrong]
---
-
-# UDM intercepts DNS
-
-The UDM answers every UDP/53 query itself.
-
-## Details
-
-More.
-"""
-
-
-class ParseTests(unittest.TestCase):
-    def test_note_text(self):
-        title, text = wi.note_text("udm-dns", NOTE)
-        self.assertEqual(title, "UDM intercepts DNS")
-        self.assertIn("udm dns UDM intercepts DNS udm dns interception", text)
-        self.assertIn("dns unifi", text)
-        self.assertTrue(text.endswith("The UDM answers every UDP/53 query itself."))
-        self.assertLessEqual(len(text), wi.EMBED_CHARS)
-
-    def test_credentials_are_stripped(self):
-        original = wi.http
-        wi.http = lambda method, url, body=None, headers=None, timeout=60.0: (
-            {"auth": {"client_token": "t"}} if url.endswith("/login")
-            else {"data": {"data": {"gitea_token": "abc\n", "webhook_secret": " s3cret\n"}}})
-        original_sa = wi.SA_TOKEN
-        try:
-            with tempfile.TemporaryDirectory() as d:
-                wi.SA_TOKEN = Path(d) / "token"
-                wi.SA_TOKEN.write_text("jwt\n")
-                self.assertEqual(wi.Credentials().fetch(), {"gitea_token": "abc", "webhook_secret": "s3cret"})
-        finally:
-            wi.http, wi.SA_TOKEN = original, original_sa
-
-    def test_tree_paging_stops_on_short_last_page(self):
-        # Gitea: truncated=true on every page including the last, "tree": null past it.
-        pages = {
-            1: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000)], "truncated": True, "total_count": 1192},
-            2: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000, 1192)] + [{"path": "raw/x.md", "type": "blob", "sha": "r"}, {"path": "wiki/sub/y.md", "type": "blob", "sha": "y"}], "truncated": True, "total_count": 1192},
-            3: {"tree": None, "truncated": False, "total_count": 1192},
-        }
-        calls = []
-        original = wi.http
-
-        def fake_http(method, url, body=None, headers=None, timeout=60.0):
-            page = int(url.rsplit("page=", 1)[1])
-            calls.append(page)
-            return pages[page]
-
-        wi.http = fake_http
-        try:
-            tree = wi.Gitea("t").tree("abc")
-        finally:
-            wi.http = original
-        self.assertEqual(calls, [1, 2])
-        self.assertEqual(len(tree), 1192)
-        self.assertNotIn("raw/x.md", tree)
-        self.assertNotIn("wiki/sub/y.md", tree)
-        self.assertEqual(tree["wiki/n1191.md"], "s1191")
-
-    def test_signature(self):
-        body = b'{"ref":"refs/heads/master"}'
-        sig = hmac.new(b"s3cret", body, hashlib.sha256).hexdigest()
-        self.assertTrue(wi.verify_signature("s3cret", body, sig))
-        self.assertFalse(wi.verify_signature("other", body, sig))
-        self.assertFalse(wi.verify_signature("s3cret", body, ""))
-
-
-class FakeGitea:
-    def __init__(self, head, tree, files):
-        self._head, self._tree, self.files = head, tree, files
-        self.raw_calls = []
-
-    def head(self):
-        return self._head
-
-    def tree(self, commit):
-        return self._tree
-
-    def raw(self, path, ref):
-        self.raw_calls.append(path)
-        return self.files[path]
-
-
-class IndexTests(unittest.TestCase):
-    def setUp(self):
-        self.tmp = tempfile.TemporaryDirectory()
-        self.path = Path(self.tmp.name) / "index.json"
-        self.original_embed = wi.embed
-        self.embed_calls = []
-
-        def fake_embed(texts, batch=2):
-            self.embed_calls.append(list(texts))
-            return [wi.unit([1.0, float(len(t) % 7)]) for t in texts]
-
-        wi.embed = fake_embed
-
-    def tearDown(self):
-        wi.embed = self.original_embed
-        self.tmp.cleanup()
-
-    def test_sync_is_incremental_and_persists(self):
-        files = {"wiki/udm-dns.md": NOTE, "wiki/ceph.md": "---\ntitle: Ceph\n---\n# Ceph\n\nOSD.\n", "wiki/_index.md": "---\ntitle: Wiki\n---\n"}
-        tree = {"wiki/udm-dns.md": "aaa", "wiki/ceph.md": "bbb", "wiki/_index.md": "ccc"}
-        g = FakeGitea("c1", tree, files)
-        idx = wi.Index(self.path)
-        s = idx.sync(g)
-        self.assertEqual((s["changed"], s["removed"], s["notes"]), (2, 0, 2))
-        self.assertEqual(set(idx.notes), {"udm-dns", "ceph"})
-        self.assertEqual(idx.notes["udm-dns"]["title"], "UDM intercepts DNS")
-        self.assertTrue(self.path.exists())
-
-        # head unchanged: nothing fetched
-        s = idx.sync(g)
-        self.assertEqual(s.get("skipped"), "head unchanged")
-        self.assertEqual(len(g.raw_calls), 2)
-
-        # one note changed, one removed, new head
-        g2 = FakeGitea("c2", {"wiki/udm-dns.md": "aaa2"}, {"wiki/udm-dns.md": NOTE.replace("itself", "itself!")})
-        s = idx.sync(g2)
-        self.assertEqual((s["changed"], s["removed"], s["notes"]), (1, 1, 1))
-        self.assertEqual(g2.raw_calls, ["wiki/udm-dns.md"])
-
-        # restart loads the persisted index
-        idx2 = wi.Index(self.path)
-        self.assertEqual(idx2.commit, "c2")
-        self.assertEqual(set(idx2.notes), {"udm-dns"})
-
-    def test_rank_orders_by_cosine(self):
-        idx = wi.Index(self.path)
-        idx.notes = {"a": {"sha": "1", "title": "A", "vector": wi.unit([1.0, 0.0])},
-                     "b": {"sha": "2", "title": "B", "vector": wi.unit([0.0, 1.0])}}
-        wi.embed = lambda texts, batch=2: [wi.unit([0.2, 1.0])]
-        rows = idx.rank("q", k=5)
-        self.assertEqual([r["slug"] for r in rows], ["b", "a"])
-        self.assertGreater(rows[0]["score"], rows[1]["score"])
-
-    def test_status_reports_age_and_errors(self):
-        idx = wi.Index(self.path)
-        self.assertFalse(idx.status()["ready"])
-        g = FakeGitea("c1", {"wiki/a.md": "x"}, {"wiki/a.md": "---\ntitle: A\n---\n# A\n\nt\n"})
-        idx.sync(g)
-        st = idx.status()
-        self.assertTrue(st["ready"])
-        self.assertEqual(st["notes"], 1)
-        self.assertEqual(st["commit"], "c1")
-
-        class Broken(FakeGitea):
-            def head(self):
-                raise wi.ApiError("gitea down")
-
-        with self.assertRaises(wi.ApiError):
-            idx.sync(Broken("c1", {}, {}))
-        self.assertIn("gitea down", idx.status()["last_error"])
-        self.assertEqual(idx.status()["notes"], 1)  # last good index kept
-        self.assertTrue(idx.status()["ready"])  # and still served
-
-
-if __name__ == "__main__":
-    unittest.main()
@@ -1,408 +0,0 @@
-#!/usr/bin/env python3
-"""wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea.
-
-Source of truth is the notes repository on Gitea. The index holds, per wiki
-note, the git blob sha, title and a bge-m3 vector of slug/title/aliases/tags/
-first paragraph. A sync fetches the repo tree, embeds every note whose blob sha
-changed (via wiki-embed, in-cluster) and drops notes that left the tree; it
-runs at start, on every push webhook from Gitea, and every RESYNC_SECONDS.
-
-HTTP:
-  POST /rank     {"query": str, "k": int}   -> {"rows": [{"slug","title","score"}], "index": {...}}
-  GET  /health   -> 200 always (liveness) with {"ready", "notes", "commit", "synced_at", "age_s", "syncing", "last_error"}
-  GET  /ready    -> 200 when an index exists, else 503 (readiness)
-  POST /webhook  Gitea push event, HMAC-SHA256 in X-Gitea-Signature -> {"queued": true}
-
-Credentials (Gitea token, webhook secret) are read from OpenBao through the
-Kubernetes auth role on every use; they never land in a Secret or a log line.
-Standard library only.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import hmac
-import json
-import logging
-import os
-import re
-import sys
-import threading
-import time
-import urllib.error
-import urllib.parse
-import urllib.request
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
-
-LOG = logging.getLogger("wiki-index")
-
-GITEA_URL = os.environ.get("GITEA_URL", "https://gitea.renner.uno").rstrip("/")
-NOTES_REPO = os.environ.get("NOTES_REPO", "k.renner/notes")
-NOTES_BRANCH = os.environ.get("NOTES_BRANCH", "master")
-WIKI_DIR = os.environ.get("WIKI_DIR", "wiki").strip("/")
-EMBED_URL = os.environ.get("EMBED_URL", "http://wiki-embed.ai.svc.cluster.local:8080").rstrip("/")
-EMBED_MODEL = os.environ.get("EMBED_MODEL", "bge-m3")
-EMBED_CHARS = int(os.environ.get("EMBED_CHARS", "1500"))
-INDEX_PATH = Path(os.environ.get("INDEX_PATH", "/data/index.json"))
-RESYNC_SECONDS = int(os.environ.get("RESYNC_SECONDS", "1800"))
-LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
-BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
-BAO_ROLE = os.environ.get("BAO_ROLE", "wiki-index")
-BAO_SECRET_PATH = os.environ.get("BAO_SECRET_PATH", "homelab/data/apps/ai/wiki-index")
-SA_TOKEN = Path(os.environ.get("SA_TOKEN", "/var/run/secrets/kubernetes.io/serviceaccount/token"))
-
-
-class ApiError(Exception):
-    pass
-
-
-# ---------------------------------------------------------------- http helpers
-
-
-def http(method: str, url: str, body=None, headers=None, timeout: float = 60.0):
-    data = None
-    hdrs = dict(headers or {})
-    if body is not None:
-        data = json.dumps(body).encode("utf-8")
-        hdrs.setdefault("Content-Type", "application/json")
-    req = urllib.request.Request(url, data=data, method=method, headers=hdrs)
-    try:
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            raw = resp.read()
-            ctype = resp.headers.get("Content-Type", "")
-    except urllib.error.HTTPError as e:
-        raise ApiError(f"{method} {url}: HTTP {e.code}: {e.read().decode('utf-8', 'replace')[:300]}") from e
-    except urllib.error.URLError as e:
-        raise ApiError(f"{method} {url}: {e.reason}") from e
-    if "json" in ctype:
-        return json.loads(raw.decode("utf-8"))
-    return raw
-
-
-# ---------------------------------------------------------------- credentials
-
-
-class Credentials:
-    """Gitea token and webhook secret from OpenBao, fetched per use."""
-
-    def __init__(self, static: dict | None = None):
-        self.static = static  # tests / local runs
-
-    def fetch(self) -> dict:
-        if self.static is not None:
-            return self.static
-        jwt = SA_TOKEN.read_text(encoding="utf-8").strip()
-        login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
-        token = login["auth"]["client_token"]
-        kv = http("GET", f"{BAO_ADDR}/v1/{BAO_SECRET_PATH}", headers={"X-Vault-Token": token})
-        data = {k: str(v).strip() for k, v in kv["data"]["data"].items()}  # `bao kv put key=@file` keeps the file's newline
-        for key in ("gitea_token", "webhook_secret"):
-            if not data.get(key):
-                raise ApiError(f"OpenBao {BAO_SECRET_PATH} has no key {key}")
-        return data
-
-
-# ---------------------------------------------------------------- gitea
-
-
-class Gitea:
-    def __init__(self, token: str, base: str = GITEA_URL, repo: str = NOTES_REPO, branch: str = NOTES_BRANCH):
-        self.base, self.repo, self.branch = base, repo, branch
-        self.headers = {"Authorization": f"token {token}"}
-
-    def api(self, path: str, **params):
-        q = ("?" + urllib.parse.urlencode(params)) if params else ""
-        return http("GET", f"{self.base}/api/v1/repos/{self.repo}{path}{q}", headers=self.headers)
-
-    def head(self) -> str:
-        return self.api(f"/branches/{urllib.parse.quote(self.branch, safe='')}")["commit"]["id"]
-
-    def tree(self, commit: str) -> dict[str, str]:
-        """path -> blob sha for every wiki note at that commit."""
-        out: dict[str, str] = {}
-        page, per_page, seen = 1, 1000, 0
-        while True:
-            t = self.api(f"/git/trees/{commit}", recursive="true", per_page=per_page, page=page)
-            entries = t.get("tree") or []  # Gitea answers "tree": null past the last page
-            for e in entries:
-                p = e.get("path", "")
-                if e.get("type") == "blob" and p.startswith(WIKI_DIR + "/") and p.endswith(".md") and "/" not in p[len(WIKI_DIR) + 1:]:
-                    out[p] = e["sha"]
-            seen += len(entries)
-            total = int(t.get("total_count") or 0)
-            if not entries or len(entries) < per_page or (total and seen >= total):
-                break
-            page += 1
-            if page > 50:
-                raise ApiError("tree paging runaway")
-        return out
-
-    def raw(self, path: str, ref: str) -> str:
-        data = http("GET", f"{self.base}/api/v1/repos/{self.repo}/raw/{urllib.parse.quote(path)}?ref={urllib.parse.quote(ref, safe='')}",
-                    headers=self.headers)
-        return data.decode("utf-8") if isinstance(data, bytes) else json.dumps(data)
-
-
-# ---------------------------------------------------------------- notes
-
-
-def parse_frontmatter(text: str) -> tuple[dict, str]:
-    m = re.match(r"^---[ \t]*\n(.*?)\n---[ \t]*(?:\n|$)", text, re.DOTALL)
-    if not m:
-        return {}, text
-    fm: dict = {}
-    for line in m.group(1).splitlines():
-        mm = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
-        if mm:
-            fm[mm.group(1)] = mm.group(2).strip()
-    return fm, text[m.end():]
-
-
-def as_list(value: str) -> list[str]:
-    s = (value or "").strip()
-    if s.startswith("[") and s.endswith("]"):
-        s = s[1:-1]
-    return [p.strip().strip("'\"") for p in s.split(",") if p.strip()]
-
-
-def first_paragraph(body: str) -> str:
-    para: list[str] = []
-    seen_h1 = False
-    for line in body.splitlines():
-        if line.startswith("# "):
-            seen_h1 = True
-            continue
-        if not seen_h1:
-            continue
-        if line.strip() == "" or line.startswith("#"):
-            if para:
-                break
-            continue
-        para.append(line.strip())
-    return " ".join(para)
-
-
-def note_text(slug: str, text: str) -> tuple[str, str]:
-    """(title, embedding text) for one note file."""
-    fm, body = parse_frontmatter(text)
-    title = (fm.get("title") or slug).strip().strip("'\"")
-    parts = [slug.replace("-", " "), title, " ".join(as_list(fm.get("aliases", ""))),
-             " ".join(as_list(fm.get("tags", ""))), first_paragraph(body)]
-    return title, " ".join(parts)[:EMBED_CHARS]
-
-
-# ---------------------------------------------------------------- embeddings
-
-
-def embed(texts: list[str], batch: int = 2) -> list[list[float]]:
-    out: list[list[float]] = []
-    for i in range(0, len(texts), batch):
-        resp = http("POST", f"{EMBED_URL}/v1/embeddings", {"model": EMBED_MODEL, "input": texts[i:i + batch]}, timeout=120)
-        rows = sorted(resp["data"], key=lambda d: d["index"])
-        out.extend(unit(d["embedding"]) for d in rows)
-    return out
-
-
-def unit(v: list[float]) -> list[float]:
-    n = sum(x * x for x in v) ** 0.5 or 1.0
-    return [round(x / n, 5) for x in v]
-
-
-# ---------------------------------------------------------------- index
-
-
-class Index:
-    def __init__(self, path: Path):
-        self.path = path
-        self.lock = threading.Lock()
-        self.notes: dict[str, dict] = {}  # slug -> {sha, title, vector}
-        self.commit = ""
-        self.synced_at = 0.0
-        self.syncing = False
-        self.last_error = ""
-        if path.exists():
-            try:
-                d = json.loads(path.read_text(encoding="utf-8"))
-                self.notes, self.commit, self.synced_at = d.get("notes", {}), d.get("commit", ""), d.get("synced_at", 0.0)
-                LOG.info("loaded index: %d notes at %s", len(self.notes), self.commit[:8])
-            except (OSError, ValueError) as e:
-                LOG.warning("index unreadable (%s); starting empty", e)
-
-    def save(self) -> None:
-        self.path.parent.mkdir(parents=True, exist_ok=True)
-        tmp = self.path.with_suffix(".tmp")
-        tmp.write_text(json.dumps({"notes": self.notes, "commit": self.commit, "synced_at": self.synced_at}), encoding="utf-8")
-        tmp.replace(self.path)
-
-    def sync(self, gitea: Gitea) -> dict:
-        """Bring the index to the branch head; returns a summary."""
-        with self.lock:
-            self.syncing = True
-        try:
-            head = gitea.head()
-            if head == self.commit and self.notes:
-                summary = {"commit": head, "changed": 0, "removed": 0, "notes": len(self.notes), "skipped": "head unchanged"}
-                self.synced_at = time.time()
-                self.save()
-                return summary
-            tree = gitea.tree(head)
-            slugs = {p[len(WIKI_DIR) + 1:-3]: (p, sha) for p, sha in tree.items() if not p.endswith("/_index.md")}
-            todo = [s for s, (p, sha) in slugs.items() if self.notes.get(s, {}).get("sha") != sha]
-            removed = [s for s in list(self.notes) if s not in slugs]
-            LOG.info("sync %s: %d notes, %d to embed, %d removed", head[:8], len(slugs), len(todo), len(removed))
-            fresh: dict[str, dict] = {}
-            for i in range(0, len(todo), 20):
-                chunk = todo[i:i + 20]
-                texts, titles = [], {}
-                for s in chunk:
-                    title, text = note_text(s, gitea.raw(slugs[s][0], head))
-                    titles[s] = title
-                    texts.append(text)
-                for s, vec in zip(chunk, embed(texts)):
-                    fresh[s] = {"sha": slugs[s][1], "title": titles[s], "vector": vec}
-            with self.lock:
-                for s in removed:
-                    self.notes.pop(s, None)
-                self.notes.update(fresh)
-                self.commit = head
-                self.synced_at = time.time()
-                self.last_error = ""
-                self.save()
-            return {"commit": head, "changed": len(todo), "removed": len(removed), "notes": len(self.notes)}
-        except Exception as e:
-            with self.lock:
-                self.last_error = str(e)[:300]
-            raise
-        finally:
-            with self.lock:
-                self.syncing = False
-
-    def rank(self, query: str, k: int = 60) -> list[dict]:
-        q = embed([query])[0]
-        with self.lock:
-            rows = [(s, n["title"], sum(a * b for a, b in zip(q, n["vector"]))) for s, n in self.notes.items()]
-        rows.sort(key=lambda r: -r[2])
-        return [{"slug": s, "title": t, "score": round(sc, 4)} for s, t, sc in rows[:k]]
-
-    def status(self) -> dict:
-        """`ready` = an index exists and can answer /rank; a failed sync keeps the last good index."""
-        with self.lock:
-            return {"ready": bool(self.notes), "notes": len(self.notes), "commit": self.commit,
-                    "synced_at": self.synced_at, "age_s": round(time.time() - self.synced_at) if self.synced_at else None,
-                    "syncing": self.syncing, "last_error": self.last_error or None}
-
-
-# ---------------------------------------------------------------- syncer
-
-
-class Syncer(threading.Thread):
-    """One worker: syncs on demand (webhook) and every RESYNC_SECONDS."""
-
-    def __init__(self, index: Index, creds: Credentials, interval: int):
-        super().__init__(daemon=True, name="syncer")
-        self.index, self.creds, self.interval = index, creds, interval
-        self.wake = threading.Event()
-
-    def trigger(self) -> None:
-        self.wake.set()
-
-    RETRY_SECONDS = 60
-
-    def run(self) -> None:
-        while True:
-            try:
-                data = self.creds.fetch()
-                summary = self.index.sync(Gitea(data["gitea_token"]))
-                LOG.info("sync done: %s", summary)
-                wait = self.interval
-            except Exception as e:  # keep serving the last good index; retry soon, not next interval
-                LOG.error("sync failed: %s", e)
-                wait = self.RETRY_SECONDS
-            self.wake.wait(wait)
-            self.wake.clear()
-
-
-def verify_signature(secret: str, body: bytes, signature: str) -> bool:
-    digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
-    return hmac.compare_digest(digest, (signature or "").strip())
-
-
-# ---------------------------------------------------------------- server
-
-
-def make_handler(index: Index, syncer: Syncer, creds: Credentials):
-    class Handler(BaseHTTPRequestHandler):
-        def log_message(self, fmt, *args):
-            LOG.info("%s %s", self.address_string(), fmt % args)
-
-        def send_json(self, code: int, obj) -> None:
-            data = json.dumps(obj).encode("utf-8")
-            self.send_response(code)
-            self.send_header("Content-Type", "application/json")
-            self.send_header("Content-Length", str(len(data)))
-            self.end_headers()
-            self.wfile.write(data)
-
-        def body(self) -> bytes:
-            n = int(self.headers.get("Content-Length") or 0)
-            return self.rfile.read(n) if n else b""
-
-        def do_GET(self):
-            if self.path == "/health":  # liveness: the process serves
-                return self.send_json(200, index.status())
-            if self.path == "/ready":  # readiness: an index exists
-                st = index.status()
-                return self.send_json(200 if st["ready"] else 503, st)
-            self.send_json(404, {"error": "not found"})
-
-        def do_POST(self):
-            if self.path == "/rank":
-                try:
-                    req = json.loads(self.body() or b"{}")
-                    query = str(req.get("query", "")).strip()
-                    k = max(1, min(int(req.get("k", 60)), 500))
-                    if not query:
-                        return self.send_json(400, {"error": "query required"})
-                    rows = index.rank(query, k)
-                    return self.send_json(200, {"rows": rows, "index": index.status()})
-                except ApiError as e:
-                    return self.send_json(502, {"error": str(e)})
-                except (ValueError, TypeError) as e:
-                    return self.send_json(400, {"error": str(e)})
-            if self.path == "/webhook":
-                raw = self.body()
-                try:
-                    secret = creds.fetch()["webhook_secret"]
-                except ApiError as e:
-                    return self.send_json(503, {"error": str(e)})
-                if not verify_signature(secret, raw, self.headers.get("X-Gitea-Signature", "")):
-                    return self.send_json(403, {"error": "bad signature"})
-                try:
-                    event = json.loads(raw or b"{}")
-                except ValueError:
-                    return self.send_json(400, {"error": "bad json"})
-                ref = event.get("ref", "")
-                if ref and ref != f"refs/heads/{NOTES_BRANCH}":
-                    return self.send_json(200, {"queued": False, "ref": ref})
-                syncer.trigger()
-                return self.send_json(200, {"queued": True})
-            self.send_json(404, {"error": "not found"})
-
-    return Handler
-
-
-def main() -> int:
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout)
-    index = Index(INDEX_PATH)
-    creds = Credentials()
-    syncer = Syncer(index, creds, RESYNC_SECONDS)
-    syncer.start()
-    server = ThreadingHTTPServer(("", LISTEN_PORT), make_handler(index, syncer, creds))
-    LOG.info("listening on :%d, repo %s@%s, embed %s", LISTEN_PORT, NOTES_REPO, NOTES_BRANCH, EMBED_URL)
-    server.serve_forever()
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
@@ -0,0 +1,976 @@
+# Paperclip Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Deploy Paperclip (server-local agent runs, option B) into the homelab cluster behind an Authentik gate, with CNPG Postgres and OpenBao-sourced secrets.
+
+**Architecture:** One Flux `Kustomization` (`paperclip`) applies `kubernetes/apps/paperclip/`: a single-replica Deployment on a ceph-block PVC, CNPG `pclp-pg` (PG17) with barman backups to Hetzner, and a Traefik Ingress that backends only the `paperclip-gate` Authentik outpost. Namespace, baseline CNP, ClusterSecretStore allowlist and image automation live in the existing `infrastructure` tree.
+
+**Tech Stack:** Flux v2 (kustomize-controller, image-reflector v1.2.5), CNPG + barman-cloud plugin, External Secrets → OpenBao, Cilium CNP, Traefik, Authentik blueprints.
+
+**Spec:** `docs/superpowers/specs/2026-09-25-paperclip-design.md`
+
+## Global Constraints
+
+- Worktree: `/home/krenner/source/homelab-worktrees/paperclip`, branch `feat/paperclip`. Never edit the main checkout.
+- Commits as claude-bot: `git -c user.name='Claude Bot' -c user.email='claude-bot@renner.uno' commit …`, message ends with `Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>`.
+- Push: `git -c credential.helper='store --file=/home/krenner/.config/claude-bot/creds' push …`. PR via `tea pr create --login claude-bot`. Never self-merge.
+- One Bash command per call: no `&&`, `;`, `|` chaining, no `2>/dev/null`, no `python -c`.
+- Namespace `paperclip`; hostname `paperclip.renner.uno`; LAN-only (no `dns.renner.uno/public` label).
+- Image `ghcr.io/paperclipai/paperclip`, tag `nightly`, digest-pinned, `digestReflectionPolicy: Always`.
+- Postgres: CNPG `pclp-pg`, `ghcr.io/cloudnative-pg/postgresql:17`, 2 instances, `ceph-block`.
+- PVC `paperclip-data`, `ceph-block`, 20Gi, mounted at `/paperclip`.
+- OpenBao key `apps/paperclip` with fields `BETTER_AUTH_SECRET`, `PAPERCLIP_TOOL_ACTION_SIGNING_SECRET`, `PAPERCLIP_SECRETS_MASTER_KEY` (64-char hex), `CLAUDE_CODE_OAUTH_TOKEN`.
+- Env: `PAPERCLIP_DEPLOYMENT_MODE=authenticated`, `PAPERCLIP_DEPLOYMENT_EXPOSURE=private`, `PAPERCLIP_PUBLIC_URL=https://paperclip.renner.uno`, `HOST=0.0.0.0`, `PORT=3100`.
+- Manifest comments: one terse factual line; rationale goes in commit messages/PR.
+- No `kubectl apply`/`exec`/`rollout restart` against the live cluster; Flux applies after merge. Server-side `--dry-run=server` is allowed.
+
+## Review Focus
+
+1. **Pod replaced mid agent run** (image bump, secret rotation via Reloader, node drain) — `Recreate` kills in-flight runs; expected: Paperclip marks them failed/recoverable on restart, not wedged. Covered in Task 9 Step 6.
+2. **Shared OAuth token expired or missing** — expected: agent runs fail with a visible auth error in Paperclip, server stays healthy. Covered in Task 9 Step 7.
+3. **Direct access bypassing the gate** from another namespace pod — expected: connection refused/dropped by CNP. Covered in Task 9 Step 4.
+4. **Unauthenticated LAN request** to `paperclip.renner.uno` — expected: redirect to Authentik, never Paperclip's UI. Covered in Task 9 Step 3.
+5. **Nightly image with a breaking DB migration** — expected: a base backup exists before the first bump lands. Covered in Task 9 Step 5.
+
+---
+
+### Task 1: Upstream facts that decide two manifest values
+
+No repo changes; produces two decisions consumed by Task 6.
+
+**Interfaces:**
+- Produces: `SIGNUP_DISABLED` (`"true"` or `"false"`), `CONFIG_PARTIAL_OK` (yes/no), `ENTRYPOINT_NEEDS_ROOT` (yes/no).
+
+- [ ] **Step 1: Sign-up vs instance claim.** Read upstream `server/src/auth/better-auth.ts`, `server/src/routes/auth.ts`, and the claim flow for `authenticated`+`private` (search `claim` under `server/src`):
+
+Run: `gh api repos/paperclipai/paperclip/contents/server/src/routes/auth.ts -H "Accept: application/vnd.github.raw"`
+
+Decide: if the first-admin claim creates the account through a path that ignores `disableSignUp` (e.g. invite/claim token handler calling the adapter directly), `SIGNUP_DISABLED="true"`. Otherwise `SIGNUP_DISABLED="false"` — the Authentik gate is the access control, and a follow-up PR flips it to `"true"` after the admin is claimed.
+
+- [ ] **Step 2: Partial config.json.** Read `server/src/config.ts` and the config-file loader it calls (search `PAPERCLIP_CONFIG` / `readConfigFile`).
+
+Run: `gh api repos/paperclipai/paperclip/contents/server/src/config.ts -H "Accept: application/vnd.github.raw"`
+
+Decide `CONFIG_PARTIAL_OK=yes` if a file containing only `{"telemetry":{"enabled":false}}` parses (every other section optional/defaulted). If not, stop and report to the user — do not invent config fields.
+
+- [ ] **Step 3: Entrypoint privilege.**
+
+Run: `gh api repos/paperclipai/paperclip/contents/docker-entrypoint.sh -H "Accept: application/vnd.github.raw"`
+
+`ENTRYPOINT_NEEDS_ROOT=yes` if it calls `usermod`/`groupmod`/`chown`/`gosu`. Record the result for Task 6.
+
+- [ ] **Step 4: Anonymous health response.**
+
+Run: `gh api repos/paperclipai/paperclip/contents/server/src/routes/health.ts -H "Accept: application/vnd.github.raw"`
+
+Confirm `GET /api/health` returns HTTP 200 to an anonymous caller when healthy (probes use it). If it returns 401 anonymously, use a `tcpSocket` probe on 3100 in Task 6 instead.
+
+---
+
+### Task 2: Namespace, baseline CNP, ClusterSecretStore allowlist
+
+**Files:**
+- Modify: `kubernetes/infrastructure/namespaces/namespaces.yaml` (append)
+- Create: `kubernetes/infrastructure/network-policies/paperclip-ns.yaml`
+- Modify: `kubernetes/infrastructure/network-policies/kustomization.yaml`
+- Modify: `kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml` (sorted list)
+
+**Interfaces:**
+- Produces: namespace `paperclip`; ESO may reference `openbao` from it.
+
+- [ ] **Step 1: Append the namespace** to the end of `namespaces.yaml`:
+
+```yaml
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: paperclip
+```
+
+- [ ] **Step 2: Create `paperclip-ns.yaml`** (copy of `paperless-ns.yaml`'s `default-deny-ingress`, ns renamed, paperless-only CNPs dropped):
+
+```yaml
+# Traefik is not admitted: the only entry is the paperclip-gate outpost (apps/paperclip/network-policy.yaml).
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: default-deny-ingress
+  namespace: paperclip
+spec:
+  endpointSelector: {}
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": database
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": monitoring
+    - fromEntities:
+        - host
+        - remote-node
+    - fromEndpoints:
+        - matchExpressions:
+            - key: io.kubernetes.pod.namespace
+              operator: Exists
+      icmps:
+        - fields:
+            - { type: 0,   family: IPv4 }
+            - { type: 3,   family: IPv4 }
+            - { type: 8,   family: IPv4 }
+            - { type: 11,  family: IPv4 }
+            - { type: 12,  family: IPv4 }
+            - { type: 1,   family: IPv6 }
+            - { type: 2,   family: IPv6 }
+            - { type: 3,   family: IPv6 }
+            - { type: 4,   family: IPv6 }
+            - { type: 128, family: IPv6 }
+            - { type: 129, family: IPv6 }
+            - { type: 133, family: IPv6 }
+            - { type: 134, family: IPv6 }
+            - { type: 135, family: IPv6 }
+            - { type: 136, family: IPv6 }
+            - { type: 143, family: IPv6 }
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": authentik
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ingress
+            app.kubernetes.io/name: traefik
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": rook-ceph
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+    - toEntities:
+        - host
+        - remote-node
+        - world
+    - toEndpoints:
+        - matchExpressions:
+            - key: io.kubernetes.pod.namespace
+              operator: Exists
+      icmps:
+        - fields:
+            - { type: 0,   family: IPv4 }
+            - { type: 3,   family: IPv4 }
+            - { type: 8,   family: IPv4 }
+            - { type: 11,  family: IPv4 }
+            - { type: 12,  family: IPv4 }
+            - { type: 1,   family: IPv6 }
+            - { type: 2,   family: IPv6 }
+            - { type: 3,   family: IPv6 }
+            - { type: 4,   family: IPv6 }
+            - { type: 128, family: IPv6 }
+            - { type: 129, family: IPv6 }
+            - { type: 133, family: IPv6 }
+            - { type: 134, family: IPv6 }
+            - { type: 135, family: IPv6 }
+            - { type: 136, family: IPv6 }
+            - { type: 143, family: IPv6 }
+```
+
+- [ ] **Step 3: Register it** — in `network-policies/kustomization.yaml` add `  - paperclip-ns.yaml` directly after `  - paperless-ns.yaml`.
+
+- [ ] **Step 4: Allowlist the namespace** — in `openbao-clustersecretstore.yaml` insert `        - paperclip` between `        - monitoring` and `        - paperless` (list is sorted).
+
+- [ ] **Step 5: Render check.**
+
+Run: `kubectl kustomize kubernetes/infrastructure/network-policies`
+Expected: output contains `namespace: paperclip`, exit 0.
+
+Run: `kubectl apply --dry-run=server -f kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml`
+Expected: `clustersecretstore.external-secrets.io/openbao configured (server dry run)`.
+
+- [ ] **Step 6: Commit** `feat(paperclip): namespace, baseline CNP and OpenBao allowlist`.
+
+---
+
+### Task 3: OpenBao secret `apps/paperclip`
+
+Out-of-repo; must exist before merge or the ExternalSecret never becomes Ready.
+
+**Interfaces:**
+- Produces: OpenBao KV `apps/paperclip` with the four fields in Global Constraints.
+
+- [ ] **Step 1:** Ask wiki-synth for the claude-bot OpenBao write procedure (note `claude-bot-openbao-write-only-identity`): login method, KV mount, exact `bao kv put`/`patch` syntax for an `apps/<name>` key. Use exactly what it returns.
+
+- [ ] **Step 2: Write the generated fields** as claude-bot (write-only identity — never read values back). Values generated inline by the command:
+
+`BETTER_AUTH_SECRET` = `openssl rand -hex 32`, `PAPERCLIP_TOOL_ACTION_SIGNING_SECRET` = `openssl rand -hex 32`, `PAPERCLIP_SECRETS_MASTER_KEY` = `openssl rand -hex 32` (64 hex chars = 32 bytes, accepted format).
+
+Generate each with a separate `openssl rand -hex 32` call and pass it directly into the `bao kv put` from Step 1; do not echo or log them.
+
+- [ ] **Step 3: Ask the user** to add the shared token themselves (never handle it): "Run `claude setup-token`, then `bao kv patch <mount>/apps/paperclip CLAUDE_CODE_OAUTH_TOKEN=<token>`." Wait for confirmation.
+
+- [ ] **Step 4: Verify field names only** (no values): `bao kv metadata get <mount>/apps/paperclip` shows a current version ≥2. If claude-bot cannot read metadata, ask the user to confirm the four keys exist.
+
+---
+
+### Task 4: Image automation + initial digest
+
+**Files:**
+- Create: `kubernetes/infrastructure/image-automation/paperclip.yaml`
+- Modify: `kubernetes/infrastructure/image-automation/kustomization.yaml`
+
+**Interfaces:**
+- Produces: ImagePolicy `flux-system:paperclip`; `INITIAL_DIGEST` (`sha256:…`) for Task 6.
+
+- [ ] **Step 1: Create `paperclip.yaml`:**
+
+```yaml
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageRepository
+metadata:
+  name: paperclip
+  namespace: flux-system
+spec:
+  image: ghcr.io/paperclipai/paperclip
+  interval: 1h
+---
+# Upstream publishes no semver tags; track the nightly tag by digest.
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImagePolicy
+metadata:
+  name: paperclip
+  namespace: flux-system
+spec:
+  imageRepositoryRef:
+    name: paperclip
+  digestReflectionPolicy: Always
+  filterTags:
+    pattern: '^nightly$'
+  policy:
+    alphabetical:
+      order: asc
+```
+
+- [ ] **Step 2: Register** — add `  - paperclip.yaml` after `  - paperless-ngx.yaml` in `image-automation/kustomization.yaml`.
+
+- [ ] **Step 3: Server dry-run.**
+
+Run: `kubectl apply --dry-run=server -f kubernetes/infrastructure/image-automation/paperclip.yaml`
+Expected: both objects `created (server dry run)`; no `unknown field "digestReflectionPolicy"` error.
+
+- [ ] **Step 4: Resolve the current nightly digest.**
+
+The workstation has `docker` only (no skopeo/crane/buildx, checked 2026-09-25); `docker manifest inspect` lists per-platform digests but not the index digest, so pull once:
+
+Run: `docker pull ghcr.io/paperclipai/paperclip:nightly`
+Run: `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/paperclipai/paperclip:nightly`
+Expected: `ghcr.io/paperclipai/paperclip@sha256:<64 hex>`. Record the `sha256:…` part as `INITIAL_DIGEST`.
+Run: `docker image rm ghcr.io/paperclipai/paperclip:nightly`
+
+- [ ] **Step 5: Commit** `feat(paperclip): Flux image automation tracks the nightly digest`.
+
+---
+
+### Task 5: Postgres (CNPG) and secrets in the app directory
+
+**Files:**
+- Create: `kubernetes/apps/paperclip/cluster.yaml`, `objectstore.yaml`, `scheduled-backup.yaml`, `hetzner-creds-externalsecret.yaml`, `externalsecret.yaml`
+
+**Interfaces:**
+- Produces: Secret `pclp-pg-app` (CNPG, key `uri`), Service `pclp-pg-rw`, Secret `paperclip-secrets`.
+
+- [ ] **Step 1: `cluster.yaml`:**
+
+```yaml
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: pclp-pg
+  namespace: paperclip
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled # data guard
+spec:
+  instances: 2
+  smartShutdownTimeout: 10
+  stopDelay: 60
+  # PG17: upstream's external Postgres target (docker/docker-compose.yml).
+  imageName: ghcr.io/cloudnative-pg/postgresql:17
+  storage:
+    storageClass: ceph-block
+    size: 10Gi
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: pclp-pg-store
+  bootstrap:
+    initdb:
+      database: paperclip
+      owner: paperclip
+  postgresql:
+    parameters:
+      shared_buffers: "256MB"
+      max_connections: "200"
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      wal_compression: "zstd"
+      archive_timeout: "30min"
+    synchronous:
+      method: any
+      number: 1
+      dataDurability: preferred
+  resources:
+    requests:
+      cpu: 200m
+      memory: 512Mi
+    limits:
+      memory: 1Gi
+```
+
+- [ ] **Step 2: `objectstore.yaml`:**
+
+```yaml
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: pclp-pg-store
+  namespace: paperclip
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled # backup target
+spec:
+  configuration:
+    destinationPath: s3://renner-uno-k8s-cnpg/pclp-pg
+    endpointURL: https://fsn1.your-objectstorage.com
+    s3Credentials:
+      accessKeyId:
+        name: hetzner-s3-creds
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: hetzner-s3-creds
+        key: ACCESS_SECRET_KEY
+      region:
+        name: hetzner-s3-creds
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  retentionPolicy: "30d"
+```
+
+- [ ] **Step 3: `scheduled-backup.yaml`:**
+
+```yaml
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: pclp-pg-daily
+  namespace: paperclip
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled # backup schedule
+spec:
+  schedule: "0 45 2 * * *"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: pclp-pg
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
+```
+
+- [ ] **Step 4: `hetzner-creds-externalsecret.yaml`:**
+
+```yaml
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hetzner-s3-creds
+  namespace: paperclip
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled # barman S3 creds
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hetzner-s3-creds
+  dataFrom:
+    - extract:
+        key: infrastructure/hetzner-s3
+```
+
+- [ ] **Step 5: `externalsecret.yaml`:**
+
+```yaml
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: paperclip-secrets
+  namespace: paperclip
+spec:
+  refreshInterval: 10m
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: paperclip-secrets
+  dataFrom:
+    - extract:
+        key: apps/paperclip
+```
+
+- [ ] **Step 6: Commit** `feat(paperclip): CNPG pclp-pg with barman backups and OpenBao secrets`. (Rendered together with Task 6's kustomization.)
+
+---
+
+### Task 6: Server workload, storage, ingress, network policy
+
+**Files:**
+- Create: `kubernetes/apps/paperclip/pvc.yaml`, `paperclip.yaml`, `gate-service.yaml`, `ingress.yaml`, `network-policy.yaml`, `kustomization.yaml`
+
+**Interfaces:**
+- Consumes: `pclp-pg-app` (`uri`), `paperclip-secrets`, ImagePolicy `flux-system:paperclip`, `INITIAL_DIGEST` (Task 4), `SIGNUP_DISABLED` / `ENTRYPOINT_NEEDS_ROOT` / probe choice (Task 1).
+- Produces: pods labelled `app.kubernetes.io/name: paperclip` listening on 3100; Service `paperclip:3100`; ExternalName `ak-outpost-paperclip-gate:9000`.
+
+- [ ] **Step 1: `pvc.yaml`:**
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: paperclip-data
+  namespace: paperclip
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled # data guard
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: ceph-block
+  resources:
+    requests:
+      storage: 20Gi
+```
+
+- [ ] **Step 2: `paperclip.yaml`** — replace `<INITIAL_DIGEST>` with the Task 4 value and `<SIGNUP_DISABLED>` with the Task 1 value. If Task 1 found `ENTRYPOINT_NEEDS_ROOT=no`, replace the container `securityContext` with `runAsNonRoot: true`, `runAsUser: 1000`, `runAsGroup: 1000`, `allowPrivilegeEscalation: false`, `capabilities: {drop: ["ALL"]}`. If Task 1 chose `tcpSocket`, replace each `httpGet` block with `tcpSocket: {port: http}`.
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: paperclip
+  namespace: paperclip
+  labels:
+    app.kubernetes.io/name: paperclip
+  annotations:
+    secret.reloader.stakater.com/reload: paperclip-secrets
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: paperclip
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: paperclip
+    spec:
+      enableServiceLinks: false
+      securityContext:
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # Telemetry is only configurable in config.json.
+        - name: disable-telemetry
+          image: ghcr.io/paperclipai/paperclip:nightly@<INITIAL_DIGEST> # {"$imagepolicy": "flux-system:paperclip"}
+          command:
+            - sh
+            - -c
+            - |
+              set -eu
+              f=/paperclip/instances/default/config.json
+              mkdir -p "$(dirname "$f")"
+              if [ -s "$f" ]; then
+                jq '.telemetry.enabled = false' "$f" > "$f.tmp"
+              else
+                printf '%s\n' '{"telemetry":{"enabled":false}}' > "$f.tmp"
+              fi
+              mv "$f.tmp" "$f"
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            runAsGroup: 1000
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: data
+              mountPath: /paperclip
+      containers:
+        - name: paperclip
+          image: ghcr.io/paperclipai/paperclip:nightly@<INITIAL_DIGEST> # {"$imagepolicy": "flux-system:paperclip"}
+          ports:
+            - name: http
+              containerPort: 3100
+          env:
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: pclp-pg-app
+                  key: uri
+            - name: HOST
+              value: "0.0.0.0"
+            - name: PORT
+              value: "3100"
+            - name: PAPERCLIP_DEPLOYMENT_MODE
+              value: authenticated
+            - name: PAPERCLIP_DEPLOYMENT_EXPOSURE
+              value: private
+            - name: PAPERCLIP_PUBLIC_URL
+              value: https://paperclip.renner.uno
+            - name: PAPERCLIP_AUTH_DISABLE_SIGN_UP
+              value: "<SIGNUP_DISABLED>"
+          envFrom:
+            - secretRef:
+                name: paperclip-secrets
+          volumeMounts:
+            - name: data
+              mountPath: /paperclip
+          startupProbe:
+            httpGet:
+              path: /api/health
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 60
+          livenessProbe:
+            httpGet:
+              path: /api/health
+              port: http
+            periodSeconds: 30
+            timeoutSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /api/health
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+          resources:
+            requests:
+              cpu: 250m
+              memory: 1Gi
+            limits:
+              memory: 4Gi
+          securityContext:
+            runAsNonRoot: false
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop: ["ALL"]
+              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: paperclip-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: paperclip
+  namespace: paperclip
+spec:
+  selector:
+    app.kubernetes.io/name: paperclip
+  ports:
+    - name: http
+      port: 3100
+      targetPort: http
+```
+
+- [ ] **Step 3: `gate-service.yaml`:**
+
+```yaml
+# ExternalName mirror so the Ingress can backend the paperclip-gate outpost in the authentik namespace.
+apiVersion: v1
+kind: Service
+metadata:
+  name: ak-outpost-paperclip-gate
+  namespace: paperclip
+spec:
+  type: ExternalName
+  externalName: ak-outpost-paperclip-gate.authentik.svc.cluster.local
+  ports:
+    - port: 9000
+      targetPort: 9000
+```
+
+- [ ] **Step 4: `ingress.yaml`:**
+
+```yaml
+# LAN-only (no dns.renner.uno/public label); backends the gate outpost, never the Service.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: paperclip
+  namespace: paperclip
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - paperclip.renner.uno
+      secretName: paperclip-tls
+  rules:
+    - host: paperclip.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ak-outpost-paperclip-gate
+                port:
+                  number: 9000
+```
+
+- [ ] **Step 5: `network-policy.yaml`** (Traefik pod-side port is 8443, verified via `kubectl get pods -n ingress -l app.kubernetes.io/name=traefik`):
+
+```yaml
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: paperclip-gate-ingress
+  namespace: paperclip
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: paperclip
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": authentik
+            goauthentik.io/outpost-name: paperclip-gate
+      toPorts:
+        - ports:
+            - port: "3100"
+              protocol: TCP
+---
+# Covers the server and every agent it spawns (same pod).
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: paperclip-egress
+  namespace: paperclip
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: paperclip
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+            cnpg.io/cluster: pclp-pg
+      toPorts:
+        - ports:
+            - port: "5432"
+              protocol: TCP
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+            - port: "53"
+              protocol: TCP
+    # Gitea, Woodpecker, HA, Loki via their *.renner.uno names.
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ingress
+            app.kubernetes.io/name: traefik
+      toPorts:
+        - ports:
+            - port: "8443"
+              protocol: TCP
+    - toEntities:
+        - kube-apiserver
+      toPorts:
+        - ports:
+            - port: "6443"
+              protocol: TCP
+    - toEntities:
+        - world
+      toPorts:
+        - ports:
+            - port: "443"
+              protocol: TCP
+```
+
+- [ ] **Step 6: `kustomization.yaml`:**
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - externalsecret.yaml
+  - hetzner-creds-externalsecret.yaml
+  - cluster.yaml
+  - objectstore.yaml
+  - scheduled-backup.yaml
+  - pvc.yaml
+  - paperclip.yaml
+  - gate-service.yaml
+  - ingress.yaml
+  - network-policy.yaml
+```
+
+- [ ] **Step 7: Render check.**
+
+Run: `kubectl kustomize kubernetes/apps/paperclip`
+Expected: exit 0; output contains `kind: Cluster`, `kind: Deployment`, `claimName: paperclip-data`, and no literal `<INITIAL_DIGEST>` or `<SIGNUP_DISABLED>`.
+
+Run: `grep -rn "<INITIAL_DIGEST>\|<SIGNUP_DISABLED>" kubernetes/apps/paperclip`
+Expected: no output, exit 1.
+
+- [ ] **Step 8: Commit** `feat(paperclip): server Deployment, PVC, gated Ingress and CNPs`.
+
+---
+
+### Task 7: Authentik gate
+
+**Files:**
+- Modify: `kubernetes/apps/authentik/blueprints-configmap.yaml` (insert before the line `      # navidrome-gate: the Navidrome web UI.`)
+- Create: `kubernetes/apps/authentik/paperclip-gate-egress.yaml`
+- Modify: `kubernetes/apps/authentik/kustomization.yaml`
+
+**Interfaces:**
+- Produces: outpost pods labelled `goauthentik.io/outpost-name: paperclip-gate`, Service `ak-outpost-paperclip-gate.authentik:9000`.
+
+- [ ] **Step 1: Insert blueprint entries** (same indentation as the neighbouring `wiki-gate` block):
+
+```yaml
+      # paperclip-gate: Paperclip UI; the CNP admits only this outpost. Paperclip keeps its own login behind it.
+      - id: paperclip-users-group
+        model: authentik_core.group
+        identifiers:
+          name: Paperclip Users
+        state: present
+        attrs:
+          is_superuser: false
+          users:
+            - !Find [authentik_core.user, [username, k.renner]]
+
+      - id: paperclip-gate-provider
+        model: authentik_providers_proxy.proxyprovider
+        identifiers:
+          name: paperclip-gate
+        state: present
+        attrs:
+          authentication_flow: !Find [authentik_flows.flow, [slug, passkey-passwordless-authentication]]
+          authorization_flow: !Find [authentik_flows.flow, [slug, passkey-stepup-authorization]]
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          mode: proxy
+          internal_host: http://paperclip.paperclip.svc.cluster.local:3100
+          internal_host_ssl_validation: false
+          external_host: https://paperclip.renner.uno
+          cookie_domain: renner.uno
+          access_token_validity: hours=24
+          refresh_token_validity: days=30
+          intercept_header_auth: true
+
+      - model: authentik_core.application
+        identifiers:
+          slug: paperclip-gate
+        state: present
+        attrs:
+          name: Paperclip
+          slug: paperclip-gate
+          provider: !KeyOf paperclip-gate-provider
+          meta_launch_url: https://paperclip.renner.uno
+          meta_description: "Paperclip agent orchestration"
+
+      - model: authentik_policies.policybinding
+        identifiers:
+          target: !Find [authentik_core.application, [slug, paperclip-gate]]
+          group: !KeyOf paperclip-users-group
+          order: 0
+        state: present
+        attrs:
+          enabled: true
+          negate: false
+
+      - model: authentik_outposts.outpost
+        identifiers:
+          name: paperclip-gate
+        state: present
+        attrs:
+          type: proxy
+          service_connection: !KeyOf k8s-conn
+          providers:
+            - !KeyOf paperclip-gate-provider
+          config:
+            log_level: info
+            authentik_host: "http://authentik-server.authentik.svc.cluster.local"
+            authentik_host_insecure: false
+            authentik_host_browser: "https://auth.renner.uno/"
+            refresh_interval: "minutes=5"
+            container_image: null
+            object_naming_template: "ak-outpost-%(name)s"
+            kubernetes_namespace: authentik
+            kubernetes_replicas: 1
+            kubernetes_service_type: ClusterIP
+            kubernetes_disabled_components: ["ingress"]
+            kubernetes_image_pull_secrets: []
+            kubernetes_ingress_annotations: {}
+            kubernetes_ingress_class_name: null
+            kubernetes_ingress_secret_name: "authentik-outpost-tls"
+            kubernetes_json_patches: null
+
+```
+
+- [ ] **Step 2: `paperclip-gate-egress.yaml`:**
+
+```yaml
+# Lets the paperclip-gate outpost reach the Paperclip server; scoped to this outpost's pods.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: paperclip-gate-egress
+  namespace: authentik
+spec:
+  endpointSelector:
+    matchLabels:
+      goauthentik.io/outpost-name: paperclip-gate
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+            app.kubernetes.io/name: paperclip
+      toPorts:
+        - ports:
+            - port: "3100"
+              protocol: TCP
+```
+
+- [ ] **Step 3: Register** — add `  - paperclip-gate-egress.yaml` after `  - paperless-gate-egress.yaml` in `apps/authentik/kustomization.yaml`.
+
+- [ ] **Step 4: Render + dry-run.**
+
+Run: `kubectl kustomize kubernetes/apps/authentik`
+Expected: exit 0 (valid YAML incl. the blueprint ConfigMap).
+
+Run: `kubectl apply --dry-run=server -f kubernetes/apps/authentik/paperclip-gate-egress.yaml`
+Expected: `ciliumnetworkpolicy.cilium.io/paperclip-gate-egress created (server dry run)`.
+
+- [ ] **Step 5: Commit** `feat(authentik): paperclip-gate outpost and Paperclip Users group`.
+
+---
+
+### Task 8: Flux Kustomization
+
+**Files:**
+- Create: `kubernetes/clusters/homelab/paperclip.yaml`
+
+- [ ] **Step 1:**
+
+```yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: paperclip
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 1m
+  timeout: 20m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/apps/paperclip
+  prune: true
+  wait: true
+  dependsOn:
+    - name: infrastructure
+    - name: infrastructure-crds
+    - name: infrastructure-ceph-cluster
+    - name: infrastructure-cnpg-barman
+  healthCheckExprs:
+    - apiVersion: postgresql.cnpg.io/v1
+      kind: Cluster
+      failed: "has(status.phase) && status.phase.startsWith('Failed')"
+      current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"
+```
+
+- [ ] **Step 2: Server dry-run.**
+
+Run: `kubectl apply --dry-run=server -f kubernetes/clusters/homelab/paperclip.yaml`
+Expected: `kustomization.kustomize.toolkit.fluxcd.io/paperclip created (server dry run)`.
+
+- [ ] **Step 3: Commit** `feat(paperclip): Flux Kustomization`.
+
+---
+
+### Task 9: PR, merge by user, live verification
+
+- [ ] **Step 1: Push and open the PR** (Task 3 must be complete).
+
+Run: `git -c credential.helper='store --file=/home/krenner/.config/claude-bot/creds' push -u origin feat/paperclip`
+Run: `tea pr create --login claude-bot --base master --head feat/paperclip --title "feat(paperclip): deploy Paperclip (server-local runs, trial)" --description "<summary of spec decisions + link to spec + bootstrap steps>\n\n🤖 Generated with [Claude Code](https://claude.com/claude-code)"`
+Then wait for the user to merge. Do not merge.
+
+- [ ] **Step 2: Reconciliation.** After merge:
+
+Run: `flux get kustomizations paperclip`
+Expected: `Ready True`. If False, read `kubectl describe kustomization paperclip -n flux-system` and the failing object's events; diagnose, don't poke.
+
+Run: `kubectl get cluster -n paperclip pclp-pg`
+Expected: `Cluster in healthy state`, 2 ready instances.
+
+Run: `kubectl get pods -n paperclip -l app.kubernetes.io/name=paperclip`
+Expected: `1/1 Running`.
+
+- [ ] **Step 3: Gate enforced (Review Focus 4).**
+
+Run: `curl -sS -o /dev/null -w '%{http_code} %{redirect_url}\n' https://paperclip.renner.uno/`
+Expected: `302` redirecting to `https://paperclip.renner.uno/outpost.goauthentik.io/…` or `auth.renner.uno`, never `200`.
+
+- [ ] **Step 4: Gate not bypassable (Review Focus 3).**
+
+`default` has no CNPs (verified 2026-09-25), so its egress is open and only paperclip's ingress policy can block the probe.
+
+Run: `kubectl get ciliumnetworkpolicies -n default`
+Expected: `No resources found` (else pick another CNP-free namespace).
+
+Run: `kubectl run cnp-probe -n default --rm -i --restart=Never --image=curlimages/curl:8.10.1 -- curl -sS -m 5 http://paperclip.paperclip.svc.cluster.local:3100/api/health`
+Expected: timeout (exit 28), no HTTP response.
+
+- [ ] **Step 5: First backup (Review Focus 5).**
+
+Run: `kubectl get backup -n paperclip`
+Expected: one `pclp-pg-daily-…` with phase `completed`.
+
+- [ ] **Step 6: Image policy.**
+
+Run: `kubectl get imagepolicy -n flux-system paperclip -o jsonpath='{.status.latestRef}'`
+Expected: `nightly` tag with a `sha256:` digest.
+
+- [ ] **Step 7: Hand-over to the user (browser steps, theirs):**
+  1. Open `https://paperclip.renner.uno`, pass Authentik, claim the instance as admin.
+  2. Create a test agent with the `claude-local` adapter and a trivial task ("reply with OK"); it must complete with the shared token (Review Focus 2: if it fails, report the run's error text — an expired token must show as an auth error, not a hang).
+  3. If `SIGNUP_DISABLED` was `"false"`, open the follow-up PR flipping `PAPERCLIP_AUTH_DISABLE_SIGN_UP` to `"true"` once the admin is claimed.
+  4. Review Focus 1 is observed on the first nightly bump: check the interrupted-run status in the UI.
+
+- [ ] **Step 8: Wiki.** Route the verified Paperclip facts (execution topology, env vars, image tags, digest-reflection usage) through wiki-ingest → wiki-distill once the deployment has proven out.
@@ -0,0 +1,206 @@
+# Paperclip in the homelab cluster — design
+
+Date: 2026-09-25
+Status: draft, awaiting review
+
+## Goal
+
+Run [Paperclip](https://github.com/paperclipai/paperclip) (MIT, agent-orchestration
+server + UI) in the homelab cluster for real use: its agents do actual work against
+homelab systems. This spec covers the **platform** only. Bot identities and their
+credentials are out of scope (see "Deferred").
+
+## Decisions
+
+| Topic | Decision |
+|---|---|
+| Execution model | **B — server-local runs** (upstream default): agent CLIs are child processes of the server, in the server container. Trial only. |
+| Upgrade path | If Paperclip is kept: **C** — `@paperclipai/plugin-kubernetes` sandbox provider + kubernetes-sigs agent-sandbox controller (`agents.x-k8s.io`), per-run Pods and Secrets. Separate spec. |
+| Claude auth | **One shared Claude subscription AI connection**, created once via Paperclip's local sign-in (`claude auth login` into `ai-local-logins/<id>`); Paperclip stores and refreshes it. Server-env `CLAUDE_CODE_OAUTH_TOKEN` is not used: `stripAiAuthBindings` blanks it for managed runs, and upstream accepts no pasted setup-token (the device-code flow needs a Daytona sandbox). |
+| Bot credentials | Out of scope. Future "AIR-Manager" Paperclip role handles permission requests. |
+| Exposure | LAN-only, behind an Authentik forward-auth gate, then Paperclip's own login. |
+
+### Known trade-off of B (accepted for the trial)
+
+Upstream `runChildProcess()` spawns every local adapter with
+`{...sanitizeInheritedPaperclipEnv(process.env), ...opts.env}`; the sanitizer strips only
+`PAPERCLIP_*` keys. Every agent therefore sees `DATABASE_URL`, `BETTER_AUTH_SECRET`,
+the shared filesystem (including every other agent's workspace and the
+`ai-local-logins` credentials) and every agent-scoped secret it can read from disk. Under B, no bot may hold
+a credential we would not tolerate any other bot using. Per-bot isolation is C's job.
+
+## Architecture
+
+```
+LAN client ─▶ Traefik ─▶ ExternalName svc ─▶ ak-outpost-paperclip-gate (authentik ns)
+                                                   │ forward-auth OK
+                                                   ▼
+                                  paperclip Deployment (ns paperclip, 1 replica)
+                                   ├─ server :3100  (+ agent CLIs as child processes)
+                                   ├─ PVC /paperclip  (ceph-block)
+                                   └─▶ pclp-pg-rw (CNPG, PG17, 2 instances) ─▶ barman ▶ Hetzner S3
+```
+
+## Components
+
+All modeled on `kubernetes/apps/paperless/` unless stated.
+
+### Namespace and baseline policy
+- `Namespace paperclip` in `kubernetes/infrastructure/namespaces/namespaces.yaml`.
+- Baseline default-deny CNP `kubernetes/infrastructure/network-policies/paperclip-ns.yaml`
+  (copy of `paperless-ns.yaml`, renamed).
+- Add `paperclip` to the `openbao` ClusterSecretStore namespace allowlist.
+
+### Flux
+- `kubernetes/clusters/homelab/paperclip.yaml`: Flux `Kustomization`,
+  `path: ./kubernetes/apps/paperclip`, `prune: true`, `wait: true`, same `dependsOn` and
+  CNPG `healthCheckExprs` as `paperless.yaml`.
+- `kubernetes/apps/paperclip/kustomization.yaml`: plain resource list.
+
+### Server Deployment (`paperclip.yaml`)
+- Image `ghcr.io/paperclipai/paperclip:nightly@sha256:<digest>` with the Flux
+  image-automation marker for digest updates.
+- `replicas: 1`, `strategy: Recreate` (single writer on an RWO PVC).
+- Starts as root: upstream `docker-entrypoint.sh` remaps the `node` user, chowns
+  `/paperclip`, then drops to uid 1000 via `gosu`. Caps limited to CHOWN,
+  DAC_OVERRIDE, FOWNER, SETGID, SETUID; `fsGroup: 1000`.
+- `serviceAccountName: paperclip-prometheus-read` with
+  `automountServiceAccountToken: true`. All agents share the pod filesystem, so all
+  of them can read that token — it is mounted anyway because the Role behind it
+  grants exactly `get` on `services/proxy` for
+  `kube-prometheus-stack-prometheus:9090` and
+  `kube-prometheus-stack-alertmanager:9093` in `monitoring`, and nothing else
+  (`kubernetes/apps/paperclip/prometheus-read-rbac.yaml`, REN-36 then REN-95).
+  It was `false` until then.
+  Three things about that grant are worth knowing before you use or debug it:
+  - **It scopes the HTTP method, not the endpoint.** The apiserver maps GET→`get`,
+    POST→`create`, PUT→`update` on a service-proxy request, so `get` alone refuses
+    `POST /-/reload` and `POST /-/quit` in the authorizer. It does *not* narrow
+    which GET endpoints are reachable: the whole read surface of both services is,
+    including `/api/v1/status/config` (scrape credentials there render as
+    `<secret>`) and the web UI.
+  - **Force GET in the caller.** Prometheus accepts both GET and POST on
+    `/api/v1/query`; under this Role only the GET form works, so a client library
+    that switches to POST for long queries will 403 for a reason that has nothing
+    to do with the query.
+  - **If reads start 403ing, check the Service name against `resourceNames`
+    first.** `resourceNames` is matched verbatim, and
+    `kubernetes/apps/monitoring/helmrelease-prometheus.yaml` floats the chart at
+    `>=72.0.0` — a release rename breaks this path silently,
+    with everything still looking correct in git. The port-qualified form is the
+    only one granted: `svc`, `svc:9090` and `http:svc:9090` are three different
+    strings to the authorizer.
+- Port 3100; `Service paperclip` ClusterIP :3100.
+- Probes: startup and readiness on `GET /api/health` with `Host: localhost` (the
+  private-hostname guard rejects the pod IP); liveness is a TCP check, since
+  `/api/health` runs `SELECT 1` and a DB outage must not restart the server.
+- Env:
+  - `DATABASE_URL` assembled from CNPG `pclp-pg-app` Secret (`uri` key).
+  - `HOST=0.0.0.0`, `PORT=3100`
+  - `PAPERCLIP_DEPLOYMENT_MODE=authenticated`
+  - `PAPERCLIP_DEPLOYMENT_EXPOSURE=private`
+  - `PAPERCLIP_PUBLIC_URL=https://paperclip.renner.uno`
+  - `PAPERCLIP_AUTH_DISABLE_SIGN_UP=false` until the first admin is claimed: the
+    claim (`/bootstrap/claim`) requires an already signed-in Better Auth account.
+    A follow-up flips it to `true`.
+  - `PAPERCLIP_TELEMETRY_DISABLED=1` (read by `resolveTelemetryConfig`).
+  - `PAPERCLIP_API_URL=http://paperclip.paperclip.svc.cluster.local:3100` plus
+    `PAPERCLIP_ALLOWED_HOSTNAMES=paperclip.paperclip.svc.cluster.local`: upstream
+    otherwise hands agents the public URL, and the gate (`intercept_header_auth`)
+    rejects their bearer tokens. The Service name also works for sandbox pods under
+    option C; agent JWTs still authenticate each call.
+  - `envFrom: secretRef paperclip-secrets`
+- Reloader: `secret.reloader.stakater.com/reload: paperclip-secrets`.
+
+### Storage
+- `PVC paperclip-data`, `ceph-block`, 20Gi, mounted at `/paperclip` (`PAPERCLIP_HOME`
+  and `HOME`): instance config, workspaces, repo checkouts, per-agent `claude-runtime`
+  dirs.
+
+### Postgres
+- CNPG `Cluster pclp-pg`: 2 instances, `imageName: ghcr.io/cloudnative-pg/postgresql:17`,
+  `ceph-block` 10Gi, `bootstrap.initdb {database: paperclip, owner: paperclip}`,
+  barman-cloud plugin as WAL archiver.
+- `ObjectStore pclp-pg-store` → `s3://renner-uno-k8s-cnpg/pclp-pg`, creds via the
+  existing `hetzner-s3-creds` ExternalSecret pattern.
+- `ScheduledBackup` nightly, `method: plugin`, `backupOwnerReference: self`.
+- `kustomize.toolkit.fluxcd.io/prune: disabled` on Cluster, ObjectStore, ScheduledBackup.
+- PG17 deviates from the cluster-wide PG16 convention because upstream's external
+  Postgres target is `postgres:17` (`docker/docker-compose.yml`) and its embedded default
+  is PG18.
+
+### Secrets
+One ExternalSecret `paperclip-secrets` ← OpenBao `apps/paperclip` (whole-object
+extract):
+- `BETTER_AUTH_SECRET`
+- `PAPERCLIP_TOOL_ACTION_SIGNING_SECRET`
+- `PAPERCLIP_SECRETS_MASTER_KEY` (64-char hex). Held in OpenBao, not on the PVC, so
+  encrypted agent secrets in Postgres survive PVC loss.
+
+### Ingress and SSO
+- Hostname `paperclip.renner.uno`, Traefik `Ingress`, `cert-manager.io/cluster-issuer:
+  letsencrypt-cluster`, **no** `dns.renner.uno/public` label (LAN-only).
+- Authentik gate, per the paperless pattern:
+  - blueprint entries in `kubernetes/apps/authentik/blueprints-configmap.yaml`:
+    proxy provider `paperclip-gate` (`internal_host:
+    http://paperclip.paperclip.svc.cluster.local:3100`, `external_host:
+    https://paperclip.renner.uno`), application, group "Paperclip Users", policy binding;
+  - outpost `ak-outpost-paperclip-gate`;
+  - `kubernetes/apps/authentik/paperclip-gate-egress.yaml` (no `auth.renner.uno`
+    outpost Ingress: like wiki/shelly gates, the outpost serves its own callback on
+    `paperclip.renner.uno`);
+  - `ExternalName` Service in `paperclip` ns pointing at the outpost; the app Ingress
+    targets it on :9000.
+- Paperclip's own email/password login stays on (no header/OIDC login upstream).
+
+### Network policy (`kubernetes/apps/paperclip/network-policy.yaml`)
+Covers the server **and every agent** (same pod under B).
+- Ingress to the app pod: only from the `paperclip-gate` outpost (by label) on 3100.
+- Egress from the app pod:
+  - `pclp-pg` pods :5432
+  - kube-dns :53
+  - `ingress` namespace (Traefik) :8443 (pod-side websecure port) — reaches Gitea, Woodpecker, HA, Loki by their
+    `*.renner.uno` names
+  - `kube-apiserver` entity :6443
+  - `world` TCP :443 (Anthropic API, GitHub, npm)
+- No SSH egress; git over HTTPS only. The namespace baseline
+  (`paperclip-ns.yaml`) also restricts `world` to TCP 443 — Cilium unions allows
+  across CNPs, so an unrestricted baseline would override this list.
+- Known gap (house baseline): pods in `monitoring`/`database` and other pods in
+  `paperclip` can reach :3100 without the gate; Paperclip's own login still applies.
+
+### Image automation
+`kubernetes/infrastructure/image-automation/paperclip.yaml`: `ImageRepository`
+(`ghcr.io/paperclipai/paperclip`) + `ImagePolicy` filtering tag `^nightly$` with
+`digestReflectionPolicy: Always` (image-reflector-controller v1.2.5 in cluster; field
+available since v0.35.0). First use of digest reflection in this repo.
+
+## Guardrails
+- No bot receives cluster-admin or any binding in `flux-system`, `authentik`, `openbao`.
+- Git work goes through PRs; bots get no merge rights on protected branches.
+- Nothing is granted under B that we would not tolerate any agent misusing.
+
+## Bootstrap (one-time)
+1. Place `apps/paperclip` values in OpenBao (the three generated secrets).
+2. Merge; Flux deploys with sign-up enabled.
+3. Sign up through the gate and claim the instance as admin; then set
+   `PAPERCLIP_AUTH_DISABLE_SIGN_UP=true`.
+4. In onboarding "Connect a model", choose Claude subscription and run the shown login
+   **as the node user** (the entrypoint makes exec sessions root, and root-owned 0600
+   credentials are unreadable to the server):
+   `kubectl exec -it -n paperclip deploy/paperclip -- gosu node sh -c 'export CLAUDE_CONFIG_DIR=<shown path> && mkdir -p "$CLAUDE_CONFIG_DIR" && claude auth login'`
+   Repeat after loss of the PVC or database.
+
+## Verification
+- Flux Kustomization `paperclip` Ready; CNPG `pclp-pg` healthy; first ScheduledBackup
+  completes.
+- `curl https://paperclip.renner.uno/api/health` through the gate returns healthy after
+  Authentik login; unauthenticated request redirects to `auth.renner.uno`.
+- Direct pod access from another namespace is denied (CNP).
+- A test agent (claude-local) completes a trivial task using the shared Claude connection.
+- Flux ImagePolicy reports a `nightly@sha256:` digest.
+
+## Deferred
+- **Option C** (sandboxed runs) — required before any per-bot credential isolation.
+- **AIR-Manager** role and all bot identities/credentials.
+- Public exposure.
@@ -1,117 +1,99 @@
 # Traffic classes on the k8s nodes

-Issue #576. Three of the four classes are separated; management is not.
+Issue #576. Five classes, each on its own VLAN.

 ## Where each class runs

 | class | link | mechanism |
 |---|---|---|
 | Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) |
-| world→pod | `ingress0`, VLAN 47 | BGP session sourced there, so the advertised next hop is that address |
-| pod→world | `ingress0`, VLAN 47 | routing rule on the pod source prefix → table 47 |
-| pod↔pod | `pod0`, VLAN 48 | routing rule on pod source **and** destination → table 48 |
-| management | `mgmt0`, VLAN 40 | not started |
+| pod↔pod | `pod0`, VLAN 48 | pod-sourced traffic is routed from table 47; bird puts the peers' pod CIDRs there via `pod0` |
+| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress takes table 47's default |
+| node↔node | `node0`, VLAN 46 | the node InternalIP and etcd live there; so do apiserver↔kubelet, cilium-health and host→remote-pod traffic |
+| world↔node | `mgmt0`, VLAN 40 | API VIP for kubectl, talosctl from outside, node egress (DNS, NTP, image pulls) |

 Hard constraint, set by the owner: inter-node and pod traffic never traverses
-the UDM. Everything above stays on the 10G switch.
+the UDM. Everything but world↔node stays on the 10G switch. VLAN 46 and 48 are
+statically addressed, without a gateway; VLAN 46 is isolated in UniFi.

-## Why policy routing rather than moving the node IP
+## Pod traffic: one table

-Cilium reaches a peer's pods at that peer's Kubernetes InternalIP, in native
-and tunnel mode alike. `--ipv4-node` is overwritten from the Node object, and
-`nodePort.directRoutingDevice` selects the BPF NodePort device, not a next hop.
-Both were established by test (#837, #838), not by reading.
+Two rules, one per family, send every packet with a pod source to table 47
+(`talos/patches/site-homelab.yaml`, priorities 1000/1001, below main). The
+table decides by longest prefix; no rule carves out exceptions.

-So the InternalIP decides where pod traffic goes — unless the lookup happens in
-a different table. The rules do that, and nothing about the node's identity
-moves. The routing decision runs before Cilium's masquerade hook, so a rule
-matching the pod source sees the pod address, and the masquerade then rewrites
-it to the egress link's.
+bird (`kubernetes/infrastructure/bird/`) owns table 47 and fills it with:

-## Rule order
+- the node's attached networks — `mgmt0`, `ingress0`, `pod0`, `node0` —
+  taken from the interfaces;
+- the peers' pod CIDRs, learned over an iBGP mesh on `pod0`, next hop the
+  peer's `pod0` address;
+- the node's own pod CIDR via `cilium_host`, originated rather than read back
+  from the kernel (reading it back made BIRD depend on the shape of Cilium's
+  route, and KRT rejects the IPv4 one);
+- the egress default via the UDM on `ingress0`, from the node annotations
+  `net.renner.uno/egress-gw-v4/v6`.

-Lower priority runs first.
+`node0` has to be in the table: a pod reaching a node address — the
+`kubernetes` ClusterIP, which socket LB rewrites to the apiserver's advertise
+address, or a hostNetwork pod — would otherwise take the default to the UDM.

-| prio | match | table |
-|---|---|---|
-| 980 | src pod, dst pod | 48 |
-| 981 | src pod, dst pod (v4) | 48 |
-| 990 | dst `fd00::/8` | main |
-| 991 | dst `10.0.0.0/8` | main |
-| 1000 | src pod | 47 |
-| 1001 | src pod (v4) | 47 |
+The routing decision runs before Cilium's masquerade hook, so the rule sees the
+pod address and the masquerade then rewrites it to the egress link's.

-980/981 match source **and** destination. Source alone would also catch the
-node's own traffic to a pod, which would then leave with a `pod0` source — an
-address `CiliumNode.spec.addresses` does not list, so Cilium classifies it as
-`world` rather than `host`.
-
-990/991 are Cilium's `ipv4/ipv6NativeRoutingCIDR`, and they are what keeps
-cluster-internal traffic out of table 47. Without them a pod's packet to a
-service matches the source rule, finds only a default route, and leaves for the
-UDM — measured, in-cluster DNS stopped resolving.
-
-Traffic to a pod on the same node falls through on its own: table 48 holds only
-the peers' CIDRs, so the lookup misses and 990/991 send it to main.
-
-## Where each table comes from
-
- **47** — two default routes declared per node in `talos/patches/k8s-*.yaml`.
-  The IPv6 gateway is the UDM's configured address on the segment, not the
-  RA-learned link-local; `accept_ra_defrtr` stays `0` on `ingress0` so the RA
-  default does not land in main.
- **48** — filled by the `bird` DaemonSet: an iBGP mesh over `pod0`, one route
-  per peer pod CIDR with the peer's `pod0` address as next hop.
-
-## The bird mesh
-
-`kubernetes/infrastructure/bird/`. Each node advertises only its own pod CIDR
-and installs what it learns into table 48. Cilium's routes in main are
-untouched, which is why the two coexist.
+Node-originated traffic does not match the rules and uses main, where Cilium
+routes each peer's pod CIDR via the peer's InternalIP — on `node0`.

 No address is written into this repo twice. Each node publishes its `pod0`
-addresses as `nodeAnnotations`, beside the `VLANConfig` that assigns them; the
-pod CIDRs come from the Node objects. An init container renders both from the
-API with `kubectl -o go-template-file`, excludes the node itself with a field
-selector, and ends with `bird -p` so BIRD's own parser gates startup.
+addresses as `nodeAnnotations` beside the `VLANConfig` that assigns them; the
+pod CIDRs come from the Node objects. bird's init container renders both from
+the API and ends with `bird -p`, so BIRD's own parser gates startup.

-The node's own CIDR is **originated**, not read back from the kernel. Reading
-it back made BIRD depend on the shape Cilium installs, and the IPv4 route is
-rejected outright: its next hop sits inside the prefix it describes, which KRT
-refuses with `strange next-hop`. The IPv6 route is a device route and passed,
-so the failure was silent on one family only.
+## Node↔node
+
+`kubelet.nodeIP.validSubnets` and `cluster.etcd.advertisedSubnets` point at
+VLAN 46. Talos picks one InternalIP per family, IPv4 first — the service CIDR
+is IPv4, and the apiserver's advertise address must match it.
+
+The API VIP stays on `mgmt0` for kubectl. No in-cluster client uses it:
+kubelet, scheduler, controller-manager and Cilium go through KubePrism
+(`localhost:7445`), which balances over every address of every control-plane
+member; pods use the ClusterIP.
+
+Talos never updates an existing etcd member's peer URLs
+(siderolabs/talos#12720). Changing `advertisedSubnets` means, per member:
+stage the config, `talosctl etcd forfeit-leadership`, `talosctl etcd leave`,
+reboot. `leave` removes only `/var/lib/etcd`; the node rejoins as a learner
+and is promoted. EPHEMERAL does not need a wipe.
+
+## Moving node IPs: the mixed state
+
+While some nodes have their InternalIP on the new network and some on the old,
+host→remote-pod traffic leaves with a node address that is not an InternalIP:
+the kernel picks the source from the route to the peer's InternalIP. Cilium
+classifies that as `world`, and every ingress policy that admits only
+`host`/`remote-node` (rook-ceph, monitoring, …) drops it. Kernel Ceph clients
+lose the OSDs on the other side; RBD mounts hang.
+
+Bridge it for the duration with a temporary `CiliumClusterwideNetworkPolicy`
+admitting the node subnets of both networks, both families, with
+`enableDefaultDeny: false` so it restricts nothing else, and delete it once
+every node has moved. Keep all OSD nodes schedulable while one is drained:
+draining two takes Ceph below its data availability.

 ## Failure behaviour

-Benign at every layer. An empty or absent table 48 makes 980/981 fall through
-to main, and pod traffic returns to VLAN 40 — unseparated, not broken. The same
-holds if the DaemonSet cannot start, which is the case during a cold rebuild
-until Gitea serves its image.
-
-## Management, the remaining class
-
-Not started. The levers differ per service and none of them is the node IP:
-
-| service | can leave the node IP? | lever |
-|---|---|---|
-| etcd | yes | `cluster.etcd.advertisedSubnets`; needs a reboot per node |
-| kube-apiserver endpoint | yes | it is a VIP — `Layer2VIPConfig.link` |
-| Talos API | n/a | `apid` answers on every node address; a firewall question |
-| kubelet | no | it *is* the node IP |
-
-Two approaches were rejected by the owner and are not to be proposed again:
-moving the node IP and etcd to `pod0` (#838), and relabelling VLAN 40 as the
-cluster VLAN while moving etcd and the API VIP to a new management VLAN.
-Replacing the CNI (#839, Antrea) was shelved.
-
-Note that the reasoning which produced that list also concluded pod↔pod was
-blocked, which policy routing disproved. Treat it as a record of decisions, not
-as an exhaustive analysis.
+If bird stops, table 47 empties, the rules find nothing and fall through to
+main: pod traffic returns to the node networks, unseparated rather than
+misrouted. The same holds during a cold rebuild until Gitea serves bird's
+image. The default route lives in table 47, not in the Talos config, so a
+default cannot outlive the specific routes and send pod traffic to the UDM.

 ## Sources

-Cilium v1.20.1 `pkg/datapath/linux/node.go`, `pkg/node/types/node.go`,
-`pkg/node/sync/local_node_sync.go`; BIRD 2.17 user manual (kernel protocol,
-`learn all`, `strict bind`, `direct`); Talos v1.13 `RoutingRuleConfig` and
-`machine.nodeAnnotations`. PRs #831, #833, #835, #836, #837, #842, #852, #854,
-#855; closed unmerged: #830, #838, #839.
+Cilium v1.20 `pkg/datapath/linux/node.go`; BIRD 2.17 user manual (kernel
+protocol, `strict bind`, `direct`); Talos v1.13 `RoutingRuleConfig`,
+`machine.nodeAnnotations`, configuration patching (lists are appended,
+`$patch: delete`), etcd `LeaveCluster`; KubePrism docs;
+siderolabs/talos#12720. PRs #831, #833, #835, #836, #837, #842, #852, #854,
+#855, #857, #858, #918, #919, #922; closed unmerged: #830, #838, #839.
@@ -0,0 +1,315 @@
+# Wiki: local web interface and agent API, as a modular .NET solution
+
+## Goal
+
+A web interface for the Notes wiki that runs entirely in the cluster, and an
+HTTP API through which agents ask the wiki typed questions and deposit facts.
+Built as one C# solution with a MudBlazor front end, and built so that every
+external dependency is replaceable: where the notes live, which model embeds
+them, what judges a typed question, what runs an agent, how the vector index
+is stored. The first adapters are what the homelab uses today. The point of
+the design is that the next one is a project and a config line, not a
+rewrite.
+
+## Where we are
+
+- `wiki-index` (Python, ns `ai`) syncs `wiki/` from `k.renner/notes` by blob
+  SHA, embeds through `wiki-embed` (bge-m3 on llama-server), answers
+  `POST /rank`. It keeps slug, title and vector, not the text. LAN-only behind
+  Traefik BasicAuth; the Gitea webhook hits the ClusterIP.
+- `~/Notes/bin/wiki_rank.py` and `wiki_ask.py` rank through the index, build a
+  document context from a local vault checkout, send it with typed questions
+  to TypeSafe, gate on `covered`. The wiki-synth agent composes answers from
+  those judgements.
+- New knowledge enters as a file in `raw/` (facts with provenance, never
+  hypotheses), distilled nightly in CI.
+- The house C# project, shelly-manager, sets the conventions: net10.0, an
+  `.slnx` with `src/` and `test/`, xUnit, MudBlazor 9, Blazor Web App with
+  interactive server rendering, minimal APIs, an Authentik proxy-header
+  authentication handler for humans and an API-key handler for machines, a
+  multi-stage Dockerfile, `dotnet test` on every push and a kaniko release
+  pipeline to the Gitea registry. Human-facing apps sit behind an Authentik
+  outpost (`<app>-gate`) with a CNP that admits only that outpost.
+
+## Architecture: ports in the core, adapters around it
+
+```
+                 ┌──────────────────────────────────────────┐
+                 │  Wiki.Host  (ASP.NET Core + Blazor/Mud)  │  composition root,
+                 │  /api/v1, /rank, /webhook, UI pages      │  config, auth
+                 └───────────────┬──────────────────────────┘
+                                 │ uses
+                 ┌───────────────▼──────────────────────────┐
+                 │  Wiki.Application  (use cases)           │  Sync, Rank, Ask,
+                 │                                          │  DepositFact, Queue-
+                 └───────────────┬──────────────────────────┘  Question, Evaluate
+                                 │ depends only on
+                 ┌───────────────▼──────────────────────────┐
+                 │  Wiki.Core  (domain + ports)             │  Note, RawSource,
+                 │  no package references                   │  Report, LinkGraph,
+                 └──────────────────────────────────────────┘  the interfaces
+   implemented by ▲            ▲            ▲            ▲            ▲
+   Adapters.Gitea  Adapters.LlamaCpp  Adapters.TypeSafe  Adapters.FileIndex  Adapters.Anthropic
+   (later: Database, Ollama/Onnx, local Jev, pgvector/Qdrant, OpenAI-compatible answer models)
+```
+
+Rules that keep it modular, enforced by project references and a test:
+
+- `Wiki.Core` references nothing. It holds the domain types and the port
+  interfaces. Adapters reference Core only; they never reference each other
+  or the host. `Wiki.Host` is the only project that knows which adapters
+  exist.
+- Every port has a **contract test suite** in `test/Wiki.Contracts` that
+  runs against every adapter of that port. A new adapter is done when the
+  suite is green. That is what makes "swap the storage" a safe operation.
+- Adapters are selected by configuration and registered by name (keyed
+  services), so several can be active at once. The UI's lab page and the
+  evaluator use that to compare, the API uses the one marked active.
+- Nothing in Core or Application knows about HTTP, Gitea, TypeSafe, files or
+  Kubernetes.
+
+### Ports
+
+| port | what it abstracts | first adapter | examples of the next ones |
+|---|---|---|---|
+| `INoteStore` | read notes, raw sources and reports at a head; create a new document (a fact becomes a raw note in the application layer, the store only moves text) | Gitea repository API (`k.renner/notes`, blob SHA) | git checkout on a volume; a database (CNPG Postgres) with the vault imported; an Obsidian sync folder |
+| `IEmbeddingModel` | text → vector, with a model id and dimension | llama-server `/v1/embeddings` (bge-m3 via wiki-embed, the model that won the German-document comparison on accuracy per cost) | the qwen3 embedding variants from that comparison; ONNX in-process; any OpenAI-compatible endpoint |
+| `IVectorIndex` | store vectors per (model id, document), query top-k | in-memory cosine with a file snapshot per model, as today | pgvector in the database; Qdrant |
+| `IChunker` | split a document into addressable units for typed questions | paragraph splitter (headings, fences), as in `wiki_ask.py` | sentence windows; semantic chunks |
+| `IJudge` | typed questions over a state: Choice, Noul, Score, with probabilities | TypeSafe System One (`jev-latest`, cloud; the wiki records no locally hosted Jev today) | a locally hosted Jev with the same request shape, should TypeSafe offer one; a constrained-output local LLM adapter, flagged as uncalibrated |
+| `IAnswerModel` | prose from a question plus evidence: the selected paragraphs with their notes and the judgements; must cite by slug; streaming | Anthropic Messages API (Claude, model from config, Sonnet by default) | any OpenAI-compatible endpoint, including the local llama-server; a different Claude model per question |
+| `IEvaluator` | measure a retrieval or judge configuration against the labelled set (questions and cited sources in `output/`) | recall@k, hit rate, MRR, as `wiki_rank_eval.py` | judge accuracy against hand-labelled paragraphs |
+| `IIdentity` | who is calling, for attribution of facts and saved answers | Authentik proxy headers (humans), API key (machines), as in shelly-manager | OIDC bearer tokens |
+
+Two ports deserve a sentence of scope each. The judge returns judgements
+with probabilities, never prose, and the UI never presents its output as an
+answer, only as ranked, quoted evidence. The answer model turns that
+evidence into words and is allowed nothing else: it sees the selected
+paragraphs and the judgements, not the whole context, and its output is
+shown next to the evidence it was given. Both are single model calls inside
+the request. The service runs no agent: no tool loop, no vault edits. The
+wiki-synth agent in a Claude session remains the path for artifacts and for
+multi-question synthesis, and it can call `/questions` itself.
+
+### Solution layout
+
+```
+Wiki.slnx
+src/
+  Wiki.Core/               domain types, ports, frontmatter and link parsing
+  Wiki.Application/        use cases; depends on Core only
+  Wiki.Adapters.Gitea/     INoteStore (read at a head, create)
+  Wiki.Adapters.LlamaCpp/  IEmbeddingModel (OpenAI-compatible embeddings)
+  Wiki.Adapters.TypeSafe/  IJudge (System One client)
+  Wiki.Adapters.FileIndex/ IVectorIndex (memory + snapshot per model)
+  Wiki.Adapters.Anthropic/ IAnswerModel (Messages API, streaming)
+  Wiki.Host/               ASP.NET Core: API, webhook, MudBlazor UI, auth,
+                           composition root, appsettings
+test/
+  Wiki.Core.Tests/         parsing, link graph, chunker
+  Wiki.Contracts/          one suite per port, parameterised over adapters
+  Wiki.Application.Tests/  use cases with fake ports
+  Wiki.Host.Tests/         API surface, auth handlers
+```
+
+Own repository `k.renner/wiki`, same shape as shelly-manager: `Dockerfile`,
+`.woodpecker/test.yml` on pull requests, `.woodpecker/release.yml` building
+the image with kaniko into the Gitea registry on push to `main`, tagged
+`build.<unix-ts>` plus the commit SHA so Flux image automation can follow
+it. The homelab repo holds only the manifests. `claude-bot` needs write
+access to the new repository.
+
+Two things the shelly-manager history settles in advance: the TypeSafe
+client is written from the documented field contract (endpoint, bearer key,
+`jev-latest`, Noul/Choice/Score, 255-option cap, 32k-token state budget),
+there is no prior C# client to copy; and the UI needs no login page, because
+identity comes from the outpost's headers, which avoids the Blazor-circuit
+sign-in constraint shelly-manager ran into.
+
+### Configuration
+
+Adapters are declared in `appsettings` and selected by name; secrets come
+through environment variables, filled by ExternalSecret, never from the file.
+
+```json
+"Wiki": {
+  "NoteStore": { "Active": "gitea", "gitea": { "Repository": "k.renner/notes", "Branch": "main" } },
+  "Embeddings": {
+    "Active": "bge-m3",
+    "Models": {
+      "bge-m3":   { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc:8080", "Model": "bge-m3" },
+      "e5-large": { "Adapter": "Ollama",   "Url": "http://ollama.ai.svc:11434",    "Model": "multilingual-e5-large" }
+    }
+  },
+  "Judges": {
+    "Active": "typesafe",
+    "Models": {
+      "typesafe":  { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" },
+      "jev-local": { "Adapter": "TypeSafe", "Url": "http://jev.ai.svc:8080/v1/systemone",  "Model": "jev" }   // illustration: same adapter, other endpoint
+    }
+  },
+  "Index": { "Active": "file", "file": { "Path": "/data/index" } },
+  "AnswerModels": {
+    "Active": "claude-sonnet",
+    "Models": {
+      "claude-sonnet": { "Adapter": "Anthropic", "Model": "claude-sonnet-5" },
+      "qwen-local":    { "Adapter": "OpenAI",    "Url": "http://llama-server.ai.svc:8080/v1", "Model": "qwen3.6-35b-a3b" }
+    }
+  }
+}
+```
+
+Every embedding model gets its own index namespace. Switching the active
+model is a config change after the new index has been built in the
+background; until then both exist and can be compared. Nothing is
+re-embedded in place.
+
+## The web interface (MudBlazor)
+
+Blazor Web App, interactive server rendering, hosted in `Wiki.Host`. Static
+assets vendored into the image, no CDN. Pages:
+
+1. **Search**: a query box, results ranked by the active index with score and
+   tags; for a phrased question, optionally the paragraph-level hits from the
+   judge with their probabilities, shown as quoted evidence.
+2. **Note**: frontmatter, rendered body with `[[wikilinks]]` resolved,
+   backlinks, sources linking to their raw file.
+3. **Browse**: by tag, by `updated`, recently changed, orphans.
+4. **Answers**: a question box. The answer streams in within seconds: rank,
+   judge, compose, all in the request. Below it the evidence: the paragraphs
+   the judge selected with their notes and probabilities, and the `covered`
+   gate; a low gate is shown as "the wiki does not cover this" instead of an
+   answer. A save button writes the answer to `output/` with the wiki-synth
+   frontmatter (`question`, `sources`, `confidence`), so it joins the
+   labelled set. Underneath, the saved answers, newest first, including the
+   ones the wiki-synth agent wrote from a session.
+5. **Maintenance**: the raw queue with ages, the last consolidation and audit
+   reports, index and sync status.
+6. **Lab**: pick any registered embedding model and judge, run the same query
+   through them side by side, and run the evaluator on the labelled set to
+   get recall@k per configuration. This is the page for "play around with
+   different models".
+7. **Settings**: read-only view of the active adapters and their health.
+
+No editing of notes in the browser. The vault stays a git repository edited
+by the consolidate agent and by hand.
+
+## API v1
+
+Reads are authenticated by the hostname's gate (humans) or API key
+(machines); writes always need an API key so a fact carries its author.
+
+| method and path | purpose |
+|---|---|
+| `GET /api/v1/status` | index commit and age, counts, active adapters |
+| `GET /api/v1/notes?q=&tag=&limit=` | list; `q` ranks through the active index |
+| `GET /api/v1/notes/{slug}` | frontmatter, body, links, backlinks, sources |
+| `GET /api/v1/tags` | tags with counts |
+| `POST /api/v1/rank` | `{question, k, model?}` → ranked slugs; also served at `/rank` for the existing scripts |
+| `POST /api/v1/ask` | `{queries, questions, k, judge?, model?}` → context, typed judgements with the answering paragraphs verbatim, `covered`. For agents and code. |
+| `POST /api/v1/questions` | `{question, k?, judge?, model?, answerModel?, save?}` → the answer text with `[[slug]]` citations, the evidence (paragraphs, probabilities), `covered`, model and token usage; streamed as server-sent events when requested; with `save`, the `output/` path and commit |
+| `POST /api/v1/facts` | deposit a raw source: title, source, `kind: fact`, observed_at, content, tags → path and commit |
+| `GET /api/v1/raw` | the distill queue with ages |
+| `GET /api/v1/reports` | `output/` documents by type, newest first |
+| `POST /api/v1/lab/eval` | run the evaluator for a (model, judge) pair |
+| `POST /webhook` | Gitea push, HMAC as today |
+
+`POST /api/v1/facts` is rejected without `source`, with a `kind` other than
+`fact`, or with content under a minimum length. The raw rule is enforced by
+shape: there is no field for a hypothesis. The file gets the vault's
+provenance frontmatter with `captured_by` set from the API key's identity.
+
+`POST /api/v1/ask` is `wiki_ask.py` moved server-side, so the scripts become
+thin clients and agents no longer need a vault checkout. Its result is
+judgements and quotations, never prose.
+
+`POST /api/v1/questions` is `/ask` plus one answer-model call, in process:
+rank the question, build the context, run the default typed questions
+(answered, where, covered), hand the selected paragraphs and judgements to
+the answer model, stream the prose back. Below the `covered` threshold it
+returns the gap instead of an answer. Time is the sum of the parts, about
+0.2 s rank, 1 to 4 s judge, a few seconds of generation. Cost is one model
+call over a few thousand tokens, a few cents on Sonnet, against a quarter
+dollar for a full agent run.
+
+## Auth and network
+
+- Humans: `wiki.renner.uno`, LAN-only (no `dns.renner.uno/public` label),
+  Ingress backend `ak-outpost-wiki-gate:9000`, a `wiki-gate-egress` CNP in
+  `authentik`, the proxy-header handler trusts `X-Authentik-*` only from that
+  outpost.
+- Machines: `wiki-index.renner.uno` with the API-key handler; the workstation
+  key stays where it is. CI reaches the ClusterIP.
+- Secrets (Gitea token, webhook secret, API keys, TypeSafe key, Anthropic
+  key) reach the
+  pod as environment variables from Kubernetes Secrets filled by
+  ExternalSecret from the `openbao` ClusterSecretStore. The pod itself has
+  no OpenBao access. The Python service fetched its secrets from OpenBao
+  directly through its own Kubernetes-auth role; that role, its policy and
+  the `ai` → `openbao` CNP entry are retired with it.
+- Egress from the pod: Gitea, wiki-embed, `api.typesafe.ai:443`,
+  `api.anthropic.com:443`. Every
+  further adapter that talks to a new endpoint gets its own CNP line, which
+  is the one place adding an adapter touches the cluster.
+
+## Health endpoints
+
+Three endpoints with three different meanings, wired to the .NET health
+checks with tags so that the probes and the humans see the right thing:
+
+| endpoint | meaning | probe |
+|---|---|---|
+| `GET /health` | the process serves HTTP. Always 200 once Kestrel listens; no dependency checks. | liveness |
+| `GET /ready` | the active note store has synced at least once and the active index is loaded; 503 while the first sync or index build runs. A failed later sync keeps the last good state and stays ready. | readiness, plus a startup probe with enough budget for the first sync (the Python service was killed mid-build when its `/health` returned 503 during the initial sync) |
+| `GET /api/v1/status` | the full picture for humans and the Settings page: index commit and age, counts, queue lengths, and one entry per registered adapter with its last check result and latency. | none |
+
+Adapter checks (Gitea reachable, embeddings endpoint answering, judge
+endpoint answering, answer model answering) are reported in `/status` and
+on the Settings page and are never part of `/ready`: a judge outage must not
+take the reader down. Their failures show up as red entries, not as
+restarts.
+
+## Phases
+
+One PR each, verified before the next. The Python `wiki-index` keeps running
+until phase 3 proves parity.
+
+0. **Skeleton.** Repository, solution, `Wiki.Core` with the ports and domain
+   types, `Wiki.Contracts` with the suites, `Wiki.Host` serving `/health`,
+   `/ready` and `/api/v1/status`. CI: test and release pipelines. Verify:
+   image in the registry, pipeline green, the Deployment's three probes
+   wired and the pod Ready.
+1. **Parity with the Python service.** Gitea, LlamaCpp and FileIndex
+   adapters, the sync use case, `/rank` and `/webhook`. Verify: the 40-query
+   recall eval gives the same numbers as the Python service on the same
+   commit; `wiki_rank.py --probe-index` against the new service passes.
+2. **Read API and UI.** Notes, tags, raw, reports; Search, Note, Browse,
+   Maintenance pages; `wiki-gate`. Verify: a note renders with resolved
+   links; an unauthenticated request is redirected to Authentik.
+3. **Ask.** Chunker, TypeSafe adapter, `/api/v1/ask`, scripts as clients.
+   Verify: `test_wiki_ask.py` passes against the API; a wiki-synth run with
+   the definition unchanged. Then the Python service is retired and the
+   `wiki-index` hostname points at the new one.
+4. **Facts.** API keys as an ExternalSecret, the deposit-fact use case, `/api/v1/facts`. Verify:
+   a fact posted from the workstation shows as a `claude-bot` commit and in
+   the next distill.
+5. **Questions.** Anthropic adapter, `/api/v1/questions` with streaming,
+   the Answers page with save. Verify: a question answered in under ten
+   seconds with citations that resolve to the shown evidence; a question the
+   wiki does not cover returns the gap; a saved answer appears in `output/`
+   with the wiki-synth frontmatter and is picked up by the evaluator.
+6. **Lab.** Evaluator, a second embedding model and a second answer model
+   registered, the Lab page. Verify: two models compared on the labelled set
+   from the UI.
+
+## Not in scope
+
+- Editing notes in the browser.
+- Replacing Obsidian; the vault stays a git repository and `notes-push.path`
+  keeps pushing.
+- Public exposure; LAN and VPN only.
+- Running or triggering agents. The service makes single model calls inside
+  a request; it never runs a tool loop or edits the vault. Distill,
+  consolidate and multi-question synthesis stay Claude processes in CI or in
+  a session; the service shows their inputs and outputs and serves their API
+  calls.
@@ -107,7 +107,7 @@ config, and the nodes are addressed by name.
 ## Step 1b — boot all three machines from install media

 ```
-https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.13.9/metal-amd64.iso
+https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.13.10/metal-amd64.iso
 ```

 Write it to a USB stick and boot each MS-01 from it. Each comes up in
@@ -0,0 +1,71 @@
+# Runbook — roll back a Home Assistant image bump
+
+Use when an automatic Home Assistant update (a
+`chore(image-automation): bump ghcr.io/home-assistant/home-assistant:<new>`
+commit on `master`) breaks Home Assistant. The owner approves the rollback
+and its timing. Atlas writes the PR; Argus merges it.
+
+## Why a plain `git revert` is not enough
+
+Image automation (`kubernetes/infrastructure/image-automation/`) re-scans
+the registry every hour and rewrites the tag every 30 minutes to the
+highest release matching the `home-assistant` ImagePolicy. A plain revert
+of the bump commit would be bumped forward again within about an hour.
+
+So the rollback commit does two things at once: it reverts the tag **and**
+drops the `# {"$imagepolicy": ...}` marker from that image line. Without
+the marker the Setters strategy has nothing to rewrite, so the old tag
+stays pinned. It is still one commit, and reverting that commit restores
+automation.
+
+## Before you roll back
+
+1. Find the bump commit:
+   ```sh
+   git log --oneline -5 origin/master -- kubernetes/apps/home-assistant/deployment.yaml
+   ```
+2. Check the release notes of the broken version for a **database schema**
+   change. Home Assistant migrates the recorder schema (CNPG cluster
+   `ha-pg`) on start and cannot downgrade it. An older version started on a
+   newer schema runs without recorder history until the schema matches
+   again. If the bump crossed a schema change, tell the owner before going
+   on: the choice is roll back without history, or restore the recorder
+   database from backup as a separate step.
+
+## Roll back (one commit, one PR)
+
+```sh
+git worktree add -b atlas/ha-rollback-<old> "$WT" origin/master && cd "$WT"
+git revert --no-commit <bump-sha>
+# In kubernetes/apps/home-assistant/deployment.yaml, delete the trailing
+#   # {"$imagepolicy": "flux-system:home-assistant"}
+# from the home-assistant image line. Leave every other line alone.
+git diff --cached --stat   # expect: 1 file, 1 line changed
+git commit -m "fix(home-assistant): pin <old>, pause image automation (bump to <new> broke HA)"
+git push -u origin HEAD
+```
+
+The PR description states: broken version, pinned version, symptom, schema
+check result, and "revert this commit to resume automatic updates".
+HA-impact review and merge follow [README.md](../README.md#2-ha-impact-review-before-merge).
+
+On merge Flux applies the Deployment (strategy `Recreate`): Home Assistant
+restarts once, about one to two minutes of downtime.
+
+## Verify
+
+```sh
+kubectl -n home-assistant get deploy home-assistant \
+  -o jsonpath='{.spec.template.spec.containers[?(@.name=="home-assistant")].image}'
+kubectl -n home-assistant rollout status deploy/home-assistant
+```
+
+Read-only commands only (see REN-240 protection rule 1). One hour after
+merge, confirm no new `chore(image-automation)` commit touched
+`deployment.yaml`.
+
+## Resume automatic updates
+
+When a fixed Home Assistant release is out, revert the rollback commit. The
+marker comes back, and image automation bumps to the newest release on its
+next pass. That bump is itself a Home Assistant restart.
@@ -70,7 +70,7 @@ spec:
              mountPath: /ca
      containers:
        - name: audiobookshelf
-          image: ghcr.io/advplyr/audiobookshelf:2.36.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
+          image: ghcr.io/advplyr/audiobookshelf:2.37.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
          ports:
            - name: http
              containerPort: 80
@@ -122,7 +122,7 @@ spec:
            claimName: audiobookshelf-metadata
        - name: books
          persistentVolumeClaim:
-            claimName: audiobookshelf-books
+            claimName: audiobookshelf-books-bulk
        - name: le-ca
          emptyDir:
            medium: Memory
@@ -35,14 +35,14 @@ spec:
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: audiobookshelf-books
+  name: audiobookshelf-books-bulk
  namespace: audiobookshelf
  annotations:
    kustomize.toolkit.fluxcd.io/prune: disabled
 spec:
  accessModes:
    - ReadWriteMany
-  storageClassName: ceph-filesystem
+  storageClassName: ceph-filesystem-bulk
  resources:
    requests:
      storage: 300Gi
@@ -976,6 +976,280 @@ data:
            kubernetes_ingress_secret_name: "authentik-outpost-tls"
            kubernetes_json_patches: null

+      # wiki-gate: the wiki service (ns ai) behind the same passkey flows as
+      # llama-gate. The service trusts X-Authentik-Username only from pod
+      # addresses, and its CNP admits only this outpost, so the gate is the
+      # whole access control for wiki.renner.uno.
+      - id: wiki-gate-provider
+        model: authentik_providers_proxy.proxyprovider
+        identifiers:
+          name: wiki-gate
+        state: present
+        attrs:
+          authentication_flow: !Find [authentik_flows.flow, [slug, passkey-passwordless-authentication]]
+          authorization_flow: !Find [authentik_flows.flow, [slug, passkey-stepup-authorization]]
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          mode: proxy
+          internal_host: http://wiki.ai.svc.cluster.local:8080
+          internal_host_ssl_validation: false
+          external_host: https://wiki.renner.uno
+          cookie_domain: renner.uno
+          access_token_validity: hours=24
+          refresh_token_validity: days=30
+          intercept_header_auth: true
+
+      - model: authentik_core.application
+        identifiers:
+          slug: wiki-gate
+        state: present
+        attrs:
+          name: Wiki
+          slug: wiki-gate
+          provider: !KeyOf wiki-gate-provider
+          meta_launch_url: https://wiki.renner.uno
+          meta_description: "Notes wiki: search, notes, answers, maintenance"
+
+      - model: authentik_outposts.outpost
+        identifiers:
+          name: wiki-gate
+        state: present
+        attrs:
+          type: proxy
+          service_connection: !KeyOf k8s-conn
+          providers:
+            - !KeyOf wiki-gate-provider
+          config:
+            log_level: info
+            authentik_host: "http://authentik-server.authentik.svc.cluster.local"
+            authentik_host_insecure: false
+            authentik_host_browser: "https://auth.renner.uno/"
+            refresh_interval: "minutes=5"
+            container_image: null
+            object_naming_template: "ak-outpost-%(name)s"
+            kubernetes_namespace: authentik
+            kubernetes_replicas: 1
+            kubernetes_service_type: ClusterIP
+            kubernetes_disabled_components: ["ingress"]
+            kubernetes_image_pull_secrets: []
+            kubernetes_ingress_annotations: {}
+            kubernetes_ingress_class_name: null
+            kubernetes_ingress_secret_name: "authentik-outpost-tls"
+            kubernetes_json_patches: null
+
+      # paperclip-gate: Paperclip UI; the CNP admits only this outpost. Paperclip keeps its own login behind it.
+      - id: paperclip-users-group
+        model: authentik_core.group
+        identifiers:
+          name: Paperclip Users
+        state: present
+        attrs:
+          is_superuser: false
+          users:
+            - !Find [authentik_core.user, [username, k.renner]]
+
+      - id: paperclip-gate-provider
+        model: authentik_providers_proxy.proxyprovider
+        identifiers:
+          name: paperclip-gate
+        state: present
+        attrs:
+          authentication_flow: !Find [authentik_flows.flow, [slug, passkey-passwordless-authentication]]
+          authorization_flow: !Find [authentik_flows.flow, [slug, passkey-stepup-authorization]]
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          mode: proxy
+          internal_host: http://paperclip.paperclip.svc.cluster.local:3100
+          internal_host_ssl_validation: false
+          external_host: https://paperclip.renner.uno
+          cookie_domain: renner.uno
+          access_token_validity: hours=24
+          refresh_token_validity: days=30
+          intercept_header_auth: true
+
+      - model: authentik_core.application
+        identifiers:
+          slug: paperclip-gate
+        state: present
+        attrs:
+          name: Paperclip
+          slug: paperclip-gate
+          provider: !KeyOf paperclip-gate-provider
+          meta_launch_url: https://paperclip.renner.uno
+          meta_description: "Paperclip agent orchestration"
+
+      - model: authentik_policies.policybinding
+        identifiers:
+          target: !Find [authentik_core.application, [slug, paperclip-gate]]
+          group: !KeyOf paperclip-users-group
+          order: 0
+        state: present
+        attrs:
+          enabled: true
+          negate: false
+
+      - model: authentik_outposts.outpost
+        identifiers:
+          name: paperclip-gate
+        state: present
+        attrs:
+          type: proxy
+          service_connection: !KeyOf k8s-conn
+          providers:
+            - !KeyOf paperclip-gate-provider
+          config:
+            log_level: info
+            authentik_host: "http://authentik-server.authentik.svc.cluster.local"
+            authentik_host_insecure: false
+            authentik_host_browser: "https://auth.renner.uno/"
+            refresh_interval: "minutes=5"
+            container_image: null
+            object_naming_template: "ak-outpost-%(name)s"
+            kubernetes_namespace: authentik
+            kubernetes_replicas: 1
+            kubernetes_service_type: ClusterIP
+            kubernetes_disabled_components: ["ingress"]
+            kubernetes_image_pull_secrets: []
+            kubernetes_ingress_annotations: {}
+            kubernetes_ingress_class_name: null
+            kubernetes_ingress_secret_name: "authentik-outpost-tls"
+            kubernetes_json_patches: null
+
+      # navidrome-gate: the Navidrome web UI. Navidrome auto-creates the
+      # X-Authentik-Username user on first login; the first user is admin.
+      # /rest and /share bypass the gate at the Traefik router.
+      - id: navidrome-gate-provider
+        model: authentik_providers_proxy.proxyprovider
+        identifiers:
+          name: navidrome-gate
+        state: present
+        attrs:
+          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          mode: proxy
+          internal_host: http://navidrome.navidrome.svc.cluster.local
+          internal_host_ssl_validation: false
+          external_host: https://navidrome.renner.uno
+          cookie_domain: renner.uno
+          access_token_validity: hours=24
+          refresh_token_validity: days=30
+          intercept_header_auth: true
+
+      - model: authentik_core.application
+        identifiers:
+          slug: navidrome-gate
+        state: present
+        attrs:
+          name: Navidrome
+          slug: navidrome-gate
+          provider: !KeyOf navidrome-gate-provider
+          meta_launch_url: https://navidrome.renner.uno
+          meta_description: "Music streaming (Subsonic)"
+
+      - model: authentik_outposts.outpost
+        identifiers:
+          name: navidrome-gate
+        state: present
+        attrs:
+          type: proxy
+          service_connection: !KeyOf k8s-conn
+          providers:
+            - !KeyOf navidrome-gate-provider
+          config:
+            log_level: info
+            authentik_host: "http://authentik-server.authentik.svc.cluster.local"
+            authentik_host_insecure: false
+            authentik_host_browser: "https://auth.renner.uno/"
+            refresh_interval: "minutes=5"
+            container_image: null
+            object_naming_template: "ak-outpost-%(name)s"
+            kubernetes_namespace: authentik
+            kubernetes_replicas: 1
+            kubernetes_service_type: ClusterIP
+            kubernetes_disabled_components: ["ingress"]
+            kubernetes_image_pull_secrets: []
+            kubernetes_ingress_annotations: {}
+            kubernetes_ingress_class_name: null
+            kubernetes_ingress_secret_name: "authentik-outpost-tls"
+            kubernetes_json_patches: null
+
+      # lidarr-gate: Lidarr runs with AuthenticationMethod=External, so the gate
+      # is its only access control.
+      - id: music-librarians-group
+        model: authentik_core.group
+        identifiers:
+          name: Music Librarians
+        state: present
+        attrs:
+          is_superuser: false
+          # Membership is owned here; blueprint apply reverts UI edits.
+          users:
+            - !Find [authentik_core.user, [username, k.renner]]
+
+      - id: lidarr-gate-provider
+        model: authentik_providers_proxy.proxyprovider
+        identifiers:
+          name: lidarr-gate
+        state: present
+        attrs:
+          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          mode: proxy
+          internal_host: http://lidarr.navidrome.svc.cluster.local
+          internal_host_ssl_validation: false
+          external_host: https://lidarr.renner.uno
+          cookie_domain: renner.uno
+          access_token_validity: hours=24
+          refresh_token_validity: days=30
+          intercept_header_auth: true
+
+      - model: authentik_core.application
+        identifiers:
+          slug: lidarr-gate
+        state: present
+        attrs:
+          name: Lidarr
+          slug: lidarr-gate
+          provider: !KeyOf lidarr-gate-provider
+          meta_launch_url: https://lidarr.renner.uno
+          meta_description: "Music library gaps: discographies, Spotify artists"
+
+      - model: authentik_policies.policybinding
+        identifiers:
+          target: !Find [authentik_core.application, [slug, lidarr-gate]]
+          group: !KeyOf music-librarians-group
+          order: 0
+        state: present
+        attrs:
+          enabled: true
+          negate: false
+          failure_result: false
+
+      - model: authentik_outposts.outpost
+        identifiers:
+          name: lidarr-gate
+        state: present
+        attrs:
+          type: proxy
+          service_connection: !KeyOf k8s-conn
+          providers:
+            - !KeyOf lidarr-gate-provider
+          config:
+            log_level: info
+            authentik_host: "http://authentik-server.authentik.svc.cluster.local"
+            authentik_host_insecure: false
+            authentik_host_browser: "https://auth.renner.uno/"
+            refresh_interval: "minutes=5"
+            container_image: null
+            object_naming_template: "ak-outpost-%(name)s"
+            kubernetes_namespace: authentik
+            kubernetes_replicas: 1
+            kubernetes_service_type: ClusterIP
+            kubernetes_disabled_components: ["ingress"]
+            kubernetes_image_pull_secrets: []
+            kubernetes_ingress_annotations: {}
+            kubernetes_ingress_class_name: null
+            kubernetes_ingress_secret_name: "authentik-outpost-tls"
+            kubernetes_json_patches: null
+
      - model: authentik_outposts.outpost
        identifiers:
          name: authentik Embedded Outpost
@@ -7,6 +7,7 @@ resources:
  - helmrelease.yaml
  - paperless-gate-ingress.yaml
  - paperless-gate-egress.yaml
+  - paperclip-gate-egress.yaml
  - vscode-gate-egress.yaml
  - shelly-manager-gate-egress.yaml
  - esphome-gate-egress.yaml
@@ -16,6 +17,9 @@ resources:
  - prometheus-gate-egress.yaml
  - wordpress-gate-egress.yaml
  - llama-gate-egress.yaml
+  - wiki-gate-egress.yaml
+  - navidrome-gate-egress.yaml
+  - lidarr-gate-egress.yaml
  - ldap-outpost-cert.yaml
  - ldap-outpost-lb.yaml
  - ldap-outpost-cnp.yaml
@@ -0,0 +1,20 @@
+# Lets the lidarr-gate outpost reach Lidarr; the matching ingress rule is in
+# network-policies/navidrome-ns.yaml.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: lidarr-gate-egress
+  namespace: authentik
+spec:
+  endpointSelector:
+    matchLabels:
+      goauthentik.io/outpost-name: lidarr-gate
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": navidrome
+            app.kubernetes.io/name: lidarr
+      toPorts:
+        - ports:
+            - port: "8686"
+              protocol: TCP
@@ -0,0 +1,20 @@
+# Lets the navidrome-gate outpost reach Navidrome; the matching ingress rule
+# is in network-policies/navidrome-ns.yaml.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: navidrome-gate-egress
+  namespace: authentik
+spec:
+  endpointSelector:
+    matchLabels:
+      goauthentik.io/outpost-name: navidrome-gate
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": navidrome
+            app.kubernetes.io/name: navidrome
+      toPorts:
+        - ports:
+            - port: "4533"
+              protocol: TCP
@@ -0,0 +1,19 @@
+# Lets the paperclip-gate outpost reach the Paperclip server; scoped to this outpost's pods.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: paperclip-gate-egress
+  namespace: authentik
+spec:
+  endpointSelector:
+    matchLabels:
+      goauthentik.io/outpost-name: paperclip-gate
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+            app.kubernetes.io/name: paperclip
+      toPorts:
+        - ports:
+            - port: "3100"
+              protocol: TCP
@@ -0,0 +1,21 @@
+# Lets the wiki-gate outpost reach the wiki service. The authentik namespace
+# baseline CNP otherwise stops any authentik pod from entering ai. Scoped to
+# this outpost's pod label; the matching ingress rule is in ai-ns.yaml.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: wiki-gate-egress
+  namespace: authentik
+spec:
+  endpointSelector:
+    matchLabels:
+      goauthentik.io/outpost-name: wiki-gate
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ai
+            app.kubernetes.io/name: wiki
+      toPorts:
+        - ports:
+            - port: "8080"
+              protocol: TCP
@@ -30,7 +30,39 @@ metadata:
 #     Web Service 9 = Music Assistant
 #     Web Service 10 = TRMNL
 #     Web Service 11 = Paperless (gate-level only, see its entry)
+#
+# CORRECTION (REN-21, 2026-09-25) to the Web Service 11 comment further down,
+# which claims that an in-cluster probe of the paperless ClusterIP "needs a
+# CiliumNetworkPolicy first — the monitoring probes get away without one only
+# because their targets share a namespace with the prober".
+#
+# That is wrong, and it was wrong when written. It is stated here rather than
+# fixed in place because the sentence lives inside the `config.yaml: |` block
+# scalar below: any edit to it changes ConfigMap DATA, and
+# `configmap.reloader.stakater.com/reload: gatus-config`
+# (deployment.yaml#reloader) would then roll the gatus Deployment — a
+# production restart for a comment. This header is outside `data:`, so it is
+# inert; the served config is byte-identical.
+#
+#   * When written, the paperless baseline admitted the whole monitoring
+#     namespace on every port, and monitoring is where blackbox-exporter runs.
+#     That admit is gone (REN-28); the probe is now admitted by the dedicated
+#     ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
+#     scoped to blackbox-exporter on the paperless pod's :8000. Cilium unions
+#     the allow rules of every CNP selecting an endpoint, so nothing else is
+#     required.
+#   * The in-cluster paperless probe therefore exists:
+#     ../monitoring-rules/probes-blackbox.yaml#paperless-app-health, target
+#     http://paperless.paperless.svc.cluster.local:80/accounts/login/
+#   * The claim does NOT generalise to gatus itself. Gatus runs in `apps`,
+#     which is a different source namespace; a gatus-sourced in-cluster check
+#     would need its own allow, exactly as
+#     cnp-cross-namespace-allow-apps-gatus-to-monitored-services describes.
+#
+# The Web Service 11 entry below stays a reachability-only check on purpose.
+# The app-health half of the pair is the blackbox probe named above.
 data:
+  # anchor: gatus-endpoints
  config.yaml: |
    storage:
      type: sqlite
@@ -6,6 +6,7 @@ metadata:
  labels:
    app: gatus
  annotations:
+    # anchor: reloader
    # Gatus reads its config once at startup and never re-reads the file, so
    # without this a ConfigMap change reconciles cleanly and then does nothing:
    # git and the running checks drift apart silently, with no failed resource
@@ -46,7 +47,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: gatus
-          image: twinproduction/gatus:v5.36.0 # {"$imagepolicy": "flux-system:gatus"}
+          image: twinproduction/gatus:v5.37.0 # {"$imagepolicy": "flux-system:gatus"}
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
@@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: gitea
-  namespace: apps
+  namespace: gitea
  labels:
    app.kubernetes.io/name: gitea
  annotations:
@@ -2,7 +2,7 @@ apiVersion: traefik.io/v1alpha1
 kind: IngressRouteTCP
 metadata:
  name: gitea-ssh
-  namespace: apps
+  namespace: gitea
  labels:
    app.kubernetes.io/name: gitea
 spec:
@@ -6,7 +6,6 @@ kind: Kustomization
 # and verified. See MIGRATION-gitea-woodpecker.md.
 resources:
  - statefulset.yaml
-  - pdb.yaml
  - service.yaml
  - ingressroute-ssh.yaml
  - ingress.yaml
@@ -1,28 +1,8 @@
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: gitea-allow-metrics-scrape
-  namespace: apps
-spec:
-  # Prometheus (monitoring ns) scrapes gitea's /metrics on :3000. The apps
-  # default-deny-ingress only allows same-ns / home-assistant / host, so the
-  # scrape is dropped (context deadline exceeded). This became necessary only
-  # after the LXC->in-cluster cutover — the old static target was external, so
-  # no in-cluster policy applied. Scoped to the gitea pod + :3000.
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: gitea
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": monitoring
-      toPorts:
-        - ports:
-            - port: "3000"
-              protocol: TCP
---
+# Prometheus scrape and Traefik are admitted by the namespace baseline in
+# infrastructure/network-policies/gitea-ns.yaml.
+#
 # Secret rotation: the flux-git rotator (openbao ns) reaches gitea to mint a new
-# flux PAT. It uses the in-cluster Service (gitea.apps.svc:3000) — the public
+# flux PAT. It uses the in-cluster Service (gitea.gitea.svc:3000) — the public
 # gitea.renner.uno name hairpins off the ingress LB from a pod (same failure as
 # pve.renner.uno). Scoped to the gitea pod :3000 and the rotator label; the
 # egress half is rotator-gitea-flux-egress in network-policies/openbao-ns.yaml.
@@ -30,7 +10,7 @@ apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
  name: gitea-allow-flux-rotator
-  namespace: apps
+  namespace: gitea
 spec:
  endpointSelector:
    matchLabels:
@@ -54,7 +34,7 @@ apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
  name: gitea-allow-review-bot-rotator
-  namespace: apps
+  namespace: gitea
 spec:
  endpointSelector:
    matchLabels:
@@ -1,23 +0,0 @@
---
-# Eviction protection for the single-replica Gitea StatefulSet.
-#
-# Gitea runs on SQLite (single writer) and its repos + DB live on a single
-# ReadWriteOnce PVC, so it is deliberately replicas:1 and cannot fail over.
-# With no PDB, a node drain / maintenance evicts the pod abruptly, mid-write,
-# with zero coordination. minAvailable:1 on a 1-replica workload makes the
-# eviction API refuse *voluntary* evictions: `kubectl drain` blocks and
-# surfaces this pod, forcing a conscious decision (drain with
-# --disable-eviction, or scale down first) instead of yanking the live DB.
-# It does NOT block involuntary disruptions (node crash, OOM).
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
-  name: gitea
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: gitea
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gitea
@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
  name: gitea
-  namespace: apps
+  namespace: gitea
  labels:
    app.kubernetes.io/name: gitea
 spec:
@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
  name: gitea
-  namespace: apps
+  namespace: gitea
  labels:
    app.kubernetes.io/name: gitea
 spec:
@@ -152,7 +152,7 @@ spec:
          # NOTE: volumeClaimTemplates is immutable on an existing StatefulSet,
          # so bumping this value cannot be `kubectl apply`-ed in place — Flux
          # will reject the whole object. Landing this size change requires a
-          # one-time `kubectl delete statefulset gitea -n apps --cascade=orphan`
+          # one-time `kubectl delete statefulset gitea -n gitea --cascade=orphan`
          # (pod + PVC keep running) so Flux re-creates the STS from this
          # template and re-adopts them. After the cleanup_packages GC reclaims
          # the ~10G of orphaned blobs, real usage is ~9G, so 40Gi is headroom.
@@ -4,7 +4,9 @@ metadata:
  name: gf-pg
  namespace: monitoring
 spec:
-  instances: 1
+  instances: 2
+  smartShutdownTimeout: 10
+  stopDelay: 60
  imageName: ghcr.io/cloudnative-pg/postgresql:16
  storage:
    storageClass: ceph-block
@@ -1,15 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: home-assistant-backup-password
-  namespace: home-assistant
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: home-assistant-backup-password # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: apps/home-assistant/backup-password
@@ -95,7 +95,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.3 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.9.4 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -30,16 +30,71 @@ metadata:
  name: ha-pg
  namespace: home-assistant
 spec:
-  instances: 1
+  instances: 2
+  # switchover, not CNPG's default `restart`: any change that rolls the pods
+  # (walStorage below did) then promotes the already-rolled replica instead of
+  # restarting the primary in place, so the recorder loses seconds of writes
+  # rather than stopDelay plus a pg_wal move plus startup (REN-176).
+  primaryUpdateMethod: switchover
+  smartShutdownTimeout: 10
+  stopDelay: 60
  imageName: ghcr.io/cloudnative-pg/postgresql:16
  storage:
    storageClass: ceph-block
-    # Sized for the migrated statistics plus 7 days of states at the current
-    # rate: statistics 10.8M rows + statistics_short_term 1.9M (~1.8 GB with
-    # indexes in Postgres) and ~18M states rows for the purge_keep_days:7 window
-    # (~4 GB). 20Gi leaves room for autovacuum bloat and index churn.
-    # ceph-block has allowVolumeExpansion: true, so this can grow in place.
-    size: 20Gi
+    # Sized for WAL headroom during an archiving stall, not for table growth.
+    # The data itself needs far less: migrated statistics 10.8M rows +
+    # statistics_short_term 1.9M (~1.8 GB with indexes in Postgres) and ~18M
+    # states rows for the purge_keep_days:7 window (~4 GB).
+    #
+    # 20Gi left only 6.63 GiB free, and that is the number that matters here.
+    # WAL archiving stalled for 10 minutes and the un-archived backlog reached
+    # 1.84 GiB — 28% of that free space (REN-129).
+    #
+    # That stall was first blamed on Velero repository maintenance. It is not
+    # the cause: REN-136 measured seven nights and the archiver's sustained
+    # drain (~14 segments/min, its serial ceiling) is flat whether Kopia is
+    # running or not, and Kopia's nearest batch drifts across 02:03-02:24Z
+    # while the burst stays pinned to 02:13Z. What is real is a nightly
+    # producer burst — 56-119 segments queued at 02:13-02:15 UTC on all seven
+    # nights, against a steady-state queue of 0. Postgres cannot recycle a WAL
+    # segment until the archiver has shipped it, so a burst that arrives while
+    # archiving is stalled walks the volume toward full, and a full PGDATA
+    # stops the recorder outright.
+    #
+    # 32Gi takes free space to ~18.6 GiB, which puts the same 1.84 GiB backlog
+    # at 10% of headroom rather than 28%. In REN-136's terms: at 6.36 GiB free
+    # one stalled night survives and two fill the volume; ~18.6 GiB carries
+    # about four. Approved by the owner on REN-135.
+    # ceph-block has allowVolumeExpansion: true, so this grows in place.
+    #
+    # Since REN-176 WAL lives on walStorage below, so the stall headroom above
+    # now belongs to that volume. 32Gi stays because a PVC cannot shrink.
+    size: 32Gi
+  # WAL on its own volume (ha-pg-N-wal), approved by the owner on REN-176.
+  #
+  # PERMANENT. The CNPG webhook rejects removing walStorage once set ("walStorage
+  # cannot be disabled once configured"), so reverting this block fails
+  # admission and leaves the home-assistant Kustomization stuck. Undoing it
+  # means a new Cluster recovered from ha-pg-store. Adding it on 1.30.1 needs
+  # no re-clone: the operator creates the PVC, rolls each pod, and the instance
+  # manager moves pg_wal onto it before Postgres starts.
+  #
+  # This does NOT keep the recorder up when WAL fills: a full WAL disk still
+  # stops Postgres. What it does is keep WAL writable if PGDATA fills (a purge
+  # that stops running), and make WAL-volume usage a direct archiver-backlog
+  # signal.
+  #
+  # Sized so stall survival is no worse than the shared 32Gi gives today.
+  # Today: ~18.6 GiB free on PGDATA / 4.66 GiB/day of WAL = ~4.0 days.
+  #   1.86 GiB  normal pg_wal footprint (max_wal_size 2GB)
+  # + 18.64 GiB four days of full archiver outage at 4.66 GiB/day
+  #             (the nightly 56-119 segment burst, REN-136, is inside that rate)
+  # = 20.5 GiB -> 21Gi: (21 - 1.86) / 4.66 = ~4.1 days. 18Gi would give ~3.5,
+  # worse than today. RBD is thin, so the ceiling costs nothing until a stall
+  # fills it. The webhook requires min_wal_size < max_wal_size < this size.
+  walStorage:
+    storageClass: ceph-block
+    size: 21Gi
  plugins:
    - name: barman-cloud.cloudnative-pg.io
      isWALArchiver: true
@@ -124,8 +179,7 @@ spec:
      # padding rather than data.
      #
      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
-      # its volume are lost outright. This is a single-instance cluster, so the
-      # archive is the only off-node copy.
+      # its volumes are lost outright.
      archive_timeout: "30min"
      # Fewer checkpoints => fewer full-page writes, since a page is only
      # fully-logged on its first modification after a checkpoint.
@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - backup-password-externalsecret.yaml
  - pvcs.yaml
  - deployment.yaml
  - service.yaml
@@ -16,6 +15,9 @@ resources:
  # health broker credential: OpenBao-backed + Class-A rotated (first of the
  # eight users to migrate off mosquitto-bootstrap.yaml).
  - mosquitto-health-credentials-externalsecret.yaml
+  # open3e-int broker credential: open3e-esp32 gateway on the Vitocal internal
+  # CAN; OpenBao-backed + Class-A rotated.
+  - mosquitto-open3e-int-credentials-externalsecret.yaml
  - mosquitto-config.yaml
  - mosquitto-certificate.yaml
  - mosquitto-service.yaml
@@ -12,7 +12,7 @@ metadata:
    #
    # Listed per-Secret rather than with the blanket auto annotation, so that
    # enrolling another broker user is a visible edit to this line.
-    secret.reloader.stakater.com/reload: mosquitto-health-credentials
+    secret.reloader.stakater.com/reload: mosquitto-health-credentials,mosquitto-open3e-int-credentials
 spec:
  replicas: 1
  strategy:
@@ -78,6 +78,7 @@ spec:
              mosquitto_passwd -b /mosquitto/auth/passwd admin "$ADMIN_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd sml2mqtt "$SML2MQTT_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd open3e "$OPEN3E_PASSWORD"
+              mosquitto_passwd -b /mosquitto/auth/passwd open3e-int "$OPEN3E_INT_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd evcc "$EVCC_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd health "$HEALTH_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd zigbee2mqtt "$ZIGBEE2MQTT_PASSWORD"
@@ -124,6 +125,12 @@ spec:
              user open3e
              topic readwrite open3e/#

+              # open3e-esp32 gateway on the Vitocal internal CAN: own subtree
+              # under open3e/ so edge-1's listener on open3e/cmnd never sees
+              # its commands.
+              user open3e-int
+              topic readwrite open3e/int/#
+
              user evcc
              topic read open3e/#
              topic readwrite evcc/#
@@ -187,6 +194,13 @@ spec:
                secretKeyRef:
                  name: mosquitto-open3e-credentials
                  key: password
+            # open3e-int: OpenBao-backed + Class-A rotated (see
+            # mosquitto-open3e-int-credentials-externalsecret.yaml).
+            - name: OPEN3E_INT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mosquitto-open3e-int-credentials
+                  key: password
            - name: EVCC_PASSWORD
              valueFrom:
                secretKeyRef:
@@ -0,0 +1,21 @@
+# Projects the mosquitto `open3e-int` broker credential (open3e-esp32 gateway
+# on the Vitocal internal CAN) from OpenBao into the Secret the broker
+# consumes. Rotator: kubernetes/infrastructure/openbao/rotation/mosquitto-open3e-int.yaml.
+#
+# creationPolicy is implicitly Owner: deleting this ExternalSecret deletes the
+# live Secret. Set creationPolicy: Orphan first to detach without destroying.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: mosquitto-open3e-int-credentials
+  namespace: home-assistant
+spec:
+  refreshInterval: 10m
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: mosquitto-open3e-int-credentials
+  dataFrom:
+    - extract:
+        key: apps/home-assistant/mosquitto-open3e-int-credentials
@@ -24,7 +24,7 @@ spec:
          # unchanged, so Kubernetes has no reason to replace anything, and
          # this image is deliberately NOT enrolled in Flux image automation
          # — see kubernetes/infrastructure/image-automation/README.md, which
-          # lists ghcr.io/konairius/ical-proxy as having "only main, latest,
+          # lists gitea.renner.uno/konairius/ical-proxy as having "only main, latest,
          # and main-<sha> commit builds; no release tags", so there is no
          # ImagePolicy watching it. The pods therefore sat on a 21-day-old
          # image while :latest moved underneath them, and the only way to
@@ -35,15 +35,26 @@ spec:
          # what is RUNNING recoverable from this repo rather than "whatever
          # :latest meant the last time a pod happened to restart".
          #
-          # main-99a912b is the merge of konairius/ical-proxy#12, which adds
-          # the summary_match/summary_replace parameters this deployment
-          # needs to strip "| Abfuhrkalender - Landkreis Amberg-Sulzbach"
-          # from every waste collection.
+          # The image comes from the Gitea package registry, built by
+          # Woodpecker (.woodpecker.yml in konairius/ical-proxy) on every
+          # push to main. It is amd64-only and pulls without credentials, so
+          # no imagePullSecret. main-4783abe adds two hardening changes on
+          # top of main-db90a59: an SSRF guard (upstreams and redirect hops
+          # resolving to private/loopback/link-local addresses are refused;
+          # public redirects are still followed, up to 5) and a cap of 20
+          # concurrent /proxy upstream fetches (over the cap: 429 +
+          # Retry-After). main-73b5d97 then raises the server WriteTimeout
+          # from 10s to 40s (the 30s upstream fetch timeout + 10s headroom),
+          # so a slow-but-legitimate upstream is no longer cut off before
+          # /proxy can answer. main-926ead0 caps the upstream response body
+          # at 2 MiB (larger feeds get 413 instead of being buffered whole,
+          # REN-230) and logs upstream fetch failures with the host,
+          # answering 502 instead of 500. The summary_match/summary_replace
+          # parameters this deployment needs are unchanged.
          #
          # Bumping this is a manual, reviewed edit. The tag format is
-          # main-<short-sha> (docker/metadata-action type=sha with the branch
-          # as prefix).
-          image: ghcr.io/konairius/ical-proxy:main-99a912b
+          # main-<short-sha>, the first 7 characters of the main commit.
+          image: gitea.renner.uno/konairius/ical-proxy:main-926ead0
          ports:
            - containerPort: 8080
              name: http
@@ -59,8 +59,7 @@ spec:
      # padding rather than data.
      #
      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
-      # its volume are lost outright. This is a single-instance cluster, so the
-      # archive is the only off-node copy.
+      # its volumes are lost outright.
      archive_timeout: "30min"
      # Write-reduction tuning. These databases are largely idle, and the
      # Postgres defaults favour throughput over storage that wears out.
@@ -32,7 +32,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.2.4 # {"$imagepolicy": "flux-system:immich-machine-learning"}
          ports:
            - name: http
              containerPort: 3003
@@ -18,6 +18,14 @@ kind: PersistentVolumeClaim
 metadata:
  name: immich-library
  namespace: immich
+  labels:
+    # Pre-migration copy: no pod mounts it since the 2026-09-25 move to
+    # immich-library-bulk, and its backup size has been frozen since. Left
+    # in the daily backup, the monthly restore-test stages it (~194 GiB x3
+    # replicas) alongside library-bulk and overruns Ceph capacity (#1089).
+    # Existing backups keep it until their 720h TTL; delete the PVC/PV by
+    # hand once no longer wanted.
+    velero.io/exclude-from-backup: "true"
  annotations:
    kustomize.toolkit.fluxcd.io/prune: disabled
 spec:
@@ -28,6 +36,38 @@ spec:
    requests:
      storage: 300Gi
 ---
+# Media root on the EC 2+1 bulk pool; thumbs mount separately below.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-library-bulk
+  namespace: immich
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ceph-filesystem-bulk
+  resources:
+    requests:
+      storage: 300Gi
+---
+# Thumbnails stay replicated: read affinity serves them from the local OSD.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-thumbs
+  namespace: immich
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ceph-filesystem
+  resources:
+    requests:
+      storage: 30Gi
+---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -87,7 +87,7 @@ spec:
              mountPath: /ca
      containers:
        - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.2.4 # {"$imagepolicy": "flux-system:immich-server"}
          ports:
            - name: http
              containerPort: 3001
@@ -110,6 +110,8 @@ spec:
          volumeMounts:
            - name: library
              mountPath: /data
+            - name: thumbs
+              mountPath: /data/thumbs
            - name: immich-config-file
              mountPath: /config
              readOnly: true
@@ -145,7 +147,10 @@ spec:
      volumes:
        - name: library
          persistentVolumeClaim:
-            claimName: immich-library
+            claimName: immich-library-bulk
+        - name: thumbs
+          persistentVolumeClaim:
+            claimName: immich-thumbs
        - name: immich-config-file
          secret:
            secretName: immich-secrets
@@ -61,6 +61,9 @@ spec:
      enabled: true
      storageClass: ceph-block
      size: 50Gi
+    # A minAvailable:1 PDB on a single replica blocks every node drain.
+    pdb:
+      create: false
    # Bootstrap admin/org/bucket on the fresh PVC. Admin password + token come
    # from the influxdb-bootstrap Secret (OpenBao via ESO), deterministic across
    # rebuilds so the bootstrap token is the live admin token. `influx setup`
@@ -3,8 +3,8 @@ kind: Kustomization
 resources:
  - gatus/
  - external-dns/
-  - gitea/
  - ical-proxy/
+  # gitea/ has its own `gitea` Kustomization (clusters/homelab/gitea.yaml, #878).
  # woodpecker/ moved to its own dedicated `woodpecker` Kustomization
  # (clusters/homelab/woodpecker.yaml) so its wp-pg CNPG cluster's barman
  # dependency + Cluster health-gating don't contaminate this shared apps KS.
@@ -4,7 +4,7 @@
 # the entrypoint exit after one pass instead of looping internally —
 # the CronJob handles scheduling.
 #
-# Books land in /audiobooks (the audiobookshelf-books PVC, shared
+# Books land in /audiobooks (the audiobookshelf-books-bulk PVC, shared
 # with the audiobookshelf Deployment). Config + DB live on
 # libation-config (RWO ceph-block, seeded once from
 # libation-bootstrap secret).
@@ -107,6 +107,9 @@ spec:
                - |
                  trap 'cp /config-internal/AccountsSettings.json /libation-state/AccountsSettings.json' EXIT
                  set -e
+                  # Strip the Audible intro/outro; re-liberated books replace their file instead of gaining a " (1)" copy.
+                  jq '.StripAudibleBrandAudio = true | .OverwriteExisting = true' /config-internal/Settings.json > /config-internal/Settings.json.tmp
+                  mv /config-internal/Settings.json.tmp /config-internal/Settings.json
                  /libation/LibationCli scan
                  /libation/LibationCli liberate
              env:
@@ -135,7 +138,7 @@ spec:
                claimName: libation-config
            - name: books
              persistentVolumeClaim:
-                claimName: audiobookshelf-books
+                claimName: audiobookshelf-books-bulk
            - name: bootstrap
              secret:
                secretName: libation-bootstrap
@@ -5,7 +5,7 @@
 # SOPS-encrypted Secret on first boot (sentinel file gate); after that
 # Libation reads and writes it directly so token rotation persists.
 #
-# Books are written to the audiobookshelf-books PVC (already provisioned
+# Books are written to the audiobookshelf-books-bulk PVC (already provisioned
 # by the audiobookshelf seed Kustomization, RWX ceph-filesystem). We
 # share the same namespace so the PVC reference is direct — k8s PVCs
 # are namespace-scoped.
@@ -1,8 +1,75 @@
+---
+# The agent metrics Service, declared here because the chart does not render
+# one under our values (REN-147).
+#
+# cilium's templates/cilium-agent/service.yaml is gated twice:
+#
+#   {{- if and .Values.agent (not .Values.preflight.enabled) .Values.prometheus.enabled }}
+#   {{- if (or .Values.prometheus.serviceMonitor.enabled .Values.prometheus.metricsService) }}
+#
+# infrastructure/cilium/helmrelease.yaml sets `prometheus.enabled: true` and
+# `prometheus.serviceMonitor.enabled: false`, and leaves `metricsService` at its
+# default of false. First gate passes, second does not. The chart's fallback
+# branch is envoy-only and needs envoy in the agent pod, but envoy runs as its
+# own DaemonSet here — so no `cilium-agent` Service was rendered at all.
+#
+# The consequence was silent: agents exported metrics on :9962 all along, the
+# ServiceMonitor below selected a Service that did not exist, and the scrape
+# pool sat at 0 active / 46 dropped targets. `cilium_*` had zero series, so
+# CiliumBPFMapPressureHigh and CiliumBPFMapFull in prometheusrule-cilium.yaml
+# evaluated an empty vector forever and could never fire.
+#
+# Why a Service here and not `prometheus.metricsService: true` in the
+# HelmRelease: that flag is the chart-native fix and its rendered diff is
+# exactly this one object (it appears in no other template — not the
+# ConfigMap, not the DaemonSet). But setting it changes the HelmRelease values,
+# which makes helm-controller run `helm upgrade` on the CNI release, and that
+# three-way-merges against live objects. PROPOSAL-cilium-single-owner.md
+# records that `cilium-config` has carried a `kubectl-patch` field manager; if
+# any of that drift is still live, the upgrade reconciles it, and the
+# `configmap.reloader.stakater.com/reload: cilium-config` annotation on the
+# HelmRelease then rolls every agent node by node. A Service applied by the
+# apps Kustomization cannot do that. If the drift is ever proven gone, delete
+# this object and set `prometheus.metricsService: true` instead.
+#
+# Name and selector follow authentik-outpost-servicemonitors.yaml: a distinct
+# name plus `app.kubernetes.io/managed-by: homelab`, so flipping either chart
+# flag later renders the chart's own `cilium-agent` Service without an
+# ownership conflict and without double-scraping the same endpoints.
+apiVersion: v1
+kind: Service
+metadata:
+  name: cilium-agent-metrics
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/managed-by: homelab
+    app.kubernetes.io/name: cilium-agent
+    app.kubernetes.io/part-of: cilium
+    # Copied onto every series by the ServiceMonitor's targetLabels below.
+    # The dashboard the chart ships queries
+    # cilium_bpf_map_pressure{k8s_app="cilium"}, so this label is load-bearing.
+    k8s-app: cilium
+spec:
+  type: ClusterIP
+  # Headless on purpose: agent metrics are per-node and every endpoint has to
+  # be scraped. A ClusterIP would hand Prometheus one random agent per scrape.
+  clusterIP: None
+  # Agent pods only. cilium-envoy carries k8s-app: cilium-envoy and exposes
+  # 9964, so it cannot match.
+  selector:
+    k8s-app: cilium
+  ports:
+    # Named and numbered exactly as the chart's own Service, so the
+    # ServiceMonitor below stays a field-for-field copy of the chart's.
+    - name: metrics
+      port: 9962
+      targetPort: prometheus
+      protocol: TCP
+---
 # Cilium's agent ServiceMonitor, declared here rather than by the chart.
 #
-# Reproduces what cilium 1.20.1 renders with prometheus.serviceMonitor.enabled,
-# taken field for field off the live object so scraping is unchanged: same name,
-# same namespace, same selector, same 10s interval and node relabeling.
+# Reproduces what cilium renders with prometheus.serviceMonitor.enabled: same
+# name, same namespace, same 10s interval and node relabeling.
 #
 # It lives here because the chart's template is gated on monitoring.coreos.com/v1
 # existing at install time, and on an empty cluster it does not — those CRDs
@@ -22,13 +89,14 @@ metadata:
    app.kubernetes.io/part-of: cilium
 spec:
  selector:
+    # Pinned to the Service above. Without managed-by this would also match a
+    # chart-rendered `cilium-agent` Service and scrape the same endpoints twice.
    matchLabels:
+      app.kubernetes.io/managed-by: homelab
      app.kubernetes.io/name: cilium-agent
  namespaceSelector:
    matchNames:
      - kube-system
-  # The chart's Service keeps rendering (prometheus.enabled), so the target and
-  # its `metrics` port are there regardless of who declares the ServiceMonitor.
  targetLabels:
    - k8s-app
  endpoints:
@@ -0,0 +1,82 @@
+# Scrape the external-secrets (ESO) controller's own metrics. Until this
+# existed, nothing in this cluster could see a failed ExternalSecret sync:
+# `externalsecret_status_condition` held 0 series, so no rule could be written
+# about a key that silently stops syncing (REN-100).
+#
+# This is the scrape only. The alert rule is deliberately NOT in this change —
+# its `for:` window has to be tuned against real data, and that data does not
+# exist until this has been live for a while. See REN-103.
+#
+# A PodMonitor, NOT the chart's `serviceMonitor.enabled`, and NOT a hand-written
+# ServiceMonitor. Both alternatives were checked against chart 2.11.0:
+#
+#   * A hand-written ServiceMonitor has nothing to select. `metrics.service.enabled`
+#     defaults to false and infrastructure/external-secrets/helmrelease.yaml
+#     overrides nothing, so templates/service.yaml renders no Service for the
+#     controller. The container port exists (`name: metrics`, containerPort 8080)
+#     but nothing fronts it — the same trap that forced a PodMonitor for the Flux
+#     controllers.
+#   * Setting `serviceMonitor.enabled: true` would work, but it triggers a Helm
+#     UPGRADE of ESO, which re-renders the Deployment and re-applies its CRDs
+#     (`crds: CreateReplace` in the HelmRelease). ESO is what all 54 ExternalSecret
+#     manifests in this repo depend on. A PodMonitor gets the same series while
+#     touching the release not at all, which is the trade this repo already made
+#     for Velero's node-agent (velero-podmonitor.yaml) and for Flux.
+#
+# THIS NEEDS A NETWORK POLICY CHANGE, unlike the Flux PodMonitor: the cluster is
+# default-deny and external-secrets-ns.yaml admitted only intra-namespace traffic,
+# host/remote-node and ICMP — no `monitoring` grant, so this scrape would have
+# been dropped and the metric would have stayed absent. The matching ingress rule
+# is added to infrastructure/network-policies/external-secrets-ns.yaml in the
+# same pull request. Without it this file is decoration.
+#
+# NO TLS AND NO AUTH ON THIS SCRAPE, and that is the chart's posture rather than
+# an omission here. In 2.11.0 `metrics.listen.secure.enabled` and
+# `metrics.listen.auth.enabled` are both false (values.yaml:365-369) and
+# infrastructure/external-secrets/helmrelease.yaml overrides neither, so the
+# controller serves /metrics over plain HTTP to anything that can reach :8080.
+# Hence no `scheme`, no `tlsConfig` and no `bearerTokenSecret` below: adding
+# them would break the scrape, not harden it. What limits exposure is the
+# network policy — a single port from a single namespace — so turning on
+# `secure.enabled` here would mean a Helm upgrade of ESO to close a hole the
+# CiliumNetworkPolicy already closes.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: external-secrets
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: external-secrets
+    # Not required for discovery — podMonitorSelector is empty, see
+    # apps/monitoring/helmrelease-prometheus.yaml — but set for consistency with
+    # the other monitors in this directory.
+    release: kube-prometheus-stack
+spec:
+  # Lives in `monitoring` rather than in `external-secrets`, matching
+  # external-dns-servicemonitor.yaml and the Flux PodMonitor. Every object in
+  # the external-secrets namespace is owned by the ESO HelmRelease; keeping this
+  # out of that namespace means this change cannot interact with the release.
+  namespaceSelector:
+    matchNames:
+      - external-secrets
+  selector:
+    matchLabels:
+      # The CONTROLLER only. The webhook and cert-controller carry
+      # `app.kubernetes.io/name: external-secrets-webhook` and
+      # `-cert-controller` respectively (see the chart's
+      # external-secrets-webhook.selectorLabels / -cert-controller.selectorLabels
+      # helpers), so an exact match on the unsuffixed name excludes both. That is
+      # intended: the `externalsecret_*` series are emitted by the controller, and
+      # the other two would only add idle targets.
+      app.kubernetes.io/name: external-secrets
+      app.kubernetes.io/instance: external-secrets
+  podMetricsEndpoints:
+    # Named container port, not a hardcoded 8080, so this follows
+    # `metrics.listen.port` if it is ever overridden in the HelmRelease.
+    - port: metrics
+      # 60s, not the chart's 30s default: `externalsecret_status_condition` is a
+      # slow-moving gauge over ~54 ExternalSecrets, and the alert built on it in
+      # REN-103 will use a multi-minute `for:` window. 30s would double the
+      # samples for no gain in detection time.
+      interval: 60s
+      scrapeTimeout: 10s
@@ -0,0 +1,91 @@
+# Scrape the Flux controllers' own metrics. Until this existed Prometheus held
+# no `gotk_*` series at all, so nothing about Flux's own behaviour was
+# queryable and no rule could be written about it (REN-97).
+#
+# THIS IS THE SCRAPE ONLY, and the readiness alert is not merely deferred —
+# it cannot be written from this endpoint. `gotk_reconcile_condition` and
+# `gotk_suspend_status` are still DECLARED by fluxcd/pkg/runtime (v0.110.3,
+# runtime/metrics/recorder.go), but at the tags pinned in gotk-components.yaml
+# no controller calls the recorder that populates them: grepping the six
+# tagged source trees for RecordReadiness / RecordCondition / RecordSuspend
+# returns zero call sites, and an unpopulated GaugeVec exports no series. So
+# the readiness metrics would stay empty however this file is written. Flux
+# moved per-object readiness to kube-state-metrics custom-resource-state
+# (`gotk_resource_info{ready=...}`); that is REN-102, and it is a prerequisite
+# for the not-Ready alert rather than a follow-up to it.
+#
+# What this scrape does yield, and why it is still worth merging on its own:
+#   * `gotk_reconcile_duration_seconds` per kind/name/namespace — reconcile
+#     latency and, by its absence, an object that has stopped being reconciled
+#     at all. RecordDuration IS called, but by FIVE of the six controllers, not
+#     all six: measured post-merge, 170 series from kustomize (33), source (43),
+#     image-reflector (68), image-automation (1) and helm (25), and ZERO from
+#     notification-controller. That is not a gap waiting to fill in — its three
+#     reconcilers are registered but never walked (controller_runtime_reconcile_
+#     total for controller=alert/provider/receiver is 0 over 56 h of uptime with
+#     2 Alerts and 1 Provider live), so the family is absent from that pod's
+#     /metrics entirely and five is the permanent value. DO NOT write a rule of
+#     the form `count(... gotk_reconcile_duration_seconds ...) < 6`, or an
+#     absent() per pod: it fires forever on notification-controller. Liveness for
+#     all six pods is `up{job="monitoring/flux-controllers"}` (below); that pod's
+#     reconcile health is `controller_runtime_reconcile_errors_total`. REN-127.
+#   * controller-runtime's own series: reconcile totals and errors per
+#     controller, workqueue depth and retry counts, leader election.
+#   * the target itself, so `up{job="monitoring/flux-controllers"}` and the
+#     chart's default TargetDown rule start covering the Flux control plane.
+#   * the scrape REN-102 needs in place anyway.
+#
+# A PodMonitor, NOT a ServiceMonitor, and that is forced rather than preferred:
+# gotk-components.yaml creates Services for only two of the six controllers
+# (source-controller, notification-controller), and both publish port 80 ->
+# targetPort `http`, which is the artifact/webhook server on :9090 — not the
+# metrics port. kustomize-, helm-, image-reflector- and image-automation-
+# controller have no Service at all, so there is nothing for a ServiceMonitor to
+# select. All six pods do carry the same two labels and the same `http-prom`
+# (:8080) container port, so one PodMonitor covers the set.
+#
+# NO NETWORK POLICY CHANGE IS NEEDED, verified rather than assumed, and unlike
+# external-secrets-podmonitor.yaml which did need one: flux-system has no
+# CiliumNetworkPolicy of its own (infrastructure/network-policies/ has no
+# flux-system file), and gotk-components.yaml ships an `allow-scraping`
+# NetworkPolicy admitting every namespace to :8080 (`from: [namespaceSelector:
+# {}]`, port 8080). So `monitoring` is already admitted, as it was for REN-21.
+# Nothing in the Flux install is touched.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: flux-controllers
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: flux
+    # Not required for discovery — podMonitorSelector is empty, see
+    # apps/monitoring/helmrelease-prometheus.yaml — but set for consistency
+    # with the other monitors in this directory.
+    release: kube-prometheus-stack
+spec:
+  # Lives in `monitoring` with a namespaceSelector rather than in flux-system,
+  # matching external-dns-servicemonitor.yaml and external-secrets-podmonitor.yaml.
+  # Flux owns every object in flux-system via gotk-components.yaml; keeping this
+  # out of that namespace means this change cannot interact with the Flux install
+  # itself.
+  namespaceSelector:
+    matchNames:
+      - flux-system
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: flux
+      app.kubernetes.io/instance: flux-system
+  podMetricsEndpoints:
+    # Matches the controllers' `http-prom` port (containerPort 8080). Present on
+    # all six Deployments in gotk-components.yaml; a future controller without
+    # it simply yields no target rather than a broken scrape config.
+    - port: http-prom
+      path: /metrics
+      # 60s, not the 30s used elsewhere here, matching the ESO PodMonitor: the
+      # useful series are slow-moving per-object gauges and histograms over a
+      # few hundred Flux objects, and any rule built on them (REN-102) will use
+      # a multi-minute `for:`. 30s would double the samples for no gain in
+      # detection time — and Prometheus is already near its retention cap
+      # (REN-107).
+      interval: 60s
+      scrapeTimeout: 10s
@@ -15,12 +15,20 @@ resources:
  - prometheusrule-netbird-availability.yaml
  - prometheusrule-netbird-ingress.yaml
  - prometheusrule-openbao.yaml
+  # Prometheus's own retention. Until REN-107 `retention: 90d` and
+  # `retentionSize: 30GB` were both set, the size cap bound first, and the
+  # effective window was a 20.5-27.2 day saw-tooth with nothing reporting it.
+  # Fires if the size cap binds before the time retention again.
+  - prometheusrule-prometheus-retention.yaml
  - prometheusrule-velero.yaml
  - prometheusrule-woodpecker-rotation.yaml
  # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
  # alerted if that stopped — HetznerBackupEndpointDown only probes the S3
  # endpoint, which proves reachability, not that anything was written.
  - prometheusrule-cnpg-backup.yaml
+  # Home Assistant, its recorder DB, Mosquitto and Zigbee2MQTT (REN-244). The
+  # only signal for them before was the generic 15m KubePodNotReady warning.
+  - prometheusrule-home-assistant.yaml
  # DNS publication. Nothing scraped external-dns, so it deleted six LAN
  # records on 2026-09-12 and stayed silent for five days.
  - prometheusrule-external-dns.yaml
@@ -30,6 +38,9 @@ resources:
  - prometheusrule-dns-resolution.yaml
  - probes-dns.yaml
  - alloy-servicemonitor.yaml
+  # Agent metrics. Ships the metrics Service as well as the ServiceMonitor:
+  # the chart renders neither under our values, so this scraped nothing and
+  # the two CiliumBPFMapPressure alerts could never fire. See the file.
  - cilium-servicemonitor.yaml
  - cnpg-podmonitors.yaml
  - crowdsec-servicemonitor.yaml
@@ -48,4 +59,22 @@ resources:
  - loki-servicemonitor.yaml
  - smartctl-exporter-servicemonitor.yaml
  - intel-gpu-exporter-servicemonitor.yaml
+  # ESO's controller metrics. Needs the `monitoring` ingress grant added to
+  # infrastructure/network-policies/external-secrets-ns.yaml in the same change,
+  # or it scrapes nothing. See the file.
+  - external-secrets-podmonitor.yaml
+  # The alert built on what that PodMonitor collects. Split from it on purpose:
+  # the `for:` window had to be chosen against real series, and Prometheus held
+  # none for this metric until the scrape landed.
+  - prometheusrule-external-secrets.yaml
+  # Flux's own control plane. The controllers were never scraped, so nothing
+  # about Flux's behaviour was queryable. Scrape only: per-object readiness is
+  # not on this endpoint at the pinned controller versions and needs
+  # kube-state-metrics instead. See the file.
+  - flux-podmonitor.yaml
+  # The readiness half the PodMonitor above cannot provide. Reads
+  # `gotk_resource_info`, which comes from the kube-state-metrics
+  # customResourceState block in apps/monitoring/helmrelease-prometheus.yaml —
+  # these two files must land and be reverted together.
+  - prometheusrule-flux.yaml
  - probes-blackbox.yaml
@@ -88,6 +88,179 @@ spec:
        # /ready answers 200 unauthenticated and is in this namespace, so the
        # same intra-namespace allowance the two above rely on covers it.
        - http://loki.monitoring.svc.cluster.local:3100/ready
+        # APP-HEALTH probes for the two gated navidrome-namespace apps. Same
+        # rule as the block above (monitoring-vantage-point-rule), but these
+        # are the first targets here that leave the monitoring namespace.
+        #
+        # Both sit entirely behind an Authentik outpost — navidrome.renner.uno
+        # on ak-outpost-navidrome-gate
+        # (../navidrome/ingress.yaml#navidrome-gate), lidarr.renner.uno on
+        # ak-outpost-lidarr-gate (../navidrome/lidarr.yaml#lidarr-gate) — so an
+        # EXTERNAL probe of either name measures the gate, not the app, and
+        # stays green while the app behind
+        # a live gate is dead. That external signal is still wanted, and REN-2
+        # adds it from UptimeRobot; these are the other half of the pair.
+        #
+        # NO CiliumNetworkPolicy change is needed — but NOT because
+        # cnp-cross-namespace-allow-apps-gatus-to-monitored-services fails to
+        # apply here. It applies: its Generalization section names
+        # blackbox-exporter by hand. The reason we are safe is that the allow
+        # that note demands is PRESENT AND DEDICATED — the navidrome baseline
+        # CNP (infrastructure/network-policies/navidrome-ns.yaml) admits
+        # blackbox-exporter by pod label on exactly the two listeners these
+        # two probes reach, :4533 and :8686. It used to admit the whole
+        # monitoring namespace on every port; REN-28 narrowed it, and the
+        # ports in that rule are the ones these probes depend on, so a probe
+        # added here against a THIRD listener in that namespace would need
+        # that rule widened first. Run that check for the next namespace you
+        # add; do not skip it on the grounds that the note "is about gatus".
+        #
+        # EGRESS — the half the wiki note is silent on, and the half that is
+        # actually load-bearing here. Cilium enforces at BOTH ends, so the
+        # destination ingress allow above is worthless if the prober itself
+        # is in egress enforcement without a matching rule. It is not: an
+        # endpoint only enters egress enforcement once some policy selecting
+        # it declares an `egress:` block, and none does for monitoring. The
+        # namespace has exactly two Cilium policies and both are ingress-only:
+        # infrastructure/network-policies/monitoring-ns.yaml
+        # (default-deny-ingress) and monitoring-alloy-syslog.yaml
+        # (allow-syslog-world-ingress). blackbox-exporter runs in that same
+        # namespace (../monitoring/helmrelease-blackbox-exporter.yaml#namespace).
+        # Re-verified at 2b71dda1 across all 635 yaml files in the repo — 73
+        # Cilium objects, 38 carrying an egress or egressDeny block, none of
+        # them in the monitoring namespace, and the single
+        # CiliumClusterwideNetworkPolicy is scoped to `reserved.health` so it
+        # cannot select a pod. 17 other namespaces here already have an egress
+        # block. If a routine hardening pass ever adds one to monitoring, it
+        # MUST list the navidrome and paperless namespaces or these three
+        # cross-namespace targets go red while the *.monitoring.svc ones above
+        # keep working — a failure mode nobody will connect back to the
+        # hardening commit. The cnp-cross-namespace-* notes document only the
+        # ingress half; that silence is what makes this the trap.
+        #
+        # BLAST RADIUS if that egress dependency is ever broken. Nothing
+        # user-facing breaks: these are Prometheus-side Probe targets, not
+        # gatus checks, so status.renner.uno is unaffected and no workload is
+        # restarted. What breaks is the signal, and it breaks misleadingly.
+        # All three targets start failing BlackboxProbeFailing
+        # (prometheusrule-blackbox.yaml#blackbox-probe-failing — severity
+        # warning, for: 10m); none of the three is in that rule's exclusion
+        # list. BlackboxProbesAbsent does NOT fire, because the other 17
+        # targets keep producing
+        # probe_success — so it reads as three apps going down, not as a
+        # policy change. WHO NOTICES: warning routes to the default `email`
+        # receiver, admin@renner.uno (helmrelease-prometheus.yaml alertmanager
+        # route, group_wait 30s, grouped by alertname), so the owner gets one
+        # email roughly 10-11 minutes after the policy lands. HOW FAST UNDONE:
+        # the REVERT recipe below is a one-line deletion; from merge it is
+        # live within ~11 minutes (GitRepository interval 1m0s,
+        # monitoring-rules Kustomization interval 10m), or immediately with
+        # `flux reconcile kustomization monitoring-rules --with-source`.
+        #
+        # /ping on both: unauthenticated, and already the pods' own kubelet
+        # probe path (../navidrome/deployment.yaml#kubelet-probes,
+        # ../navidrome/lidarr.yaml#kubelet-probes). For lidarr that it is
+        # ungated is proven independently of the gate — its reconcile script
+        # (../navidrome/lidarr.yaml#reconcile-script) curls /ping in a
+        # loop BEFORE it has read the API key out of config.xml.
+        #
+        # STATUS CODE, unproven for all three of the new targets: module
+        # http_2xx follows redirects and passes only on a FINAL 2xx, while
+        # every piece of repo evidence behind these paths establishes only
+        # `< 400` — a kubelet httpGet accepts 200-399, and that reconcile
+        # script's `curl -f` without -L exits 0 on a 302. That evidence proves the
+        # paths are UNGATED, not that they answer 200. Check
+        # probe_http_status_code on the first scrapes; a target that settles
+        # on a 3xx gets REVERTED, not widened to accept 3xx.
+        #
+        # REVERT — an executed recipe, not a claim. Every command below was
+        # actually run against master 2b71dda1 on 2026-09-25; the results are
+        # what is recorded here.
+        #
+        # NO `git revert` works. All four candidates conflict in THIS file at
+        # 2b71dda1, because the comment blocks around these targets were
+        # rewritten after the probes landed (#979):
+        #
+        #   git revert -m 1 0a82d61f   merge of #977, paperless     CONFLICT
+        #   git revert -m 1 16238d39   merge of #976, nav + lidarr  CONFLICT
+        #   git revert cb9e73bd        the paperless commit         CONFLICT
+        #   git revert 2219c928        the nav + lidarr commit      CONFLICT
+        #
+        # Hand-resolving any of them is the trap: the natural --theirs
+        # resolution of the 16238d39 conflict also drops the paperless target
+        # below, leaving NO app-health probes and no signal that it happened.
+        #
+        # Delete the target lines forward instead. Only the target lines are
+        # part of the object; the comments are inert. Both of these were run
+        # and verified:
+        #
+        #   # all three app-health targets — blackbox-http 20 -> 17
+        #   sed -i '/svc\.cluster\.local:80\/ping$/d;/paperless\.paperless\.svc\.cluster\.local:80\/accounts\/login\/$/d' \
+        #     kubernetes/apps/monitoring-rules/probes-blackbox.yaml
+        #
+        #   # the paperless target alone — blackbox-http 20 -> 19
+        #   sed -i '/paperless\.paperless\.svc\.cluster\.local:80\/accounts\/login\/$/d' \
+        #     kubernetes/apps/monitoring-rules/probes-blackbox.yaml
+        #
+        # Verified after each run: the file still parses, blackbox-http drops
+        # to 17 and to 19 respectively, blackbox-edge-wan (3) and
+        # blackbox-tcp-relays (2) are untouched, and `git diff` contains
+        # nothing but the deleted target lines. Then drop whichever comment
+        # block is left orphaned, open a PR, and merge — timing as in the
+        # BLAST RADIUS paragraph above.
+        - http://navidrome.navidrome.svc.cluster.local:80/ping
+        - http://lidarr.navidrome.svc.cluster.local:80/ping
+        # anchor: paperless-app-health
+        # APP-HEALTH probe for paperless. Same rule as the block above
+        # (monitoring-vantage-point-rule); the blind spot it closes is the one
+        # gatus already documents in its Web Service 11 comment
+        # (../gatus/configmap.yaml#gatus-endpoints) — through the
+        # paperless-gate outpost EVERY path, /api/ included,
+        # answers 302 to an unauthenticated caller, so no external probe of
+        # paperless.renner.uno can tell a live app from a live gate. That
+        # external signal is still wanted, and REN-2 adds it from UptimeRobot;
+        # this is the other half of the pair.
+        #
+        # That gatus comment says the in-cluster probe "needs a
+        # CiliumNetworkPolicy first". That sentence was WRONG when written,
+        # and still is, on different grounds. The probe is admitted by the
+        # dedicated policy
+        # ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
+        # scoped to blackbox-exporter on the paperless pod's :8000 (REN-28);
+        # the namespace-wide `monitoring` admit this probe used to rely on is
+        # gone. Cilium unions the allow rules of every CNP selecting an
+        # endpoint (the mechanism ../paperless/network-policy.yaml#cnp-union
+        # relies on), so that policy is all the probe needs. The EGRESS
+        # paragraph in the block above applies here identically.
+        #
+        # The stale sentence is deliberately left in place in
+        # apps/gatus/configmap.yaml: it sits INSIDE the `config.yaml: |`
+        # block scalar, so editing it changes ConfigMap data and
+        # `configmap.reloader.stakater.com/reload`
+        # (../gatus/deployment.yaml#reloader) rolls the gatus Deployment. The
+        # correction is written at the top of
+        # that file instead, outside `data:`, where it costs no restart.
+        #
+        # /accounts/login/ because paperless-ngx has no unauthenticated health
+        # endpoint at all. It is the app's own established "the webserver
+        # answers" signal — the same path its startup, liveness and readiness
+        # probes use (../paperless/paperless.yaml#http-probes) — and it
+        # renders a Django view
+        # rather than a static file. It does NOT cover the consumer/celery
+        # workers: see process-liveness-vs-protocol-liveness.
+        #
+        # The cluster-local Host header is accepted only because
+        # PAPERLESS_ALLOWED_HOSTS is "*"
+        # (../paperless/paperless.yaml#allowed-hosts). Narrow that and
+        # Django answers 400 here and this probe goes permanently red.
+        #
+        # This is the target most likely to fail the STATUS CODE check in the
+        # block above — a Django login view can answer 302 — so check it
+        # first, and revert rather than widen if it does. Use the REVERT
+        # recipe in that block: `git revert -m 1 0a82d61f` conflicts here too
+        # at 2b71dda1, and the tested one-line deletion for this target alone
+        # is the second command listed there (20 -> 19 targets).
+        - http://paperless.paperless.svc.cluster.local:80/accounts/login/
 # WAN-path probes of the public edge BY PROXY IP. The name-based
 # home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway)
 # and never leaves the LAN — the vantage-point blind spot that let the
@@ -150,3 +323,103 @@ spec:
      static:
        - proxy-1.renner.uno:33080
        - proxy-2.renner.uno:33080
+---
+# anchor: negative-control-paperless
+# NEGATIVE CONTROLS for the paperless ingress scoping (REN-28 PR 2a).
+#
+# These four targets are here TO BE REFUSED. They are not availability
+# monitoring: every one of them is a workload in the `paperless` namespace that
+# blackbox-exporter has no business reaching, and each must read
+# `probe_success == 1` before REN-28's PR 2b and `0` after it. That flip is the
+# test. Without them, "the whole-namespace admit is gone" is an assertion about
+# a CNP diff rather than a measured property of the cluster, and a later
+# re-widening of the grant would be silent.
+#
+# Why this can only be the ingress policy under test: blackbox-exporter runs in
+# `monitoring`, whose baseline CNP (infrastructure/network-policies/
+# monitoring-ns.yaml) is ingress-only — it declares no `egress:` at all — and
+# the cluster's default-deny.yaml is scoped `namespace: apps`, which covers
+# neither `monitoring` nor `paperless`. So the only thing that can turn these
+# red is paperless's own ingress policy, which is exactly the variable. Nothing
+# else can make the test pass or fail.
+#
+# WHY A SEPARATE jobName, and why these must NEVER be folded into the
+# blackbox-http Probe above: after PR 2b these targets fail permanently and BY
+# DESIGN. BlackboxProbeFailing and BlackboxProbesAbsent
+# (prometheusrule-blackbox.yaml#blackbox-probe-failing and
+# prometheusrule-blackbox.yaml#blackbox-probes-absent) are both scoped to
+# job="blackbox-http", so a distinct job is outside both and stays silent — and
+# the hand-maintained instance!~ exclusion regex in the first of them, whose
+# fragility that file warns about, does not have to grow a fifth entry. Adding
+# these to blackbox-http instead would
+# page the on-call forever.
+#
+# The inverted alert that gives these targets teeth — CNPNegativeControlReachable
+# (prometheusrule-blackbox.yaml#cnp-negative-control-reachable),
+# `probe_success{job="blackbox-negative-control"} == 1` — landed with PR 2b,
+# together with the policy change, not before: until then these controls were
+# SUPPOSED to read 1, so the alert would have fired on the correct state.
+#
+# TWO WAYS A BROKEN HARNESS READS AS A PASSING TEST. The inverted alert fires
+# only on `1`, so anything that forces `0` for a reason other than the policy
+# disarms it silently. Both of these are load-bearing:
+#
+#   * The IPv4 fallback. The shared tcp_connect module
+#     (monitoring/helmrelease-blackbox-exporter.yaml) prefers ip6 with
+#     ip_protocol_fallback: true. All four Services are SingleStack IPv4, so
+#     ip6 never resolves and every `1` these controls ever read came through
+#     the fallback (probe_ip_protocol == 4). Set the fallback to false and all
+#     four drop to `0` with the policy wide open. Do not change the module for
+#     these targets — blackbox-tcp-relays pins ip6-first on purpose; give this
+#     job its own module instead.
+#   * scrapeTimeout: 10s below. Cilium drops the SYN, so a refused connect runs
+#     to the module's own 5s timeout. 5s < 10s is what keeps the SCRAPE
+#     succeeding, so `up` stays 1 and probe_success reads 0 instead of going
+#     absent. Do not trim scrapeTimeout toward 5s: a scrape that times out
+#     leaves no probe_success series for the inverted alert to read, and sets
+#     `up` to 0, which the chart's job-agnostic TargetDown (defaultRules are on
+#     in monitoring/helmrelease-prometheus.yaml) can page on.
+#
+# A bare TCP connect is the right probe: it completes or it times out, with no
+# application-layer ambiguity about whether a 403 meant the policy or the app.
+# Known, accepted side effect: the pprl-pg-rw connect opens and closes without a
+# startup packet, so Postgres logs "incomplete startup packet" at LOG level once
+# a minute until PR 2b lands and the connect stops arriving.
+apiVersion: monitoring.coreos.com/v1
+kind: Probe
+metadata:
+  name: blackbox-negative-control-paperless
+  namespace: monitoring
+  labels:
+    # Load-bearing, as noted at the top of this file: probeSelector matches on
+    # it. Without it this Probe is never discovered and the control reads
+    # `absent`, not `1` — which would look like a passing negative test.
+    release: kube-prometheus-stack
+spec:
+  jobName: blackbox-negative-control
+  prober:
+    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
+  module: tcp_connect
+  interval: 60s
+  scrapeTimeout: 10s
+  targets:
+    staticConfig:
+      static:
+        # Proves BOTH halves of the pprl-pg scoping PR 2b will introduce: the
+        # source (blackbox is not Prometheus, and the pprl-pg-allow-prometheus
+        # rule 2b adds admits only Prometheus) and the port (5432 is not the
+        # exporter's 9187). Both that rule and paperless-allow-blackbox
+        # (../../infrastructure/network-policies/paperless-ns.yaml#pprl-pg-allow-prometheus,
+        # ...#paperless-allow-blackbox) now exist, which is why this target
+        # must read 0. Service is CNPG-generated from Cluster `pprl-pg`
+        # (../paperless/cluster.yaml#pprl-pg) and is the name paperless itself
+        # dials (../paperless/paperless.yaml#db-host).
+        - pprl-pg-rw.paperless.svc.cluster.local:5432
+        # Three witnesses that the whole-namespace `monitoring` admit is gone
+        # for workloads no new rule selects. Ports from the Services themselves:
+        # ../paperless/redis.yaml#service-port,
+        # ../paperless/tika.yaml#service-port,
+        # ../paperless/gotenberg.yaml#service-port.
+        - redis.paperless.svc.cluster.local:6379
+        - tika.paperless.svc.cluster.local:9998
+        - gotenberg.paperless.svc.cluster.local:3000
@@ -31,6 +31,7 @@ spec:
        # target means. A target whose loss deserves a critical page gets its
        # own rule — and lands in the list above. That is what keeps the
        # exclusion list from being arbitrary.
+        # anchor: blackbox-probe-failing
        - alert: BlackboxProbeFailing
          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
          for: 10m
@@ -46,6 +47,7 @@ spec:
        # The check that the checks are running. Every rule above reads
        # probe_success; if the series stop being produced at all, none of them
        # can fire and the silence looks exactly like health.
+        # anchor: blackbox-probes-absent
        - alert: BlackboxProbesAbsent
          expr: absent(probe_success{job="blackbox-http"})
          for: 15m
@@ -57,3 +59,42 @@ spec:
              they are all blind until it returns.
          labels:
            severity: critical
+    # The inverted alert for the paperless negative controls (REN-28 §5.6).
+    # A separate group so nothing above can grow a matcher for it by accident.
+    - name: blackbox-negative-control.rules
+      rules:
+        # The four targets in probes-blackbox.yaml#negative-control-paperless are
+        # paperless-namespace workloads that blackbox-exporter must NOT reach
+        # since REN-28 PR 2b scoped the paperless ingress to per-workload
+        # policies. `0` is healthy here and `1` is the failure: a control that
+        # connects means the grant was re-widened — a namespace-wide
+        # `monitoring` admit back on the paperless baseline, or a per-workload
+        # policy whose selector grew — and blackbox, a pod with no business
+        # there, can open TCP into the namespace whose paperless pod trusts
+        # X-Authentik-Username from anything that connects.
+        #
+        # Scoped to job="blackbox-negative-control" ONLY. Never blackbox-http,
+        # whose targets are meant to read 1.
+        #
+        # This rule reads 0 as silence, so anything that forces 0 for another
+        # reason disarms it without a sound. The two known ways are commented
+        # on the Probe: the IPv4 fallback of the shared tcp_connect module, and
+        # scrapeTimeout staying above the module's 5s timeout.
+        #
+        # for: 15m rides out a short policy churn during a Cilium restart or a
+        # CNP re-apply; a real re-widening is permanent and fires regardless.
+        # anchor: cnp-negative-control-reachable
+        - alert: CNPNegativeControlReachable
+          expr: probe_success{job="blackbox-negative-control"} == 1
+          for: 15m
+          annotations:
+            summary: "Negative control {{ $labels.instance }} is reachable from blackbox-exporter"
+            description: >-
+              blackbox-exporter in `monitoring` completed a TCP connect to
+              {{ $labels.instance }}, which the paperless ingress policy must
+              refuse (REN-28). The paperless ingress grant has been re-widened.
+              Check infrastructure/network-policies/paperless-ns.yaml for a
+              namespace-wide `monitoring` admit on default-deny-ingress, or a
+              per-workload policy whose selector now matches this workload.
+          labels:
+            severity: warning
@@ -52,19 +52,123 @@ spec:
        - alert: CephPoolLostRedundancy
          # Names the pool, which POOL_NO_REDUNDANCY does not.
          #
-          # There is no pool-size metric, but bytes_used/stored IS the
-          # replication factor. The >1e6 floor drops pools too small to measure
-          # meaningfully, empty ones included.
+          # There is no pool-size metric, but for a REPLICATED pool
+          # bytes_used/stored IS the replication factor. The >1e6 floor drops
+          # pools too small to measure meaningfully, empty ones included.
          #
-          # No pool is exempt.
+          # ERASURE-CODED POOLS ARE EXCLUDED, AND MUST STAY EXCLUDED (REN-88).
+          #
+          # That ratio is not a replication factor for an EC pool, it is the
+          # coding overhead: k+m shards over k data shards. For the two ec:2+1
+          # pools here that is 3/2 = exactly 1.5 — sitting precisely on this
+          # rule's own threshold, on the wrong side of it half the time.
+          # Measured on `cephfs-bulk` (pool 15): used 480042086400 / stored
+          # 320028049408 = 1.5000000384, a margin of 12 KiB in 480 GB, so which
+          # way the comparison lands is decided by the last bits of a float
+          # division. It paged critical for 199 minutes on 2026-09-25 (12:37 to
+          # 15:56 UTC, two emails) with the cluster HEALTH_OK, zero degraded
+          # PGs and every OSD up, then cleared on its own. 50 of the 210
+          # samples in the preceding 30h were below 1.5; only that run held the
+          # 30m `for`. `migration.rgw.buckets.data` (pool 12) is the same ec:2+1
+          # shape and was one drift away from doing the same thing.
+          #
+          # The premise fails for EC too, not just the arithmetic: the old
+          # description claimed a single OSD loss destroys data, but 2+1 carries
+          # a parity shard and survives one OSD loss, same as replica-2.
+          #
+          # Do NOT "fix" this by nudging the threshold to 1.4 or 1.49. That
+          # keeps a knife-edge, just a quieter one, and it also narrows the real
+          # replica-2 -> replica-1 signal this rule exists to catch (1.0).
+          #
+          # EC pools keep full redundancy coverage without this rule: a real
+          # loss raises POOL_NO_REDUNDANCY, which is a live series in
+          # `ceph_health_detail` and is deliberately in scope for
+          # CephHealthWarn above. That check reads Ceph's own verdict instead of
+          # inferring one from a byte ratio, which is the right instrument here
+          # — an EC pool that loses a shard does not change its stored/used
+          # ratio at all, because Ceph does not delete the surviving shards.
+          # So there is no gap to reopen, and no second EC-specific threshold
+          # worth inventing.
+          #
+          # `on(pool_id)` is deliberately the whole matcher. ceph_pool_metadata
+          # carries labels ceph_pool_bytes_used does not (name, type,
+          # description, application, compression_mode), so default all-label
+          # matching would never join. Adding `cluster` would be defensive
+          # against a second Ceph cluster that does not exist, at the cost of
+          # the rule silently matching nothing if that label is ever dropped —
+          # an alert that fails open invisibly is the worse trade.
          expr: |
-            ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6
+            ceph_pool_bytes_used / ceph_pool_stored < 1.5
+              and on(pool_id) ceph_pool_metadata{type="replicated"} == 1
+              and ceph_pool_stored > 1e6
          for: 30m
          labels:
            severity: critical
          annotations:
-            summary: "Ceph pool {{ $labels.pool_id }} is running unreplicated"
-            description: "Pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning a single OSD loss destroys data. No pool is meant to be unreplicated — an unreplicated pool also blocks ALL OSD maintenance cluster-wide, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."
+            summary: "Ceph replicated pool {{ $labels.pool_id }} is running unreplicated"
+            description: "Replicated pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning it is down to a single copy and one OSD loss destroys data. No replicated pool is meant to be unreplicated — it also blocks OSD maintenance for as long as it lasts, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Erasure-coded pools are out of scope for this rule and are covered by POOL_NO_REDUNDANCY via CephHealthWarn (REN-88). Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."
+
+        - alert: CephPoolLostRedundancyBlind
+          # Canary for the rule directly above. Keep the two adjacent.
+          #
+          # CephPoolLostRedundancy is `critical` and it fails CLOSED and
+          # SILENTLY. Its expression joins three metrics on one label and
+          # filters on one specific label VALUE, and if any of those stops
+          # matching the expression returns nothing, which is indistinguishable
+          # from "every pool is fine". Health stays ok, no email is sent, and
+          # nothing anywhere reports that a critical storage alert has gone
+          # dark. Measured at REN-88 review time: `type` carried exactly two
+          # values, `replicated` and `erasure`, across 15 series — so the
+          # dependency is real, and a Rook or Ceph upgrade that renames the
+          # label, renames the value, or drops it in a relabel rule is enough.
+          #
+          # This expression is CephPoolLostRedundancy's with both thresholds
+          # removed, wrapped in absent(): "can that rule produce any series at
+          # all". It goes to 1 if either metric disappears, if `pool_id` is
+          # dropped from either side of the join, if the `type` label is
+          # renamed, or if `replicated` stops being its value.
+          #
+          # The one deliberate difference is that `ceph_pool_stored > 1e6` is
+          # NOT mirrored here. That floor is about data volume, not structure —
+          # replicated pools that are all legitimately tiny is a quiet state,
+          # not a broken dependency, and copying it in would make this fire for
+          # a benign reason.
+          #
+          # ONE CANARY PER FRAGILE JOIN, NOT ONE PER RULE GROUP. Judged on
+          # drift: a group-level canary has to enumerate every metric the group
+          # reads, and adding a rule to the group does not update it. Six months
+          # on it silently under-covers while LOOKING comprehensive, which is the
+          # same failure shape as a discovery step that passes having checked
+          # nothing. This one's scope is exactly one rule sitting six lines
+          # above it, so "is it still true" is answerable by reading the two
+          # blocks together. It can still drift — someone edits the expr above
+          # and not this — but that drift shows up in the same diff hunk, and
+          # when it does bite it FIRES rather than going quiet. For a guard,
+          # loud and wrong beats silent and wrong.
+          #
+          # 1h, not the 30m above. The exporter moved through 17 distinct mgr
+          # instances in 30d, and every one of those failovers makes all ceph_*
+          # metrics absent for as long as it takes the new mgr to be scraped;
+          # anything under ~15m would page on routine failover, and noise is
+          # what REN-88 was about. A long `for` costs nothing in detection value
+          # because the failure this catches is a schema change — permanent, not
+          # transient — and a blindness that self-heals inside an hour is not
+          # worth an email. Transient exporter loss is already covered by the
+          # chart's own TargetDown.
+          #
+          # `warning`, not `critical`: an alert stopped being able to fire.
+          # Storage is not broken.
+          expr: |
+            absent(
+              ceph_pool_bytes_used / ceph_pool_stored
+                and on(pool_id) ceph_pool_metadata{type="replicated"} == 1
+            )
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            summary: "CephPoolLostRedundancy cannot fire — its metrics no longer join"
+            description: "The CephPoolLostRedundancy alert has been unable to produce any series for an hour, so a `critical` storage alert is silently dark. It joins ceph_pool_bytes_used / ceph_pool_stored against ceph_pool_metadata{type=\"replicated\"} on `pool_id`; one of those metrics, the `pool_id` label, the `type` label, or the `replicated` value has stopped matching — most likely a Rook or Ceph upgrade, or a relabel rule. Check what the exporter emits now with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail` and by querying `ceph_pool_metadata` directly, then fix the join in kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml — do not delete this canary (REN-93)."

        - alert: CephHealthError
          # HEALTH_ERR means data availability or redundancy is at risk.
@@ -95,3 +199,116 @@ spec:
          annotations:
            summary: "{{ $labels.instance }} /var below 30% free — Ceph mon will warn"
            description: "/var on {{ $labels.instance }} is under 30% free, which is Ceph's mon_data_avail_warn threshold, so the mon on this node will report MON_DISK_LOW. Find the actual consumer with `talosctl -n <node> usage /var/lib --depth 1` and then `/var/lib/containerd --depth 1` — note that overlayfs snapshots belong to running containers and are not reclaimable by image GC."
+
+        - alert: CephOSDNearfullApproaching
+          # Warns BEFORE Ceph's own nearfull check, which is the incident and
+          # not a warning. mon_osd_nearfull_ratio is 0.85 and is not
+          # overridden here.
+          #
+          # There is no rebalance to hide behind: failureDomain is host with
+          # size 3 on exactly three OSDs, so every OSD holds a copy of every
+          # replicated object and all three cross 0.85 within seconds of each
+          # other. Measured 2026-09-26: 81.86 / 81.87 / 81.83%.
+          #
+          # 0.75 is 0.85 minus the largest bulk-tier burst on record. Measured
+          # at 30s resolution over the 14d to 2026-09-26: the fullest OSD went
+          # 71.63% -> 81.87% between 05:18 and 05:43Z (+10.24 pp, ~275 GiB raw)
+          # and reclaimed within the hour. At a SUSTAINED 75%, one more burst of
+          # that size reaches nearfull, which is the point at which the level
+          # itself is the finding.
+          #
+          # `for: 30m` is the shortest wait that outlives a self-reclaiming
+          # burst. That same burst held >= 75% for 19.0 min, so 30m clears it by
+          # 11 min; a permanent step holds it indefinitely. Do NOT shorten this
+          # to 15m — the 09-26 burst then pages, with the cluster reclaiming on
+          # its own while someone reads the mail.
+          #
+          # Nor should it be lengthened to 1h. The fastest observed SUSTAINED
+          # climb was +11.44 pp in one hour (2026-09-25 08:02-09:02Z, a real
+          # ingest that never reclaimed). From 75% that reaches 0.85 in ~53 min,
+          # so a 1h pending window would fire after nearfull, not before it.
+          # 30m fires at ~80.7% in that worst case — about 22 min of lead — and
+          # at 2.24 pp/h (the worst 6h-sustained rate) it fires at ~76.1%, about
+          # 4h of lead. The 30m/1h choice is the whole trade: a burst reclaims
+          # in ~20 min and a step does not, so ~30m is the earliest honest
+          # verdict. REN-162 is measuring the burst envelope; if a burst is ever
+          # seen holding >= 75% for more than ~25 min, this `for` moves out, not
+          # the threshold.
+          #
+          # Replay over the 14d to 2026-09-26 08:00Z: the worst 30m-sustained
+          # floor on any OSD was 71.632% (09-26 04:31Z), so this rule fires ZERO
+          # times on the observed normal shape, including the 09-26 excursion to
+          # 81.87% and the 09-25 step from 59% to 71%. Margin 3.37 pp.
+          #
+          # `max by (ceph_daemon)` is not cosmetic. rook-ceph-exporter pods are
+          # replaced often — 32 distinct series for three OSDs in 14 days — and
+          # the raw expression carries `pod` and `instance`, so every restart
+          # would change the alert identity and reset a pending window.
+          expr: |
+            max by (ceph_daemon) (
+              ceph_osd_stat_bytes_used / ceph_osd_stat_bytes
+            ) >= 0.75
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ceph {{ $labels.ceph_daemon }} sustained above 75% — one bulk burst from nearfull"
+            description: "{{ $labels.ceph_daemon }} has been at or above 75% utilisation for 30 minutes ({{ $value | humanizePercentage }} now). That is 0.85 nearfull minus the largest bulk-tier burst measured here (+10.24 pp in 25 minutes), so a single repeat of that burst trips mon_osd_nearfull_ratio — and with failureDomain host, size 3 and three OSDs, it trips on all three at once with no rebalance available. This is sustained, not a spike: a burst clears in about 20 minutes and never holds this for 30. Find what grew with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph df` and `ceph osd df tree`; the bulk tier (cephfs-bulk, pool 15) is the usual mover."
+
+        - alert: CephReplicapoolHeadroomLow
+          # The `replicapool` counterpart to the OSD rule above, in the units a
+          # reader can act on: GiB that still fit.
+          #
+          # These two are NOT independent signals and must not be read as
+          # corroboration. `ceph_pool_max_avail` for a size-3 pool on three
+          # equal OSDs is exactly (full_ratio - fullest_OSD_utilisation) x
+          # osd_size; verified against live data, the identity holds at 0.95 to
+          # five decimals. So this rule is the OSD series rescaled — a second
+          # TIER, deliberately set to trip after the OSD rule, not a second
+          # opinion. Ceph's own nearfull line of 0.85 sits at 89.4 GiB here.
+          #
+          # 160 GiB is what must still fit, measured:
+          #   89 GiB  max_avail at the 0.85 nearfull line, i.e. the part of
+          #           max_avail that is not ours to spend
+          #   38 GiB  ha-pg's unconsumed allocation after the approved 20Gi ->
+          #           32Gi expansion: 2 instances x (31.33 GiB provisioned -
+          #           12.4 GiB used) as of 2026-09-26
+          #   35 GiB  replicapool churn allowance — the worst observed single
+          #           day (+14.3 GiB, 09-23) plus eleven days at the 14d mean
+          #           of 1.9 GiB/day
+          #   = 162 GiB, rounded DOWN to 160 so the rounding costs quiet, not
+          #     headroom.
+          #
+          # Selecting on the pool NAME through ceph_pool_metadata, not on a
+          # hardcoded pool_id: ids are assignable and replicapool is only id 1
+          # by history. `on(pool_id)` is the whole matcher for the same reason
+          # given on CephPoolLostRedundancy above — ceph_pool_metadata carries
+          # labels ceph_pool_max_avail does not, so default matching never
+          # joins.
+          #
+          # `max by (pool_id)` twice over: it collapses the churning mgr `pod`
+          # and `instance` labels (8 series in 14 days), and during an mgr
+          # failover it takes the live value rather than a stale low one.
+          #
+          # `for: 30m` also covers a real exporter artefact: ceph_pool_max_avail
+          # reads exactly 0 for a single 30s sample after an mgr restart, twice
+          # in the 14d window (e.g. 09-23 12:27Z), on every mgr series at once.
+          # A `for` shorter than a few minutes would page on that alone. Note
+          # the glitch is one-directional, so it cannot cancel a genuine breach
+          # — 0 is still below the floor.
+          #
+          # Replay over the 14d to 2026-09-26 08:00Z: the lowest 30m-sustained
+          # ceiling was 208.97 GiB (09-26 04:31Z), so ZERO fires on the observed
+          # normal shape. The 09-26 burst took max_avail to 117.4 GiB for about
+          # 10 minutes and did not come close to holding it.
+          expr: |
+            max by (pool_id) (
+              ceph_pool_max_avail
+                and on(pool_id) ceph_pool_metadata{name="replicapool"}
+            ) < 160 * 1024^3
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "replicapool usable headroom below 160 GiB"
+            description: "replicapool has had under 160 GiB usable for 30 minutes ({{ $value | humanize1024 }}B now). 160 GiB is the 89 GiB that max_avail still holds above Ceph's 0.85 nearfull line, plus the 38 GiB ha-pg has not yet consumed of its 32Gi expansion, plus 35 GiB of observed replicapool churn — below this, what is already committed no longer fits before nearfull. This is the same series as CephOSDNearfullApproaching rescaled (max_avail = (0.95 - fullest OSD) x OSD size), so it is a second tier and not independent confirmation. Check `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph df`; growth here is usually the bulk tier on the shared OSDs, not replicapool itself."
@@ -34,9 +34,20 @@ spec:
          # query parameter, and an exact match would break silently if the
          # instance label ever encodes it differently.
          expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
-          # Two consecutive failed probes: enough to skip a single lost probe
-          # through Traefik, not enough to sit on a sealed store.
-          for: 1m
+          # Five minutes of continuously failing probes — about ten consecutive
+          # samples, since the probe and the rule evaluation both run at 30s
+          # (probes-blackbox.yaml, helmrelease-prometheus.yaml). Matches
+          # WoodpeckerDown above.
+          #
+          # This was 1m, the tightest probe window in the cluster by 5x, and it
+          # went pending on a single 5s HTTPS timeout with OpenBao healthy
+          # (REN-131). Isolated blips are ambient on this probe fleet at roughly
+          # one per two hours, so a window that clears on three samples pages on
+          # jitter. The widening costs nothing real: per the description below
+          # nothing breaks the moment OpenBao goes away, and OpenBaoQuorumLost
+          # (prometheusrule-openbao.yaml, for: 2m) still covers the fast
+          # failure, which is quorum loss rather than a lost probe.
+          for: 5m
          labels:
            severity: critical
          annotations:
@@ -13,7 +13,7 @@ spec:
      rules:
        # The Postgres clusters back up to Hetzner. HetznerBackupEndpointDown
        # probes that S3 endpoint, which proves the bucket is reachable and
-        # nothing more — these two rules check that something is actually being
+        # nothing more — these rules check that something is actually being
        # written to it.

        - alert: CNPGWalArchiveBacklog
@@ -23,22 +23,79 @@ spec:
          # that moment, and pg_wal keeps growing until it fills the PVC and
          # takes Postgres down.
          #
-          # This is the load-bearing alert of the two. It reads the
+          # This is the load-bearing alert of the group. It reads the
          # instance-level collector directly, so unlike the backup-age metric
          # discussed below it does not depend on Cluster status being
          # populated.
          #
-          # Threshold: >5 segments sustained for 30m. Brief non-zero values are
-          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
-          # 80 MiB of un-archived WAL, well below anything that threatens the
-          # volume but far above idle noise.
-          expr: cnpg_collector_pg_wal_archive_status{value="ready"} > 5
-          for: 30m
+          # Threshold: >5 segments sustained for 20m. A segment is 16 MiB, so 5
+          # is roughly 80 MiB of un-archived WAL — well below anything that
+          # threatens the volume, but far above idle noise.
+          #
+          # WHAT THIS RULE KEYS ON: persistence, not depth. The failure that
+          # costs something is an archiver that has *stopped*, and a stopped
+          # archiver can sit at 60 segments indefinitely without ever reaching
+          # some larger number. A depth threshold tuned above the nightly
+          # envelope below would miss exactly that case, and would need raising
+          # again every time the recorder grows. So there is one rule here, and
+          # the tuning knob is `for` (REN-152).
+          #
+          # THE NIGHTLY BURST, which this rule must stay silent through.
+          # ha-pg-1 queues WAL every night on the same clock:
+          #
+          #   flat 0 until 02:12Z; fill from 02:13Z; peak 02:14-02:15Z;
+          #   drained by 02:26Z at the worst. 02:12Z is 04:12 local, which is
+          #   when Home Assistant's recorder purge runs.
+          #
+          # It is a write burst, not IO contention. Drain is a flat
+          # 13-15 segments/min every night — including nights when Kopia's
+          # repository-maintenance batch was saturating the RBD devices, and
+          # nights when it did not start until after the burst had already
+          # drained. That batch drifts across 02:03-02:24Z while the peak stays
+          # pinned to 02:15Z, and a drifting cause cannot produce an effect
+          # fixed to the clock. What varies is the fill (19-40 segments/min),
+          # so the height of the backlog is set by how much WAL is produced
+          # against an archiver with a flat ~14 segments/min ceiling.
+          #
+          # WHY 20m. Measured against Prometheus' full 27-night retention
+          # (2026-08-30..2026-09-26), at the real 30s scrape resolution, across
+          # all 12 CNPG instances. Only ha-pg-1 ever exceeds 5 at all; every
+          # other instance peaks at 2. Longest contiguous excursion above 5 on
+          # any night: 13.0 minutes, on 2026-09-11, at a peak of 141 segments.
+          # Firing evaluations over those 27 nights: 0 at 20m, and also 0 at
+          # 15m — but 15m leaves only 2 minutes of margin against the worst
+          # night, and the nightly peak climbed from 116 to 141 inside the
+          # measured month as the recorder grew. 20m leaves 7 minutes and does
+          # not have to be re-derived every time the database gets bigger.
+          #
+          # The 5 minutes 20m costs in detection latency buy nothing back: a
+          # real stall lasts hours, and at 32Gi (REN-141) the volume holds
+          # roughly four nights of stalled WAL before free space is in danger.
+          #
+          # `for` was 30m before REN-129. Outside the nightly windows the
+          # metric is a hard 0 on all 12 instances, so there was never a noise
+          # floor that needed 30m to wait out — the extra 10m only delayed the
+          # page.
+          #
+          # WHEN IT FIRES, the fix is drain rate or headroom, never a higher
+          # threshold here. ObjectStore `wal.maxParallel`
+          # (kubernetes/apps/home-assistant/ha-pg-objectstore.yaml) is unset,
+          # which the barman-cloud plugin reads as one WAL segment at a time;
+          # that is where the ~14 segments/min ceiling comes from. Headroom is
+          # `spec.storage.size` in ha-pg-cluster.yaml. Both are REN-135.
+          #
+          # `max by (namespace, pod)` is not cosmetic. The raw series carries
+          # `role`, which flips on failover; a changed label is a new series,
+          # and a new series restarts the `for` timer. Aggregating `role` away
+          # means a backlog that persists across a failover still pages 20m
+          # after the backlog started rather than 20m after the failover.
+          expr: max by (namespace, pod) (cnpg_collector_pg_wal_archive_status{value="ready"}) > 5
+          for: 20m
          labels:
            severity: warning
          annotations:
            summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
-            description: "{{ $value }} WAL segments have been waiting to archive for over 30 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."
+            description: "{{ $value }} WAL segments have been waiting to archive for over 20 minutes. NOTE ON TIMING: a nightly write burst on ha-pg-1 at 02:14Z (04:14 local) is normal and drains itself within about 13 minutes, and this rule is tuned to stay silent through it — so if this fired, the backlog did NOT clear and the archiver is stuck. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."

        - alert: CNPGBackupFailed
          # cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0
@@ -11,6 +11,31 @@ spec:
  groups:
    - name: disk-wear.rules
      rules:
+        # The drives carrying Talos's EPHEMERAL volume (/var), as (node, device)
+        # in smartctl-exporter's labels. EPHEMERAL sits on retired Ceph drives
+        # whose contents rebuild, and the owner's call is that they run to
+        # failure rather than be replaced on wear (REN-419, REN-420). So the two
+        # wear alerts below skip them. The failure alerts — spare, critical
+        # warning, media errors — still cover them: run to failure still means
+        # knowing when it failed.
+        #
+        # Resolved from node-exporter's /var mount, not a list of serials, so it
+        # follows a drive swap without an edit. If the join stops matching (a
+        # relabel, an encrypted /var on a dm device) the series is empty, and
+        # the wear alerts return for EPHEMERAL drives: it fails loud, not
+        # silent.
+        - record: node_device:nvme_ephemeral_volume:info
+          expr: >-
+            count by (node, device) (
+              label_replace(
+                label_replace(
+                  node_filesystem_size_bytes{mountpoint="/var"}
+                    * on (instance) group_left (nodename) node_uname_info,
+                  "node", "$1", "nodename", "(.+)"
+                ),
+                "device", "$1", "device", "/dev/(nvme[0-9]+)n[0-9]+(p[0-9]+)?"
+              )
+            )
        # Wear percentages are self-reported: 100 means the rated endurance is
        # spent, not that the drive stops.
        #
@@ -45,6 +70,7 @@ spec:
              )
            ) < 180
            and smartctl_device_percentage_used >= 80
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
          # Long, because a projection off a 7d rate should not react to a day.
          for: 6h
          labels:
@@ -55,13 +81,17 @@ spec:
              The drive is past 80% of its rated endurance and the write rate
              of the last week would spend the rest within six months. This is
              a lead-time warning, not a fault: decide whether to replace the
-              drive or move what is writing to it. The Disk Health dashboard
+              drive or move what is writing to it. Drives carrying EPHEMERAL
+              are excluded; they run to failure. The Disk Health dashboard
              shows which drives carry which role.
        # Kept as a flat threshold on purpose: past rated endurance the drive
        # runs outside what the manufacturer stands behind regardless of how
-        # fast it got there, and the alert clears by replacing it.
+        # fast it got there, and the alert clears by replacing it. EPHEMERAL
+        # drives are excluded for the same reason as above.
        - alert: NvmeDriveWearCritical
-          expr: smartctl_device_percentage_used >= 95
+          expr: >-
+            smartctl_device_percentage_used >= 95
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
          for: 1h
          labels:
            severity: critical
@@ -0,0 +1,227 @@
+# Alert when an ExternalSecret stops syncing.
+#
+# This is the second half of REN-103. The first half (#994) added the PodMonitor
+# and the `monitoring` ingress grant that make the series exist at all; before
+# that, `externalsecret_status_condition` held 0 series and a key that silently
+# stopped refreshing was invisible to Prometheus, to Alertmanager and to anyone
+# looking at a dashboard. The alert is split out because its `for:` window had to
+# be chosen against real data rather than guessed.
+#
+# `exported_namespace`, NOT `namespace`. The metric carries its own `namespace`
+# label (the ExternalSecret's), and the PodMonitor attaches a target label of the
+# same name (the controller pod's, always `external-secrets`). Prometheus resolves
+# that collision by renaming the metric's label to `exported_namespace`, so on
+# every one of the ~105 live series `namespace="external-secrets"`. Grouping by
+# `namespace` would therefore still fire correctly and still be useless: every
+# failing ExternalSecret in the cluster would be reported as living in
+# `external-secrets`. ESO's own shipped Grafana dashboard does group by
+# `namespace` — that is right for the chart's ServiceMonitor, which scrapes ESO
+# through its own Service and injects no conflicting target label, and wrong here.
+# This is a general trap for the PodMonitor-instead-of-ServiceMonitor trade this
+# repo keeps making: any metric carrying its own `namespace`, `pod` or `container`
+# label gets it shadowed.
+#
+# `honorLabels: true` on the PodMonitor endpoint would keep the clean label name
+# instead, and is deliberately not used: it drops the target labels, so nothing
+# records which controller replica reported, and it would make this the only
+# monitor in the directory doing so.
+#
+# A consequence worth knowing rather than fixing: aggregating by
+# `exported_namespace` leaves these alerts with no `namespace` label at all, and
+# Alertmanager's route groups on ['alertname', 'cluster', 'namespace']. So every
+# failing ExternalSecret arrives in ONE email listing all of them, each named by
+# its own annotation, instead of one email per namespace. For a fleet this size
+# that is the better shape.
+#
+# The obvious worry about that shape — that a SECOND ExternalSecret failing while
+# the first alert is already firing gets swallowed by the open group — does not
+# hold, and the setting that decides it is `group_interval: 5m` on the root route
+# in apps/monitoring/helmrelease-prometheus.yaml. `group_interval` governs
+# notification of a group whose alert SET has changed, which is a different thing
+# from `repeat_interval: 12h`, which governs re-sending an unchanged group. A
+# second failure therefore produces a fresh email naming both within 5 minutes.
+# This rule routes to the root `email` receiver: it is `severity: warning` and
+# carries no `source = "flux"`, so it matches none of the four child routes.
+#
+# The inhibit rules are also checked and do not touch these alerts. All three key
+# on `equal: ['namespace', 'alertname']` (or `['namespace']`), and no critical or
+# warning alert in the cluster shares this `alertname`. In particular
+# OpenBaoQuorumLost does NOT inhibit this rule — different alertname — so an
+# OpenBao outage can surface both. That is the intended order rather than a
+# problem: OpenBaoQuorumLost is critical at 2m and arrives first, well before
+# this rule's 15m.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: external-secrets
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's ruleSelector matches on this label; without it the
+    # rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: external-secrets.rules
+      rules:
+        # Measured against the live Prometheus before this rule was proposed for
+        # merge, through the apiserver services/proxy read path:
+        #
+        #   count(externalsecret_status_condition)                       = 105
+        #   count(count by (exported_namespace,name) (...))              =  53
+        #   count(count by (exported_namespace,name) (...{condition="Ready"})) = 52
+        #   count(count by (exported_namespace) (...))                   =  20
+        #   count(...{condition="Ready",status="False"})                 =  52
+        #   the ExternalSecretSyncFailing expression below                = EMPTY
+        #
+        # So 105 = 52 ExternalSecrets x 2 Ready polarities + 1 Deleted tombstone,
+        # across 20 namespaces, and THIS RULE ARRIVES SILENT: it matches nothing
+        # in the cluster as it stands, which is what makes it safe to load. The
+        # 52 on the line below is a count, not an estimate.
+        #
+        # `== 1` IS LOAD-BEARING, not defensive. ESO writes BOTH polarities of the
+        # Ready condition on every reconcile — the true one at 1 and the opposite
+        # at 0 (esmetrics.UpdateExternalSecretCondition) — so
+        # `{condition="Ready",status="False"}` is present with value 0 for every
+        # healthy ExternalSecret. A bare selector would match all 52 of them.
+        #
+        # `max`, not the `sum` the vendor dashboard uses. Both give 1 today,
+        # because ESO runs a single controller replica. During a Deployment
+        # rollout Prometheus can briefly scrape the old and the new pod at once,
+        # and `sum(...) == 1` would then evaluate to 2 and silently STOP matching
+        # — the alert would resolve itself in the middle of a failure. `max` is 0
+        # or 1 whatever the replica count.
+        #
+        # The gauge, not `externalsecret_sync_calls_error`. The counter was the
+        # other candidate, on the theory that a counter cannot flap the way a
+        # condition gauge might. Reading the controller says the opposite on both
+        # halves: the counter increments on strictly MORE paths than the condition
+        # does — a failed Get of the ExternalSecret itself, a failed Get or label
+        # Patch of the target Secret, an unresolvable SecretStore — and none of
+        # those call markAsFailed(), so none of them set Ready=False. A cache blip
+        # or an apiserver hiccup moves the counter and nothing else. It is the
+        # noisier signal, not the quieter one. And the gauge does
+        # not flap, because it is set to 1 only by markAsFailed() and cleared by
+        # the next successful reconcile. There is no unset state in between.
+        #
+        # The live numbers say the same thing, measured after #994 landed:
+        # `externalsecret_sync_calls_error` has exactly one series in the whole
+        # cluster, `ai/wiki-anthropic`, at 8 errors over 47 calls — and that
+        # ExternalSecret is Ready=True and has never been otherwise in the
+        # observation window. So on today's data the counter carries 8 events that
+        # correspond to no sync failure at all, and the gauge carries none.
+        #
+        # That observation window is now long enough to mean something, and it is
+        # the answer to the only open question this rule was held back for — how
+        # much the gauge flaps in ordinary operation. Measured 2026-09-26T08:02Z,
+        # over the 464 consecutive scrapes since #994 started collecting at
+        # 00:18Z:
+        #
+        #   min_over_time(up{job="monitoring/external-secrets"}[8h])     = 1
+        #   sum(increase(externalsecret_sync_calls_total[8h]))           = 959
+        #   count(increase(externalsecret_sync_calls_total[8h]) > 0)     = 52
+        #   count(max_over_time(
+        #     externalsecret_status_condition{
+        #       condition="Ready",status="False"}[8h]) == 1)             = EMPTY
+        #   sum(increase(externalsecret_sync_calls_error[8h]))           = 0
+        #
+        # Every one of the 52 ExternalSecrets reconciled at least once in that
+        # window — 959 syncs, so the 1h and 10m refreshInterval cycles both turned
+        # over repeatedly — and across all of them the Ready gauge never reached 1
+        # in a single scrape. Zero flap amplitude, against real reconcile traffic
+        # rather than against silence. The counter was flat over the same window
+        # too, which is why 8 stays 8 above. `for: 15m` is therefore not damping
+        # anything measurable; it is derived from the failure modes below, and the
+        # data says no shorter window would have been noisy either.
+        #
+        # The 2026-09-20 `ai/wiki-embed-basic-auth` incident is evidence FOR this
+        # rule, not against it, though it reads the other way at first. That key
+        # sat in ESO retry backoff through ~10 failures and NEEDED A MANUAL
+        # force-sync (the incident is written up in
+        # clusters/homelab/uptime-monitor.yaml, as the in-repo proof for REN-100).
+        # Because it never self-healed, Ready=False was continuous from the first
+        # failure until a human intervened — hours, not minutes. `for: 15m` catches
+        # that with a wide margin. It was a stuck condition, not a flap; the flap
+        # reading would only hold if the retries had eventually succeeded on their
+        # own, and they did not.
+        - alert: ExternalSecretSyncFailing
+          expr: >-
+            max by (exported_namespace, name) (
+              externalsecret_status_condition{condition="Ready",status="False"}
+            ) == 1
+          # 15m, and the reason it can be this relaxed is that a failed sync is
+          # not an outage: consumers keep using the last Secret ESO wrote, so what
+          # degrades is freshness, not availability. Nothing here needs catching
+          # inside a minute.
+          #
+          # It needs to be at least this long because the provider-error path
+          # returns the error to controller-runtime, which retries with
+          # exponential backoff rather than waiting out `refreshInterval`. A
+          # momentary OpenBao blip is therefore cleared within seconds and never
+          # reaches 15m, while a rolling restart of the 3-replica OpenBao
+          # StatefulSet is ridden out entirely — and that outage has its own
+          # alert (OpenBaoQuorumLost, critical at 2m), so nothing is hidden by
+          # waiting. 15m of continuous Ready=False means repeated failure.
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "ExternalSecret {{ $labels.exported_namespace }}/{{ $labels.name }} is not syncing"
+            description: >-
+              ESO has been unable to sync this ExternalSecret for 15 minutes. The
+              Secret it manages still holds its last successfully synced value, so
+              consumers keep working — what breaks is that a rotation at the
+              provider no longer propagates, which surfaces much later as a
+              credential that stopped being accepted. Get the reason with
+              `kubectl -n {{ $labels.exported_namespace }} get externalsecret
+              {{ $labels.name }} -o jsonpath='{.status.conditions}'`; the common
+              causes are a key that was renamed or removed in OpenBao, a
+              ClusterSecretStore that cannot authenticate, and a target Secret
+              owned by something else.
+
+        # Dead-man's-switch, mirroring ExternalDNSMetricsMissing and
+        # OpenBaoMetricsMissing. The alert above can only fire on series that
+        # exist, so deleting the PodMonitor, reverting the `monitoring` ingress
+        # grant in infrastructure/network-policies/external-secrets-ns.yaml, or
+        # bumping the ESO chart onto a metrics port that no longer matches would
+        # all disarm it silently — which is the exact failure mode REN-103 exists
+        # to end. TargetDown does not cover this: it reports a target that exists
+        # and is failing, not the absence of a target.
+        - alert: ExternalSecretMetricsMissing
+          expr: absent(externalsecret_status_condition)
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "No ExternalSecret metrics are reaching Prometheus"
+            description: >-
+              `externalsecret_status_condition` has no series. Either the ESO
+              controller is gone or its scrape stopped being collected, and
+              ExternalSecretSyncFailing is blind for as long as this is true.
+              Check the PodMonitor `monitoring/external-secrets` still selects the
+              controller pods, and that external-secrets-ns.yaml still admits
+              `monitoring` on TCP/8080.
+
+        # NOT covered here, deliberately, so nobody assumes it is:
+        #
+        #   * `condition="Deleted"`. In chart 2.11.0 the only writer of that
+        #     condition is the NotFound branch of Reconcile, which fabricates
+        #     `Deleted=False` purely to make UpdateExternalSecretCondition drop
+        #     the object's Ready series. It is a tombstone for an ExternalSecret
+        #     that no longer exists, not a health state, and `Deleted=True` is
+        #     never emitted at all — a rule on it would be dead code. The one such
+        #     series in the cluster today (`ai/wiki-index-basic-auth`) is leftover
+        #     process state for an ExternalSecret that moved namespace; it clears
+        #     when the controller next restarts.
+        #   * An ExternalSecret that exists in the cluster but was never
+        #     reconciled emits no series at all, so nothing above can see it. That
+        #     needs object-existence data — kube-state-metrics custom resource
+        #     state — rather than ESO's own metrics, and is not in this change.
+        #   * A sync that fails and then succeeds on its own inside 15m. This is a
+        #     choice, not an oversight: the retry left the Secret holding a freshly
+        #     synced value, so nothing degraded and there is nothing for a human to
+        #     do. Alerting on it would mean paging for a self-healed event. The case
+        #     that would justify revisiting is the same failure recurring for days
+        #     without any single occurrence lasting 15m, which would show up as a
+        #     climbing `externalsecret_sync_calls_error` on an ExternalSecret that
+        #     is Ready=True. Nothing in the cluster shows that shape today, and the
+        #     rule for it needs a measured recurrence rate rather than a guess.
@@ -0,0 +1,274 @@
+# Per-object Flux readiness, and the groundwork for the Ready-but-frozen
+# ImagePolicy signal (REN-102, closing REN-97).
+#
+# EVERYTHING HERE READS `gotk_resource_info`, WHICH IS A kube-state-metrics
+# SERIES, NOT A FLUX CONTROLLER SERIES. apps/monitoring-rules/flux-podmonitor.yaml
+# scrapes the controllers and yields reconcile durations and controller-runtime
+# counters, but readiness is not on that endpoint at the pinned controller
+# versions: `gotk_reconcile_condition` and `gotk_suspend_status` are declared by
+# fluxcd/pkg/runtime and populated by nothing. The series these rules read are
+# created by the `customResourceState` block in
+# apps/monitoring/helmrelease-prometheus.yaml. If that block is removed, every
+# rule below goes blind — which is what FluxImagePolicyMetricsMissing exists to
+# say out loud.
+#
+# THE STALENESS HALF DELIBERATELY SHIPS WITHOUT AN ALERT. See the second group.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: flux
+  namespace: monitoring
+  labels:
+    # ruleSelector matches on this label; without it the rule is created but
+    # never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: flux.rules
+      rules:
+        # Why `max by (...)` and not the bare selector: it collapses the KSM
+        # target labels (`pod`, `instance`, `endpoint`, ...). Without it, a
+        # kube-state-metrics rollout changes `pod`, which resolves the alert on
+        # the old series and re-fires it on the new one — a fresh mail for an
+        # unchanged failure. KSM rolled 11 times in the last 30 days, so this is
+        # routine, not hypothetical.
+        #
+        # Why `ready=~"False|Unknown"` and not `ready="False"`: Flux sets Ready
+        # False on failure but Unknown while a reconcile is in flight, and an
+        # object wedged in Unknown is stuck just as hard as one that is False —
+        # this repo has already had a Progressing CephCluster deadlock the whole
+        # apps layer (see rook-servicemonitors.yaml). Matching the two values
+        # explicitly, rather than `ready!="True"`, keeps an object whose `ready`
+        # label is absent entirely (freshly created, condition not yet written)
+        # out of the alert.
+        #
+        # Why `suspended!="true"` and not an `unless` join: `suspended` is a
+        # label on the same Info series, so one matcher does what the withdrawn
+        # draft needed a second vector for. The `!=` form also covers kinds where
+        # the label is absent. There is no deleted-object guard because KSM stops
+        # emitting the series when the object goes away — under `gotk_*` that
+        # guard had to be written, here it is structural.
+        #
+        # for: 2h — 31 of the 34 ImageRepositories scan at `interval: 1h`
+        # (2 at 30m: shelly-manager, vsftpd; 1 at 10m: wiki), so 91% of the
+        # population cannot recover from a transient registry failure faster
+        # than one hour whatever this value says. Anything under 1h mails about
+        # a single self-healing missed scan. 2h means two consecutive failed
+        # scans. Detection latency added by going through KSM is one 30s scrape.
+        - alert: FluxImageAutomationNotReady
+          expr: >-
+            max by (customresource_kind, name, exported_namespace) (
+              gotk_resource_info{
+                customresource_kind=~"ImageRepository|ImagePolicy|ImageUpdateAutomation",
+                ready=~"False|Unknown",
+                suspended!="true"
+              }
+            )
+          for: 2h
+          labels:
+            severity: warning
+          annotations:
+            summary: "Flux {{ $labels.customresource_kind }}/{{ $labels.name }} has not been Ready for 2h"
+            description: >-
+              An image-automation object has been not-Ready for two reconcile
+              intervals. Nothing is broken in the running cluster, but the image
+              it governs has stopped moving, security rebuilds included.
+              `flux get image all -n {{ $labels.exported_namespace }}` gives the
+              reason; the usual causes are registry credentials, a rate limit, or
+              a tag format the policy's `pattern`/`extract` no longer matches.
+              flux-notifications/alerts.yaml already mailed once at the moment of
+              transition — this is the "still broken two hours later" signal that
+              edge-triggered mail cannot give.
+
+        # The exact negation of the matcher above, so a kind added to
+        # helmrelease-prometheus.yaml's customResourceState config is covered by
+        # one of these two rules without anyone remembering to widen a list.
+        #
+        # for: 1h, derived from the intervals actually in this repo: all 33
+        # Kustomizations are at 10m (six retries inside the hour), all 25
+        # HelmReleases at 30m (two), and the flux-system GitRepository at 1m0s
+        # (sixty). The dominant non-failure source of ready="False" is a
+        # `dependsOn` wait, whose `reason` is not a label and so cannot be
+        # filtered out; an hour clears a whole-cluster reconcile chain.
+        #
+        # KNOWN TRADE, stated rather than hidden: this matcher also picks up the
+        # 17 HelmRepository and 4 OCIRepository objects, and every one of those is
+        # at `interval: 24h`. A failed index or manifest fetch there cannot
+        # self-heal inside an hour, so a transient failure yields up to two mails
+        # (repeat_interval is 12h) before the next scheduled reconcile. That is
+        # accepted as correct — a chart source stale for an hour is worth knowing
+        # — and it is real information rather than a blip. If it turns out to be
+        # the noisy one, the fix is a third rule for the 24h-interval kinds, not
+        # a global bump of this value, which would make Kustomization alerts
+        # useless.
+        - alert: FluxResourceNotReady
+          expr: >-
+            max by (customresource_kind, name, exported_namespace) (
+              gotk_resource_info{
+                customresource_kind!~"ImageRepository|ImagePolicy|ImageUpdateAutomation",
+                ready=~"False|Unknown",
+                suspended!="true"
+              }
+            )
+          for: 1h
+          labels:
+            severity: critical
+          annotations:
+            summary: "Flux {{ $labels.customresource_kind }}/{{ $labels.name }} has not been Ready for 1h"
+            description: >-
+              A Flux object outside image automation has been not-Ready for an
+              hour, so the desired state in git is not reaching the cluster.
+              `flux get all -A --status-selector ready=false` lists them with a
+              reason. For a Kustomization or HelmRelease this means a rollout is
+              stuck; for a HelmRepository or OCIRepository it means the chart or
+              artifact index is stale and dependent releases will not upgrade.
+
+        # THE DEAD MAN'S SWITCH, and the reason it is safe here and was not in
+        # #992. This asserts that the 34 ImagePolicy series exist — and they
+        # exist only because this change created them. Merged against `gotk_*`
+        # alone, as the withdrawn draft had it, this rule would have been
+        # permanently true and mailed admin@renner.uno every 12h forever.
+        #
+        # Its value is catching the removal of the customResourceState block, or
+        # a malformed edit to it, which would otherwise take all three rules
+        # above silently blind. `KubeStateMetricsDown` and `TargetDown` overlap it
+        # when the whole pod is down; neither notices a config that parses but
+        # emits nothing.
+        #
+        # for: 30m rides out the kube-state-metrics pod roll that applying this
+        # change itself causes, and any later roll.
+        - alert: FluxImagePolicyMetricsMissing
+          expr: absent(gotk_resource_info{customresource_kind="ImagePolicy"})
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Flux per-object readiness metrics have stopped being exported"
+            description: >-
+              kube-state-metrics is exporting no ImagePolicy series, so
+              FluxImageAutomationNotReady and FluxResourceNotReady cannot fire and
+              Flux readiness is unmonitored. Check the `customResourceState` block
+              in apps/monitoring/helmrelease-prometheus.yaml, then the
+              kube-state-metrics pod log — a custom-resource-state config that
+              fails to parse is logged there and yields no series.
+
+    # The Ready-but-frozen ImagePolicy signal: a policy whose `filterTags` or
+    # semver range excludes everything new keeps resolving the tag it resolved
+    # months ago, holds Ready=True the whole time, and no readiness rule can see
+    # it. `tika.yaml`'s own comment names the case.
+    #
+    # THIS GROUP DELIBERATELY CONTAINS NO ALERT, AND THAT IS A MEASURED DECISION,
+    # NOT AN OMISSION. Every candidate threshold was tested against the real
+    # release history and the real registries on 2026-09-26:
+    #
+    #   * A flat "frozen for N days" duration does not exist. Over the 191
+    #     automated bump commits in the 147 days to 2026-09-25, gatus went 127
+    #     days between bumps and paperless-ngx 126, both provably healthy — they
+    #     bumped. Six policies have never bumped at all. Any N below 127 fires on
+    #     the healthy pair; any N above 147 is longer than the whole observation
+    #     window.
+    #
+    #   * The "tag count grew >= 10%" comparator from this card's first design is
+    #     wrong, and the registries say so. Of the 18 policies frozen 14 days or
+    #     more, a 10% threshold fires on exactly three — gotenberg (+87 tags/14d
+    #     on 526), valkey (+68 on 301), eclipse-mosquitto (+13 on 90) — and all
+    #     three churn variant tags their filters reject on purpose. It does NOT
+    #     fire on tika (+4 on 136 = 3.0%), which is the canonical case this card
+    #     was opened for.
+    #
+    #   * Relaxing to "grew by >= 1 tag" does catch tika, but then also fires on
+    #     those same three plus velero-plugin-for-aws, whose `1.13.x` pin is
+    #     deliberate. Alertmanager routes every severity to the same email
+    #     receiver with repeat_interval: 12h, so that is four standing mails
+    #     every twelve hours, indefinitely, for four non-faults.
+    #
+    #   * `.status.lastScanResult.latestTags` looks like the way out and is not:
+    #     the CRD defines it as "the first 10 tags when sorting all the tags in
+    #     descending ALPHABETICAL order", which is not newest-first for any tag
+    #     scheme in this repo.
+    #
+    # So the recorded series below materialise the candidate set and, more to the
+    # point, START THE HISTORY that a threshold has to be derived from — a real
+    # time series of tagCount per repository, which has never existed here. The
+    # alert is deferred to a follow-up card that reads 14+ days of it. Recording
+    # rules cannot mail anyone, so this costs nothing while it warms up.
+    #
+    # interval: 15m is LOAD-BEARING and is declared here rather than inherited.
+    # Global evaluationInterval is 30s and no other rule group in this directory
+    # sets one (45 of the 46 loaded groups run at 30s). The `>= 1250` warm-up
+    # guard below counts samples, so at 30s it would be satisfied in 10.4 hours
+    # instead of 13 days and the frozen series would read true for policies that
+    # had not been observed frozen at all.
+    - name: flux-image-policy-staleness.rules
+      interval: 15m
+      rules:
+        # Stage one exists only to strip the KSM target labels. Counting distinct
+        # tags means counting distinct series, and `gotk_resource_info` carries
+        # `pod` and `instance` — there were 11 distinct kube-state-metrics pod
+        # identities in the last 30 days, so counting the raw series would report
+        # "several tags seen" after every roll, including the roll this change
+        # itself causes, and would keep reporting it for the whole window.
+        - record: flux:image_policy_tag:info
+          expr: >-
+            max by (name, exported_namespace, source_name, tag) (
+              gotk_resource_info{customresource_kind="ImagePolicy"}
+            )
+
+        # Distinct resolved tags seen in a trailing day. A tag change retires one
+        # series and starts another, so `count by (...)` over `count_over_time`
+        # is a count of tag values, and `== 1` means the tag did not move.
+        - record: flux:image_policy_tags_seen:1d
+          expr: >-
+            count by (name, exported_namespace, source_name) (
+              count_over_time(flux:image_policy_tag:info[1d])
+            )
+
+        # The repository side, keyed by `source_name` so it joins on the actual
+        # `spec.imageRepositoryRef.name` reference rather than on the convention
+        # that a policy is named after its repository. All 34 pairs match by name
+        # today, so this is drift insurance, not a bug fix.
+        - record: flux:image_repository_tag_count:1d
+          expr: >-
+            max by (source_name, exported_namespace) (
+              label_replace(
+                max_over_time(gotk_image_repository_tag_count[1d]),
+                "source_name", "$1", "name", "(.+)"
+              )
+            )
+
+        # The candidate set: resolved tag unmoved for 14 days while the
+        # repository's tag count is genuinely higher than it was.
+        #
+        # 14d, not the 30d this card was first designed around, because 30d does
+        # not fit. `retentionSize: 30GB` binds long before `retention: 90d`:
+        # (time() - prometheus_tsdb_lowest_timestamp/1000)/86400 measured 26.63
+        # days on 2026-09-26, at 84.5% of the size cap and growing ~1.09 GB/day
+        # (REN-107). A 30d window can never be satisfied. 14d uses 53% of that
+        # floor.
+        #
+        # >= 1250 is the warm-up guard and its arithmetic is: at interval 15m a
+        # full 14d window holds 14*24*4 = 1344 samples, so 1250 is 93% of a full
+        # window = 13.02 days of history, leaving ~23h of slack for Prometheus
+        # restarts and KSM rolls. Without it the series would read true on day
+        # one, when "unmoved for 14 days" is simply not yet knowable. 13.02 days
+        # against a 26.63-day floor is 2.0x margin; re-check it with the
+        # lowest_timestamp query above if retention is ever cut.
+        #
+        # `last_over_time - min_over_time >= 1` and not a max/min ratio: a ratio
+        # is direction-agnostic and is satisfied by a tag count that DROPPED
+        # (registry pruning, an edited exclusionList, a partial scan) exactly as
+        # by one that grew. Comparing the current value against the window
+        # minimum only counts net growth.
+        - record: flux:image_policy_tag_frozen:14d
+          expr: >-
+            (
+              max_over_time(flux:image_policy_tags_seen:1d[14d]) == 1
+              and
+              count_over_time(flux:image_policy_tags_seen:1d[14d]) >= 1250
+            )
+            and on (source_name, exported_namespace) (
+              (
+                last_over_time(flux:image_repository_tag_count:1d[1h])
+                - min_over_time(flux:image_repository_tag_count:1d[14d])
+              ) >= 1
+            )
--- a/Show More
+++ b/Show More