zigbee2mqtt: raise memory limit to 1Gi

Node derives the V8 heap from the container limit: 512Mi gave a 259 MB heap, and Z2M crashed with "JavaScript heap out of memory" (exit 134) at 2026-09-28 17:56Z and 2026-09-29 07:36Z. Request raised to 256Mi to cover the measured 232Mi working set. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Merge pull request 'navidrome: beets writes the retried partial-album matches (REN-350)' (#1073 ) from atlas/ren-350-beets-write-retry into master
2026-09-30 07:30:17 +02:00 · 2026-09-30 03:04:40 +00:00 · 2026-09-30 03:02:24 +00:00 · 2026-09-30 00:06:06 +00:00 · 2026-09-29 23:40:31 +00:00 · 2026-09-29 13:42:16 +00:00
92 changed files with 4051 additions and 764 deletions
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -0,0 +1,66 @@
+# Paperclip agent runtime images (REN-284 / REN-286): the stock
+# agent-runtime-claude plus a tool set, one image per sandbox environment.
+# Built with kaniko like build-ci-images.yml and pushed with the same
+# ci_images_token (package read+write only, see the header there).
+#
+# Own workflow, not a step in build-ci-images.yml. A change to that file
+# rebuilds all of its images at once and fills the 24-CPU woodpecker-ci-quota;
+# on the REN-286 merge (pipeline 11306) this step then got no pod
+# ("exceeded quota") and the go image was never built. Here it runs only when
+# its own sources change, and it can be re-run alone with a deployment to
+# target `build-agent-runtime`, without rebuilding the other images.
+#
+# The name looks odd on purpose. The kubernetes sandbox plugin keeps the image
+# name and tag `agent-runtime-claude:v1` and swaps only the registry prefix
+# (`imageRegistry`), replacing host + org. So each variant needs its own
+# prefix with the same name under it:
+#   imageRegistry gitea.renner.uno/konairius/agent-runtime-go
+#   -> gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1
+# `v1` is what the plugin pulls and moves on every build; the date tag is the
+# rollback point. konairius/ because the ci_images_token can push there and
+# it is the target of the gitea_admin/* migration (#330).
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/agent-runtime-*/**"
+      - ".woodpecker/build-agent-runtime.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-agent-runtime"'
+
+steps:
+  - name: build-agent-runtime
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images
+    # Context is ci/images (not the variant dir) so both Containerfiles can
+    # COPY the shared agent-runtime-common/ files.
+    # One pod, both images one after the other (kaniko --cleanup between
+    # builds), at 1 CPU so it still fits if it overlaps another pipeline.
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: "1"
+            memory: 4Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        for VARIANT in go ops; do
+          /kaniko/executor \
+            --cleanup \
+            --context . \
+            --dockerfile "agent-runtime-$VARIANT/Containerfile" \
+            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:$TAG" \
+            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:v1" \
+            || exit 1
+        done
@@ -26,8 +26,14 @@
 # 38-reference change tracked separately in #330 rather than bundled here.
 # Note this pipeline runs no LLM agent, so the exfil path that motivated the
 # other cutovers does not apply to it.
+#
+# Push builds run on master only. Without the branch filter a feature branch
+# that touches this file started all steps at once: it pushed unreviewed
+# images and used the whole woodpecker-ci-quota, so the PR checks could not
+# get a pod (REN-286).
 when:
  - event: push
+    branch: master
    path:
      - "ci/images/**"
      - "secret-rotator/**"
@@ -36,6 +36,7 @@ when:
      - "kubernetes/**"
      - ".woodpecker/**"
      - "scripts/ci/promtool-rules.sh"
+      - "scripts/ci/yaml-citations.sh"

 # THE THIRD DISCOVERY CHECK WENT TO A SCRIPT. kustomize-build and shellcheck
 # both hand-roll the same shape — `find`, a floor guard against empty discovery,
@@ -50,10 +51,10 @@ when:
 # ci-scripts-test.yml. A FOURTH inline copy should go the same way; do not add
 # one here.
 #
-# Every step carries `depends_on: []` so all five run in PARALLEL and every
+# Every step carries `depends_on: []` so all six run in PARALLEL and every
 # failure shows in one run. Woodpecker's default is sequential, which would stop
 # at the first failure and make a contributor fix-and-repush to discover the
-# next — and these five are independent, cheap, offline checks with no ordering
+# next — and these six are independent, cheap, offline checks with no ordering
 # between them. Keep it on every step: `depends_on` is per-step, so dropping it
 # from one silently re-serialises that step behind the others.
 steps:
@@ -285,7 +286,8 @@ steps:
  # and 04:22 local for a backlog that clears itself. Nothing here would have
  # caught it, and the arithmetic that did catch it was hand-written from scratch
  # twice, in two runs, and thrown away both times because there was nowhere in
-  # the repo to commit it. It is now mutations.txt:surge-for-2m.
+  # the repo to commit it. REN-152 retired that rule; its `for` margin is now
+  # guarded by mutations.txt:backlog-for-12m.
  #
  # Offline, like the rest of this pipeline: no cluster, no registry, no secrets,
  # no network. promtool reads files and exits.
@@ -320,3 +322,22 @@ steps:
    image: prom/prometheus:v3.15.0
    commands:
      - sh scripts/ci/promtool-rules.sh
+
+  # Comments cite other YAML blocks by anchor name, never by line number
+  # (REN-183; the exact syntax is in CONTRIBUTING.md, not repeated here because
+  # an example written in this comment would itself be checked). Line numbers drift on any edit
+  # above them and nothing noticed: three drift fixes were merged by hand
+  # before this step existed. This fails on a new line-number citation and on
+  # a citation whose file or anchor marker does not exist.
+  # Existence only — it never judges whether the anchored block still says
+  # what the citing comment claims, so ordinary edits cannot turn it red.
+  #
+  # It scans every YAML file in the checkout, but this pipeline only runs on
+  # the paths in `when:` above, so an edit confined to ansible/ or esphome/ is
+  # checked on the next PR that does touch kubernetes/ or .woodpecker/.
+  # Self-test: scripts/ci/test-yaml-citations.sh, in ci-scripts-test.yml.
+  - name: yaml-citations
+    depends_on: [] # see note above `steps:`
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh scripts/ci/yaml-citations.sh
@@ -23,24 +23,30 @@
 # only runs when someone remembers to touch the right directory is not a guard.
 #
 # Only in homelab (scripts/ci/ lives here); the file is not deployed to the
-# mirrored repos, so it does not run on their PRs. Both tests are offline (no
+# mirrored repos, so it does not run on their PRs. All tests here are offline (no
 # token, no network, no cluster) — the ci-claude image just provides sh + jq, and
 # it is already pulled by pr-review.yml on the same event, so the runner has it
 # cached.
 when:
  - event: pull_request

-# `depends_on: []` on both so they run in parallel and one failure does not hide
+# `depends_on: []` on every step so they run in parallel and one failure does not hide
 # the other. Same reasoning as the note above `steps:` in ci-manifests-test.yml.
 steps:
  - name: model-selection-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    commands:
      - sh scripts/ci/test-model-selection.sh

  - name: promtool-rules-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    commands:
      - sh scripts/ci/test-promtool-rules.sh
+
+  - name: yaml-citations-guard-tests
+    depends_on: []
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    commands:
+      - sh scripts/ci/test-yaml-citations.sh
@@ -26,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -70,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -17,7 +17,7 @@ when:

 steps:
  - name: checkout-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -54,7 +54,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: distill
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -89,14 +89,14 @@ steps:
          -p

  - name: audit
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    commands:
      # Findings are report lines in output/, not a pipeline failure.
      - python3 -B notes/bin/wiki_audit.py --vault "$$PWD/notes"

  - name: consolidate
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -145,7 +145,7 @@ steps:
        exit $$rc

  - name: push-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -0,0 +1,7 @@
+# Contributing
+
+## Citing other YAML from a comment
+
+- Mark the block you cite with a comment line `# anchor: <name>` (kebab-case, unique in that file). If the block is inside a `|` block scalar, put the marker just above the key, never inside the scalar.
+- Cite it as `path.yaml#<name>`, with the path relative to the citing file's directory or to the repo root. Never cite `path.yaml:<line>`, because line numbers go stale on any edit above them.
+- `scripts/ci/yaml-citations.sh` enforces this in `ci-manifests-test.yml`: it fails on line-number citations and on citations whose file or anchor does not exist.
@@ -0,0 +1,70 @@
+# homelab
+
+GitOps source for the Renner homelab cluster. Flux applies `master` to
+production, so every merge is a production change. Changes go in by pull
+request only; see [CONTRIBUTING.md](CONTRIBUTING.md) for YAML conventions.
+
+## Change rules that protect Home Assistant
+
+Owner decision (REN-240, REN-243). Home Assistant runs the house, so it
+gets two rules on top of normal review.
+
+### 1. Disruptive work runs only 02:00–05:00 Europe/Berlin
+
+Planned work that can restart Home Assistant or disrupt the cluster is
+merged and applied only between **02:00 and 05:00 Europe/Berlin**. That
+covers:
+
+- node drain, node reboot, Talos upgrade
+- CNI (Cilium, Multus) and cluster-wide network policy
+- storage (Rook/Ceph, CSI, snapshot controller, CNPG operator)
+- Flux core (`kubernetes/clusters/*/flux-system/`, Flux controllers)
+- Authentik
+- OpenBao, including secret rotation that restarts consumers
+- anything in `kubernetes/apps/home-assistant/`
+
+A PR in this list states in its description that it waits for the window.
+Flux applies on merge, so "wait for the window" means **merge** in the
+window, not just apply. If unsure whether a change is disruptive, treat it
+as disruptive.
+
+Not covered: Home Assistant image bumps by Flux image automation. They stay
+fully automatic by owner decision. If one breaks Home Assistant, roll it
+back with [RUNBOOK-home-assistant-image-rollback.md](kubernetes/RUNBOOK-home-assistant-image-rollback.md).
+An incident rollback is not planned work; the owner decides its timing.
+
+### 2. HA-impact review before merge
+
+Every PR that touches `kubernetes/apps/home-assistant/`, or makes a
+cluster-wide disruptive change from the list above, needs an **HA-impact
+review** before merge: a named reviewer states in the PR what Home
+Assistant will notice (restart, lost integration, lost history, none) and
+approves.
+
+- Reviewer: Hestia, the Home Assistant maintainer.
+- Until Hestia exists: the owner does the HA-impact review.
+
+This is in addition to the normal review and merge by Argus. Argus does
+not merge such a PR without the HA-impact approval.
+
+## Agent runtime images
+
+Pod images for the Paperclip Kubernetes sandbox (REN-284). Each is the stock
+`ghcr.io/paperclipai/agent-runtime-claude` (pinned by digest) plus a tool
+set. Base user (1000, non-root), entrypoint and cmd are unchanged; the
+images run with a read-only root FS. Source in `ci/images/agent-runtime-*`,
+built by `.woodpecker/build-agent-runtime.yml` (re-run alone: deployment to
+target `build-agent-runtime`), versions bumped by Renovate. Both also
+carry `claude-agent-acp` for the ACP engine, pinned and patched like
+Paperclip ships it, SDK override included
+(`ci/images/agent-runtime-common/`); bump it by hand
+together with Paperclip. All tools are on the default `PATH`.
+
+| Image | `imageRegistry` | Adds |
+|---|---|---|
+| `gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-go` | Go toolchain, golangci-lint |
+| `gitea.renner.uno/konairius/agent-runtime-ops/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-ops` | kubectl, flux, kustomize, helm |
+
+The plugin keeps the name `agent-runtime-claude:v1` and swaps only host and
+org, hence the nested path. Every build also pushes a date tag
+(`:YYYY-MM-DD`) as the rollback point. `konairius` is public: no pull secret.
@@ -0,0 +1,66 @@
+diff --git a/dist/acp-agent.js b/dist/acp-agent.js
+index 0214a509eb5e7b777a69e4e34637941c275d27b1..403be02df66e25eee29e7d5bef14627011e02b30 100644
+--- a/dist/acp-agent.js
+++ b/dist/acp-agent.js
+@@ -2922,9 +2922,19 @@ export class ClaudeAcpAgent {
+                                             amount: message.total_cost_usd,
+                                             currency: "USD",
+                                         },
+-                                        ...(message.origin && {
+-                                            _meta: { "_claude/origin": message.origin },
+-                                        }),
+                                        _meta: {
+                                            ...(message.origin && { "_claude/origin": message.origin }),
+                                            // ACP's core Usage shape exposes context occupancy and
+                                            // cost, but not the billable input/output split. Keep the
+                                            // SDK's bounded aggregate in extension metadata so ACPX can
+                                            // normalize it without persisting a raw provider message.
+                                            usage: {
+                                                input_tokens: message.usage.input_tokens,
+                                                output_tokens: message.usage.output_tokens,
+                                                cache_read_input_tokens: message.usage.cache_read_input_tokens,
+                                                cache_creation_input_tokens: message.usage.cache_creation_input_tokens,
+                                            },
+                                        },
+                                     },
+                                 });
+                             }
+@@ -5311,6 +5321,10 @@ export class ClaudeAcpAgent {
+             settingSources: ["user", "project", "local"],
+             ...(thinking !== undefined && { thinking }),
+             ...userProvidedOptions,
+            // Paperclip Runner owns the complete session context. Its isolated
+            // user root contains only assigned skills; project/local settings
+            // would reintroduce host prompts, plugins, skills, and MCP servers.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && { settingSources: ["user"] }),
+             ...(settings && { settings }),
+             env,
+             // Override certain fields that must be controlled by ACP
+@@ -5318,7 +5332,9 @@ export class ClaudeAcpAgent {
+             includePartialMessages: true,
+             forwardSubagentText,
+             mcpServers: {
+-                ...(userProvidedOptions?.mcpServers || {}),
+                ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1"
+                    ? {}
+                    : (userProvidedOptions?.mcpServers || {})),
+                 ...mcpServers,
+                 ...(fileChangeAuditSupport
+                     ? { [FILE_CHANGE_AUDIT_SERVER_NAME]: fileChangeAuditSupport.mcpServer }
+@@ -5329,6 +5345,16 @@ export class ClaudeAcpAgent {
+             allowDangerouslySkipPermissions: ALLOW_BYPASS,
+             permissionMode: initialPermissionMode,
+             canUseTool: this.canUseTool(sessionId),
+            // The runner pins this actual MCP endpoint before provider launch.
+            // Exempt only task delivery/control tools at the SDK dispatch boundary,
+            // not arbitrary provider permission metadata or MCP name prefixes.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && {
+                allowedTools: mcpServers.paperclip?.type === "http"
+                    && mcpServers.paperclip.url === process.env.PAPERCLIP_ACPX_TASK_TOOL_BRIDGE_URL
+                    ? ["paperclip_finish", "paperclip_block", "read_current_wake_comments", "request_human_input"]
+                        .map((tool) => `mcp__paperclip__${tool}`)
+                    : [],
+            }),
+             // Forward MCP elicitation requests onto ACP elicitation. Only attached
+             // when the client advertised support, so non-supporting clients keep the
+             // SDK's default (auto-decline) behavior. (AskUserQuestion is handled in
@@ -0,0 +1,9 @@
+{
+  "private": true,
+  "dependencies": {
+    "@agentclientprotocol/claude-agent-acp": "0.73.0"
+  },
+  "overrides": {
+    "@anthropic-ai/claude-agent-sdk": "0.3.280"
+  }
+}
@@ -0,0 +1,60 @@
+# Paperclip agent runtime + Go toolchain (REN-284 / REN-286). Runs as the pod
+# image of the `k8s-go` sandbox environment.
+#
+# Every tool comes from its upstream image, pinned by tag AND digest, so the
+# dockerfile manager in Renovate bumps version and digest together. A curl +
+# sha256 pin (as in ci/images/k8s) would break on every Renovate bump.
+#
+# Both binaries are static; they run on the Ubuntu 22.04 base unchanged.
+FROM golang:1.27.1-bookworm@sha256:69a7b9788769bec032d238959b61854e9ae87f57be9029ec04e9885fabf99195 AS go
+FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS golangci-lint
+
+# upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
+# digest current but cannot order commit tags; a newer base is a manual bump.
+FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518742e33ce2f@sha256:33ad285f9476b8637d6d940c08d577797030125675bdf4f497a8ed1f79261c8b
+
+COPY --from=go /usr/local/go /usr/local/go
+COPY --from=golangci-lint /usr/bin/golangci-lint /usr/local/bin/golangci-lint
+# go and gofmt also in /usr/local/bin: a process started with the default PATH
+# (not the image ENV) still finds them.
+USER root
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/go \
+    && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
+USER 1000:1000
+
+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000
+
+# The sandbox pod has a read-only root FS. Writable mounts: $HOME
+# (/home/paperclip, 1Gi), $HOME/.cache (1Gi), /tmp (2Gi), /workspace.
+# - GOPATH (module cache) goes to /tmp, the biggest one.
+# - GOCACHE and the golangci-lint cache keep their default under $HOME/.cache.
+# - GOTOOLCHAIN=local: a go.mod asking for a newer Go fails clearly instead of
+#   downloading a toolchain through the egress allow-list.
+ENV PATH=/usr/local/go/bin:$PATH \
+    GOPATH=/tmp/go \
+    GOTOOLCHAIN=local
+
+# Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
+# The PATH check uses the base image's default PATH, not the ENV above.
+RUN go version && golangci-lint version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in go gofmt golangci-lint claude-agent-acp; do command -v "$t"; done'
@@ -0,0 +1,51 @@
+# Paperclip agent runtime + cluster tools (REN-284 / REN-286). Runs as the pod
+# image of the `k8s-ops` sandbox environment.
+#
+# Every tool comes from its upstream image, pinned by tag AND digest, so the
+# dockerfile manager in Renovate bumps version and digest together. A curl +
+# sha256 pin (as in ci/images/k8s) would break on every Renovate bump.
+#
+# kubectl and flux match the ci-k8s image. All four binaries are static; they
+# run on the Ubuntu 22.04 base unchanged.
+FROM registry.k8s.io/kubectl:v1.36.5@sha256:0667e3f3e6ef435d7cd4c16684244703ebd67314c17511c2a9f87ea44f6cdf6b AS kubectl
+FROM ghcr.io/fluxcd/flux-cli:v2.9.5@sha256:704d5529535570a495249b643159c93a9aa88ffca07813c974a54b7ba6edbd86 AS flux
+FROM registry.k8s.io/kustomize/kustomize:v5.8.1@sha256:899fcd3bc898160e62bcaf82932b0cb29ba38d16272353db2e7acbba82129429 AS kustomize
+FROM alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2 AS helm
+
+# upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
+# digest current but cannot order commit tags; a newer base is a manual bump.
+FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518742e33ce2f@sha256:33ad285f9476b8637d6d940c08d577797030125675bdf4f497a8ed1f79261c8b
+
+COPY --from=kubectl /bin/kubectl /usr/local/bin/kubectl
+COPY --from=flux /usr/local/bin/flux /usr/local/bin/flux
+COPY --from=kustomize /app/kustomize /usr/local/bin/kustomize
+COPY --from=helm /usr/bin/helm /usr/local/bin/helm
+
+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000
+
+# Read-only root FS: kubectl (~/.kube/cache) and helm (~/.cache, ~/.config)
+# write only under $HOME, which the sandbox pod mounts writable. No ENV needed.
+
+# Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
+RUN kubectl version --client && flux version --client && kustomize version && helm version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in kubectl flux kustomize helm claude-agent-acp; do command -v "$t"; done'
@@ -1,4 +1,4 @@
-FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2
+FROM node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1

 # Default to the latest Claude Code. Pinning a fixed version is the wrong
 # default here: old releases eventually stop working against the API, so a
@@ -0,0 +1,71 @@
+# Runbook — roll back a Home Assistant image bump
+
+Use when an automatic Home Assistant update (a
+`chore(image-automation): bump ghcr.io/home-assistant/home-assistant:<new>`
+commit on `master`) breaks Home Assistant. The owner approves the rollback
+and its timing. Atlas writes the PR; Argus merges it.
+
+## Why a plain `git revert` is not enough
+
+Image automation (`kubernetes/infrastructure/image-automation/`) re-scans
+the registry every hour and rewrites the tag every 30 minutes to the
+highest release matching the `home-assistant` ImagePolicy. A plain revert
+of the bump commit would be bumped forward again within about an hour.
+
+So the rollback commit does two things at once: it reverts the tag **and**
+drops the `# {"$imagepolicy": ...}` marker from that image line. Without
+the marker the Setters strategy has nothing to rewrite, so the old tag
+stays pinned. It is still one commit, and reverting that commit restores
+automation.
+
+## Before you roll back
+
+1. Find the bump commit:
+   ```sh
+   git log --oneline -5 origin/master -- kubernetes/apps/home-assistant/deployment.yaml
+   ```
+2. Check the release notes of the broken version for a **database schema**
+   change. Home Assistant migrates the recorder schema (CNPG cluster
+   `ha-pg`) on start and cannot downgrade it. An older version started on a
+   newer schema runs without recorder history until the schema matches
+   again. If the bump crossed a schema change, tell the owner before going
+   on: the choice is roll back without history, or restore the recorder
+   database from backup as a separate step.
+
+## Roll back (one commit, one PR)
+
+```sh
+git worktree add -b atlas/ha-rollback-<old> "$WT" origin/master && cd "$WT"
+git revert --no-commit <bump-sha>
+# In kubernetes/apps/home-assistant/deployment.yaml, delete the trailing
+#   # {"$imagepolicy": "flux-system:home-assistant"}
+# from the home-assistant image line. Leave every other line alone.
+git diff --cached --stat   # expect: 1 file, 1 line changed
+git commit -m "fix(home-assistant): pin <old>, pause image automation (bump to <new> broke HA)"
+git push -u origin HEAD
+```
+
+The PR description states: broken version, pinned version, symptom, schema
+check result, and "revert this commit to resume automatic updates".
+HA-impact review and merge follow [README.md](../README.md#2-ha-impact-review-before-merge).
+
+On merge Flux applies the Deployment (strategy `Recreate`): Home Assistant
+restarts once, about one to two minutes of downtime.
+
+## Verify
+
+```sh
+kubectl -n home-assistant get deploy home-assistant \
+  -o jsonpath='{.spec.template.spec.containers[?(@.name=="home-assistant")].image}'
+kubectl -n home-assistant rollout status deploy/home-assistant
+```
+
+Read-only commands only (see REN-240 protection rule 1). One hour after
+merge, confirm no new `chore(image-automation)` commit touched
+`deployment.yaml`.
+
+## Resume automatic updates
+
+When a fixed Home Assistant release is out, revert the rollback commit. The
+marker comes back, and image automation bumps to the newest release on its
+next pass. That bump is itself a Home Assistant restart.
@@ -70,7 +70,7 @@ spec:
              mountPath: /ca
      containers:
        - name: audiobookshelf
-          image: ghcr.io/advplyr/audiobookshelf:2.36.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
+          image: ghcr.io/advplyr/audiobookshelf:2.37.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
          ports:
            - name: http
              containerPort: 80
@@ -40,17 +40,19 @@ metadata:
 # fixed in place because the sentence lives inside the `config.yaml: |` block
 # scalar below: any edit to it changes ConfigMap DATA, and
 # `configmap.reloader.stakater.com/reload: gatus-config`
-# (gatus/deployment.yaml:13) would then roll the gatus Deployment — a
+# (deployment.yaml#reloader) would then roll the gatus Deployment — a
 # production restart for a comment. This header is outside `data:`, so it is
 # inert; the served config is byte-identical.
 #
-#   * infrastructure/network-policies/paperless-ns.yaml:34-36 ALREADY admits
-#     the whole monitoring namespace on every port, and monitoring is where
-#     blackbox-exporter runs. Cilium unions the allow rules of every CNP
-#     selecting an endpoint, so no extra policy is required.
-#   * The in-cluster paperless probe therefore exists, added with no change
-#     to the paperless namespace:
-#     monitoring-rules/probes-blackbox.yaml, target
+#   * When written, the paperless baseline admitted the whole monitoring
+#     namespace on every port, and monitoring is where blackbox-exporter runs.
+#     That admit is gone (REN-28); the probe is now admitted by the dedicated
+#     ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
+#     scoped to blackbox-exporter on the paperless pod's :8000. Cilium unions
+#     the allow rules of every CNP selecting an endpoint, so nothing else is
+#     required.
+#   * The in-cluster paperless probe therefore exists:
+#     ../monitoring-rules/probes-blackbox.yaml#paperless-app-health, target
 #     http://paperless.paperless.svc.cluster.local:80/accounts/login/
 #   * The claim does NOT generalise to gatus itself. Gatus runs in `apps`,
 #     which is a different source namespace; a gatus-sourced in-cluster check
@@ -60,6 +62,7 @@ metadata:
 # The Web Service 11 entry below stays a reachability-only check on purpose.
 # The app-health half of the pair is the blackbox probe named above.
 data:
+  # anchor: gatus-endpoints
  config.yaml: |
    storage:
      type: sqlite
@@ -6,6 +6,7 @@ metadata:
  labels:
    app: gatus
  annotations:
+    # anchor: reloader
    # Gatus reads its config once at startup and never re-reads the file, so
    # without this a ConfigMap change reconciles cleanly and then does nothing:
    # git and the running checks drift apart silently, with no failed resource
@@ -1,15 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: home-assistant-backup-password
-  namespace: home-assistant
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: home-assistant-backup-password # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: apps/home-assistant/backup-password
@@ -95,7 +95,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.3 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.9.4 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -31,6 +31,11 @@ metadata:
  namespace: home-assistant
 spec:
  instances: 2
+  # switchover, not CNPG's default `restart`: any change that rolls the pods
+  # (walStorage below did) then promotes the already-rolled replica instead of
+  # restarting the primary in place, so the recorder loses seconds of writes
+  # rather than stopDelay plus a pg_wal move plus startup (REN-176).
+  primaryUpdateMethod: switchover
  smartShutdownTimeout: 10
  stopDelay: 60
  imageName: ghcr.io/cloudnative-pg/postgresql:16
@@ -61,7 +66,35 @@ spec:
    # one stalled night survives and two fill the volume; ~18.6 GiB carries
    # about four. Approved by the owner on REN-135.
    # ceph-block has allowVolumeExpansion: true, so this grows in place.
+    #
+    # Since REN-176 WAL lives on walStorage below, so the stall headroom above
+    # now belongs to that volume. 32Gi stays because a PVC cannot shrink.
    size: 32Gi
+  # WAL on its own volume (ha-pg-N-wal), approved by the owner on REN-176.
+  #
+  # PERMANENT. The CNPG webhook rejects removing walStorage once set ("walStorage
+  # cannot be disabled once configured"), so reverting this block fails
+  # admission and leaves the home-assistant Kustomization stuck. Undoing it
+  # means a new Cluster recovered from ha-pg-store. Adding it on 1.30.1 needs
+  # no re-clone: the operator creates the PVC, rolls each pod, and the instance
+  # manager moves pg_wal onto it before Postgres starts.
+  #
+  # This does NOT keep the recorder up when WAL fills: a full WAL disk still
+  # stops Postgres. What it does is keep WAL writable if PGDATA fills (a purge
+  # that stops running), and make WAL-volume usage a direct archiver-backlog
+  # signal.
+  #
+  # Sized so stall survival is no worse than the shared 32Gi gives today.
+  # Today: ~18.6 GiB free on PGDATA / 4.66 GiB/day of WAL = ~4.0 days.
+  #   1.86 GiB  normal pg_wal footprint (max_wal_size 2GB)
+  # + 18.64 GiB four days of full archiver outage at 4.66 GiB/day
+  #             (the nightly 56-119 segment burst, REN-136, is inside that rate)
+  # = 20.5 GiB -> 21Gi: (21 - 1.86) / 4.66 = ~4.1 days. 18Gi would give ~3.5,
+  # worse than today. RBD is thin, so the ceiling costs nothing until a stall
+  # fills it. The webhook requires min_wal_size < max_wal_size < this size.
+  walStorage:
+    storageClass: ceph-block
+    size: 21Gi
  plugins:
    - name: barman-cloud.cloudnative-pg.io
      isWALArchiver: true
@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - backup-password-externalsecret.yaml
  - pvcs.yaml
  - deployment.yaml
  - service.yaml
@@ -194,12 +194,15 @@ spec:
            periodSeconds: 30
            timeoutSeconds: 5
            failureThreshold: 4
+          # Node sizes the V8 heap from this limit: 512Mi gave a 259 MB heap,
+          # and Z2M died on "JavaScript heap out of memory" (exit 134) at
+          # 2026-09-28 17:56Z and 2026-09-29 07:36Z.
          resources:
            requests:
              cpu: 50m
-              memory: 128Mi
+              memory: 256Mi
            limits:
-              memory: 512Mi
+              memory: 1Gi
        # Fires a forced interview whenever a still-un-interviewed device
        # transmits — i.e. at the only moment a sleepy battery device can
        # actually complete one. Z2M itself only retries interviews on a ZDO
@@ -24,7 +24,7 @@ spec:
          # unchanged, so Kubernetes has no reason to replace anything, and
          # this image is deliberately NOT enrolled in Flux image automation
          # — see kubernetes/infrastructure/image-automation/README.md, which
-          # lists ghcr.io/konairius/ical-proxy as having "only main, latest,
+          # lists gitea.renner.uno/konairius/ical-proxy as having "only main, latest,
          # and main-<sha> commit builds; no release tags", so there is no
          # ImagePolicy watching it. The pods therefore sat on a 21-day-old
          # image while :latest moved underneath them, and the only way to
@@ -35,15 +35,26 @@ spec:
          # what is RUNNING recoverable from this repo rather than "whatever
          # :latest meant the last time a pod happened to restart".
          #
-          # main-99a912b is the merge of konairius/ical-proxy#12, which adds
-          # the summary_match/summary_replace parameters this deployment
-          # needs to strip "| Abfuhrkalender - Landkreis Amberg-Sulzbach"
-          # from every waste collection.
+          # The image comes from the Gitea package registry, built by
+          # Woodpecker (.woodpecker.yml in konairius/ical-proxy) on every
+          # push to main. It is amd64-only and pulls without credentials, so
+          # no imagePullSecret. main-4783abe adds two hardening changes on
+          # top of main-db90a59: an SSRF guard (upstreams and redirect hops
+          # resolving to private/loopback/link-local addresses are refused;
+          # public redirects are still followed, up to 5) and a cap of 20
+          # concurrent /proxy upstream fetches (over the cap: 429 +
+          # Retry-After). main-73b5d97 then raises the server WriteTimeout
+          # from 10s to 40s (the 30s upstream fetch timeout + 10s headroom),
+          # so a slow-but-legitimate upstream is no longer cut off before
+          # /proxy can answer. main-926ead0 caps the upstream response body
+          # at 2 MiB (larger feeds get 413 instead of being buffered whole,
+          # REN-230) and logs upstream fetch failures with the host,
+          # answering 502 instead of 500. The summary_match/summary_replace
+          # parameters this deployment needs are unchanged.
          #
          # Bumping this is a manual, reviewed edit. The tag format is
-          # main-<short-sha> (docker/metadata-action type=sha with the branch
-          # as prefix).
-          image: ghcr.io/konairius/ical-proxy:main-99a912b
+          # main-<short-sha>, the first 7 characters of the main commit.
+          image: gitea.renner.uno/konairius/ical-proxy:main-926ead0
          ports:
            - containerPort: 8080
              name: http
@@ -32,7 +32,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.2.4 # {"$imagepolicy": "flux-system:immich-machine-learning"}
          ports:
            - name: http
              containerPort: 3003
@@ -87,7 +87,7 @@ spec:
              mountPath: /ca
      containers:
        - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.2.4 # {"$imagepolicy": "flux-system:immich-server"}
          ports:
            - name: http
              containerPort: 3001
@@ -15,12 +15,20 @@ resources:
  - prometheusrule-netbird-availability.yaml
  - prometheusrule-netbird-ingress.yaml
  - prometheusrule-openbao.yaml
+  # Prometheus's own retention. Until REN-107 `retention: 90d` and
+  # `retentionSize: 30GB` were both set, the size cap bound first, and the
+  # effective window was a 20.5-27.2 day saw-tooth with nothing reporting it.
+  # Fires if the size cap binds before the time retention again.
+  - prometheusrule-prometheus-retention.yaml
  - prometheusrule-velero.yaml
  - prometheusrule-woodpecker-rotation.yaml
  # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
  # alerted if that stopped — HetznerBackupEndpointDown only probes the S3
  # endpoint, which proves reachability, not that anything was written.
  - prometheusrule-cnpg-backup.yaml
+  # Home Assistant, its recorder DB, Mosquitto and Zigbee2MQTT (REN-244). The
+  # only signal for them before was the generic 15m KubePodNotReady warning.
+  - prometheusrule-home-assistant.yaml
  # DNS publication. Nothing scraped external-dns, so it deleted six LAN
  # records on 2026-09-12 and stayed silent for five days.
  - prometheusrule-external-dns.yaml
@@ -55,6 +63,10 @@ resources:
  # infrastructure/network-policies/external-secrets-ns.yaml in the same change,
  # or it scrapes nothing. See the file.
  - external-secrets-podmonitor.yaml
+  # The alert built on what that PodMonitor collects. Split from it on purpose:
+  # the `for:` window had to be chosen against real series, and Prometheus held
+  # none for this metric until the scrape landed.
+  - prometheusrule-external-secrets.yaml
  # Flux's own control plane. The controllers were never scraped, so nothing
  # about Flux's behaviour was queryable. Scrape only: per-object readiness is
  # not on this endpoint at the pinned controller versions and needs
@@ -66,7 +78,3 @@ resources:
  # these two files must land and be reverted together.
  - prometheusrule-flux.yaml
  - probes-blackbox.yaml
-  # TEMPORARY, REN-140. Negative-test probes for the REN-28 §7 bar, taking the
-  # "reachable before" reading that stops being takeable once the paperless PR
-  # merges. Delete this line and the file once the "after" reading is recorded.
-  - probes-ren28-negative.yaml
@@ -93,10 +93,11 @@ spec:
        # are the first targets here that leave the monitoring namespace.
        #
        # Both sit entirely behind an Authentik outpost — navidrome.renner.uno
-        # on ak-outpost-navidrome-gate (navidrome/ingress.yaml:21),
-        # lidarr.renner.uno on ak-outpost-lidarr-gate
-        # (navidrome/lidarr.yaml:218) — so an EXTERNAL probe of either name
-        # measures the gate, not the app, and stays green while the app behind
+        # on ak-outpost-navidrome-gate
+        # (../navidrome/ingress.yaml#navidrome-gate), lidarr.renner.uno on
+        # ak-outpost-lidarr-gate (../navidrome/lidarr.yaml#lidarr-gate) — so an
+        # EXTERNAL probe of either name measures the gate, not the app, and
+        # stays green while the app behind
        # a live gate is dead. That external signal is still wanted, and REN-2
        # adds it from UptimeRobot; these are the other half of the pair.
        #
@@ -124,7 +125,7 @@ spec:
        # infrastructure/network-policies/monitoring-ns.yaml
        # (default-deny-ingress) and monitoring-alloy-syslog.yaml
        # (allow-syslog-world-ingress). blackbox-exporter runs in that same
-        # namespace (apps/monitoring/helmrelease-blackbox-exporter.yaml:5).
+        # namespace (../monitoring/helmrelease-blackbox-exporter.yaml#namespace).
        # Re-verified at 2b71dda1 across all 635 yaml files in the repo — 73
        # Cilium objects, 38 carrying an egress or egressDeny block, none of
        # them in the monitoring namespace, and the single
@@ -142,9 +143,10 @@ spec:
        # gatus checks, so status.renner.uno is unaffected and no workload is
        # restarted. What breaks is the signal, and it breaks misleadingly.
        # All three targets start failing BlackboxProbeFailing
-        # (prometheusrule-blackbox.yaml:34 — severity warning, for: 10m); none
-        # of the three is in that rule's exclusion list. BlackboxProbesAbsent
-        # does NOT fire, because the other 17 targets keep producing
+        # (prometheusrule-blackbox.yaml#blackbox-probe-failing — severity
+        # warning, for: 10m); none of the three is in that rule's exclusion
+        # list. BlackboxProbesAbsent does NOT fire, because the other 17
+        # targets keep producing
        # probe_success — so it reads as three apps going down, not as a
        # policy change. WHO NOTICES: warning routes to the default `email`
        # receiver, admin@renner.uno (helmrelease-prometheus.yaml alertmanager
@@ -156,16 +158,17 @@ spec:
        # `flux reconcile kustomization monitoring-rules --with-source`.
        #
        # /ping on both: unauthenticated, and already the pods' own kubelet
-        # probe path (navidrome/deployment.yaml:80-93,
-        # navidrome/lidarr.yaml:114-127). For lidarr that it is ungated is
-        # proven independently of the gate — lidarr.yaml:29 curls /ping in a
+        # probe path (../navidrome/deployment.yaml#kubelet-probes,
+        # ../navidrome/lidarr.yaml#kubelet-probes). For lidarr that it is
+        # ungated is proven independently of the gate — its reconcile script
+        # (../navidrome/lidarr.yaml#reconcile-script) curls /ping in a
        # loop BEFORE it has read the API key out of config.xml.
        #
        # STATUS CODE, unproven for all three of the new targets: module
        # http_2xx follows redirects and passes only on a FINAL 2xx, while
        # every piece of repo evidence behind these paths establishes only
-        # `< 400` — a kubelet httpGet accepts 200-399, and lidarr.yaml:29's
-        # `curl -f` without -L exits 0 on a 302. That evidence proves the
+        # `< 400` — a kubelet httpGet accepts 200-399, and that reconcile
+        # script's `curl -f` without -L exits 0 on a 302. That evidence proves the
        # paths are UNGATED, not that they answer 200. Check
        # probe_http_status_code on the first scrapes; a target that settles
        # on a 3xx gets REVERTED, not widened to accept 3xx.
@@ -207,41 +210,48 @@ spec:
        # BLAST RADIUS paragraph above.
        - http://navidrome.navidrome.svc.cluster.local:80/ping
        - http://lidarr.navidrome.svc.cluster.local:80/ping
+        # anchor: paperless-app-health
        # APP-HEALTH probe for paperless. Same rule as the block above
        # (monitoring-vantage-point-rule); the blind spot it closes is the one
-        # gatus already documents at apps/gatus/configmap.yaml:205-214 —
-        # through the paperless-gate outpost EVERY path, /api/ included,
+        # gatus already documents in its Web Service 11 comment
+        # (../gatus/configmap.yaml#gatus-endpoints) — through the
+        # paperless-gate outpost EVERY path, /api/ included,
        # answers 302 to an unauthenticated caller, so no external probe of
        # paperless.renner.uno can tell a live app from a live gate. That
        # external signal is still wanted, and REN-2 adds it from UptimeRobot;
        # this is the other half of the pair.
        #
        # That gatus comment says the in-cluster probe "needs a
-        # CiliumNetworkPolicy first". That sentence is simply WRONG, not
-        # merely inapplicable: paperless-ns.yaml:34-36 already admits the
-        # whole monitoring namespace, which is where blackbox-exporter runs,
-        # and Cilium unions the allow rules of every CNP selecting an
-        # endpoint (the mechanism apps/paperless/network-policy.yaml:22-26
-        # relies on). The EGRESS paragraph in the block above applies here
-        # identically. Nothing in the paperless namespace is edited by this
-        # change.
+        # CiliumNetworkPolicy first". That sentence was WRONG when written,
+        # and still is, on different grounds. The probe is admitted by the
+        # dedicated policy
+        # ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
+        # scoped to blackbox-exporter on the paperless pod's :8000 (REN-28);
+        # the namespace-wide `monitoring` admit this probe used to rely on is
+        # gone. Cilium unions the allow rules of every CNP selecting an
+        # endpoint (the mechanism ../paperless/network-policy.yaml#cnp-union
+        # relies on), so that policy is all the probe needs. The EGRESS
+        # paragraph in the block above applies here identically.
        #
        # The stale sentence is deliberately left in place in
        # apps/gatus/configmap.yaml: it sits INSIDE the `config.yaml: |`
        # block scalar, so editing it changes ConfigMap data and
-        # `configmap.reloader.stakater.com/reload` (gatus/deployment.yaml:13)
-        # rolls the gatus Deployment. The correction is written at the top of
+        # `configmap.reloader.stakater.com/reload`
+        # (../gatus/deployment.yaml#reloader) rolls the gatus Deployment. The
+        # correction is written at the top of
        # that file instead, outside `data:`, where it costs no restart.
        #
        # /accounts/login/ because paperless-ngx has no unauthenticated health
        # endpoint at all. It is the app's own established "the webserver
        # answers" signal — the same path its startup, liveness and readiness
-        # probes use (paperless.yaml:252-272) — and it renders a Django view
+        # probes use (../paperless/paperless.yaml#http-probes) — and it
+        # renders a Django view
        # rather than a static file. It does NOT cover the consumer/celery
        # workers: see process-liveness-vs-protocol-liveness.
        #
        # The cluster-local Host header is accepted only because
-        # PAPERLESS_ALLOWED_HOSTS is "*" (paperless.yaml:27). Narrow that and
+        # PAPERLESS_ALLOWED_HOSTS is "*"
+        # (../paperless/paperless.yaml#allowed-hosts). Narrow that and
        # Django answers 400 here and this probe goes permanently red.
        #
        # This is the target most likely to fail the STATUS CODE check in the
@@ -314,12 +324,13 @@ spec:
        - proxy-1.renner.uno:33080
        - proxy-2.renner.uno:33080
 ---
+# anchor: negative-control-paperless
 # NEGATIVE CONTROLS for the paperless ingress scoping (REN-28 PR 2a).
 #
 # These four targets are here TO BE REFUSED. They are not availability
 # monitoring: every one of them is a workload in the `paperless` namespace that
 # blackbox-exporter has no business reaching, and each must read
-# `probe_success == 1` today and `0` once REN-28's PR 2b lands. That flip is the
+# `probe_success == 1` before REN-28's PR 2b and `0` after it. That flip is the
 # test. Without them, "the whole-namespace admit is gone" is an assertion about
 # a CNP diff rather than a measured property of the cluster, and a later
 # re-widening of the grant would be silent.
@@ -335,16 +346,39 @@ spec:
 # WHY A SEPARATE jobName, and why these must NEVER be folded into the
 # blackbox-http Probe above: after PR 2b these targets fail permanently and BY
 # DESIGN. BlackboxProbeFailing and BlackboxProbesAbsent
-# (prometheusrule-blackbox.yaml:35,50) are both scoped to job="blackbox-http",
-# so a distinct job is outside both and stays silent — and the hand-maintained
-# instance!~ exclusion regex at :35, whose fragility that file warns about, does
-# not have to grow a fifth entry. Adding these to blackbox-http instead would
+# (prometheusrule-blackbox.yaml#blackbox-probe-failing and
+# prometheusrule-blackbox.yaml#blackbox-probes-absent) are both scoped to
+# job="blackbox-http", so a distinct job is outside both and stays silent — and
+# the hand-maintained instance!~ exclusion regex in the first of them, whose
+# fragility that file warns about, does not have to grow a fifth entry. Adding
+# these to blackbox-http instead would
 # page the on-call forever.
 #
-# The inverted alert that gives these targets teeth — CNPNegativeControlReachable,
-# `probe_success{job="blackbox-negative-control"} == 1` — ships in PR 2b, NOT
-# here. Between 2a and 2b these controls are SUPPOSED to read 1, so the alert
-# would fire on the correct state. Do not add it before the policy change.
+# The inverted alert that gives these targets teeth — CNPNegativeControlReachable
+# (prometheusrule-blackbox.yaml#cnp-negative-control-reachable),
+# `probe_success{job="blackbox-negative-control"} == 1` — landed with PR 2b,
+# together with the policy change, not before: until then these controls were
+# SUPPOSED to read 1, so the alert would have fired on the correct state.
+#
+# TWO WAYS A BROKEN HARNESS READS AS A PASSING TEST. The inverted alert fires
+# only on `1`, so anything that forces `0` for a reason other than the policy
+# disarms it silently. Both of these are load-bearing:
+#
+#   * The IPv4 fallback. The shared tcp_connect module
+#     (monitoring/helmrelease-blackbox-exporter.yaml) prefers ip6 with
+#     ip_protocol_fallback: true. All four Services are SingleStack IPv4, so
+#     ip6 never resolves and every `1` these controls ever read came through
+#     the fallback (probe_ip_protocol == 4). Set the fallback to false and all
+#     four drop to `0` with the policy wide open. Do not change the module for
+#     these targets — blackbox-tcp-relays pins ip6-first on purpose; give this
+#     job its own module instead.
+#   * scrapeTimeout: 10s below. Cilium drops the SYN, so a refused connect runs
+#     to the module's own 5s timeout. 5s < 10s is what keeps the SCRAPE
+#     succeeding, so `up` stays 1 and probe_success reads 0 instead of going
+#     absent. Do not trim scrapeTimeout toward 5s: a scrape that times out
+#     leaves no probe_success series for the inverted alert to read, and sets
+#     `up` to 0, which the chart's job-agnostic TargetDown (defaultRules are on
+#     in monitoring/helmrelease-prometheus.yaml) can page on.
 #
 # A bare TCP connect is the right probe: it completes or it times out, with no
 # application-layer ambiguity about whether a 403 meant the policy or the app.
@@ -374,15 +408,18 @@ spec:
        # Proves BOTH halves of the pprl-pg scoping PR 2b will introduce: the
        # source (blackbox is not Prometheus, and the pprl-pg-allow-prometheus
        # rule 2b adds admits only Prometheus) and the port (5432 is not the
-        # exporter's 9187). Neither that rule nor paperless-allow-blackbox
-        # exists yet at this commit — that is the point, and why this target
-        # must read 1 now. Service is CNPG-generated from Cluster `pprl-pg`
-        # (paperless/cluster.yaml:4) and is the name paperless itself dials
-        # (paperless.yaml:46).
+        # exporter's 9187). Both that rule and paperless-allow-blackbox
+        # (../../infrastructure/network-policies/paperless-ns.yaml#pprl-pg-allow-prometheus,
+        # ...#paperless-allow-blackbox) now exist, which is why this target
+        # must read 0. Service is CNPG-generated from Cluster `pprl-pg`
+        # (../paperless/cluster.yaml#pprl-pg) and is the name paperless itself
+        # dials (../paperless/paperless.yaml#db-host).
        - pprl-pg-rw.paperless.svc.cluster.local:5432
        # Three witnesses that the whole-namespace `monitoring` admit is gone
        # for workloads no new rule selects. Ports from the Services themselves:
-        # redis.yaml:76, tika.yaml:57, gotenberg.yaml:63.
+        # ../paperless/redis.yaml#service-port,
+        # ../paperless/tika.yaml#service-port,
+        # ../paperless/gotenberg.yaml#service-port.
        - redis.paperless.svc.cluster.local:6379
        - tika.paperless.svc.cluster.local:9998
        - gotenberg.paperless.svc.cluster.local:3000
@@ -1,98 +0,0 @@
-# TEMPORARY — negative-test probes for REN-28 §7. DELETE AFTER THE "AFTER"
-# READING IS TAKEN. This file exists to make two of the four §7 negative tests
-# measurable without granting `pods/exec` in `monitoring`, which the owner
-# declined on REN-140 as a wider grant than the change it would verify.
-#
-# WHAT IT IS. blackbox-exporter is already an HTTP/TCP prober running in the
-# `monitoring` namespace. Pointing it at targets that the REN-28 paperless PR
-# is supposed to CLOSE turns "is this path still open?" into a number in
-# Prometheus, readable through the existing read-only REN-36 grant. Every
-# target below answers today and MUST stop answering after the PR lands. A
-# `probe_success` that stays 1 after the merge means the rule does not do what
-# §5 claims.
-#
-# WHY IT DOES NOT PAGE ANYONE. jobName is `blackbox-ren28-negative`, not
-# `blackbox-http`. Every rule in prometheusrule-blackbox.yaml is scoped to
-# job="blackbox-http" (BlackboxProbeFailing) or absent(...{job="blackbox-http"})
-# (BlackboxProbesAbsent), so neither can match these series — before or after.
-# TargetDown (general.rules) cannot match either: the scrape target of a Probe
-# is blackbox-exporter itself, which answers 200 with probe_success 0 on a
-# failed probe, so `up` stays 1 regardless of the result. Verified against the
-# live rule set on 2026-09-26. This is deliberate: a monitor that pages for an
-# expected failure is worse than no monitor.
-#
-# NO POLICY CHANGE IS NEEDED TO MAKE THESE WORK TODAY. Ingress: the paperless
-# baseline still admits the whole `monitoring` namespace on every port
-# (infrastructure/network-policies/paperless-ns.yaml:34-36) — that admission is
-# exactly what REN-28 removes. Egress: the `monitoring` namespace is in no
-# egress enforcement at all; its only two Cilium policies, monitoring-ns.yaml
-# and monitoring-alloy-syslog.yaml, are ingress-only. Same dependency the three
-# cross-namespace targets in probes-blackbox.yaml already rest on.
-#
-# TWO POD-IP TARGETS. pprl-pg's :9187 metrics port is on the POD, not on any
-# Service — that is why it is scraped by a PodMonitor and not a ServiceMonitor
-# (cnpg-podmonitors.yaml). There is no stable name for it, so the literal pod
-# IPs are used. They were read live on 2026-09-26; if either pod is rescheduled
-# the probe goes red for the wrong reason. BEFORE TRUSTING AN "AFTER" READING,
-# re-confirm the IP still belongs to the same pod:
-#
-#   kube_pod_info{namespace="paperless", pod=~"pprl-pg-.*"}   -> pod_ip
-#
-# The pprl-pg-r:5432 target below is the stable-name version of the same
-# statement and survives a reschedule; it is on the database port rather than
-# the metrics port, so the two are kept together on purpose.
-#
-# REVERT. Delete this file and its line in kustomization.yaml. Nothing else
-# references it; no workload is touched; Prometheus drops the job on the next
-# operator reconcile.
-apiVersion: monitoring.coreos.com/v1
-kind: Probe
-metadata:
-  name: blackbox-ren28-negative
-  namespace: monitoring
-  labels:
-    # probeSelector is NOT loosened on this cluster (helmrelease-prometheus.yaml
-    # keeps ruleSelector/probeSelector/scrapeConfigSelector strict), so this
-    # label is load-bearing, not decorative. Without it the Probe is created and
-    # never discovered — and a test that silently does not run reads exactly
-    # like a test that passed.
-    release: kube-prometheus-stack
-spec:
-  jobName: blackbox-ren28-negative
-  prober:
-    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
-  module: tcp_connect
-  interval: 60s
-  # 10s matches blackbox-tcp-relays. It is also the discriminator: a Cilium
-  # deny DROPS the SYN, so a refused path completes in ~1ms and a denied path
-  # burns the full timeout. probe_duration_seconds distinguishes "nothing is
-  # listening" from "policy stopped it" — probe_success alone does not.
-  scrapeTimeout: 10s
-  targets:
-    staticConfig:
-      static:
-        # REN-28 §7 test 4 — "from a monitoring pod that is not Prometheus, a
-        # connection to pprl-pg :9187 must time out". blackbox-exporter is
-        # precisely that pod. Both CNPG instances, because the PR's
-        # pprl-pg-allow-prometheus rule selects on a label both carry: if the
-        # selector were wrong in a way that left one instance open, one target
-        # would tell us and the other would not.
-        - 10.244.4.166:9187 # pprl-pg-2, node k8s-2
-        - 10.244.2.220:9187 # pprl-pg-3, node k8s-3
-        # Same statement on a stable name and the port that actually matters:
-        # after the PR the only monitoring→pprl-pg allow is Prometheus on
-        # :9187, so a non-Prometheus monitoring pod must not reach Postgres.
-        - pprl-pg-r.paperless.svc.cluster.local:5432
-        # REN-28 §7 test 2, workload dimension — "from blackbox-exporter, a
-        # paperless target other than the one it is scoped to must time out".
-        # The test as written in §7 says "a paperless POD port other than
-        # 8000"; there is no such port. The paperless pod's only other listener
-        # is the bridge-tls-proxy socat, bound to 127.0.0.1:11143
-        # (apps/paperless/paperless.yaml:289), so every other port on that pod
-        # is closed today and answers RST — there is no "reachable before" to
-        # record and the port half of paperless-allow-blackbox (:8000) cannot
-        # be tested at all. gotenberg is the half that CAN be: it is one of the
-        # other eight workloads the endpointSelector:{} baseline currently
-        # exposes to all of `monitoring`, and the PR's per-workload selector is
-        # what closes it.
-        - gotenberg.paperless.svc.cluster.local:3000
@@ -31,6 +31,7 @@ spec:
        # target means. A target whose loss deserves a critical page gets its
        # own rule — and lands in the list above. That is what keeps the
        # exclusion list from being arbitrary.
+        # anchor: blackbox-probe-failing
        - alert: BlackboxProbeFailing
          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
          for: 10m
@@ -46,6 +47,7 @@ spec:
        # The check that the checks are running. Every rule above reads
        # probe_success; if the series stop being produced at all, none of them
        # can fire and the silence looks exactly like health.
+        # anchor: blackbox-probes-absent
        - alert: BlackboxProbesAbsent
          expr: absent(probe_success{job="blackbox-http"})
          for: 15m
@@ -57,3 +59,42 @@ spec:
              they are all blind until it returns.
          labels:
            severity: critical
+    # The inverted alert for the paperless negative controls (REN-28 §5.6).
+    # A separate group so nothing above can grow a matcher for it by accident.
+    - name: blackbox-negative-control.rules
+      rules:
+        # The four targets in probes-blackbox.yaml#negative-control-paperless are
+        # paperless-namespace workloads that blackbox-exporter must NOT reach
+        # since REN-28 PR 2b scoped the paperless ingress to per-workload
+        # policies. `0` is healthy here and `1` is the failure: a control that
+        # connects means the grant was re-widened — a namespace-wide
+        # `monitoring` admit back on the paperless baseline, or a per-workload
+        # policy whose selector grew — and blackbox, a pod with no business
+        # there, can open TCP into the namespace whose paperless pod trusts
+        # X-Authentik-Username from anything that connects.
+        #
+        # Scoped to job="blackbox-negative-control" ONLY. Never blackbox-http,
+        # whose targets are meant to read 1.
+        #
+        # This rule reads 0 as silence, so anything that forces 0 for another
+        # reason disarms it without a sound. The two known ways are commented
+        # on the Probe: the IPv4 fallback of the shared tcp_connect module, and
+        # scrapeTimeout staying above the module's 5s timeout.
+        #
+        # for: 15m rides out a short policy churn during a Cilium restart or a
+        # CNP re-apply; a real re-widening is permanent and fires regardless.
+        # anchor: cnp-negative-control-reachable
+        - alert: CNPNegativeControlReachable
+          expr: probe_success{job="blackbox-negative-control"} == 1
+          for: 15m
+          annotations:
+            summary: "Negative control {{ $labels.instance }} is reachable from blackbox-exporter"
+            description: >-
+              blackbox-exporter in `monitoring` completed a TCP connect to
+              {{ $labels.instance }}, which the paperless ingress policy must
+              refuse (REN-28). The paperless ingress grant has been re-widened.
+              Check infrastructure/network-policies/paperless-ns.yaml for a
+              namespace-wide `monitoring` admit on default-deny-ingress, or a
+              per-workload policy whose selector now matches this workload.
+          labels:
+            severity: warning
@@ -108,6 +108,68 @@ spec:
            summary: "Ceph replicated pool {{ $labels.pool_id }} is running unreplicated"
            description: "Replicated pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning it is down to a single copy and one OSD loss destroys data. No replicated pool is meant to be unreplicated — it also blocks OSD maintenance for as long as it lasts, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Erasure-coded pools are out of scope for this rule and are covered by POOL_NO_REDUNDANCY via CephHealthWarn (REN-88). Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."

+        - alert: CephPoolLostRedundancyBlind
+          # Canary for the rule directly above. Keep the two adjacent.
+          #
+          # CephPoolLostRedundancy is `critical` and it fails CLOSED and
+          # SILENTLY. Its expression joins three metrics on one label and
+          # filters on one specific label VALUE, and if any of those stops
+          # matching the expression returns nothing, which is indistinguishable
+          # from "every pool is fine". Health stays ok, no email is sent, and
+          # nothing anywhere reports that a critical storage alert has gone
+          # dark. Measured at REN-88 review time: `type` carried exactly two
+          # values, `replicated` and `erasure`, across 15 series — so the
+          # dependency is real, and a Rook or Ceph upgrade that renames the
+          # label, renames the value, or drops it in a relabel rule is enough.
+          #
+          # This expression is CephPoolLostRedundancy's with both thresholds
+          # removed, wrapped in absent(): "can that rule produce any series at
+          # all". It goes to 1 if either metric disappears, if `pool_id` is
+          # dropped from either side of the join, if the `type` label is
+          # renamed, or if `replicated` stops being its value.
+          #
+          # The one deliberate difference is that `ceph_pool_stored > 1e6` is
+          # NOT mirrored here. That floor is about data volume, not structure —
+          # replicated pools that are all legitimately tiny is a quiet state,
+          # not a broken dependency, and copying it in would make this fire for
+          # a benign reason.
+          #
+          # ONE CANARY PER FRAGILE JOIN, NOT ONE PER RULE GROUP. Judged on
+          # drift: a group-level canary has to enumerate every metric the group
+          # reads, and adding a rule to the group does not update it. Six months
+          # on it silently under-covers while LOOKING comprehensive, which is the
+          # same failure shape as a discovery step that passes having checked
+          # nothing. This one's scope is exactly one rule sitting six lines
+          # above it, so "is it still true" is answerable by reading the two
+          # blocks together. It can still drift — someone edits the expr above
+          # and not this — but that drift shows up in the same diff hunk, and
+          # when it does bite it FIRES rather than going quiet. For a guard,
+          # loud and wrong beats silent and wrong.
+          #
+          # 1h, not the 30m above. The exporter moved through 17 distinct mgr
+          # instances in 30d, and every one of those failovers makes all ceph_*
+          # metrics absent for as long as it takes the new mgr to be scraped;
+          # anything under ~15m would page on routine failover, and noise is
+          # what REN-88 was about. A long `for` costs nothing in detection value
+          # because the failure this catches is a schema change — permanent, not
+          # transient — and a blindness that self-heals inside an hour is not
+          # worth an email. Transient exporter loss is already covered by the
+          # chart's own TargetDown.
+          #
+          # `warning`, not `critical`: an alert stopped being able to fire.
+          # Storage is not broken.
+          expr: |
+            absent(
+              ceph_pool_bytes_used / ceph_pool_stored
+                and on(pool_id) ceph_pool_metadata{type="replicated"} == 1
+            )
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            summary: "CephPoolLostRedundancy cannot fire — its metrics no longer join"
+            description: "The CephPoolLostRedundancy alert has been unable to produce any series for an hour, so a `critical` storage alert is silently dark. It joins ceph_pool_bytes_used / ceph_pool_stored against ceph_pool_metadata{type=\"replicated\"} on `pool_id`; one of those metrics, the `pool_id` label, the `type` label, or the `replicated` value has stopped matching — most likely a Rook or Ceph upgrade, or a relabel rule. Check what the exporter emits now with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail` and by querying `ceph_pool_metadata` directly, then fix the join in kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml — do not delete this canary (REN-93)."
+
        - alert: CephHealthError
          # HEALTH_ERR means data availability or redundancy is at risk.
          expr: ceph_health_status == 2
@@ -137,3 +199,116 @@ spec:
          annotations:
            summary: "{{ $labels.instance }} /var below 30% free — Ceph mon will warn"
            description: "/var on {{ $labels.instance }} is under 30% free, which is Ceph's mon_data_avail_warn threshold, so the mon on this node will report MON_DISK_LOW. Find the actual consumer with `talosctl -n <node> usage /var/lib --depth 1` and then `/var/lib/containerd --depth 1` — note that overlayfs snapshots belong to running containers and are not reclaimable by image GC."
+
+        - alert: CephOSDNearfullApproaching
+          # Warns BEFORE Ceph's own nearfull check, which is the incident and
+          # not a warning. mon_osd_nearfull_ratio is 0.85 and is not
+          # overridden here.
+          #
+          # There is no rebalance to hide behind: failureDomain is host with
+          # size 3 on exactly three OSDs, so every OSD holds a copy of every
+          # replicated object and all three cross 0.85 within seconds of each
+          # other. Measured 2026-09-26: 81.86 / 81.87 / 81.83%.
+          #
+          # 0.75 is 0.85 minus the largest bulk-tier burst on record. Measured
+          # at 30s resolution over the 14d to 2026-09-26: the fullest OSD went
+          # 71.63% -> 81.87% between 05:18 and 05:43Z (+10.24 pp, ~275 GiB raw)
+          # and reclaimed within the hour. At a SUSTAINED 75%, one more burst of
+          # that size reaches nearfull, which is the point at which the level
+          # itself is the finding.
+          #
+          # `for: 30m` is the shortest wait that outlives a self-reclaiming
+          # burst. That same burst held >= 75% for 19.0 min, so 30m clears it by
+          # 11 min; a permanent step holds it indefinitely. Do NOT shorten this
+          # to 15m — the 09-26 burst then pages, with the cluster reclaiming on
+          # its own while someone reads the mail.
+          #
+          # Nor should it be lengthened to 1h. The fastest observed SUSTAINED
+          # climb was +11.44 pp in one hour (2026-09-25 08:02-09:02Z, a real
+          # ingest that never reclaimed). From 75% that reaches 0.85 in ~53 min,
+          # so a 1h pending window would fire after nearfull, not before it.
+          # 30m fires at ~80.7% in that worst case — about 22 min of lead — and
+          # at 2.24 pp/h (the worst 6h-sustained rate) it fires at ~76.1%, about
+          # 4h of lead. The 30m/1h choice is the whole trade: a burst reclaims
+          # in ~20 min and a step does not, so ~30m is the earliest honest
+          # verdict. REN-162 is measuring the burst envelope; if a burst is ever
+          # seen holding >= 75% for more than ~25 min, this `for` moves out, not
+          # the threshold.
+          #
+          # Replay over the 14d to 2026-09-26 08:00Z: the worst 30m-sustained
+          # floor on any OSD was 71.632% (09-26 04:31Z), so this rule fires ZERO
+          # times on the observed normal shape, including the 09-26 excursion to
+          # 81.87% and the 09-25 step from 59% to 71%. Margin 3.37 pp.
+          #
+          # `max by (ceph_daemon)` is not cosmetic. rook-ceph-exporter pods are
+          # replaced often — 32 distinct series for three OSDs in 14 days — and
+          # the raw expression carries `pod` and `instance`, so every restart
+          # would change the alert identity and reset a pending window.
+          expr: |
+            max by (ceph_daemon) (
+              ceph_osd_stat_bytes_used / ceph_osd_stat_bytes
+            ) >= 0.75
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ceph {{ $labels.ceph_daemon }} sustained above 75% — one bulk burst from nearfull"
+            description: "{{ $labels.ceph_daemon }} has been at or above 75% utilisation for 30 minutes ({{ $value | humanizePercentage }} now). That is 0.85 nearfull minus the largest bulk-tier burst measured here (+10.24 pp in 25 minutes), so a single repeat of that burst trips mon_osd_nearfull_ratio — and with failureDomain host, size 3 and three OSDs, it trips on all three at once with no rebalance available. This is sustained, not a spike: a burst clears in about 20 minutes and never holds this for 30. Find what grew with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph df` and `ceph osd df tree`; the bulk tier (cephfs-bulk, pool 15) is the usual mover."
+
+        - alert: CephReplicapoolHeadroomLow
+          # The `replicapool` counterpart to the OSD rule above, in the units a
+          # reader can act on: GiB that still fit.
+          #
+          # These two are NOT independent signals and must not be read as
+          # corroboration. `ceph_pool_max_avail` for a size-3 pool on three
+          # equal OSDs is exactly (full_ratio - fullest_OSD_utilisation) x
+          # osd_size; verified against live data, the identity holds at 0.95 to
+          # five decimals. So this rule is the OSD series rescaled — a second
+          # TIER, deliberately set to trip after the OSD rule, not a second
+          # opinion. Ceph's own nearfull line of 0.85 sits at 89.4 GiB here.
+          #
+          # 160 GiB is what must still fit, measured:
+          #   89 GiB  max_avail at the 0.85 nearfull line, i.e. the part of
+          #           max_avail that is not ours to spend
+          #   38 GiB  ha-pg's unconsumed allocation after the approved 20Gi ->
+          #           32Gi expansion: 2 instances x (31.33 GiB provisioned -
+          #           12.4 GiB used) as of 2026-09-26
+          #   35 GiB  replicapool churn allowance — the worst observed single
+          #           day (+14.3 GiB, 09-23) plus eleven days at the 14d mean
+          #           of 1.9 GiB/day
+          #   = 162 GiB, rounded DOWN to 160 so the rounding costs quiet, not
+          #     headroom.
+          #
+          # Selecting on the pool NAME through ceph_pool_metadata, not on a
+          # hardcoded pool_id: ids are assignable and replicapool is only id 1
+          # by history. `on(pool_id)` is the whole matcher for the same reason
+          # given on CephPoolLostRedundancy above — ceph_pool_metadata carries
+          # labels ceph_pool_max_avail does not, so default matching never
+          # joins.
+          #
+          # `max by (pool_id)` twice over: it collapses the churning mgr `pod`
+          # and `instance` labels (8 series in 14 days), and during an mgr
+          # failover it takes the live value rather than a stale low one.
+          #
+          # `for: 30m` also covers a real exporter artefact: ceph_pool_max_avail
+          # reads exactly 0 for a single 30s sample after an mgr restart, twice
+          # in the 14d window (e.g. 09-23 12:27Z), on every mgr series at once.
+          # A `for` shorter than a few minutes would page on that alone. Note
+          # the glitch is one-directional, so it cannot cancel a genuine breach
+          # — 0 is still below the floor.
+          #
+          # Replay over the 14d to 2026-09-26 08:00Z: the lowest 30m-sustained
+          # ceiling was 208.97 GiB (09-26 04:31Z), so ZERO fires on the observed
+          # normal shape. The 09-26 burst took max_avail to 117.4 GiB for about
+          # 10 minutes and did not come close to holding it.
+          expr: |
+            max by (pool_id) (
+              ceph_pool_max_avail
+                and on(pool_id) ceph_pool_metadata{name="replicapool"}
+            ) < 160 * 1024^3
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "replicapool usable headroom below 160 GiB"
+            description: "replicapool has had under 160 GiB usable for 30 minutes ({{ $value | humanize1024 }}B now). 160 GiB is the 89 GiB that max_avail still holds above Ceph's 0.85 nearfull line, plus the 38 GiB ha-pg has not yet consumed of its 32Gi expansion, plus 35 GiB of observed replicapool churn — below this, what is already committed no longer fits before nearfull. This is the same series as CephOSDNearfullApproaching rescaled (max_avail = (0.95 - fullest OSD) x OSD size), so it is a second tier and not independent confirmation. Check `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph df`; growth here is usually the bulk tier on the shared OSDs, not replicapool itself."
@@ -28,101 +28,74 @@ spec:
          # discussed below it does not depend on Cluster status being
          # populated.
          #
-          # Threshold: >5 segments sustained for 15m. Brief non-zero values are
-          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
-          # 80 MiB of un-archived WAL, well below anything that threatens the
-          # volume but far above idle noise.
+          # Threshold: >5 segments sustained for 20m. A segment is 16 MiB, so 5
+          # is roughly 80 MiB of un-archived WAL — well below anything that
+          # threatens the volume, but far above idle noise.
          #
-          # `for` was 30m and is now 15m (REN-129). To be clear about what that
-          # does and does not buy: this rule does NOT fire on the nightly write
-          # burst documented under the Surge rule below, at 30m or at 15m, and
-          # that is correct — across seven consecutive measured nights the
-          # longest contiguous excursion above 5 was 10 minutes and every one
-          # of them drained itself. What 15m is for is the case that actually
-          # costs something: an archiver that has stopped. Outside those
-          # nightly windows the metric is a hard 0 on all 12 CNPG instances in
-          # this cluster, so there was never a noise floor that needed 30m to
-          # wait out — the extra 15m only delayed that page.
+          # WHAT THIS RULE KEYS ON: persistence, not depth. The failure that
+          # costs something is an archiver that has *stopped*, and a stopped
+          # archiver can sit at 60 segments indefinitely without ever reaching
+          # some larger number. A depth threshold tuned above the nightly
+          # envelope below would miss exactly that case, and would need raising
+          # again every time the recorder grows. So there is one rule here, and
+          # the tuning knob is `for` (REN-152).
          #
-          # `max by (namespace, pod)` is not cosmetic. The raw series carries
-          # `role`, which flips on failover; a changed label is a new series,
-          # and a new series restarts the `for` timer. Aggregating `role` away
-          # means a backlog that persists across a failover still pages 15m
-          # after the backlog started rather than 15m after the failover.
-          expr: max by (namespace, pod) (cnpg_collector_pg_wal_archive_status{value="ready"}) > 5
-          for: 15m
-          labels:
-            severity: warning
-          annotations:
-            summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
-            description: "{{ $value }} WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."
-
-        - alert: CNPGWalArchiveBacklogSurge
-          # Companion to CNPGWalArchiveBacklog above, for the opposite failure
-          # shape: not a stuck archiver, but a working archiver that cannot
-          # drain as fast as Postgres is producing.
-          #
-          # CNPGWalArchiveBacklog cannot see that shape at any `for` value that
-          # is also safe against a flapping archiver, because the event is over
-          # before the window closes.
-          #
-          # What is normal here, measured on ha-pg-1 at 1-minute steps over the
-          # seven nights 2026-09-19 to 2026-09-26 (REN-129). Every night is the
-          # same burst, and nothing in that week exceeds 5 at any other hour:
+          # THE NIGHTLY BURST, which this rule must stay silent through.
+          # ha-pg-1 queues WAL every night on the same clock:
          #
          #   flat 0 until 02:12Z; fill from 02:13Z; peak 02:14-02:15Z;
-          #   drained by 02:22Z. Nightly peaks 56, 91, 100, 101, 106, 113, 119.
+          #   drained by 02:26Z at the worst. 02:12Z is 04:12 local, which is
+          #   when Home Assistant's recorder purge runs.
          #
-          # This is a write burst, not IO contention. Drain was 13.0-14.9
-          # segments/min on all seven nights — including the night whose Kopia
-          # repository-maintenance batch did not start until after the burst had
-          # already drained, and the night when two such jobs were saturating
-          # the RBD devices. That batch drifts across 02:03-02:24Z while the
-          # peak stays pinned to 02:15Z, and a drifting cause cannot produce an
-          # effect fixed to the clock. What varies is the fill (18.7-39.7
-          # segments/min), so the height of the backlog is set by how much WAL
-          # is produced against an archiver with a flat ~14 segments/min
-          # ceiling. 02:12Z is 04:12 local, which is when Home Assistant's
-          # recorder purge runs.
+          # It is a write burst, not IO contention. Drain is a flat
+          # 13-15 segments/min every night — including nights when Kopia's
+          # repository-maintenance batch was saturating the RBD devices, and
+          # nights when it did not start until after the burst had already
+          # drained. That batch drifts across 02:03-02:24Z while the peak stays
+          # pinned to 02:15Z, and a drifting cause cannot produce an effect
+          # fixed to the clock. What varies is the fill (19-40 segments/min),
+          # so the height of the backlog is set by how much WAL is produced
+          # against an archiver with a flat ~14 segments/min ceiling.
          #
-          # So the nightly burst is a benign, self-clearing fact of this
-          # cluster and this rule has to stay silent through it. Threshold:
-          # >64 segments (1 GiB of un-archived WAL) for 8m.
+          # WHY 20m. Measured against Prometheus' full 27-night retention
+          # (2026-08-30..2026-09-26), at the real 30s scrape resolution, across
+          # all 12 CNPG instances. Only ha-pg-1 ever exceeds 5 at all; every
+          # other instance peaks at 2. Longest contiguous excursion above 5 on
+          # any night: 13.0 minutes, on 2026-09-11, at a peak of 141 segments.
+          # Firing evaluations over those 27 nights: 0 at 20m, and also 0 at
+          # 15m — but 15m leaves only 2 minutes of margin against the worst
+          # night, and the nightly peak climbed from 116 to 141 inside the
+          # measured month as the recorder grew. 20m leaves 7 minutes and does
+          # not have to be re-derived every time the database gets bigger.
          #
-          #   - 64, not something nearer the >5 above, because this rule is
-          #     deliberately not a faster copy of that one. 1 GiB is where the
-          #     backlog becomes a material fraction of a CNPG PVC's free space
-          #     (ha-pg's volume is 20Gi, holding steady at 6.4-7.7 GiB free),
-          #     and a rule that trips on every write spike is how the
-          #     load-bearing rule above gets ignored.
-          #   - 8m because the measured envelope spends at most 5 consecutive
-          #     minutes above 64 on any of the seven nights, so 8m clears the
-          #     worst observed night by 3 minutes. An earlier revision of this
-          #     rule used 2m; that fires at ~02:16Z on 6 of the 7 nights, which
-          #     is two owner emails a night at 04:16 and 04:22 local for a
-          #     condition that repairs itself.
+          # The 5 minutes 20m costs in detection latency buy nothing back: a
+          # real stall lasts hours, and at 32Gi (REN-141) the volume holds
+          # roughly four nights of stalled WAL before free space is in danger.
          #
-          # What 8m still catches: 1 GiB queued that is not moving. From 119,
-          # the highest peak in the week, a healthy archiver is back under 64
-          # inside 5 minutes. Sitting above it for 8 means the archiver has
-          # lost most of its throughput, and that is worth a 04:00 look.
+          # `for` was 30m before REN-129. Outside the nightly windows the
+          # metric is a hard 0 on all 12 instances, so there was never a noise
+          # floor that needed 30m to wait out — the extra 10m only delayed the
+          # page.
          #
-          # When it does fire, the fix is drain rate or headroom, never a
-          # higher threshold here. ObjectStore `wal.maxParallel`
+          # WHEN IT FIRES, the fix is drain rate or headroom, never a higher
+          # threshold here. ObjectStore `wal.maxParallel`
          # (kubernetes/apps/home-assistant/ha-pg-objectstore.yaml) is unset,
          # which the barman-cloud plugin reads as one WAL segment at a time;
          # that is where the ~14 segments/min ceiling comes from. Headroom is
          # `spec.storage.size` in ha-pg-cluster.yaml. Both are REN-135.
          #
-          # `max by (namespace, pod)` for the same reason as the rule above:
-          # `role` flips on failover and would restart the `for` timer.
-          expr: max by (namespace, pod) (cnpg_collector_pg_wal_archive_status{value="ready"}) > 64
-          for: 8m
+          # `max by (namespace, pod)` is not cosmetic. The raw series carries
+          # `role`, which flips on failover; a changed label is a new series,
+          # and a new series restarts the `for` timer. Aggregating `role` away
+          # means a backlog that persists across a failover still pages 20m
+          # after the backlog started rather than 20m after the failover.
+          expr: max by (namespace, pod) (cnpg_collector_pg_wal_archive_status{value="ready"}) > 5
+          for: 20m
          labels:
            severity: warning
          annotations:
-            summary: "CNPG WAL archiving cannot keep up on {{ $labels.namespace }}/{{ $labels.pod }}"
-            description: "{{ $value }} WAL segments (>1 GiB) have been queued for archiving for 8 minutes, which is longer than a healthy archiver needs to clear that much — from a backlog of 119 it takes under 5. Point-in-time recovery is degraded for everything after the last archived segment, and pg_wal grows against the PVC's free space until it clears. Start with the archiver: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`, and check whether its ObjectStore sets `wal.maxParallel` — unset means one segment at a time, a ceiling of about 14 segments/min (REN-129)."
+            summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "{{ $value }} WAL segments have been waiting to archive for over 20 minutes. NOTE ON TIMING: a nightly write burst on ha-pg-1 at 02:14Z (04:14 local) is normal and drains itself within about 13 minutes, and this rule is tuned to stay silent through it — so if this fired, the backlog did NOT clear and the archiver is stuck. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."

        - alert: CNPGBackupFailed
          # cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0
@@ -0,0 +1,227 @@
+# Alert when an ExternalSecret stops syncing.
+#
+# This is the second half of REN-103. The first half (#994) added the PodMonitor
+# and the `monitoring` ingress grant that make the series exist at all; before
+# that, `externalsecret_status_condition` held 0 series and a key that silently
+# stopped refreshing was invisible to Prometheus, to Alertmanager and to anyone
+# looking at a dashboard. The alert is split out because its `for:` window had to
+# be chosen against real data rather than guessed.
+#
+# `exported_namespace`, NOT `namespace`. The metric carries its own `namespace`
+# label (the ExternalSecret's), and the PodMonitor attaches a target label of the
+# same name (the controller pod's, always `external-secrets`). Prometheus resolves
+# that collision by renaming the metric's label to `exported_namespace`, so on
+# every one of the ~105 live series `namespace="external-secrets"`. Grouping by
+# `namespace` would therefore still fire correctly and still be useless: every
+# failing ExternalSecret in the cluster would be reported as living in
+# `external-secrets`. ESO's own shipped Grafana dashboard does group by
+# `namespace` — that is right for the chart's ServiceMonitor, which scrapes ESO
+# through its own Service and injects no conflicting target label, and wrong here.
+# This is a general trap for the PodMonitor-instead-of-ServiceMonitor trade this
+# repo keeps making: any metric carrying its own `namespace`, `pod` or `container`
+# label gets it shadowed.
+#
+# `honorLabels: true` on the PodMonitor endpoint would keep the clean label name
+# instead, and is deliberately not used: it drops the target labels, so nothing
+# records which controller replica reported, and it would make this the only
+# monitor in the directory doing so.
+#
+# A consequence worth knowing rather than fixing: aggregating by
+# `exported_namespace` leaves these alerts with no `namespace` label at all, and
+# Alertmanager's route groups on ['alertname', 'cluster', 'namespace']. So every
+# failing ExternalSecret arrives in ONE email listing all of them, each named by
+# its own annotation, instead of one email per namespace. For a fleet this size
+# that is the better shape.
+#
+# The obvious worry about that shape — that a SECOND ExternalSecret failing while
+# the first alert is already firing gets swallowed by the open group — does not
+# hold, and the setting that decides it is `group_interval: 5m` on the root route
+# in apps/monitoring/helmrelease-prometheus.yaml. `group_interval` governs
+# notification of a group whose alert SET has changed, which is a different thing
+# from `repeat_interval: 12h`, which governs re-sending an unchanged group. A
+# second failure therefore produces a fresh email naming both within 5 minutes.
+# This rule routes to the root `email` receiver: it is `severity: warning` and
+# carries no `source = "flux"`, so it matches none of the four child routes.
+#
+# The inhibit rules are also checked and do not touch these alerts. All three key
+# on `equal: ['namespace', 'alertname']` (or `['namespace']`), and no critical or
+# warning alert in the cluster shares this `alertname`. In particular
+# OpenBaoQuorumLost does NOT inhibit this rule — different alertname — so an
+# OpenBao outage can surface both. That is the intended order rather than a
+# problem: OpenBaoQuorumLost is critical at 2m and arrives first, well before
+# this rule's 15m.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: external-secrets
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's ruleSelector matches on this label; without it the
+    # rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: external-secrets.rules
+      rules:
+        # Measured against the live Prometheus before this rule was proposed for
+        # merge, through the apiserver services/proxy read path:
+        #
+        #   count(externalsecret_status_condition)                       = 105
+        #   count(count by (exported_namespace,name) (...))              =  53
+        #   count(count by (exported_namespace,name) (...{condition="Ready"})) = 52
+        #   count(count by (exported_namespace) (...))                   =  20
+        #   count(...{condition="Ready",status="False"})                 =  52
+        #   the ExternalSecretSyncFailing expression below                = EMPTY
+        #
+        # So 105 = 52 ExternalSecrets x 2 Ready polarities + 1 Deleted tombstone,
+        # across 20 namespaces, and THIS RULE ARRIVES SILENT: it matches nothing
+        # in the cluster as it stands, which is what makes it safe to load. The
+        # 52 on the line below is a count, not an estimate.
+        #
+        # `== 1` IS LOAD-BEARING, not defensive. ESO writes BOTH polarities of the
+        # Ready condition on every reconcile — the true one at 1 and the opposite
+        # at 0 (esmetrics.UpdateExternalSecretCondition) — so
+        # `{condition="Ready",status="False"}` is present with value 0 for every
+        # healthy ExternalSecret. A bare selector would match all 52 of them.
+        #
+        # `max`, not the `sum` the vendor dashboard uses. Both give 1 today,
+        # because ESO runs a single controller replica. During a Deployment
+        # rollout Prometheus can briefly scrape the old and the new pod at once,
+        # and `sum(...) == 1` would then evaluate to 2 and silently STOP matching
+        # — the alert would resolve itself in the middle of a failure. `max` is 0
+        # or 1 whatever the replica count.
+        #
+        # The gauge, not `externalsecret_sync_calls_error`. The counter was the
+        # other candidate, on the theory that a counter cannot flap the way a
+        # condition gauge might. Reading the controller says the opposite on both
+        # halves: the counter increments on strictly MORE paths than the condition
+        # does — a failed Get of the ExternalSecret itself, a failed Get or label
+        # Patch of the target Secret, an unresolvable SecretStore — and none of
+        # those call markAsFailed(), so none of them set Ready=False. A cache blip
+        # or an apiserver hiccup moves the counter and nothing else. It is the
+        # noisier signal, not the quieter one. And the gauge does
+        # not flap, because it is set to 1 only by markAsFailed() and cleared by
+        # the next successful reconcile. There is no unset state in between.
+        #
+        # The live numbers say the same thing, measured after #994 landed:
+        # `externalsecret_sync_calls_error` has exactly one series in the whole
+        # cluster, `ai/wiki-anthropic`, at 8 errors over 47 calls — and that
+        # ExternalSecret is Ready=True and has never been otherwise in the
+        # observation window. So on today's data the counter carries 8 events that
+        # correspond to no sync failure at all, and the gauge carries none.
+        #
+        # That observation window is now long enough to mean something, and it is
+        # the answer to the only open question this rule was held back for — how
+        # much the gauge flaps in ordinary operation. Measured 2026-09-26T08:02Z,
+        # over the 464 consecutive scrapes since #994 started collecting at
+        # 00:18Z:
+        #
+        #   min_over_time(up{job="monitoring/external-secrets"}[8h])     = 1
+        #   sum(increase(externalsecret_sync_calls_total[8h]))           = 959
+        #   count(increase(externalsecret_sync_calls_total[8h]) > 0)     = 52
+        #   count(max_over_time(
+        #     externalsecret_status_condition{
+        #       condition="Ready",status="False"}[8h]) == 1)             = EMPTY
+        #   sum(increase(externalsecret_sync_calls_error[8h]))           = 0
+        #
+        # Every one of the 52 ExternalSecrets reconciled at least once in that
+        # window — 959 syncs, so the 1h and 10m refreshInterval cycles both turned
+        # over repeatedly — and across all of them the Ready gauge never reached 1
+        # in a single scrape. Zero flap amplitude, against real reconcile traffic
+        # rather than against silence. The counter was flat over the same window
+        # too, which is why 8 stays 8 above. `for: 15m` is therefore not damping
+        # anything measurable; it is derived from the failure modes below, and the
+        # data says no shorter window would have been noisy either.
+        #
+        # The 2026-09-20 `ai/wiki-embed-basic-auth` incident is evidence FOR this
+        # rule, not against it, though it reads the other way at first. That key
+        # sat in ESO retry backoff through ~10 failures and NEEDED A MANUAL
+        # force-sync (the incident is written up in
+        # clusters/homelab/uptime-monitor.yaml, as the in-repo proof for REN-100).
+        # Because it never self-healed, Ready=False was continuous from the first
+        # failure until a human intervened — hours, not minutes. `for: 15m` catches
+        # that with a wide margin. It was a stuck condition, not a flap; the flap
+        # reading would only hold if the retries had eventually succeeded on their
+        # own, and they did not.
+        - alert: ExternalSecretSyncFailing
+          expr: >-
+            max by (exported_namespace, name) (
+              externalsecret_status_condition{condition="Ready",status="False"}
+            ) == 1
+          # 15m, and the reason it can be this relaxed is that a failed sync is
+          # not an outage: consumers keep using the last Secret ESO wrote, so what
+          # degrades is freshness, not availability. Nothing here needs catching
+          # inside a minute.
+          #
+          # It needs to be at least this long because the provider-error path
+          # returns the error to controller-runtime, which retries with
+          # exponential backoff rather than waiting out `refreshInterval`. A
+          # momentary OpenBao blip is therefore cleared within seconds and never
+          # reaches 15m, while a rolling restart of the 3-replica OpenBao
+          # StatefulSet is ridden out entirely — and that outage has its own
+          # alert (OpenBaoQuorumLost, critical at 2m), so nothing is hidden by
+          # waiting. 15m of continuous Ready=False means repeated failure.
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "ExternalSecret {{ $labels.exported_namespace }}/{{ $labels.name }} is not syncing"
+            description: >-
+              ESO has been unable to sync this ExternalSecret for 15 minutes. The
+              Secret it manages still holds its last successfully synced value, so
+              consumers keep working — what breaks is that a rotation at the
+              provider no longer propagates, which surfaces much later as a
+              credential that stopped being accepted. Get the reason with
+              `kubectl -n {{ $labels.exported_namespace }} get externalsecret
+              {{ $labels.name }} -o jsonpath='{.status.conditions}'`; the common
+              causes are a key that was renamed or removed in OpenBao, a
+              ClusterSecretStore that cannot authenticate, and a target Secret
+              owned by something else.
+
+        # Dead-man's-switch, mirroring ExternalDNSMetricsMissing and
+        # OpenBaoMetricsMissing. The alert above can only fire on series that
+        # exist, so deleting the PodMonitor, reverting the `monitoring` ingress
+        # grant in infrastructure/network-policies/external-secrets-ns.yaml, or
+        # bumping the ESO chart onto a metrics port that no longer matches would
+        # all disarm it silently — which is the exact failure mode REN-103 exists
+        # to end. TargetDown does not cover this: it reports a target that exists
+        # and is failing, not the absence of a target.
+        - alert: ExternalSecretMetricsMissing
+          expr: absent(externalsecret_status_condition)
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "No ExternalSecret metrics are reaching Prometheus"
+            description: >-
+              `externalsecret_status_condition` has no series. Either the ESO
+              controller is gone or its scrape stopped being collected, and
+              ExternalSecretSyncFailing is blind for as long as this is true.
+              Check the PodMonitor `monitoring/external-secrets` still selects the
+              controller pods, and that external-secrets-ns.yaml still admits
+              `monitoring` on TCP/8080.
+
+        # NOT covered here, deliberately, so nobody assumes it is:
+        #
+        #   * `condition="Deleted"`. In chart 2.11.0 the only writer of that
+        #     condition is the NotFound branch of Reconcile, which fabricates
+        #     `Deleted=False` purely to make UpdateExternalSecretCondition drop
+        #     the object's Ready series. It is a tombstone for an ExternalSecret
+        #     that no longer exists, not a health state, and `Deleted=True` is
+        #     never emitted at all — a rule on it would be dead code. The one such
+        #     series in the cluster today (`ai/wiki-index-basic-auth`) is leftover
+        #     process state for an ExternalSecret that moved namespace; it clears
+        #     when the controller next restarts.
+        #   * An ExternalSecret that exists in the cluster but was never
+        #     reconciled emits no series at all, so nothing above can see it. That
+        #     needs object-existence data — kube-state-metrics custom resource
+        #     state — rather than ESO's own metrics, and is not in this change.
+        #   * A sync that fails and then succeeds on its own inside 15m. This is a
+        #     choice, not an oversight: the retry left the Secret holding a freshly
+        #     synced value, so nothing degraded and there is nothing for a human to
+        #     do. Alerting on it would mean paging for a self-healed event. The case
+        #     that would justify revisiting is the same failure recurring for days
+        #     without any single occurrence lasting 15m, which would show up as a
+        #     climbing `externalsecret_sync_calls_error` on an ExternalSecret that
+        #     is Ready=True. Nothing in the cluster shows that shape today, and the
+        #     rule for it needs a measured recurrence rate rather than a guess.
@@ -0,0 +1,119 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: home-assistant
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: home-assistant.rules
+      rules:
+        # The four services the house stops working without (REN-244). Before
+        # this group the only signal for any of them was the cluster-wide
+        # KubePodNotReady: 15m, severity warning, and not named per service.
+        #
+        # SHAPE. One alert per service, with `or` joining the ways it can be
+        # down, so one outage is one alert and not one per symptom. All four are
+        # `for: 5m`, severity critical. Alertmanager has one receiver today
+        # (email to the owner, apps/monitoring/helmrelease-prometheus.yaml), so
+        # "critical" does not change where these go today. It does keep them
+        # out of reach of the inhibit rules, which only silence warning/info.
+        #
+        # 5m IS SHORTER THAN A SLOW HA UPGRADE. Home Assistant can take up to
+        # 10 min to come back after an automatic image bump. REN-244 asks for
+        # detection in 5 min on purpose; a HomeAssistantDown that resolves on
+        # its own a few minutes later, right after an upgrade, is that case.
+        #
+        # Tests: tests/home-assistant_test.yaml.
+
+        - alert: HomeAssistantDown
+          # `absent` covers the target vanishing entirely (pod gone, Service
+          # has no endpoints during a Recreate rollout); `== 0` covers a target
+          # that exists but does not answer /api/prometheus.
+          #
+          # `max by (job, namespace)` gives both branches the same label set.
+          # Without it the alert changes identity when a pod that was gone
+          # comes back failing, and the new identity restarts the `for` timer.
+          expr: |
+            max by (job, namespace) (up{job="home-assistant", namespace="home-assistant"}) == 0
+            or absent(up{job="home-assistant", namespace="home-assistant"})
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Home Assistant has been down for 5 minutes"
+            description: "The home-assistant scrape target in namespace home-assistant is down or missing. A restart after an automatic image upgrade can take up to 10 minutes, so if an upgrade just rolled out, give it until then before acting. Rollback: kubernetes/RUNBOOK-home-assistant-image-rollback.md."
+
+        - alert: HomeAssistantRecorderDBUnhealthy
+          # ha-pg, the recorder's CNPG cluster. Fires when no instance is the
+          # primary, or when fewer than all instances have Postgres up.
+          #
+          # cnpg_collector_up, not `up`: the exporter lives in the instance
+          # manager, which keeps serving /metrics while Postgres itself is down.
+          # `up` would read 1 through exactly the outage this rule is for.
+          #
+          # `role` comes from podTargetLabels on the cnpg-ha-pg PodMonitor
+          # (cnpg-podmonitors.yaml). It flips on failover, so the instance count
+          # is taken across roles. Both branches come out as {} (`absent` of a
+          # comparison carries no labels), so it is one alert whichever of them
+          # is true, and a failover does not start a second one.
+          #
+          # THE 2 IS `spec.instances` in
+          # kubernetes/apps/home-assistant/ha-pg-cluster.yaml. Change both
+          # together, or this pages forever (more instances is fine; fewer is
+          # not).
+          expr: |
+            count(cnpg_collector_up{job="home-assistant/cnpg-ha-pg"} == 1) < 2
+            or absent(cnpg_collector_up{job="home-assistant/cnpg-ha-pg", role="primary"} == 1)
+          for: 5m
+          labels:
+            severity: critical
+            namespace: home-assistant
+          annotations:
+            summary: "Recorder database ha-pg has no primary or is missing an instance"
+            description: "In CNPG cluster home-assistant/ha-pg either no instance reports role=primary with Postgres up, or fewer than 2 instances have Postgres up. Without a primary the Home Assistant recorder cannot write history. Check `kubectl -n home-assistant get cluster ha-pg` and the pods' logs."
+
+        - alert: MosquittoDown
+          # Pod readiness only: there is no broker exporter, and no mosquitto_*
+          # or mqtt_* series in Prometheus.
+          #
+          # The pod regex is the Deployment's name shape
+          # (mosquitto-<replicaset hash>-<suffix>) and not `mosquitto-.*` on
+          # purpose: mosquitto-bootstrap.yaml runs six Jobs named
+          # mosquitto-*-credentials-bootstrap, and a bootstrap pod that is
+          # briefly Ready must not hide a broker that is down.
+          expr: |
+            absent(kube_pod_status_ready{namespace="home-assistant", pod=~"mosquitto-[a-z0-9]{6,10}-[a-z0-9]{5}", condition="true"} == 1)
+          for: 5m
+          labels:
+            severity: critical
+            # `absent` of a comparison carries no labels; set it for routing.
+            namespace: home-assistant
+          annotations:
+            summary: "Mosquitto (MQTT broker) has not been ready for 5 minutes"
+            description: "No mosquitto pod in namespace home-assistant is Ready. Zigbee2MQTT, Shelly and every other MQTT device lose their link to Home Assistant while this lasts, so Zigbee2MQTTDownOrCoordinatorOffline may fire alongside it for the same cause."
+
+        - alert: Zigbee2MQTTDownOrCoordinatorOffline
+          # Two ways Zigbee stops working: the pod is not Ready, or it runs but
+          # has lost the coordinator / MQTT bridge. The second is read from
+          # Home Assistant's own bridge sensor, so it disappears when HA is down;
+          # HomeAssistantDown covers that case and this rule stays silent
+          # instead of paging twice.
+          #
+          # Both branches come out as {} (`absent` of a comparison carries no
+          # labels; the sensor branch is reduced with `max`), for the same
+          # reason as in HomeAssistantDown: pod-not-ready turning into
+          # bridge-offline is one outage and keeps one `for` timer.
+          expr: |
+            absent(kube_pod_status_ready{namespace="home-assistant", pod=~"zigbee2mqtt-[a-z0-9]{6,10}-[a-z0-9]{5}", condition="true"} == 1)
+            or max(homeassistant_binary_sensor_state{entity="binary_sensor.zigbee2mqtt_bridge_connection_state"}) == 0
+          for: 5m
+          labels:
+            severity: critical
+            namespace: home-assistant
+          annotations:
+            summary: "Zigbee2MQTT is down or its coordinator/bridge is offline"
+            description: "Either the zigbee2mqtt pod in namespace home-assistant is not Ready, or Home Assistant's binary_sensor.zigbee2mqtt_bridge_connection_state reads off. If MosquittoDown is firing too, fix the broker first; the bridge sensor goes off whenever MQTT is gone."
@@ -0,0 +1,153 @@
+# Catches a Prometheus retention setting that is configured but NOT IN FORCE.
+#
+# kube-prometheus-stack takes both `retention` (time) and `retentionSize`
+# (bytes) and passes both to Prometheus, which honours WHICHEVER BINDS FIRST.
+# When the size cap binds first, the time setting silently becomes decoration:
+# Prometheus drops its oldest blocks, every query still succeeds, and the only
+# symptom is that a range longer than the real window quietly returns less data
+# than asked for. Nothing in the 182 alert rules this cluster already loads
+# reports that — verified against the live rule set when this was written.
+#
+# That is not hypothetical here. Measured through the read-only proxy on
+# 2026-09-26 with `retention: 90d` and `retentionSize: 30GB` in
+# apps/monitoring/helmrelease-prometheus.yaml (REN-107):
+#
+#   storage.tsdb.retention.size = 30GiB    -> 32,212,254,720 bytes
+#   storage.tsdb.retention.time = 90d      -> 7,776,000 s
+#   actual window held                     -> 2,291,239 s  (26.5 days, 29% of 90d)
+#
+# and size eviction has been firing on a ~7-day cycle for at least as long as
+# the TSDB retains history. At 2026-09-19 23:00->23:30 the store went 28.16 GB
+# -> 21.14 GB, `prometheus_tsdb_size_retentions_total` went 0 -> 1, and the
+# window held collapsed 27.21 -> 20.48 days. The effective retention is a
+# 20.5-27.2 day saw-tooth, not 90 days and not any single number.
+#
+# WHY NOT JUST ALERT ON prometheus_tsdb_size_retentions_total > 0
+#
+# Because that is a PROCESS-LIFETIME counter and it lies by omission. It resets
+# to 0 on every Prometheus restart, and this Prometheus restarts every few days
+# (seven distinct pod IPs in the 26 days of history the TSDB holds — the chart
+# is pinned `>=72.0.0`, so Flux upgrades it whenever upstream releases). Each
+# fresh pod reads 0 for the first ~6 days of its life even though the cluster
+# has been evicting data continuously for a month. REN-107 was originally filed
+# as "the cap will bind in ~3 days" on exactly that reading, taken on a pod that
+# had started 5 days earlier. A spot read of this counter cannot distinguish
+# "never evicted" from "evicted, then restarted".
+#
+# So this rule does not read the counter at all. It compares the two things
+# that are true regardless of process age: the window actually on disk, and the
+# limits the running process reports.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-retention
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: prometheus-retention.rules
+      rules:
+        - alert: PrometheusRetentionNotInForce
+          # NO HARDCODED 90d / 30GB. Both limits are read from the running
+          # process via `prometheus_tsdb_retention_limit_seconds` and
+          # `prometheus_tsdb_retention_limit_bytes`, which Prometheus exports
+          # from the flags it actually started with. Edit the HelmRelease and
+          # this rule follows; it cannot drift out of sync with the config the
+          # way a literal threshold would. That matters because a literal here
+          # would be a second copy of the very setting whose accuracy this rule
+          # exists to police.
+          #
+          # TWO CLAUSES, AND THE SECOND IS THE GUARD
+          #
+          # Clause 1 — the window on disk is materially shorter than the
+          # configured time retention (<90% of it).
+          #
+          # Clause 2 — the store has reached 80% of the configured SIZE cap at
+          # some point in the last 7 days.
+          #
+          # Clause 1 alone would fire on a legitimately young TSDB: a freshly
+          # provisioned PVC holds one day of data against a 90d setting and
+          # nothing is wrong. Clause 2 is what distinguishes "not full yet" from
+          # "full, and the size cap is what is actually in force" — the store can
+          # only be near the byte cap AND short of the time window if bytes are
+          # the binding constraint. Dropping either clause breaks the rule in a
+          # different direction; verified both ways against the live TSDB before
+          # merge (see the PR).
+          #
+          # max_over_time(...[7d:1h]) rather than the instantaneous ratio,
+          # because size eviction is a SAW-TOOTH: the store climbs to ~87% of
+          # the cap, sheds a compacted block, and drops back to ~66%. An instant
+          # ratio would resolve and re-fire on that ~7-day cycle and email twice
+          # a week about an unchanging condition. The 7d window spans a full
+          # tooth, so the alert is continuous while the condition is.
+          #
+          # The 7d PEAK IS DIVIDED BY THE CAP IN FORCE NOW, not by the cap in
+          # force when the peak was recorded. The difference is the day the
+          # config is fixed: REN-107 moves 90d/30GiB to 30d/40GiB, the window
+          # on disk is still ~21 days (so clause 1 stays true until it grows
+          # past 27), and last week's 28 GB peak was 87% of the OLD cap. A
+          # per-step ratio would keep this firing for up to 7 days after the
+          # fix that resolves it; against the new cap the same peak is 66% and
+          # it clears at once. tests/prometheus-retention_test.yaml holds both.
+          #
+          # Everything is aggregated (min/sum/max) so the expression carries NO
+          # instance or pod label. That is deliberate and load-bearing: it makes
+          # the 7d lookback survive the pod restarts described in the header,
+          # which would otherwise start a new series and reset the window.
+          # Confirmed on the live TSDB — the 0.874 peak this returns today was
+          # recorded by a pod that no longer exists. There is exactly one
+          # Prometheus here, so losing the per-instance breakdown costs nothing.
+          expr: |
+            (
+              min(time() - prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus"})
+                < 0.9 * min(prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus"})
+            )
+            and
+            (
+                max_over_time(
+                  sum(
+                      prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus"}
+                    + prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus"}
+                    + prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus"}
+                  )[7d:1h]
+                )
+              / max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})
+              > 0.8
+            )
+          # Long, because this is a configuration fact rather than an incident.
+          # Nothing degrades at the moment it becomes true and nothing is fixed
+          # by waiting, so there is no value in a fast signal — only in not
+          # paging during the minutes after a restart while the aggregation
+          # settles. It clears within one evaluation once the config is honest.
+          for: 1h
+          labels:
+            severity: warning
+            # Static, and true: the Prometheus this describes runs in
+            # `monitoring`. The expression aggregates the real label away (see
+            # above), and the chart's inhibit rules plus Alertmanager's
+            # `group_by` both key on `namespace` — without it this alert groups
+            # under the empty namespace, unlike every other alert here.
+            namespace: monitoring
+          annotations:
+            summary: "Prometheus retention setting is not in force — size cap binds first"
+            # $value is clause 1's left-hand side: the window actually on disk,
+            # in seconds. The configured pair is deliberately NOT templated in
+            # via `query` — that runs on every evaluation and can fail the rule
+            # itself. `runtimeinfo.storageRetention` reports both as one string
+            # ("90d or 30GiB"), which is the cheaper and more direct read.
+            description: >-
+              Prometheus is holding only {{ $value | humanizeDuration }} of history while its
+              configured time retention is longer, and the store has been within 20% of its
+              configured size cap in the last 7 days. The size cap is therefore what is actually
+              in force and the `retention` value in
+              kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured —
+              queries over a longer range return less data than requested, silently and
+              successfully. Confirm the running pair with
+              `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise
+              `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention`
+              to a figure the storage budget can actually hold. Do not read
+              `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets
+              on every restart. Background: REN-107.
@@ -18,8 +18,8 @@ value. Against Iris's seven measured nights it fires on six of them — two owne
 emails a night, 04:16 and 04:22 local, for a WAL backlog that clears itself in
 under ten minutes. It was caught by a human writing the arithmetic out by hand,
 twice, in two separate runs, and throwing the work away both times because
-there was nowhere in the repo to put it. That rule is now
-`mutations.txt:surge-for-2m`.
+there was nowhere in the repo to put it. [REN-152](/REN/issues/REN-152) then
+retired the depth rule in favour of one persistence rule (`> 5` for `20m`).

 ## Layout

@@ -27,6 +27,9 @@ there was nowhere in the repo to put it. That rule is now
 | --- | --- |
 | `cnpg-backup_test.yaml` | The suite. **Generated** — see below. |
 | `generate-cnpg-backup-test.py` | Regenerates it from the REN-129 measurement table. |
+| `external-secrets_test.yaml` | Hand-written suite for `prometheusrule-external-secrets.yaml` (REN-103). |
+| `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). |
+| `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |

 ## Adding a suite
@@ -90,10 +93,10 @@ the assertions reach the rules — both failure modes above were green. So
 breaks one value the rules depend on and requires the suites to **fail**. A
 mutation nothing notices is reported as a hole in the suite.

-This is also the answer to "why is that case in there". `surge-for-5m` is
-silent on all seven real nights and fires only on the adversarial envelope
-case, so deleting the envelope case makes `for: 8m` reducible to `5m` with the
-suite still green. The control is what makes the case load-bearing.
+This is also the answer to "why is that case in there". `backlog-for-12m` is
+silent on all seven reconstructed nights and fires only on the raw 2026-09-11
+worst-night case, so deleting that case makes `for: 20m` reducible to `12m`
+with the suite still green. The control is what makes the case load-bearing.

 Mutations are `name|manifest-path|sed-program`, applied to the stripped copy in
 the mirror (two spaces shallower than the manifest on disk — anchor on
@@ -105,10 +108,9 @@ otherwise renaming a rule quietly turns its control into a no-op.
 Margin is **pending** time, which is one minute less than the run length. An
 alert goes pending at the first evaluation where the expression is true and
 fires when `now - activeAt >= for`, so N consecutive one-minute samples above a
-threshold accumulate N-1 minutes. A trace with 5 samples above 64 does not fire
-`for: 5m`; it fires `for: 4m`. The `>5 pend` and `>64 pend` columns in
-`--report` are the numbers to compare a `for` against — not the sample counts
-next to them.
+threshold accumulate N-1 minutes. A trace with 5 samples above 5 does not fire
+`for: 5m`; it fires `for: 4m`. The `>5 pend` column in `--report` is the number
+to compare a `for` against — not the sample count next to it.

 ## Running it locally

@@ -0,0 +1,176 @@
+# promtool test rules for the CephPoolLostRedundancyBlind canary in
+# prometheusrule-ceph.yaml (REN-93).
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh, which strips `spec:` from the manifest into a
+# mirrored tree — hence `../` below, resolved from this file's directory. See
+# README.md here. The negative controls for this suite are the `ceph-*` entries
+# in mutations.txt.
+#
+# WHAT IT IS FOR. `promtool check rules` proves the canary parses. It cannot
+# prove the canary fires when the join breaks, or stays quiet when the join is
+# fine — every scenario below is well-formed PromQL in both the passing and the
+# failing case; only the synthetic data changes, and only evaluation sees it.
+#
+# `exp_annotations` must match the RENDERED annotations exactly, so the strings
+# below are the rule's with `{{ $labels.pool_id }}` substituted. Tedious to keep
+# in sync, and that is the point: an annotation edited into nonsense fails here
+# instead of in an email.
+rule_files:
+  - ../prometheusrule-ceph.yaml
+evaluation_interval: 1m
+tests:
+  # A. Healthy replicated pool, metadata intact. Neither alert may fire.
+  #    used/stored = 3.0, so well clear of the 1.5 threshold, and the join
+  #    resolves — so the canary has nothing to report either. This is the
+  #    scenario that fails if someone inverts a comparison.
+  - interval: 1m
+    input_series:
+      - series: 'ceph_pool_bytes_used{pool_id="1"}'
+        values: '6000000+0x200'
+      - series: 'ceph_pool_stored{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_metadata{pool_id="1",type="replicated",name="rbd"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancyBlind
+        exp_alerts: []
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancy
+        exp_alerts: []
+
+  # B. Real redundancy loss, metadata intact: used/stored = 1.0.
+  #    The critical alert must fire AND the canary must stay quiet. The second
+  #    half is the load-bearing one — a canary that also fires on a real
+  #    redundancy loss would mask the alert it exists to protect, and
+  #    `absent()` over an expression that IS producing series is the only thing
+  #    stopping that. Also pins the `for: 30m`: 120m is well past it.
+  - interval: 1m
+    input_series:
+      - series: 'ceph_pool_bytes_used{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_stored{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_metadata{pool_id="1",type="replicated",name="rbd"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancy
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              pool_id: "1"
+            exp_annotations:
+              summary: "Ceph replicated pool 1 is running unreplicated"
+              description: "Replicated pool 1 has a replication factor below 1.5, meaning it is down to a single copy and one OSD loss destroys data. No replicated pool is meant to be unreplicated — it also blocks OSD maintenance for as long as it lasts, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Erasure-coded pools are out of scope for this rule and are covered by POOL_NO_REDUNDANCY via CephHealthWarn (REN-88). Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancyBlind
+        exp_alerts: []
+
+  # C, D and E are the same failure seen three ways: the pool is genuinely at
+  # replication 1.0, and CephPoolLostRedundancy produces NOTHING. Each asserts
+  # `exp_alerts: []` for the critical alert — that empty list is the silent
+  # failure being surfaced, not an absence of interest — and asserts the canary
+  # fires instead. Remove the canary from the rule file and all three fail.
+  #
+  # C. `type` renamed to `pool_type` by a Rook or Ceph upgrade.
+  - interval: 1m
+    input_series:
+      - series: 'ceph_pool_bytes_used{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_stored{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_metadata{pool_id="1",pool_type="replicated",name="rbd"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancy
+        exp_alerts: []
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancyBlind
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "CephPoolLostRedundancy cannot fire — its metrics no longer join"
+              description: "The CephPoolLostRedundancy alert has been unable to produce any series for an hour, so a `critical` storage alert is silently dark. It joins ceph_pool_bytes_used / ceph_pool_stored against ceph_pool_metadata{type=\"replicated\"} on `pool_id`; one of those metrics, the `pool_id` label, the `type` label, or the `replicated` value has stopped matching — most likely a Rook or Ceph upgrade, or a relabel rule. Check what the exporter emits now with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail` and by querying `ceph_pool_metadata` directly, then fix the join in kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml — do not delete this canary (REN-93)."
+
+  # D. Label VALUE changed rather than the label name: `replicated` ->
+  #    `replica`. Every label the rule names still exists, which is why a check
+  #    on label presence alone would not catch this.
+  - interval: 1m
+    input_series:
+      - series: 'ceph_pool_bytes_used{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_stored{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_metadata{pool_id="1",type="replica",name="rbd"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancy
+        exp_alerts: []
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancyBlind
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "CephPoolLostRedundancy cannot fire — its metrics no longer join"
+              description: "The CephPoolLostRedundancy alert has been unable to produce any series for an hour, so a `critical` storage alert is silently dark. It joins ceph_pool_bytes_used / ceph_pool_stored against ceph_pool_metadata{type=\"replicated\"} on `pool_id`; one of those metrics, the `pool_id` label, the `type` label, or the `replicated` value has stopped matching — most likely a Rook or Ceph upgrade, or a relabel rule. Check what the exporter emits now with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail` and by querying `ceph_pool_metadata` directly, then fix the join in kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml — do not delete this canary (REN-93)."
+
+  # E. `pool_id` renamed to `pool` on the METADATA side only. Both metrics
+  #    exist, both label values are unchanged, and `type="replicated"` still
+  #    matches — only the `on(pool_id)` join key is gone from one side, so the
+  #    join matches nothing. The subtlest of the three, and the one a human
+  #    reading the expression is least likely to spot.
+  - interval: 1m
+    input_series:
+      - series: 'ceph_pool_bytes_used{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_stored{pool_id="1"}'
+        values: '2000000+0x200'
+      - series: 'ceph_pool_metadata{pool="1",type="replicated",name="rbd"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancy
+        exp_alerts: []
+      - eval_time: 120m
+        alertname: CephPoolLostRedundancyBlind
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "CephPoolLostRedundancy cannot fire — its metrics no longer join"
+              description: "The CephPoolLostRedundancy alert has been unable to produce any series for an hour, so a `critical` storage alert is silently dark. It joins ceph_pool_bytes_used / ceph_pool_stored against ceph_pool_metadata{type=\"replicated\"} on `pool_id`; one of those metrics, the `pool_id` label, the `type` label, or the `replicated` value has stopped matching — most likely a Rook or Ceph upgrade, or a relabel rule. Check what the exporter emits now with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail` and by querying `ceph_pool_metadata` directly, then fix the join in kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml — do not delete this canary (REN-93)."
+
+  # F. mgr FAILOVER, which is the noise case the canary's `for: 1h` exists for.
+  #    All ceph_* metrics stop at t=30m and never return; with the 5m lookback
+  #    they read as present until t=35m, so the canary goes pending at ~t=36m.
+  #    It must NOT be firing at 75m (a failover-length gap — the exporter moved
+  #    through 17 mgr instances in 30d) and MUST be firing by 95m. Shorten the
+  #    `for:` and the 75m assertion fails; lengthen it and the 95m one does.
+  #    Both eval_times were checked against promtool 3.15.0 rather than derived,
+  #    because the exact ActiveAt depends on lookback and evaluation alignment.
+  - interval: 1m
+    input_series:
+      - series: 'ceph_pool_bytes_used{pool_id="1"}'
+        values: '6000000+0x30'
+      - series: 'ceph_pool_stored{pool_id="1"}'
+        values: '2000000+0x30'
+      - series: 'ceph_pool_metadata{pool_id="1",type="replicated",name="rbd"}'
+        values: '1+0x30'
+    alert_rule_test:
+      - eval_time: 75m
+        alertname: CephPoolLostRedundancyBlind
+        exp_alerts: []
+      - eval_time: 95m
+        alertname: CephPoolLostRedundancyBlind
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "CephPoolLostRedundancy cannot fire — its metrics no longer join"
+              description: "The CephPoolLostRedundancy alert has been unable to produce any series for an hour, so a `critical` storage alert is silently dark. It joins ceph_pool_bytes_used / ceph_pool_stored against ceph_pool_metadata{type=\"replicated\"} on `pool_id`; one of those metrics, the `pool_id` label, the `type` label, or the `replicated` value has stopped matching — most likely a Rook or Ceph upgrade, or a relabel rule. Check what the exporter emits now with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail` and by querying `ceph_pool_metadata` directly, then fix the join in kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml — do not delete this canary (REN-93)."
@@ -3,9 +3,10 @@
 #
 # Unit tests for kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml.
 # Every assertion below is either one of Iris's seven measured nights
-# (REN-129) rebuilt from its published (peak, fill, drain) numbers, an
+# (REN-129) rebuilt from its published (peak, fill, drain) numbers, the
+# worst night in Prometheus' retention as raw samples (REN-152), an
 # adversarial combination of the extremes of those numbers, or a failure
-# this cluster has not had but the rules exist to catch.
+# this cluster has not had but the rule exists to catch.
 #
 # `rule_files` entries are resolved RELATIVE TO THIS FILE'S DIRECTORY, not
 # to the working directory promtool was invoked from — promtool joins
@@ -24,13 +25,12 @@ evaluation_interval: 1m

 tests:
  # ---------------------------------------------------------------------
-  # 1-7. THE SEVEN MEASURED NIGHTS. Both rules must stay silent through
-  # every one of them. This is the half of the suite that has already
-  # caught something: the surge rule shipped for review at `for: 2m` and
-  # fires here on 6 of these 7 nights (mutations.txt:surge-for-2m).
+  # 1-7. THE SEVEN MEASURED NIGHTS. The rule must stay silent through
+  # every one of them. The retired depth rule (`> 64`) shipped for review
+  # at `for: 2m` and fired on 6 of these 7 nights; REN-152 removed it.
  # ---------------------------------------------------------------------
  # 2026-09-20: the nightly Home Assistant recorder purge at 04:12 local.
-  # peak 106 at 02:15Z, fill 35.3/min, drain 13.2/min -> 10 samples above 5 (9m pending vs for: 15m), 5 above 64 (4m pending vs for: 8m).
+  # peak 106 at 02:15Z, fill 35.3/min, drain 13.2/min -> 10 samples above 5 (9m pending vs for: 20m).
  - name: nightly burst 2026-09-20 — recorder purge, benign, self-clearing
    interval: 1m
    input_series:
@@ -46,37 +46,17 @@ tests:
      - eval_time: 7m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:15Z, backlog 106
-      - eval_time: 7m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      # 02:18Z, backlog 66
-      - eval_time: 10m
-        alertname: CNPGWalArchiveBacklog
-        exp_alerts: []
-      # 02:18Z, backlog 66
-      - eval_time: 10m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:22Z, backlog 14
      - eval_time: 14m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:22Z, backlog 14
-      - eval_time: 14m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:24Z, backlog 0
      - eval_time: 16m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:24Z, backlog 0
-      - eval_time: 16m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # 2026-09-21: the nightly Home Assistant recorder purge at 04:12 local.
-  # peak 100 at 02:15Z, fill 33.3/min, drain 14.3/min -> 9 samples above 5 (8m pending vs for: 15m), 4 above 64 (3m pending vs for: 8m).
+  # peak 100 at 02:15Z, fill 33.3/min, drain 14.3/min -> 9 samples above 5 (8m pending vs for: 20m).
  - name: nightly burst 2026-09-21 — recorder purge, benign, self-clearing
    interval: 1m
    input_series:
@@ -92,37 +72,17 @@ tests:
      - eval_time: 7m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:15Z, backlog 100
-      - eval_time: 7m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      # 02:17Z, backlog 71
-      - eval_time: 9m
-        alertname: CNPGWalArchiveBacklog
-        exp_alerts: []
-      # 02:17Z, backlog 71
-      - eval_time: 9m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:21Z, backlog 14
      - eval_time: 13m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:21Z, backlog 14
-      - eval_time: 13m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:23Z, backlog 0
      - eval_time: 15m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:23Z, backlog 0
-      - eval_time: 15m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # 2026-09-22: the nightly Home Assistant recorder purge at 04:12 local.
-  # peak 101 at 02:15Z, fill 33.7/min, drain 14.4/min -> 9 samples above 5 (8m pending vs for: 15m), 4 above 64 (3m pending vs for: 8m).
+  # peak 101 at 02:15Z, fill 33.7/min, drain 14.4/min -> 9 samples above 5 (8m pending vs for: 20m).
  - name: nightly burst 2026-09-22 — recorder purge, benign, self-clearing
    interval: 1m
    input_series:
@@ -138,37 +98,17 @@ tests:
      - eval_time: 7m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:15Z, backlog 101
-      - eval_time: 7m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      # 02:17Z, backlog 72
-      - eval_time: 9m
-        alertname: CNPGWalArchiveBacklog
-        exp_alerts: []
-      # 02:17Z, backlog 72
-      - eval_time: 9m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:21Z, backlog 15
      - eval_time: 13m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:21Z, backlog 15
-      - eval_time: 13m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:23Z, backlog 0
      - eval_time: 15m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:23Z, backlog 0
-      - eval_time: 15m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # 2026-09-23: the nightly Home Assistant recorder purge at 04:12 local.
-  # peak 119 at 02:15Z, fill 39.7/min, drain 14.9/min -> 10 samples above 5 (9m pending vs for: 15m), 5 above 64 (4m pending vs for: 8m).
+  # peak 119 at 02:15Z, fill 39.7/min, drain 14.9/min -> 10 samples above 5 (9m pending vs for: 20m).
  - name: nightly burst 2026-09-23 — recorder purge, benign, self-clearing
    interval: 1m
    input_series:
@@ -184,37 +124,17 @@ tests:
      - eval_time: 7m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:15Z, backlog 119
-      - eval_time: 7m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      # 02:18Z, backlog 74
-      - eval_time: 10m
-        alertname: CNPGWalArchiveBacklog
-        exp_alerts: []
-      # 02:18Z, backlog 74
-      - eval_time: 10m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:22Z, backlog 15
      - eval_time: 14m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:22Z, backlog 15
-      - eval_time: 14m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:24Z, backlog 0
      - eval_time: 16m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:24Z, backlog 0
-      - eval_time: 16m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # 2026-09-24: the nightly Home Assistant recorder purge at 04:12 local.
-  # peak 56 at 02:14Z, fill 18.7/min, drain 14.0/min -> 6 samples above 5 (5m pending vs for: 15m), 0 above 64 (0m pending vs for: 8m).
+  # peak 56 at 02:14Z, fill 18.7/min, drain 14.0/min -> 6 samples above 5 (5m pending vs for: 20m).
  - name: nightly burst 2026-09-24 — recorder purge, benign, self-clearing
    interval: 1m
    input_series:
@@ -230,29 +150,17 @@ tests:
      - eval_time: 6m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:14Z, backlog 56
-      - eval_time: 6m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:17Z, backlog 14
      - eval_time: 9m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:17Z, backlog 14
-      - eval_time: 9m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:19Z, backlog 0
      - eval_time: 11m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:19Z, backlog 0
-      - eval_time: 11m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # 2026-09-25: the nightly Home Assistant recorder purge at 04:12 local.
-  # peak 91 at 02:15Z, fill 22.8/min, drain 13.0/min -> 10 samples above 5 (9m pending vs for: 15m), 4 above 64 (3m pending vs for: 8m).
+  # peak 91 at 02:15Z, fill 22.8/min, drain 13.0/min -> 10 samples above 5 (9m pending vs for: 20m).
  - name: nightly burst 2026-09-25 — recorder purge, benign, self-clearing
    interval: 1m
    input_series:
@@ -268,37 +176,17 @@ tests:
      - eval_time: 7m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:15Z, backlog 91
-      - eval_time: 7m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      # 02:17Z, backlog 65
-      - eval_time: 9m
-        alertname: CNPGWalArchiveBacklog
-        exp_alerts: []
-      # 02:17Z, backlog 65
-      - eval_time: 9m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:21Z, backlog 13
      - eval_time: 13m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:21Z, backlog 13
-      - eval_time: 13m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:23Z, backlog 0
      - eval_time: 15m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:23Z, backlog 0
-      - eval_time: 15m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # 2026-09-26: the nightly Home Assistant recorder purge at 04:12 local.
-  # peak 113 at 02:15Z, fill 28.5/min, drain 14.1/min -> 11 samples above 5 (10m pending vs for: 15m), 5 above 64 (4m pending vs for: 8m).
+  # peak 113 at 02:15Z, fill 28.5/min, drain 14.1/min -> 11 samples above 5 (10m pending vs for: 20m).
  - name: nightly burst 2026-09-26 — recorder purge, benign, self-clearing
    interval: 1m
    input_series:
@@ -314,61 +202,23 @@ tests:
      - eval_time: 7m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:15Z, backlog 113
-      - eval_time: 7m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      # 02:18Z, backlog 71
-      - eval_time: 10m
-        alertname: CNPGWalArchiveBacklog
-        exp_alerts: []
-      # 02:18Z, backlog 71
-      - eval_time: 10m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:22Z, backlog 14
      - eval_time: 14m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:22Z, backlog 14
-      - eval_time: 14m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:24Z, backlog 0
      - eval_time: 16m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:24Z, backlog 0
-      - eval_time: 16m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # ---------------------------------------------------------------------
  # 8. THE ADVERSARIAL ENVELOPE. A night that did not happen, built from
  # the worst fill (39.7/min, 09-23), the highest peak (119, 09-23) and
-  # the slowest drain (13.0/min, 09-25) in the week. Inside what the
-  # cluster has demonstrated, worse than anything it actually did.
-  #
-  # This is where the two `for` windows get measured for margin rather
-  # than asserted about. Margin is PENDING time, which is one less than
-  # the run length — an alert goes pending on the first true evaluation
-  # and fires at activeAt + for, so 6 consecutive samples above 64
-  # accumulate 5 minutes, not 6:
-  #
-  #   above 5:   11 samples ->  10m pending  vs `for: 15m`  -> 5m margin
-  #   above 64:   6 samples ->   5m pending  vs `for: 8m`   -> 3m margin
-  #
-  # The worst MEASURED night accumulates 4m above 64 (5 samples, 09-20
-  # and 09-23), so the envelope is one minute tighter than anything that
-  # actually happened. 3 minutes is the number to defend if anyone
-  # proposes lowering `for: 8m` — and mutations.txt:surge-for-5m is the
-  # control that makes this case load-bearing rather than decorative: at
-  # `for: 5m` the surge rule is still silent on all seven real nights and
-  # fires only here. Delete this case and that mutation stops being
-  # caught.
+  # the slowest drain (13.0/min, 09-25) in the week. 11 samples above 5,
+  # 10m pending against `for: 20m`.
  # ---------------------------------------------------------------------
  # Synthetic: max fill 39.7 + max peak 119 + min drain 13.0.
-  # peak 119 at 02:15Z, fill 39.7/min, drain 13.0/min -> 11 samples above 5 (10m pending vs for: 15m), 6 above 64 (5m pending vs for: 8m).
+  # peak 119 at 02:15Z, fill 39.7/min, drain 13.0/min -> 11 samples above 5 (10m pending vs for: 20m).
  - name: adversarial envelope — worst measured fill and peak, slowest drain
    interval: 1m
    input_series:
@@ -384,98 +234,94 @@ tests:
      - eval_time: 7m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:15Z, backlog 119
-      - eval_time: 7m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      # 02:19Z, backlog 67
-      - eval_time: 11m
-        alertname: CNPGWalArchiveBacklog
-        exp_alerts: []
-      # 02:19Z, backlog 67
-      - eval_time: 11m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:23Z, backlog 15
      - eval_time: 15m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:23Z, backlog 15
-      - eval_time: 15m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
      # 02:26Z, backlog 0
      - eval_time: 18m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      # 02:26Z, backlog 0
-      - eval_time: 18m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # ---------------------------------------------------------------------
-  # 9. THE FAILURE BOTH RULES EXIST FOR: an archiver that stopped with
-  # 70 segments (1.1 GiB) queued. Not a burst — it does not drain.
-  # Surge fires first (8m after the backlog crosses 64), the
-  # load-bearing rule 7 minutes later (15m after it crosses 5). Both
-  # assertions bracket the transition: one minute early nothing is
+  # 9. THE WORST NIGHT ON RECORD, 2026-09-11, as raw 30s samples (see
+  # WORST_NIGHT in the generator). Peak 141, 26 samples above 5 = 12.5m
+  # of true expression. At `evaluation_interval: 1m` the rule sees it
+  # true from 02:13Z to 02:25Z: 12m pending against `for: 20m`,
+  # 8m of margin. This is the case that makes the margin
+  # load-bearing: at `for: 12m` it fires at 02:25Z, and nothing else in
+  # the suite notices (mutations.txt:backlog-for-12m).
+  # ---------------------------------------------------------------------
+  - name: worst night on record 2026-09-11 — peak 141, raw samples
+    interval: 30s
+    input_series:
+      - series: 'cnpg_collector_pg_wal_archive_status{namespace="home-assistant",pod="ha-pg-1",container="postgres",role="primary",value="ready"}'
+        # 02:08:00Z .. 02:47:00Z, one sample per 30s
+        values: '0 0 0 0 0 0 0 0 0 28 53 80 110 141 136 133 130 125 117 117 115 112 102 91 81 75 68 60 52 45 37 29 22 15 8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0'
+      - series: 'cnpg_collector_pg_wal_archive_status{namespace="home-assistant",pod="ha-pg-1",container="postgres",role="primary",value="done"}'
+        values: '148000 148007 148014 148021 148028 148035 148042 148049 148056 148063 148070 148077 148084 148091 148098 148105 148112 148119 148126 148133 148140 148147 148154 148161 148168 148175 148182 148189 148196 148203 148210 148217 148224 148231 148238 148245 148252 148259 148266 148273 148280 148287 148294 148301 148308 148315 148322 148329 148336 148343 148350 148357 148364 148371 148378 148385 148392 148399 148406 148413 148420 148427 148434 148441 148448 148455 148462 148469 148476 148483 148490 148497 148504 148511 148518 148525 148532 148539 148546'
+    alert_rule_test:
+      # 02:14Z, backlog 110
+      - eval_time: 6m
+        alertname: CNPGWalArchiveBacklog
+        exp_alerts: []
+      # 02:25Z, backlog 8
+      - eval_time: 17m
+        alertname: CNPGWalArchiveBacklog
+        exp_alerts: []
+      # 02:47Z, backlog 0
+      - eval_time: 39m
+        alertname: CNPGWalArchiveBacklog
+        exp_alerts: []
+
+  # ---------------------------------------------------------------------
+  # 10. THE FAILURE THE RULE EXISTS FOR: an archiver that stopped with
+  # 70 segments (1.1 GiB) queued. Not a burst — it does not drain. The
+  # backlog crosses 5 at t=5m, so `for: 20m` elapses at t=25m.
+  # The assertions bracket the transition: one minute early nothing is
  # firing, on the minute it is.
  # ---------------------------------------------------------------------
-  - name: stalled archiver at 70 segments — both rules must fire
+  - name: stalled archiver at 70 segments — must fire
    interval: 1m
    input_series:
      - series: 'cnpg_collector_pg_wal_archive_status{namespace="home-assistant",pod="ha-pg-1",container="postgres",role="primary",value="ready"}'
        values: '0 0 0 0 0 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70'
      - series: 'cnpg_collector_pg_wal_archive_status{namespace="home-assistant",pod="ha-pg-1",container="postgres",role="primary",value="done"}'
-        # Flat, because a stalled archiver archives nothing. The rules do
+        # Flat, because a stalled archiver archives nothing. The rule does
        # not read this; it is here so the input stays realistic.
        values: '148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000 148000'
    alert_rule_test:
-      # Backlog crosses 64 at t=5m, so `for: 8m` elapses at t=13m.
-      - eval_time: 12m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
-      - eval_time: 13m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts:
-          # No `role` label: proof that `max by (namespace, pod)`
-          # aggregated it away. See mutations.txt:no-max-by.
-          - exp_labels:
-              severity: warning
-              namespace: home-assistant
-              pod: ha-pg-1
-            exp_annotations:
-              summary: "CNPG WAL archiving cannot keep up on home-assistant/ha-pg-1"
-              description: "70 WAL segments (>1 GiB) have been queued for archiving for 8 minutes, which is longer than a healthy archiver needs to clear that much — from a backlog of 119 it takes under 5. Point-in-time recovery is degraded for everything after the last archived segment, and pg_wal grows against the PVC's free space until it clears. Start with the archiver: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`, and check whether its ObjectStore sets `wal.maxParallel` — unset means one segment at a time, a ceiling of about 14 segments/min (REN-129)."
-      # Backlog crosses 5 at t=5m, so `for: 15m` elapses at t=20m.
-      - eval_time: 19m
+      - eval_time: 24m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      - eval_time: 20m
+      - eval_time: 25m
        alertname: CNPGWalArchiveBacklog
        exp_alerts:
+          # No `role` label: proof that `max by (namespace, pod)` aggregated it away. See mutations.txt:no-max-by.
          - exp_labels:
              severity: warning
              namespace: home-assistant
              pod: ha-pg-1
            exp_annotations:
              summary: "CNPG WAL archiving is falling behind on home-assistant/ha-pg-1"
-              description: "70 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."
+              description: "70 WAL segments have been waiting to archive for over 20 minutes. NOTE ON TIMING: a nightly write burst on ha-pg-1 at 02:14Z (04:14 local) is normal and drains itself within about 13 minutes, and this rule is tuned to stay silent through it — so if this fired, the backlog did NOT clear and the archiver is stuck. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."

  # ---------------------------------------------------------------------
-  # 10. THE STALL THE SURGE RULE CANNOT SEE: 12 segments, stuck. Under
-  # 1 GiB forever, so the surge rule is right to stay silent — and this
-  # is exactly the shape that makes the >5 rule load-bearing rather than
-  # a slower copy of it. Point-in-time recovery is just as broken at 12
-  # segments as at 70.
+  # 11. THE STALL A DEPTH THRESHOLD CANNOT SEE: 8 segments, stuck. It
+  # would never reach any number tuned above the nightly envelope, and
+  # point-in-time recovery is just as broken at 8 segments as at 70. This
+  # is why REN-152 keyed the rule on persistence and retired the depth rule.
  # ---------------------------------------------------------------------
-  - name: low stall at 12 segments — only the >5 rule may fire
+  - name: low stall at 8 segments — must fire
    interval: 1m
    input_series:
      - series: 'cnpg_collector_pg_wal_archive_status{namespace="home-assistant",pod="ha-pg-1",container="postgres",role="primary",value="ready"}'
-        values: '0 0 0 0 0 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12'
+        values: '0 0 0 0 0 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8'
    alert_rule_test:
-      - eval_time: 20m
+      - eval_time: 24m
+        alertname: CNPGWalArchiveBacklog
+        exp_alerts: []
+      - eval_time: 25m
        alertname: CNPGWalArchiveBacklog
        exp_alerts:
          - exp_labels:
@@ -484,14 +330,10 @@ tests:
              pod: ha-pg-1
            exp_annotations:
              summary: "CNPG WAL archiving is falling behind on home-assistant/ha-pg-1"
-              description: "12 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."
-      # 34 minutes at 12 segments and the surge rule is still quiet.
-      - eval_time: 39m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []
+              description: "8 WAL segments have been waiting to archive for over 20 minutes. NOTE ON TIMING: a nightly write burst on ha-pg-1 at 02:14Z (04:14 local) is normal and drains itself within about 13 minutes, and this rule is tuned to stay silent through it — so if this fired, the backlog did NOT clear and the archiver is stuck. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."

  # ---------------------------------------------------------------------
-  # 11. PINNED AT EXACTLY 5 for the whole window. `> 5` is strict, so
+  # 12. PINNED AT EXACTLY 5 for the whole window. `> 5` is strict, so
  # this is silence. A rule written `>= 5` would page on it, and a rule
  # that had lost its `value="ready"` selector would page on the `done`
  # counter regardless of what `ready` does — which is what the second
@@ -508,19 +350,16 @@ tests:
      - eval_time: 39m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      - eval_time: 39m
-        alertname: CNPGWalArchiveBacklogSurge
-        exp_alerts: []

  # ---------------------------------------------------------------------
-  # 12. FAILOVER DURING A STALL — the case `max by (namespace, pod)` was
+  # 13. FAILOVER DURING A STALL — the case `max by (namespace, pod)` was
  # added for. The backlog starts at t=5m on the primary; at t=11m the
  # instance is promoted and `role` changes, which is a NEW SERIES to
  # Prometheus and restarts the `for` timer on the raw expression.
  # Aggregating `role` away keeps one continuous series, so the page
-  # still lands 15m after the backlog started (t=20m) rather than 15m
-  # after the failover. Without `max by` this case is silent at t=20m —
-  # see mutations.txt:no-max-by.
+  # still lands 20m after the backlog started (t=25m) rather
+  # than 20m after the failover. Without `max by` this case is
+  # silent at t=25m — see mutations.txt:no-max-by.
  # ---------------------------------------------------------------------
  - name: role flip mid-stall — for timer must not restart
    interval: 1m
@@ -532,10 +371,10 @@ tests:
        # Starts at t=11m. `_` is a gap, not a zero.
        values: '_ _ _ _ _ _ _ _ _ _ _ 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70 70'
    alert_rule_test:
-      - eval_time: 19m
+      - eval_time: 24m
        alertname: CNPGWalArchiveBacklog
        exp_alerts: []
-      - eval_time: 20m
+      - eval_time: 25m
        alertname: CNPGWalArchiveBacklog
        exp_alerts:
          - exp_labels:
@@ -544,10 +383,10 @@ tests:
              pod: ha-pg-1
            exp_annotations:
              summary: "CNPG WAL archiving is falling behind on home-assistant/ha-pg-1"
-              description: "70 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."
+              description: "70 WAL segments have been waiting to archive for over 20 minutes. NOTE ON TIMING: a nightly write burst on ha-pg-1 at 02:14Z (04:14 local) is normal and drains itself within about 13 minutes, and this rule is tuned to stay silent through it — so if this fired, the backlog did NOT clear and the archiver is stuck. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."

  # ---------------------------------------------------------------------
-  # 13. ONE STALLED INSTANCE AMONG HEALTHY ONES. There are 12 CNPG
+  # 14. ONE STALLED INSTANCE AMONG HEALTHY ONES. There are 12 CNPG
  # instances in this cluster. `by (namespace, pod)` has to keep them
  # apart: exactly one alert, naming the pod that is actually stuck, so
  # the 04:00 reader is not sent to look at the wrong Postgres.
@@ -562,7 +401,7 @@ tests:
      - series: 'cnpg_collector_pg_wal_archive_status{namespace="immich",pod="immich-pg-1",container="postgres",role="primary",value="ready"}'
        values: '0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0'
    alert_rule_test:
-      - eval_time: 20m
+      - eval_time: 25m
        alertname: CNPGWalArchiveBacklog
        exp_alerts:
          - exp_labels:
@@ -571,10 +410,10 @@ tests:
              pod: ha-pg-1
            exp_annotations:
              summary: "CNPG WAL archiving is falling behind on home-assistant/ha-pg-1"
-              description: "70 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."
+              description: "70 WAL segments have been waiting to archive for over 20 minutes. NOTE ON TIMING: a nightly write burst on ha-pg-1 at 02:14Z (04:14 local) is normal and drains itself within about 13 minutes, and this rule is tuned to stay silent through it — so if this fired, the backlog did NOT clear and the archiver is stuck. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`."

  # ---------------------------------------------------------------------
-  # 14. THE EPOCH-ZERO GUARD on CNPGBackupFailed. The metric is a unix
+  # 15. THE EPOCH-ZERO GUARD on CNPGBackupFailed. The metric is a unix
  # timestamp and reads 0 when no failure was ever recorded, so the rule
  # carries a `> 0` guard to stop that default being read as "failed in
  # 1970". Held at 0 for the whole window: silence. Nothing else in this
@@ -0,0 +1,115 @@
+# promtool test rules for prometheusrule-external-secrets.yaml.
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh, which strips `spec:` from the manifest into a
+# mirrored tree — hence `../` below, resolved from this file's directory. See
+# README.md here. The negative controls for this suite are the `eso-*` entries
+# in mutations.txt.
+#
+# WHAT IT IS FOR. Every claim in the rule file that a reviewer would otherwise
+# have to take on trust is asserted here instead: that the tombstone series does
+# not alert, that `for: 15m` really is 15m, that the annotation names the
+# ExternalSecret's own namespace and not the controller's, and that `max`
+# survives a two-pod scrape where `sum` does not.
+#
+# THE REJECTED FORM, for the record. REN-100 and ESO's shipped dashboard both
+# use `sum(...) by (namespace, name) == 1`. Run against the identical input
+# series below, that expression produces exactly one alert,
+# {namespace="external-secrets", name="gitea-db"} — the wrong namespace on the
+# one it catches, and nothing at all for monitoring/loki-basic-auth, whose two
+# scraped pods sum to 2 and stop matching `== 1`. Both defects, one expression.
+rule_files:
+  - ../prometheusrule-external-secrets.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # Series shapes copied from the live cluster after #994 landed: the metric's
+  # own `namespace` has already been shadowed to `exported_namespace` by the
+  # PodMonitor's target label, and `namespace` is the controller pod's.
+  - interval: 1m
+    input_series:
+      # A healthy ExternalSecret. ESO writes BOTH polarities of the Ready
+      # condition on every reconcile, the inactive one at 0 — which is why the
+      # rule needs `== 1` and not a bare selector.
+      - series: 'externalsecret_status_condition{condition="Ready",status="True",name="uptime-robot-api-key",exported_namespace="uptime-monitor",namespace="external-secrets",job="monitoring/external-secrets",pod="external-secrets-56c649799b-t65gf"}'
+        values: '1+0x60'
+      - series: 'externalsecret_status_condition{condition="Ready",status="False",name="uptime-robot-api-key",exported_namespace="uptime-monitor",namespace="external-secrets",job="monitoring/external-secrets",pod="external-secrets-56c649799b-t65gf"}'
+        values: '0+0x60'
+
+      # Starts failing at t=10m and stays failed.
+      - series: 'externalsecret_status_condition{condition="Ready",status="False",name="gitea-db",exported_namespace="gitea",namespace="external-secrets",job="monitoring/external-secrets",pod="external-secrets-56c649799b-t65gf"}'
+        values: '0+0x9 1+0x50'
+      - series: 'externalsecret_status_condition{condition="Ready",status="True",name="gitea-db",exported_namespace="gitea",namespace="external-secrets",job="monitoring/external-secrets",pod="external-secrets-56c649799b-t65gf"}'
+        values: '1+0x9 0+0x50'
+
+      # Tombstone left behind for an ExternalSecret that no longer exists. Held
+      # at 1 forever until the controller restarts, so a rule that matched
+      # `condition!="Ready"` or ignored `status` would page on it every day.
+      - series: 'externalsecret_status_condition{condition="Deleted",status="False",name="wiki-index-basic-auth",exported_namespace="ai",namespace="external-secrets",job="monitoring/external-secrets",pod="external-secrets-56c649799b-t65gf"}'
+        values: '1+0x60'
+
+      # The same failing ExternalSecret reported by TWO controller pods, as
+      # happens for the seconds of a Deployment rollout when Prometheus still
+      # has the old target and already has the new one.
+      - series: 'externalsecret_status_condition{condition="Ready",status="False",name="loki-basic-auth",exported_namespace="monitoring",namespace="external-secrets",job="monitoring/external-secrets",pod="external-secrets-56c649799b-t65gf"}'
+        values: '0+0x9 1+0x50'
+      - series: 'externalsecret_status_condition{condition="Ready",status="False",name="loki-basic-auth",exported_namespace="monitoring",namespace="external-secrets",job="monitoring/external-secrets",pod="external-secrets-7d4f9c6b88-qq2xv"}'
+        values: '0+0x9 1+0x50'
+
+    alert_rule_test:
+      # 5m into the failure: pending, not firing. Asserts the window is real —
+      # a `for:` accidentally set to 0 passes every other case here and fails
+      # only this one.
+      - eval_time: 15m
+        alertname: ExternalSecretSyncFailing
+        exp_alerts: []
+
+      # 20m into the failure. Both failing ExternalSecrets fire, ONCE each; the
+      # healthy one and the tombstone stay silent. The annotations are asserted
+      # in full because the whole point of REN-103's label finding is that the
+      # text names `gitea`, not `external-secrets`.
+      - eval_time: 30m
+        alertname: ExternalSecretSyncFailing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              exported_namespace: gitea
+              name: gitea-db
+            exp_annotations:
+              summary: "ExternalSecret gitea/gitea-db is not syncing"
+              description: "ESO has been unable to sync this ExternalSecret for 15 minutes. The Secret it manages still holds its last successfully synced value, so consumers keep working — what breaks is that a rotation at the provider no longer propagates, which surfaces much later as a credential that stopped being accepted. Get the reason with `kubectl -n gitea get externalsecret gitea-db -o jsonpath='{.status.conditions}'`; the common causes are a key that was renamed or removed in OpenBao, a ClusterSecretStore that cannot authenticate, and a target Secret owned by something else."
+          - exp_labels:
+              severity: warning
+              exported_namespace: monitoring
+              name: loki-basic-auth
+            exp_annotations:
+              summary: "ExternalSecret monitoring/loki-basic-auth is not syncing"
+              description: "ESO has been unable to sync this ExternalSecret for 15 minutes. The Secret it manages still holds its last successfully synced value, so consumers keep working — what breaks is that a rotation at the provider no longer propagates, which surfaces much later as a credential that stopped being accepted. Get the reason with `kubectl -n monitoring get externalsecret loki-basic-auth -o jsonpath='{.status.conditions}'`; the common causes are a key that was renamed or removed in OpenBao, a ClusterSecretStore that cannot authenticate, and a target Secret owned by something else."
+
+      # Metrics are present, so the dead-man's-switch stays quiet.
+      - eval_time: 30m
+        alertname: ExternalSecretMetricsMissing
+        exp_alerts: []
+
+  # The scrape disappears entirely: PodMonitor deleted, selector broken by a
+  # chart bump, or the `monitoring` ingress grant reverted out of
+  # external-secrets-ns.yaml. ExternalSecretSyncFailing cannot fire on series
+  # that do not exist — this is the case that made REN-103 necessary, so it is
+  # the one the dead-man's-switch has to cover.
+  - interval: 1m
+    input_series:
+      - series: 'up{job="monitoring/external-secrets"}'
+        values: '0+0x60'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: ExternalSecretSyncFailing
+        exp_alerts: []
+      - eval_time: 30m
+        alertname: ExternalSecretMetricsMissing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "No ExternalSecret metrics are reaching Prometheus"
+              description: "`externalsecret_status_condition` has no series. Either the ESO controller is gone or its scrape stopped being collected, and ExternalSecretSyncFailing is blind for as long as this is true. Check the PodMonitor `monitoring/external-secrets` still selects the controller pods, and that external-secrets-ns.yaml still admits `monitoring` on TCP/8080."
@@ -63,6 +63,25 @@ NIGHTS = [
 # has demonstrated it can do. Nothing here is extrapolated past the measurements.
 ENVELOPE = ("envelope", 119, "02:15", 39.7, 13.0, None)

+# The worst night in Prometheus' 27-night retention (2026-08-30..09-26):
+# 2026-09-11, peak 141, the longest excursion above 5 on record. Not rebuilt
+# from a model like NIGHTS — these are the raw samples, read back through the
+# API server's services/proxy with
+#   max by (namespace, pod) (cnpg_collector_pg_wal_archive_status{
+#     value="ready", pod="ha-pg-1"})
+# over 02:08:00Z..02:47:00Z at the real 30s scrape step (REN-152). 26
+# consecutive samples above 5 = 12.5 minutes pending, which is the number the
+# `for: 20m` margin is measured against, and why mutations.txt:backlog-for-12m
+# is caught here and nowhere else.
+WORST_NIGHT = ("2026-09-11", [
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 53, 80, 110, 141, 136, 133, 130, 125, 117,
+    117, 115, 112, 102, 91, 81, 75, 68, 60, 52, 45, 37, 29, 22, 15, 8, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+])
+
+FOR_MIN = 20  # `for:` on CNPGWalArchiveBacklog, in minutes
+
 # Series labels. `container`, `role` and `instance` are on the real series and
 # are carried here on purpose: they are what `max by (namespace, pod)` has to
 # aggregate away, and a test whose input is already aggregated cannot see whether
@@ -185,10 +204,9 @@ def eval_minutes(vals):
    run above it (the moment maximum pending time has accumulated). Plus the
    peak, plus two minutes after the backlog has settled back to 0."""
    picks = set()
-    for thr in (5, 64):
-        _, last = longest_run_above(vals, thr)
-        if last is not None:
-            picks.add(last)
+    _, last = longest_run_above(vals, 5)
+    if last is not None:
+        picks.add(last)
    picks.add(max(range(len(vals)), key=lambda i: vals[i]))
    settled = next(
        (i for i in range(len(vals) - 1, -1, -1) if vals[i] > 0),
@@ -209,13 +227,11 @@ def burst_case(name, date, peak, peak_min_s, fill, drain, why):
    peak_min = parse_hhmm(peak_min_s)
    vals = trace(peak, peak_min, fill, drain)
    n5, _ = longest_run_above(vals, 5)
-    n64, _ = longest_run_above(vals, 64)
    w(f"  # {why}")
    w(
        f"  # peak {peak} at {peak_min_s}Z, fill {fill}/min, drain {drain}/min"
        f" -> {n5} samples above 5 ({pending_minutes(vals, 5)}m pending vs"
-        f" for: 15m), {n64} above 64 ({pending_minutes(vals, 64)}m pending vs"
-        f" for: 8m)."
+        f" for: {FOR_MIN}m)."
    )
    w(f"  - name: {name}")
    w("    interval: 1m")
@@ -230,13 +246,47 @@ def burst_case(name, date, peak, peak_min_s, fill, drain, why):
    w("    alert_rule_test:")
    for i in eval_minutes(vals):
        clock = hhmm(T0 + i)
-        for alert in ("CNPGWalArchiveBacklog", "CNPGWalArchiveBacklogSurge"):
-            w(f"      # {clock}Z, backlog {vals[i]}")
-            w(f"      - eval_time: {i}m")
-            w(f"        alertname: {alert}")
-            w("        exp_alerts: []")
+        w(f"      # {clock}Z, backlog {vals[i]}")
+        w(f"      - eval_time: {i}m")
+        w("        alertname: CNPGWalArchiveBacklog")
+        w("        exp_alerts: []")
    w("")
-    return vals, n5, n64
+    return vals, n5
+
+
+def desc(n, pod=POD, ns=NS):
+    """The rule's `description` as promtool renders it for `n` segments."""
+    return (
+        f"{n} WAL segments have been waiting to archive for over {FOR_MIN} minutes."
+        " NOTE ON TIMING: a nightly write burst on ha-pg-1 at 02:14Z (04:14 local)"
+        " is normal and drains itself within about 13 minutes, and this rule is"
+        " tuned to stay silent through it — so if this fired, the backlog did NOT"
+        " clear and the archiver is stuck. Point-in-time recovery is broken for"
+        " anything after the last archived segment, and pg_wal will grow until the"
+        " PVC fills. Check the barman-cloud sidecar logs and the ObjectStore"
+        f" credentials: `kubectl -n {ns} logs {pod} -c plugin-barman-cloud`."
+    )
+
+
+def firing(eval_time, n, pod=POD, ns=NS, note=None):
+    w(f"      - eval_time: {eval_time}")
+    w("        alertname: CNPGWalArchiveBacklog")
+    w("        exp_alerts:")
+    if note:
+        w(f"          # {note}")
+    w("          - exp_labels:")
+    w("              severity: warning")
+    w(f"              namespace: {ns}")
+    w(f"              pod: {pod}")
+    w("            exp_annotations:")
+    w(f'              summary: "CNPG WAL archiving is falling behind on {ns}/{pod}"')
+    w(f'              description: "{desc(n, pod, ns)}"')
+
+
+def silent(eval_time, alert="CNPGWalArchiveBacklog"):
+    w(f"      - eval_time: {eval_time}")
+    w(f"        alertname: {alert}")
+    w("        exp_alerts: []")


 def main():
@@ -251,23 +301,22 @@ def main():
    assert_window_covers_bursts()

    if args.report:
-        # `pend` columns are what the `for` windows are actually compared
-        # against; see pending_minutes(). `>5 pub` is Iris's published column
-        # and is the only number here that is not derived by this script.
+        # `pend` is what the `for` window is actually compared against; see
+        # pending_minutes(). `>5 pub` is Iris's published column and is the
+        # only number here that is not derived by this script.
        print(f"{'night':12} {'peak':>5} {'fill':>6} {'drain':>6} "
-              f"{'>5 pub':>7} {'>5 n':>6} {'>5 pend':>8} "
-              f"{'>64 n':>6} {'>64 pend':>9}")
+              f"{'>5 pub':>7} {'>5 n':>6} {'>5 pend':>8}")
        for date, peak, pm, fill, drain, dur in NIGHTS + [ENVELOPE]:
            vals = trace(peak, parse_hhmm(pm), fill, drain)
            n5, _ = longest_run_above(vals, 5)
-            n64, _ = longest_run_above(vals, 64)
            print(f"{date:12} {peak:5} {fill:6} {drain:6} "
                  f"{('-' if dur is None else dur):>7} {n5:>6} "
-                  f"{pending_minutes(vals, 5):>8} "
-                  f"{n64:>6} {pending_minutes(vals, 64):>9}")
+                  f"{pending_minutes(vals, 5):>8}")
+        wn, _ = longest_run_above(WORST_NIGHT[1], 5)
+        print(f"{WORST_NIGHT[0]:12} {max(WORST_NIGHT[1]):5} {'raw':>6} {'30s':>6} "
+              f"{'-':>7} {wn:>6} {max(0, wn - 1) / 2:>8}")
        print()
-        print("for: 15m on >5  must exceed every '>5 pend'  above.")
-        print("for: 8m  on >64 must exceed every '>64 pend' above.")
+        print(f"for: {FOR_MIN}m on >5 must exceed every '>5 pend' above.")
        return 0

    w("# GENERATED by generate-cnpg-backup-test.py — see README.md in this")
@@ -275,9 +324,10 @@ def main():
    w("#")
    w("# Unit tests for kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml.")
    w("# Every assertion below is either one of Iris's seven measured nights")
-    w("# (REN-129) rebuilt from its published (peak, fill, drain) numbers, an")
+    w("# (REN-129) rebuilt from its published (peak, fill, drain) numbers, the")
+    w("# worst night in Prometheus' retention as raw samples (REN-152), an")
    w("# adversarial combination of the extremes of those numbers, or a failure")
-    w("# this cluster has not had but the rules exist to catch.")
+    w("# this cluster has not had but the rule exists to catch.")
    w("#")
    w("# `rule_files` entries are resolved RELATIVE TO THIS FILE'S DIRECTORY, not")
    w("# to the working directory promtool was invoked from — promtool joins")
@@ -296,14 +346,13 @@ def main():
    w("")
    w("tests:")
    w("  # ---------------------------------------------------------------------")
-    w("  # 1-7. THE SEVEN MEASURED NIGHTS. Both rules must stay silent through")
-    w("  # every one of them. This is the half of the suite that has already")
-    w("  # caught something: the surge rule shipped for review at `for: 2m` and")
-    w("  # fires here on 6 of these 7 nights (mutations.txt:surge-for-2m).")
+    w("  # 1-7. THE SEVEN MEASURED NIGHTS. The rule must stay silent through")
+    w("  # every one of them. The retired depth rule (`> 64`) shipped for review")
+    w("  # at `for: 2m` and fired on 6 of these 7 nights; REN-152 removed it.")
    w("  # ---------------------------------------------------------------------")
    checks = []
    for date, peak, pm, fill, drain, dur in NIGHTS:
-        vals, n5, n64 = burst_case(
+        vals, n5 = burst_case(
            f"nightly burst {date} — recorder purge, benign, self-clearing",
            date, peak, pm, fill, drain,
            f"{date}: the nightly Home Assistant recorder purge at 04:12 local.",
@@ -313,26 +362,8 @@ def main():
    w("  # ---------------------------------------------------------------------")
    w("  # 8. THE ADVERSARIAL ENVELOPE. A night that did not happen, built from")
    w("  # the worst fill (39.7/min, 09-23), the highest peak (119, 09-23) and")
-    w("  # the slowest drain (13.0/min, 09-25) in the week. Inside what the")
-    w("  # cluster has demonstrated, worse than anything it actually did.")
-    w("  #")
-    w("  # This is where the two `for` windows get measured for margin rather")
-    w("  # than asserted about. Margin is PENDING time, which is one less than")
-    w("  # the run length — an alert goes pending on the first true evaluation")
-    w("  # and fires at activeAt + for, so 6 consecutive samples above 64")
-    w("  # accumulate 5 minutes, not 6:")
-    w("  #")
-    w("  #   above 5:   11 samples ->  10m pending  vs `for: 15m`  -> 5m margin")
-    w("  #   above 64:   6 samples ->   5m pending  vs `for: 8m`   -> 3m margin")
-    w("  #")
-    w("  # The worst MEASURED night accumulates 4m above 64 (5 samples, 09-20")
-    w("  # and 09-23), so the envelope is one minute tighter than anything that")
-    w("  # actually happened. 3 minutes is the number to defend if anyone")
-    w("  # proposes lowering `for: 8m` — and mutations.txt:surge-for-5m is the")
-    w("  # control that makes this case load-bearing rather than decorative: at")
-    w("  # `for: 5m` the surge rule is still silent on all seven real nights and")
-    w("  # fires only here. Delete this case and that mutation stops being")
-    w("  # caught.")
+    w("  # the slowest drain (13.0/min, 09-25) in the week. 11 samples above 5,")
+    w("  # 10m pending against `for: 20m`.")
    w("  # ---------------------------------------------------------------------")
    burst_case(
        "adversarial envelope — worst measured fill and peak, slowest drain",
@@ -340,92 +371,80 @@ def main():
        why="Synthetic: max fill 39.7 + max peak 119 + min drain 13.0.",
    )

-    # ---- 9. stalled archiver, above the surge threshold -------------------
-    stalled = [0] * 5 + [70] * 35
+    date, raw = WORST_NIGHT
+    wn, wlast = longest_run_above(raw, 5)
+    peak_i = max(range(len(raw)), key=lambda i: raw[i])
    w("  # ---------------------------------------------------------------------")
-    w("  # 9. THE FAILURE BOTH RULES EXIST FOR: an archiver that stopped with")
-    w("  # 70 segments (1.1 GiB) queued. Not a burst — it does not drain.")
-    w("  # Surge fires first (8m after the backlog crosses 64), the")
-    w("  # load-bearing rule 7 minutes later (15m after it crosses 5). Both")
-    w("  # assertions bracket the transition: one minute early nothing is")
+    w(f"  # 9. THE WORST NIGHT ON RECORD, {date}, as raw 30s samples (see")
+    w("  # WORST_NIGHT in the generator). Peak 141, 26 samples above 5 = 12.5m")
+    w("  # of true expression. At `evaluation_interval: 1m` the rule sees it")
+    w(f"  # true from 02:13Z to 02:25Z: 12m pending against `for: {FOR_MIN}m`,")
+    w(f"  # {FOR_MIN - 12}m of margin. This is the case that makes the margin")
+    w("  # load-bearing: at `for: 12m` it fires at 02:25Z, and nothing else in")
+    w("  # the suite notices (mutations.txt:backlog-for-12m).")
+    w("  # ---------------------------------------------------------------------")
+    w(f"  - name: worst night on record {date} — peak 141, raw samples")
+    w("    interval: 30s")
+    w("    input_series:")
+    w(f"      - series: '{labels()}'")
+    w(f"        # {hhmm(T0)}:00Z .. {hhmm(T0 + 39)}:00Z, one sample per 30s")
+    w(f"        values: '{fmt(raw)}'")
+    w(f"      - series: '{labels(value='done')}'")
+    w(f"        values: '{fmt(counter(148_000, 7, len(raw)))}'")
+    w("    alert_rule_test:")
+    for i in (peak_i, wlast, len(raw) - 1):
+        # eval times in whole minutes: the group evaluates on the minute.
+        m = (i // 2)
+        w(f"      # {hhmm(T0 + m)}Z, backlog {raw[m * 2]}")
+        silent(f"{m}m")
+    w("")
+
+    # ---- 10. stalled archiver, well above threshold ------------------------
+    stalled = [0] * 5 + [70] * 35
+    fire_at = 5 + FOR_MIN
+    w("  # ---------------------------------------------------------------------")
+    w("  # 10. THE FAILURE THE RULE EXISTS FOR: an archiver that stopped with")
+    w("  # 70 segments (1.1 GiB) queued. Not a burst — it does not drain. The")
+    w(f"  # backlog crosses 5 at t=5m, so `for: {FOR_MIN}m` elapses at t={fire_at}m.")
+    w("  # The assertions bracket the transition: one minute early nothing is")
    w("  # firing, on the minute it is.")
    w("  # ---------------------------------------------------------------------")
-    w("  - name: stalled archiver at 70 segments — both rules must fire")
+    w("  - name: stalled archiver at 70 segments — must fire")
    w("    interval: 1m")
    w("    input_series:")
    w(f"      - series: '{labels()}'")
    w(f"        values: '{fmt(stalled)}'")
    w(f"      - series: '{labels(value='done')}'")
-    w("        # Flat, because a stalled archiver archives nothing. The rules do")
+    w("        # Flat, because a stalled archiver archives nothing. The rule does")
    w("        # not read this; it is here so the input stays realistic.")
    w(f"        values: '{fmt([148_000] * SAMPLES)}'")
    w("    alert_rule_test:")
-    w("      # Backlog crosses 64 at t=5m, so `for: 8m` elapses at t=13m.")
-    w("      - eval_time: 12m")
-    w("        alertname: CNPGWalArchiveBacklogSurge")
-    w("        exp_alerts: []")
-    w("      - eval_time: 13m")
-    w("        alertname: CNPGWalArchiveBacklogSurge")
-    w("        exp_alerts:")
-    w("          # No `role` label: proof that `max by (namespace, pod)`")
-    w("          # aggregated it away. See mutations.txt:no-max-by.")
-    w("          - exp_labels:")
-    w("              severity: warning")
-    w(f"              namespace: {NS}")
-    w(f"              pod: {POD}")
-    w("            exp_annotations:")
-    w(f'              summary: "CNPG WAL archiving cannot keep up on {NS}/{POD}"')
-    w("              description: \"70 WAL segments (>1 GiB) have been queued for archiving for 8 minutes, which is longer than a healthy archiver needs to clear that much — from a backlog of 119 it takes under 5. Point-in-time recovery is degraded for everything after the last archived segment, and pg_wal grows against the PVC's free space until it clears. Start with the archiver: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`, and check whether its ObjectStore sets `wal.maxParallel` — unset means one segment at a time, a ceiling of about 14 segments/min (REN-129).\"")
-    w("      # Backlog crosses 5 at t=5m, so `for: 15m` elapses at t=20m.")
-    w("      - eval_time: 19m")
-    w("        alertname: CNPGWalArchiveBacklog")
-    w("        exp_alerts: []")
-    w("      - eval_time: 20m")
-    w("        alertname: CNPGWalArchiveBacklog")
-    w("        exp_alerts:")
-    w("          - exp_labels:")
-    w("              severity: warning")
-    w(f"              namespace: {NS}")
-    w(f"              pod: {POD}")
-    w("            exp_annotations:")
-    w(f'              summary: "CNPG WAL archiving is falling behind on {NS}/{POD}"')
-    w("              description: \"70 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`.\"")
+    silent(f"{fire_at - 1}m")
+    firing(f"{fire_at}m", 70, note="No `role` label: proof that `max by (namespace, pod)` "
+           "aggregated it away. See mutations.txt:no-max-by.")
    w("")

-    # ---- 10. low stall, below the surge threshold -------------------------
-    low = [0] * 5 + [12] * 35
+    # ---- 11. low stall ------------------------------------------------------
+    low = [0] * 5 + [8] * 35
    w("  # ---------------------------------------------------------------------")
-    w("  # 10. THE STALL THE SURGE RULE CANNOT SEE: 12 segments, stuck. Under")
-    w("  # 1 GiB forever, so the surge rule is right to stay silent — and this")
-    w("  # is exactly the shape that makes the >5 rule load-bearing rather than")
-    w("  # a slower copy of it. Point-in-time recovery is just as broken at 12")
-    w("  # segments as at 70.")
+    w("  # 11. THE STALL A DEPTH THRESHOLD CANNOT SEE: 8 segments, stuck. It")
+    w("  # would never reach any number tuned above the nightly envelope, and")
+    w("  # point-in-time recovery is just as broken at 8 segments as at 70. This")
+    w("  # is why REN-152 keyed the rule on persistence and retired the depth rule.")
    w("  # ---------------------------------------------------------------------")
-    w("  - name: low stall at 12 segments — only the >5 rule may fire")
+    w("  - name: low stall at 8 segments — must fire")
    w("    interval: 1m")
    w("    input_series:")
    w(f"      - series: '{labels()}'")
    w(f"        values: '{fmt(low)}'")
    w("    alert_rule_test:")
-    w("      - eval_time: 20m")
-    w("        alertname: CNPGWalArchiveBacklog")
-    w("        exp_alerts:")
-    w("          - exp_labels:")
-    w("              severity: warning")
-    w(f"              namespace: {NS}")
-    w(f"              pod: {POD}")
-    w("            exp_annotations:")
-    w(f'              summary: "CNPG WAL archiving is falling behind on {NS}/{POD}"')
-    w("              description: \"12 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`.\"")
-    w("      # 34 minutes at 12 segments and the surge rule is still quiet.")
-    w("      - eval_time: 39m")
-    w("        alertname: CNPGWalArchiveBacklogSurge")
-    w("        exp_alerts: []")
+    silent(f"{fire_at - 1}m")
+    firing(f"{fire_at}m", 8)
    w("")

-    # ---- 11. exactly at the threshold ------------------------------------
+    # ---- 12. exactly at the threshold ------------------------------------
    w("  # ---------------------------------------------------------------------")
-    w("  # 11. PINNED AT EXACTLY 5 for the whole window. `> 5` is strict, so")
+    w("  # 12. PINNED AT EXACTLY 5 for the whole window. `> 5` is strict, so")
    w("  # this is silence. A rule written `>= 5` would page on it, and a rule")
    w("  # that had lost its `value=\"ready\"` selector would page on the `done`")
    w("  # counter regardless of what `ready` does — which is what the second")
@@ -439,24 +458,19 @@ def main():
    w(f"      - series: '{labels(value='done')}'")
    w(f"        values: '{fmt(counter(148_000, 14))}'")
    w("    alert_rule_test:")
-    w("      - eval_time: 39m")
-    w("        alertname: CNPGWalArchiveBacklog")
-    w("        exp_alerts: []")
-    w("      - eval_time: 39m")
-    w("        alertname: CNPGWalArchiveBacklogSurge")
-    w("        exp_alerts: []")
+    silent("39m")
    w("")

-    # ---- 12. role flip ---------------------------------------------------
+    # ---- 13. role flip ---------------------------------------------------
    w("  # ---------------------------------------------------------------------")
-    w("  # 12. FAILOVER DURING A STALL — the case `max by (namespace, pod)` was")
+    w("  # 13. FAILOVER DURING A STALL — the case `max by (namespace, pod)` was")
    w("  # added for. The backlog starts at t=5m on the primary; at t=11m the")
    w("  # instance is promoted and `role` changes, which is a NEW SERIES to")
    w("  # Prometheus and restarts the `for` timer on the raw expression.")
    w("  # Aggregating `role` away keeps one continuous series, so the page")
-    w("  # still lands 15m after the backlog started (t=20m) rather than 15m")
-    w("  # after the failover. Without `max by` this case is silent at t=20m —")
-    w("  # see mutations.txt:no-max-by.")
+    w(f"  # still lands {FOR_MIN}m after the backlog started (t={fire_at}m) rather")
+    w(f"  # than {FOR_MIN}m after the failover. Without `max by` this case is")
+    w(f"  # silent at t={fire_at}m — see mutations.txt:no-max-by.")
    w("  # ---------------------------------------------------------------------")
    w("  - name: role flip mid-stall — for timer must not restart")
    w("    interval: 1m")
@@ -468,24 +482,13 @@ def main():
    w("        # Starts at t=11m. `_` is a gap, not a zero.")
    w(f"        values: '{'_ ' * 11}{fmt([70] * 29)}'")
    w("    alert_rule_test:")
-    w("      - eval_time: 19m")
-    w("        alertname: CNPGWalArchiveBacklog")
-    w("        exp_alerts: []")
-    w("      - eval_time: 20m")
-    w("        alertname: CNPGWalArchiveBacklog")
-    w("        exp_alerts:")
-    w("          - exp_labels:")
-    w("              severity: warning")
-    w(f"              namespace: {NS}")
-    w(f"              pod: {POD}")
-    w("            exp_annotations:")
-    w(f'              summary: "CNPG WAL archiving is falling behind on {NS}/{POD}"')
-    w("              description: \"70 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`.\"")
+    silent(f"{fire_at - 1}m")
+    firing(f"{fire_at}m", 70)
    w("")

-    # ---- 13. one stalled pod among healthy ones --------------------------
+    # ---- 14. one stalled pod among healthy ones --------------------------
    w("  # ---------------------------------------------------------------------")
-    w("  # 13. ONE STALLED INSTANCE AMONG HEALTHY ONES. There are 12 CNPG")
+    w("  # 14. ONE STALLED INSTANCE AMONG HEALTHY ONES. There are 12 CNPG")
    w("  # instances in this cluster. `by (namespace, pod)` has to keep them")
    w("  # apart: exactly one alert, naming the pod that is actually stuck, so")
    w("  # the 04:00 reader is not sent to look at the wrong Postgres.")
@@ -500,21 +503,12 @@ def main():
    w(f"      - series: '{labels(pod='immich-pg-1', ns='immich')}'")
    w(f"        values: '{fmt([0] * SAMPLES)}'")
    w("    alert_rule_test:")
-    w("      - eval_time: 20m")
-    w("        alertname: CNPGWalArchiveBacklog")
-    w("        exp_alerts:")
-    w("          - exp_labels:")
-    w("              severity: warning")
-    w(f"              namespace: {NS}")
-    w("              pod: ha-pg-1")
-    w("            exp_annotations:")
-    w(f'              summary: "CNPG WAL archiving is falling behind on {NS}/ha-pg-1"')
-    w("              description: \"70 WAL segments have been waiting to archive for over 15 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n home-assistant logs ha-pg-1 -c plugin-barman-cloud`.\"")
+    firing(f"{fire_at}m", 70, pod="ha-pg-1")
    w("")

-    # ---- 14. CNPGBackupFailed --------------------------------------------
+    # ---- 15. CNPGBackupFailed --------------------------------------------
    w("  # ---------------------------------------------------------------------")
-    w("  # 14. THE EPOCH-ZERO GUARD on CNPGBackupFailed. The metric is a unix")
+    w("  # 15. THE EPOCH-ZERO GUARD on CNPGBackupFailed. The metric is a unix")
    w("  # timestamp and reads 0 when no failure was ever recorded, so the rule")
    w("  # carries a `> 0` guard to stop that default being read as \"failed in")
    w("  # 1970\". Held at 0 for the whole window: silence. Nothing else in this")
@@ -527,9 +521,7 @@ def main():
    w(f"      - series: 'cnpg_collector_last_failed_backup_timestamp{{namespace=\"{NS}\",pod=\"{POD}\"}}'")
    w(f"        values: '{fmt([0] * SAMPLES)}'")
    w("    alert_rule_test:")
-    w("      - eval_time: 39m")
-    w("        alertname: CNPGBackupFailed")
-    w("        exp_alerts: []")
+    silent("39m", "CNPGBackupFailed")

    print("\n".join(L))

@@ -0,0 +1,223 @@
+# promtool test rules for prometheusrule-home-assistant.yaml (REN-245).
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh — see README.md here. Negative controls are the
+# `ha-*` entries in mutations.txt.
+#
+# WHAT IT IS FOR. Each rule claims three things in its comments: it fires 5m
+# after the outage starts and not before, it stays silent on a healthy system,
+# and one outage is one alert even when the outage changes shape. Each claim
+# is asserted here. Series names and labels follow the live shapes recorded in
+# the REN-244 "findings" document.
+rule_files:
+  - ../prometheusrule-home-assistant.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # --- HomeAssistantDown --------------------------------------------------
+  # Target answers for 10m, fails for 5m, disappears (Recreate rollout) for
+  # 3m, then comes back still failing. One outage from t=10m: the alert must
+  # fire at t=15m and still be the same alert after the target changed shape.
+  - interval: 1m
+    input_series:
+      - series: 'up{job="home-assistant",namespace="home-assistant",instance="10.0.0.1:8124",pod="home-assistant-6d9c7b5f4-abcde"}'
+        values: '1x9 0x4'
+      - series: 'up{job="home-assistant",namespace="home-assistant",instance="10.0.0.2:8124",pod="home-assistant-6d9c7b5f4-fghij"}'
+        values: '_x22 0x10'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: HomeAssistantDown
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: HomeAssistantDown
+        exp_alerts:
+          - exp_labels: {severity: critical, job: home-assistant, namespace: home-assistant}
+            exp_annotations:
+              summary: "Home Assistant has been down for 5 minutes"
+              description: "The home-assistant scrape target in namespace home-assistant is down or missing. A restart after an automatic image upgrade can take up to 10 minutes, so if an upgrade just rolled out, give it until then before acting. Rollback: kubernetes/RUNBOOK-home-assistant-image-rollback.md."
+      # Old target gone (stale after 5m lookback), new one failing: same alert.
+      - eval_time: 25m
+        alertname: HomeAssistantDown
+        exp_alerts:
+          - exp_labels: {severity: critical, job: home-assistant, namespace: home-assistant}
+            exp_annotations:
+              summary: "Home Assistant has been down for 5 minutes"
+              description: "The home-assistant scrape target in namespace home-assistant is down or missing. A restart after an automatic image upgrade can take up to 10 minutes, so if an upgrade just rolled out, give it until then before acting. Rollback: kubernetes/RUNBOOK-home-assistant-image-rollback.md."
+
+  # Healthy HA stays silent.
+  - interval: 1m
+    input_series:
+      - series: 'up{job="home-assistant",namespace="home-assistant",instance="10.0.0.1:8124",pod="home-assistant-6d9c7b5f4-abcde"}'
+        values: '1x30'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: HomeAssistantDown
+        exp_alerts: []
+
+  # --- HomeAssistantRecorderDBUnhealthy ----------------------------------
+  # Two instances as in ha-pg-cluster.yaml. The replica's Postgres goes down at
+  # t=10m (its exporter keeps answering with cnpg_collector_up=0).
+  - interval: 1m
+    input_series:
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-2",role="primary",cluster="ha-pg"}'
+        values: '1x30'
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-1",role="replica",cluster="ha-pg"}'
+        values: '1x9 0x20'
+    alert_rule_test:
+      - eval_time: 9m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts: []
+      - eval_time: 14m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts:
+          - exp_labels: {severity: critical, namespace: home-assistant}
+            exp_annotations:
+              summary: "Recorder database ha-pg has no primary or is missing an instance"
+              description: "In CNPG cluster home-assistant/ha-pg either no instance reports role=primary with Postgres up, or fewer than 2 instances have Postgres up. Without a primary the Home Assistant recorder cannot write history. Check `kubectl -n home-assistant get cluster ha-pg` and the pods' logs."
+
+  # Primary's Postgres goes down at t=10m. Both branches are true at once (no
+  # primary AND only one instance up); still exactly one alert.
+  - interval: 1m
+    input_series:
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-2",role="primary",cluster="ha-pg"}'
+        values: '1x9 0x20'
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-1",role="replica",cluster="ha-pg"}'
+        values: '1x30'
+    alert_rule_test:
+      - eval_time: 15m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts:
+          - exp_labels: {severity: critical, namespace: home-assistant}
+            exp_annotations:
+              summary: "Recorder database ha-pg has no primary or is missing an instance"
+              description: "In CNPG cluster home-assistant/ha-pg either no instance reports role=primary with Postgres up, or fewer than 2 instances have Postgres up. Without a primary the Home Assistant recorder cannot write history. Check `kubectl -n home-assistant get cluster ha-pg` and the pods' logs."
+
+  # Both instances up but neither is primary (failover stuck): fires.
+  - interval: 1m
+    input_series:
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-2",role="replica",cluster="ha-pg"}'
+        values: '1x30'
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-1",role="replica",cluster="ha-pg"}'
+        values: '1x30'
+    alert_rule_test:
+      - eval_time: 4m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts:
+          - exp_labels: {severity: critical, namespace: home-assistant}
+            exp_annotations:
+              summary: "Recorder database ha-pg has no primary or is missing an instance"
+              description: "In CNPG cluster home-assistant/ha-pg either no instance reports role=primary with Postgres up, or fewer than 2 instances have Postgres up. Without a primary the Home Assistant recorder cannot write history. Check `kubectl -n home-assistant get cluster ha-pg` and the pods' logs."
+
+  # Clean switchover at t=10m: ha-pg-1 takes primary, ha-pg-2 becomes replica.
+  # The role label flips, so both old series go stale and new ones start. Must
+  # stay silent throughout.
+  - interval: 1m
+    input_series:
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-2",role="primary",cluster="ha-pg"}'
+        values: '1x9'
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-1",role="replica",cluster="ha-pg"}'
+        values: '1x9'
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-1",role="primary",cluster="ha-pg"}'
+        values: '_x10 1x20'
+      - series: 'cnpg_collector_up{job="home-assistant/cnpg-ha-pg",namespace="home-assistant",pod="ha-pg-2",role="replica",cluster="ha-pg"}'
+        values: '_x10 1x20'
+    alert_rule_test:
+      - eval_time: 12m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts: []
+      - eval_time: 30m
+        alertname: HomeAssistantRecorderDBUnhealthy
+        exp_alerts: []
+
+  # --- MosquittoDown ------------------------------------------------------
+  # The broker goes not-Ready at t=10m while a credentials-bootstrap Job pod is
+  # Ready. The Job pod must not mask the broker.
+  - interval: 1m
+    input_series:
+      - series: 'kube_pod_status_ready{namespace="home-assistant",pod="mosquitto-7f8b9c6d5-x2k9q",condition="true"}'
+        values: '1x9 0x20'
+      - series: 'kube_pod_status_ready{namespace="home-assistant",pod="mosquitto-credentials-bootstrap-q7w2e",condition="true"}'
+        values: '1x30'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: MosquittoDown
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: MosquittoDown
+        exp_alerts:
+          - exp_labels: {severity: critical, namespace: home-assistant}
+            exp_annotations:
+              summary: "Mosquitto (MQTT broker) has not been ready for 5 minutes"
+              description: "No mosquitto pod in namespace home-assistant is Ready. Zigbee2MQTT, Shelly and every other MQTT device lose their link to Home Assistant while this lasts, so Zigbee2MQTTDownOrCoordinatorOffline may fire alongside it for the same cause."
+
+  # --- Zigbee2MQTTDownOrCoordinatorOffline -------------------------------
+  # Pod Ready throughout; HA's bridge sensor reads 0 from t=10m (coordinator
+  # unplugged).
+  - interval: 1m
+    input_series:
+      - series: 'kube_pod_status_ready{namespace="home-assistant",pod="zigbee2mqtt-5c4d8f7b6-m3n4p",condition="true"}'
+        values: '1x30'
+      - series: 'homeassistant_binary_sensor_state{entity="binary_sensor.zigbee2mqtt_bridge_connection_state",domain="binary_sensor",friendly_name="Zigbee2MQTT Bridge Connection state",job="home-assistant",namespace="home-assistant",instance="10.0.0.1:8124"}'
+        values: '1x9 0x20'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: Zigbee2MQTTDownOrCoordinatorOffline
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: Zigbee2MQTTDownOrCoordinatorOffline
+        exp_alerts:
+          - exp_labels: {severity: critical, namespace: home-assistant}
+            exp_annotations:
+              summary: "Zigbee2MQTT is down or its coordinator/bridge is offline"
+              description: "Either the zigbee2mqtt pod in namespace home-assistant is not Ready, or Home Assistant's binary_sensor.zigbee2mqtt_bridge_connection_state reads off. If MosquittoDown is firing too, fix the broker first; the bridge sensor goes off whenever MQTT is gone."
+
+  # Pod not Ready from t=10m with the bridge sensor at 0; a replacement pod is
+  # Ready from t=18m but the bridge stays offline. One outage that changes
+  # shape: fires at t=15m as one alert, and is still that alert once only the
+  # sensor branch is true.
+  - interval: 1m
+    input_series:
+      - series: 'kube_pod_status_ready{namespace="home-assistant",pod="zigbee2mqtt-5c4d8f7b6-m3n4p",condition="true"}'
+        values: '1x9 0x20'
+      - series: 'kube_pod_status_ready{namespace="home-assistant",pod="zigbee2mqtt-5c4d8f7b6-r5s6t",condition="true"}'
+        values: '_x17 1x12'
+      - series: 'homeassistant_binary_sensor_state{entity="binary_sensor.zigbee2mqtt_bridge_connection_state",domain="binary_sensor",friendly_name="Zigbee2MQTT Bridge Connection state",job="home-assistant",namespace="home-assistant",instance="10.0.0.1:8124"}'
+        values: '1x9 0x20'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: Zigbee2MQTTDownOrCoordinatorOffline
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: Zigbee2MQTTDownOrCoordinatorOffline
+        exp_alerts:
+          - exp_labels: {severity: critical, namespace: home-assistant}
+            exp_annotations:
+              summary: "Zigbee2MQTT is down or its coordinator/bridge is offline"
+              description: "Either the zigbee2mqtt pod in namespace home-assistant is not Ready, or Home Assistant's binary_sensor.zigbee2mqtt_bridge_connection_state reads off. If MosquittoDown is firing too, fix the broker first; the bridge sensor goes off whenever MQTT is gone."
+      - eval_time: 20m
+        alertname: Zigbee2MQTTDownOrCoordinatorOffline
+        exp_alerts:
+          - exp_labels: {severity: critical, namespace: home-assistant}
+            exp_annotations:
+              summary: "Zigbee2MQTT is down or its coordinator/bridge is offline"
+              description: "Either the zigbee2mqtt pod in namespace home-assistant is not Ready, or Home Assistant's binary_sensor.zigbee2mqtt_bridge_connection_state reads off. If MosquittoDown is firing too, fix the broker first; the bridge sensor goes off whenever MQTT is gone."
+
+  # HA down: the bridge sensor goes stale and disappears, the Z2M pod is fine.
+  # HomeAssistantDown owns this; the Zigbee rule must stay silent.
+  - interval: 1m
+    input_series:
+      - series: 'kube_pod_status_ready{namespace="home-assistant",pod="zigbee2mqtt-5c4d8f7b6-m3n4p",condition="true"}'
+        values: '1x30'
+      - series: 'homeassistant_binary_sensor_state{entity="binary_sensor.zigbee2mqtt_bridge_connection_state",domain="binary_sensor",friendly_name="Zigbee2MQTT Bridge Connection state",job="home-assistant",namespace="home-assistant",instance="10.0.0.1:8124"}'
+        values: '1x9'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: Zigbee2MQTTDownOrCoordinatorOffline
+        exp_alerts: []
@@ -28,45 +28,124 @@
 # `detected by` notes below are for the reader. The machine check is only "the
 # suite must fail"; nothing parses them.

-# The defect this whole exercise exists because of. The surge rule was written
-# `for: 2m` and reviewed at that value. Six of Iris's seven measured nights
-# accumulate 3-4 minutes above 64, so it fires on all six — two owner emails a
-# night, 04:16 and 04:22 local, for a backlog that clears itself.
-# Detected by: cases 1-7, the measured nights.
-surge-for-2m|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|s/^\([[:space:]]*\)for: 8m$/\1for: 2m/
-
-# The margin control, and the only reason case 8 is in the suite. At `for: 5m`
-# the surge rule is still silent on all seven REAL nights (worst is 4m pending)
-# and fires only on the adversarial envelope (5m pending). Without the envelope
-# case, `for: 8m` could be cut to 5m with the suite staying green.
-# Detected by: case 8, the adversarial envelope.
-surge-for-5m|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|s/^\([[:space:]]*\)for: 8m$/\1for: 5m/
+# The margin control, and the only reason case 9 is in the suite. The worst
+# night in Prometheus' retention (2026-09-11, peak 141) holds the expression
+# true for 12 minutes at a 1m evaluation step, so at `for: 12m` it fires. The
+# seven reconstructed nights peak at 10m pending and would stay silent — without
+# the raw worst-night case, `for: 20m` could be cut to 12m with the suite green.
+# Detected by: case 9, the worst night on record.
+backlog-for-12m|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|s/^\([[:space:]]*\)for: 20m$/\1for: 12m/

 # The value REN-129 changed, in the direction it changed it from. At 30m the
-# load-bearing rule does not page until 35 minutes into a stall that started at
-# t=5m, so the assertion at t=20m sees nothing. This is the control that stops
-# the 15m being quietly walked back.
-# Detected by: cases 9, 10, 12, 13 — every case that expects a page.
-backlog-for-30m|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|s/^\([[:space:]]*\)for: 15m$/\1for: 30m/
+# rule does not page until 35 minutes into a stall that started at t=5m, so the
+# assertion at t=25m sees nothing. This is the control that stops the 20m being
+# quietly walked back.
+# Detected by: cases 10, 11, 13, 14 — every case that expects a page.
+backlog-for-30m|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|s/^\([[:space:]]*\)for: 20m$/\1for: 30m/

 # Drop the aggregation, keeping the selector. `role` flips on failover; a
 # changed label is a new series to Prometheus and restarts the `for` timer, so
-# a backlog that persists across a promotion pages 15m after the promotion
-# instead of 15m after the backlog. Scoped to `expr:` lines because
+# a backlog that persists across a promotion pages 20m after the promotion
+# instead of 20m after the backlog. Scoped to `expr:` lines because
 # `max by (namespace, pod)` also appears in this manifest's comments, where
 # rewriting it would mutate the documentation and nothing else.
-# Detected by: case 12, the role flip.
+# Detected by: case 13, the role flip.
 no-max-by|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|/^[[:space:]]*expr:/s/max by (namespace, pod) (\(cnpg_collector_pg_wal_archive_status{value="ready"}\))/\1/

 # Drop the `value="ready"` selector. The same metric name also carries
-# `value="done"`, a monotonic counter in the hundreds of thousands, so both
-# rules would fire permanently on every CNPG instance in the cluster. The
+# `value="done"`, a monotonic counter in the hundreds of thousands, so the
+# rule would fire permanently on every CNPG instance in the cluster. The
 # measured-night cases carry that counter series in their input specifically so
 # this is detectable rather than a silent no-op.
-# Detected by: cases 1-8 and 11 — every case with a `done` series.
+# Detected by: cases 1-9 and 12 — every case with a `done` series.
 no-value-selector|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|/^[[:space:]]*expr:/s/{value="ready"}//

-# Loosen the comparison on the load-bearing rule. `> 5` is strict; `>= 5` pages
+# Loosen the comparison. `> 5` is strict; `>= 5` pages
 # on a backlog sitting at exactly five segments, which is 80 MiB and normal.
-# Detected by: case 11, pinned at exactly 5.
+# Detected by: case 12, pinned at exactly 5.
 backlog-ge-5|kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml|/^[[:space:]]*expr:/s/> 5$/>= 5/
+
+# --- prometheusrule-external-secrets.yaml (REN-103) -------------------------
+# Scoped to the ExternalSecretSyncFailing expression with /max by/ or /== 1/
+# where it matters: the manifest's comments quote the rejected forms, and
+# rewriting those would mutate the documentation and nothing else.
+
+# The window. At 5m the failure that starts at t=10m is already firing at the
+# t=15m assertion, which expects pending.
+# Detected by: the eval_time 15m case.
+eso-for-5m|kubernetes/apps/monitoring-rules/prometheusrule-external-secrets.yaml|s/^\([[:space:]]*\)for: 15m$/\1for: 5m/
+
+# REN-100's aggregation. `sum` over the two-pod rollout case is 2, fails
+# `== 1`, and loki-basic-auth never fires.
+# Detected by: the eval_time 30m case, loki-basic-auth.
+eso-sum-not-max|kubernetes/apps/monitoring-rules/prometheusrule-external-secrets.yaml|s/^\([[:space:]]*\)max by (exported_namespace, name) ($/\1sum by (exported_namespace, name) (/
+
+# The label Argus found shadowed on #994. Grouping by `namespace` names the
+# controller's namespace on every alert.
+# Detected by: the eval_time 30m case, exp_labels and the rendered summary.
+eso-by-namespace|kubernetes/apps/monitoring-rules/prometheusrule-external-secrets.yaml|s/^\([[:space:]]*\)max by (exported_namespace, name) ($/\1max by (namespace, name) (/
+
+# Drop `== 1`. ESO writes the inactive Ready polarity at 0, so every healthy
+# ExternalSecret matches the bare selector and fires.
+# Detected by: the eval_time 30m case, uptime-robot-api-key.
+eso-no-eq-1|kubernetes/apps/monitoring-rules/prometheusrule-external-secrets.yaml|s/^\([[:space:]]*\)) == 1$/\1)/
+
+# Drop the condition selector. The Deleted tombstone is status="False" at 1
+# forever, so it pages until the controller restarts.
+# Detected by: the eval_time 30m case, wiki-index-basic-auth.
+eso-any-condition|kubernetes/apps/monitoring-rules/prometheusrule-external-secrets.yaml|s/^\([[:space:]]*\)externalsecret_status_condition{condition="Ready",status="False"}$/\1externalsecret_status_condition{status="False"}/
+
+# prometheusrule-home-assistant.yaml (REN-245). Detected by the matching
+# `eval_time: 14m` / `15m` pairs in home-assistant_test.yaml.
+ha-down-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-home-assistant.yaml|/alert: HomeAssistantDown/,/for:/s/for: 5m/for: 10m/
+# Drops the label normalisation: the alert changes identity when the target
+# comes back failing, and the t=25m assertion loses it.
+ha-down-no-max|kubernetes/apps/monitoring-rules/prometheusrule-home-assistant.yaml|s/max by (job, namespace) (up{job="home-assistant", namespace="home-assistant"}) == 0/up{job="home-assistant", namespace="home-assistant"} == 0/
+# The instance floor pinned to spec.instances. At 3 the healthy two-instance
+# cluster pages. Detected by the switchover case, which must stay silent.
+ha-recorder-count-3|kubernetes/apps/monitoring-rules/prometheusrule-home-assistant.yaml|s/== 1) < 2$/== 1) < 3/
+# The loose regex that lets a Ready bootstrap Job pod mask a down broker.
+ha-mosquitto-loose-regex|kubernetes/apps/monitoring-rules/prometheusrule-home-assistant.yaml|s/pod=~"mosquitto-\[a-z0-9\]{6,10}-\[a-z0-9\]{5}"/pod=~"mosquitto-.*"/
+# Drops the bridge-sensor branch. Detected by the coordinator-offline case.
+ha-z2m-no-bridge|kubernetes/apps/monitoring-rules/prometheusrule-home-assistant.yaml|s/bridge_connection_state"}) == 0$/bridge_connection_state"}) == 2/
+
+# --- prometheusrule-ceph.yaml (REN-93) --------------------------------------
+# The canary and the critical rule it guards share the `{type="replicated"}`
+# join text, so these scope by the canary's `absent(` ... `)` block: inside it
+# for the canary, `!` for everything else. The annotation quotes the selector
+# with escaped quotes and so never matches.
+
+# The window, shortened. At 15m the canary pages on a mgr-failover-length gap.
+# Detected by: case F, the eval_time 75m silence.
+ceph-blind-for-15m|kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml|s/^\([[:space:]]*\)for: 1h$/\1for: 15m/
+
+# The window, lengthened. At 2h a real blindness that started at t=30m is
+# still not reported at 95m.
+# Detected by: case F, the eval_time 95m page.
+ceph-blind-for-2h|kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml|s/^\([[:space:]]*\)for: 1h$/\1for: 2h/
+
+# Drop the value filter from the canary only. It then watches a join the
+# critical rule does not use, and stays quiet through a renamed `type` label
+# or value while the critical rule is dark.
+# Detected by: cases C and D.
+ceph-blind-any-type|kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml|/^[[:space:]]*absent($/,/^[[:space:]]*)$/s/{type="replicated"}//
+
+# Invert the canary: the bare join instead of its absence. It then fires on
+# every healthy cluster and on every real redundancy loss, i.e. it masks the
+# alert it exists to protect.
+# Detected by: cases A and B.
+ceph-blind-inverted|kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml|s/^\([[:space:]]*\)absent($/\1(/
+
+# REN-88's gate, removed from the critical rule. Erasure-coded and renamed
+# pools then match the critical rule again. Proves this suite loads and asserts
+# the critical rule, not only the canary.
+# Detected by: cases C, D and E, which assert the critical rule is silent.
+ceph-critical-any-type|kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml|/^[[:space:]]*absent($/,/^[[:space:]]*)$/!s/{type="replicated"}//
+
+# prometheusrule-prometheus-retention.yaml (REN-107). Detected by the cases in
+# prometheus-retention_test.yaml.
+retention-no-time-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/< 0.9 \* min(prometheus_tsdb_retention_limit_seconds/< 100 * min(prometheus_tsdb_retention_limit_seconds/
+retention-no-size-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0/
+retention-ratio-0.9|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0.9/
+retention-lookback-1d|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/)\[7d:1h\]$/)[1d:1h]/
+retention-cap-at-peak-time|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/\/ max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})$/\/ min_over_time(max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})[7d:1h])/
@@ -0,0 +1,218 @@
+# promtool test rules for prometheusrule-prometheus-retention.yaml (REN-107).
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh — see README.md here. Negative controls are the
+# `retention-*` entries in mutations.txt.
+#
+# WHAT IT IS FOR. PrometheusRetentionNotInForce makes five claims in its
+# comments, and each scenario below asserts one of them:
+#
+#   1. It fires on the state measured on 2026-09-26 (90d/30GiB, 20.5-27.2 day
+#      saw-tooth), across a pod restart and across the post-eviction trough.
+#   2. It goes silent the moment the REN-107 fix (30d/40GiB) is in force,
+#      although the window on disk is still ~21 days and last week's peak was
+#      87% of the OLD cap.
+#   3. It stays silent at 30d steady state, where the store sits above 80% of
+#      the 40GiB cap but the time retention is what binds (clause 1 guards).
+#   4. It stays silent on a freshly provisioned PVC (clause 2 guards).
+#   5. It fires if the size cap binds under the new config.
+#
+# All series are hourly (the rule's subquery step) and evaluated hourly.
+# `lowest_timestamp_seconds` is written as (t - window), so `time() - lowest`
+# reads back the window: `-X+3600xN` holds a constant X-second window while
+# Prometheus deletes old data, a constant value is a window that only grows.
+# Byte figures follow the live pod: WAL ~0.5 GB, head chunks ~0.2 GB.
+rule_files:
+  - ../prometheusrule-prometheus-retention.yaml
+
+evaluation_interval: 1h
+
+tests:
+  # --- 1. Today: 90d / 30GiB, size cap binds ------------------------------
+  # Pod a holds a 26.5-day window at 87.5% of the cap. At t=40h it is
+  # replaced (a chart upgrade) and pod b starts after an eviction: 20.5 GB,
+  # 66% of the cap, same 26.5-day window. The alert must stay up across the
+  # restart and the trough for as long as the peak is inside 7 days, and
+  # clear once it is not — the 7d window is what spans a saw-tooth tooth.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '7776000x39'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '32212254720x39'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-2291239+3600x39'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '27500000000x39'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x39'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x39'
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 7776000x180'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 32212254720x180'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 -2147239+3600x180'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 20500000000x180'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 500000000x180'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 200000000x180'
+    alert_rule_test:
+      - eval_time: 0h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: &retention_firing_26d
+          - exp_labels: {severity: warning, namespace: monitoring}
+            exp_annotations:
+              summary: "Prometheus retention setting is not in force — size cap binds first"
+              description: "Prometheus is holding only 26d 12h 27m 19s of history while its configured time retention is longer, and the store has been within 20% of its configured size cap in the last 7 days. The size cap is therefore what is actually in force and the `retention` value in kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured — queries over a longer range return less data than requested, silently and successfully. Confirm the running pair with `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention` to a figure the storage budget can actually hold. Do not read `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets on every restart. Background: REN-107."
+      # After the restart and in the trough: the peak was pod a's.
+      - eval_time: 60h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: *retention_firing_26d
+      - eval_time: 150h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: *retention_firing_26d
+      # Pod a's last sample (39h) is out of the 7d subquery from 208h on.
+      - eval_time: 210h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 2. The fix day: 90d/30GiB -> 30d/40GiB -----------------------------
+  # Pod a is today's state just after an eviction (peak 28.2 GB = 87.5% of
+  # 30GiB until t=79h, then 20.5 GB). The fix rolls out at t=100h; pod b holds
+  # the data it inherited and deletes nothing, so its window grows from 20.8
+  # days and is still under 27 days (90% of 30d) until t~=250h. Clause 1 is
+  # therefore true throughout; the rule must be silent anyway, because the
+  # 28.2 GB peak is 66% of the cap now in force.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '7776000x99'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '32212254720x99'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-1771200+3600x99'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '27500000000x79 20500000000x19'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x99'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x99'
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 2592000x160'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 42949672960x160'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 -1411200x160'
+      # ~1.02 GB/day of ingest, nothing deleted.
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 20500000000+42500000x160'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 500000000x160'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 200000000x160'
+    alert_rule_test:
+      # Before the fix it fires (window 20.5 days, peak within 7 days).
+      - eval_time: 99h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts:
+          - exp_labels: {severity: warning, namespace: monitoring}
+            exp_annotations:
+              summary: "Prometheus retention setting is not in force — size cap binds first"
+              description: "Prometheus is holding only 20d 12h 0m 0s of history while its configured time retention is longer, and the store has been within 20% of its configured size cap in the last 7 days. The size cap is therefore what is actually in force and the `retention` value in kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured — queries over a longer range return less data than requested, silently and successfully. Confirm the running pair with `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention` to a figure the storage budget can actually hold. Do not read `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets on every restart. Background: REN-107."
+      - eval_time: 100h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 110h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 170h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 250h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 3. 30d steady state -------------------------------------------------
+  # Time retention binds: the window never drops below 30 days (it runs to
+  # ~31.5 while a block waits for deletion) and the store sits at 38 GB,
+  # 88.5% of 40GiB. Clause 2 is true all the time; clause 1 must keep the
+  # rule silent.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '2592000x200'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '42949672960x200'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-2721600+3600x200'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '37300000000x200'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x200'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x200'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 180h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 4. Fresh PVC --------------------------------------------------------
+  # A new volume holds hours of data against a 30d setting: clause 1 is true
+  # for 27 days. Clause 2 (7.6 GB at t=180h, 18% of the cap) must keep it
+  # silent.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '2592000x200'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '42949672960x200'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '0x200'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '0+42500000x200'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x200'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x200'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 180h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 5. The size cap binds again under 30d/40GiB ------------------------
+  # Ingest grew: the store is at 38 GB (88.5%) and holds only 25 days.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '2592000x10'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '42949672960x10'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-2160000+3600x10'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '37300000000x10'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x10'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x10'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts:
+          - exp_labels: {severity: warning, namespace: monitoring}
+            exp_annotations:
+              summary: "Prometheus retention setting is not in force — size cap binds first"
+              description: "Prometheus is holding only 25d 0h 0m 0s of history while its configured time retention is longer, and the store has been within 20% of its configured size cap in the last 7 days. The size cap is therefore what is actually in force and the `retention` value in kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured — queries over a longer range return less data than requested, silently and successfully. Confirm the running pair with `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention` to a figure the storage budget can actually hold. Do not read `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets on every restart. Background: REN-107."
@@ -2,6 +2,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
  name: blackbox-exporter
+  # anchor: namespace
  namespace: monitoring
 spec:
  interval: 30m
@@ -17,7 +17,7 @@ spec:
  chart:
    spec:
      chart: kube-prometheus-stack
-      version: ">=72.0.0"
+      version: "91.7.0"
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
@@ -35,7 +35,7 @@ spec:
    prometheus:
      # prometheus.renner.uno is served by a standalone, Authentik-gated
      # Ingress (prometheus-gate-ingress.yaml), NOT this Helm-generated one.
-      # Prometheus had no auth of its own — its full 90d metrics DB is an
+      # Prometheus had no auth of its own — its full 30d metrics DB is an
      # infra-recon oracle (hostnames, IPs, service inventory). Gated behind
      # the prometheus-gate proxy outpost, same as alertmanager-gate (#146).
      # The chart Ingress is disabled here — two Ingresses for one host would
@@ -76,8 +76,17 @@ spec:
        # Rewrites the in-cluster URL embedded in alert generatorURLs and
        # status pages so click-through from emails works.
        externalUrl: https://prometheus.renner.uno
-        retention: 90d
-        retentionSize: 30GB
+        # Time retention is the one in force; the size cap is a backstop
+        # (REN-107, owner decision: 30 days on the existing disk). This was
+        # 90d/30GB, and at ~1 GB/day of ingest the 30GiB cap bound first: the
+        # real window was a 20.5-27 day saw-tooth while this line said 90d.
+        # 30d peaks near 38 GB (30d plus up to one 3-day block awaiting
+        # deletion, plus WAL), under 40GiB (42.9 GB), under the 50Gi PVC.
+        # Raising either number means re-doing that arithmetic against the
+        # PVC; PrometheusRetentionNotInForce (monitoring-rules) fires if the
+        # size cap starts binding again.
+        retention: 30d
+        retentionSize: 40GiB
        scrapeInterval: 30s
        evaluationInterval: 30s
        storageSpec:
@@ -0,0 +1,210 @@
+# Beets fixes the tags of the music library in place, once a night (REN-349).
+# It never moves, copies, renames or deletes a file: Lidarr and Navidrome
+# paths stay valid. Navidrome picks up the new tags on its hourly scan.
+#
+# REN-350: two dry runs with write: no were checked (match report and a
+# 5-album spot check), so beets now writes tags. Art plugins stay off: they
+# would add or embed cover files, a separate change.
+#
+# Retry: the first write run matched 74 of 560 albums; most skipped folders
+# hold only part of an album. Partial-album matches are allowed now (see
+# max_rec). A -W trial matched 158 albums, all at 96% or better, and none of
+# the 74 written albums changed release, so beets now writes them. Muse "Black
+# Holes and Revelations" matched an instrumental release: it is ignored and
+# dropped from the DB (DB only, no file change).
+#
+# Lidarr does not write tags: lidarr-reconcile sets writeAudioTags "no" on
+# every pod start (lidarr.yaml), so it cannot undo the beets tags.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: beets-config
+  namespace: navidrome
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ceph-block
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: beets-config
+  namespace: navidrome
+data:
+  config.yaml: |
+    directory: /music
+    library: /config/library.db
+    statefile: /config/state.pickle
+    plugins: [musicbrainz, fetchart, embedart, lastgenre]
+    # Beets defaults plus folders whose only match is a wrong release.
+    ignore: ['.*', '*~', 'System Volume Information', 'lost+found',
+             'Black Holes and Revelations']
+    # Keep the first-release date in year; reissues must not change it.
+    original_date: yes
+    match:
+      preferred:
+        countries: ['DE', 'XE', 'XW', 'US', 'GB|UK']
+        media: ['CD', 'Digital Media|File']
+        original_year: yes
+      # Partial albums may match; a missing track costs less than a wrong
+      # title or artist (defaults: medium cap, weight 0.9).
+      max_rec:
+        missing_tracks: strong
+      distance_weights:
+        missing_tracks: 0.2
+    import:
+      write: yes
+      copy: no
+      move: no
+      link: no
+      hardlink: no
+      reflink: no
+      delete: no
+      quiet: yes
+      quiet_fallback: skip
+      incremental: yes
+      # Do not record skipped folders: they are retried the next night.
+      incremental_skip_later: yes
+      resume: no
+      # Never "remove": that deletes files.
+      duplicate_action: skip
+    # Both add or change cover art in /music; not part of the tag fix.
+    fetchart:
+      auto: no
+    embedart:
+      auto: no
+    lastgenre:
+      auto: yes
+  run.sh: |
+    set -u
+    beet() { /lsiopy/bin/beet -c /etc/beets/config.yaml "$@"; }
+    mkdir -p /config/logs
+    # One-time reset: the first dry run stored reissue matches in the DB.
+    # The DB is beets state only; no file in /music is touched.
+    if [ ! -f /config/.reset-ren350 ]; then
+      rm -f /config/library.db /config/state.pickle
+      touch /config/.reset-ren350
+    fi
+    # One-time: forget the skipped folders so they are retried. The 74
+    # written albums are re-imported too: beets re-matches them under the
+    # looser weights and replaces their DB entry (no duplicate skip).
+    if [ ! -f /config/.retry-ren350 ]; then
+      rm -f /config/state.pickle
+      touch /config/.retry-ren350
+    fi
+    # One-time: the trial stored an instrumental release for this album.
+    # `rm` without -d drops the DB entry only; the file is not touched.
+    if [ ! -f /config/.drop-ren350 ]; then
+      beet rm -a -f 'path:/music/Muse/Black Holes and Revelations'
+      touch /config/.drop-ren350
+    fi
+    beet ls -a -f '$path $mb_albumid' | sort > /config/logs/albums-before.txt
+    LOG=/config/logs/import-$(date +%Y%m%d-%H%M).log
+    beet import -l "$LOG" /music
+    rc=$?
+    beet ls -a -f '$path $mb_albumid' | sort > /config/logs/albums-after.txt
+    comm -23 /config/logs/albums-before.txt /config/logs/albums-after.txt \
+      > /config/logs/albums-changed.txt
+    echo "== import exit $rc"
+    echo "== albums with a changed release: $(wc -l < /config/logs/albums-changed.txt)"
+    cat /config/logs/albums-changed.txt
+    echo "== albums in library: $(beet ls -a | wc -l)"
+    echo "== skipped this run: $(grep -c '^skip ' "$LOG")"
+    echo "== duplicates skipped: $(grep -c '^duplicate-skip ' "$LOG")"
+    # The trial (-W) stored matches in the DB only; `beet write` puts them
+    # into the files (tags only, no move).
+    beet write -p > /config/logs/write-preview.txt 2>&1
+    echo "== items to write: $(grep -vc '^ ' /config/logs/write-preview.txt)"
+    beet write
+    wrc=$?
+    echo "== write exit $wrc"
+    beet write -p > /config/logs/write-after.txt 2>&1
+    echo "== items still differing after write: $(grep -vc '^ ' /config/logs/write-after.txt)"
+    echo "== skipped albums"
+    grep '^skip ' "$LOG"
+    echo "== written changes (first 2000 lines)"
+    head -n 2000 /config/logs/write-preview.txt
+    [ "$rc" -eq 0 ] && rc=$wrc
+    exit $rc
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: beets
+  namespace: navidrome
+  labels:
+    app.kubernetes.io/name: beets
+spec:
+  schedule: "30 3 * * *"
+  timeZone: Europe/Berlin
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  startingDeadlineSeconds: 3600
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      # The first full import is rate-limited by MusicBrainz (1 req/s). An
+      # interrupted run is safe: incremental import resumes the next night.
+      activeDeadlineSeconds: 72000
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: beets
+        spec:
+          restartPolicy: Never
+          enableServiceLinks: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            runAsGroup: 1000
+            fsGroup: 1000
+            fsGroupChangePolicy: OnRootMismatch
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: beets
+              image: lscr.io/linuxserver/beets:2.14.1-ls354
+              # Bypass the s6 init: it needs root, beet does not.
+              command: ["sh", "/etc/beets/run.sh"]
+              env:
+                - name: TZ
+                  value: Europe/Berlin
+                - name: BEETSDIR
+                  value: /config
+                - name: HOME
+                  value: /config
+              volumeMounts:
+                - name: config
+                  mountPath: /config
+                - name: beets-config
+                  mountPath: /etc/beets
+                  readOnly: true
+                # Writable for tag writes; move/copy/delete stay off above.
+                - name: music
+                  mountPath: /music
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 256Mi
+                limits:
+                  memory: 1Gi
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+          volumes:
+            - name: config
+              persistentVolumeClaim:
+                claimName: beets-config
+            - name: beets-config
+              configMap:
+                name: beets-config
+            - name: music
+              persistentVolumeClaim:
+                claimName: navidrome-music-bulk
@@ -83,6 +83,7 @@ spec:
            - name: music
              mountPath: /music
              readOnly: true
+          # anchor: kubelet-probes
          livenessProbe:
            httpGet:
              path: /ping
@@ -16,6 +16,7 @@ spec:
        paths:
          - path: /
            pathType: Prefix
+            # anchor: navidrome-gate
            backend:
              service:
                name: ak-outpost-navidrome-gate
@@ -67,3 +68,26 @@ spec:
                name: navidrome
                port:
                  number: 80
+---
+# Native API (/auth, /api) for clients that cannot follow the SSO redirect
+# (Koda on Garmin logs in with POST /auth/login). Requests carrying an
+# authentik_proxy_ session cookie (the web UI) keep going through the gate.
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: navidrome-native-api
+  namespace: navidrome
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - kind: Rule
+      match: Host(`navidrome.renner.uno`) && (PathPrefix(`/auth`) || PathPrefix(`/api`)) && !HeaderRegexp(`Cookie`, `authentik_proxy_`)
+      priority: 1000
+      middlewares:
+        - name: navidrome-strip-identity
+      services:
+        - name: navidrome
+          port: 80
+  tls:
+    secretName: navidrome-tls
@@ -6,3 +6,4 @@ resources:
  - gate-service.yaml
  - ingress.yaml
  - lidarr.yaml
+  - beets.yaml
@@ -23,6 +23,7 @@ metadata:
  name: lidarr-reconcile
  namespace: navidrome
 data:
+  # anchor: reconcile-script
  reconcile.sh: |
    set -eu
    API=http://localhost:8686/api/v1
@@ -111,6 +112,7 @@ spec:
              mountPath: /config
            - name: music
              mountPath: /music
+          # anchor: kubelet-probes
          livenessProbe:
            httpGet:
              path: /ping
@@ -213,6 +215,7 @@ spec:
        paths:
          - path: /
            pathType: Prefix
+            # anchor: lidarr-gate
            backend:
              service:
                name: ak-outpost-lidarr-gate
@@ -58,7 +58,8 @@
 # aggregation did not already cover them. It does, so none is added:
 # `flux-view-flux-system` carries `rbac.authorization.k8s.io/aggregate-to-view:
 # "true"` and lists get/list/watch on `*` across all six API groups —
-# `../../clusters/homelab/flux-system/gotk-components.yaml:232-254`. The
+# the `flux-view-flux-system` ClusterRole in
+# `../../clusters/homelab/flux-system/gotk-components.yaml`. The
 # aggregation controller merges those rules into `view` itself, so this one
 # binding picks up kustomizations, helmreleases, gitrepositories, imagepolicies,
 # imagerepositories and imageupdateautomations. Read from git, not from a live
@@ -9,6 +9,7 @@ resources:
  - pvc.yaml
  - prometheus-read-rbac.yaml
  - cluster-view-rbac.yaml
+  - sandbox-rbac.yaml
  - paperclip.yaml
  - gate-service.yaml
  - ingress.yaml
@@ -73,6 +73,15 @@ spec:
        - ports:
            - port: "8080"
              protocol: TCP
+    # Lidarr API (REN-350); pod port, the Service maps 80 -> 8686.
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": navidrome
+            app.kubernetes.io/name: lidarr
+      toPorts:
+        - ports:
+            - port: "8686"
+              protocol: TCP
    - toEntities:
        - kube-apiserver
      toPorts:
@@ -34,7 +34,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: paperclip
-          image: ghcr.io/paperclipai/paperclip:nightly@sha256:9222efbf11863160669e13d5559e6a1be881886dd1758650b74c3993dc3ef2a6 # {"$imagepolicy": "flux-system:paperclip"}
+          image: ghcr.io/paperclipai/paperclip:nightly@sha256:c02c7a8385fe5e3e9999ec03ecb45f29d60b8bf44dac3fedf032d7ef2046045d # {"$imagepolicy": "flux-system:paperclip"}
          ports:
            - name: http
              containerPort: 3100
@@ -0,0 +1,121 @@
+# Write rights for the Paperclip Kubernetes plugin in namespace
+# `paperclip-renner` (REN-285, owner-approved plan REN-284 step 1).
+#
+# The plugin (backend `sandbox-cr`, `inCluster: true`) runs inside the
+# paperclip server and uses the pod's mounted token. For each agent run it
+# creates a Sandbox CR (agent-sandbox controller, infrastructure/agent-sandbox),
+# a NetworkPolicy/CiliumNetworkPolicy, Secrets, and then execs the agent CLI
+# into the pod.
+#
+# WHICH ServiceAccount
+#
+# The card names ServiceAccount `paperclip`. There is none: the paperclip pod
+# runs as `paperclip-prometheus-read` (paperclip.yaml, `serviceAccountName`), so
+# that is the account whose token the plugin holds, and the one bound here.
+# See prometheus-read-rbac.yaml for why it keeps that name.
+#
+# WHAT IT COSTS
+#
+# Every process in the paperclip pod can read the token, so this is a grant to
+# every agent in that pod: create pods, exec into them, read and write Secrets —
+# all only in `paperclip-renner`. It cannot touch any other namespace.
+#
+# WHY IT CANNOT ESCAPE THE NAMESPACE
+#
+#   * All namespaced rights are a Role in `paperclip-renner`, not a ClusterRole,
+#     so no later binding can widen them to another namespace.
+#   * No `escalate`, no `bind`: the account can create Roles and RoleBindings
+#     only with rights it already holds there (apiserver escalation check).
+#   * No `patch`/`update` on namespaces. The namespace carries PSS `restricted`
+#     (infrastructure/namespaces), and that is what stops a privileged pod.
+#   * The one cluster-scoped right is `create namespaces`. Kubernetes cannot
+#     limit `create` by name. A namespace created this way is empty and the
+#     account holds no rights inside it. `paperclip-renner` itself is declared
+#     in git, so the plugin's create is a no-op AlreadyExists here.
+#
+# PRODUCTION
+#
+# RBAC only. No pod restarts, no traffic change; RBAC is evaluated per request.
+# Revert: delete this file and its line in kustomization.yaml.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: paperclip-sandbox-manager
+  namespace: paperclip-renner
+  labels:
+    app.kubernetes.io/name: paperclip
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - secrets
+      - serviceaccounts
+      - resourcequotas
+      - limitranges
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get", "list", "watch"]
+  # WebSocket exec is an HTTP GET; the apiserver authorizes it as `create`
+  # (kube 1.31+). `get` is kept for older SPDY clients.
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["get", "create"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources: ["roles", "rolebindings"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["networkpolicies"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["cilium.io"]
+    resources: ["ciliumnetworkpolicies"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["agents.x-k8s.io"]
+    resources: ["sandboxes"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["agents.x-k8s.io"]
+    resources: ["sandboxes/status"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: paperclip-sandbox-manager
+  namespace: paperclip-renner
+  labels:
+    app.kubernetes.io/name: paperclip
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: paperclip-sandbox-manager
+subjects:
+  - kind: ServiceAccount
+    name: paperclip-prometheus-read
+    # The subject lives in `paperclip`, the binding in `paperclip-renner`.
+    namespace: paperclip
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: paperclip-sandbox-namespace-create
+  labels:
+    app.kubernetes.io/name: paperclip
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: paperclip-sandbox-namespace-create
+  labels:
+    app.kubernetes.io/name: paperclip
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: paperclip-sandbox-namespace-create
+subjects:
+  - kind: ServiceAccount
+    name: paperclip-prometheus-read
+    namespace: paperclip
@@ -1,6 +1,7 @@
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
+  # anchor: pprl-pg
  name: pprl-pg
  namespace: paperless
  annotations:
@@ -59,6 +59,7 @@ metadata:
 spec:
  selector:
    app.kubernetes.io/name: gotenberg
+  # anchor: service-port
  ports:
    - port: 3000
      targetPort: http
@@ -5,20 +5,27 @@
 # any pod that can talk directly to paperless's HTTP listener can claim any
 # username.
 #
-# WHAT IS ENFORCED IS WIDER THAN THIS RULE. default-deny-ingress in
+# WHAT IS ENFORCED IS WIDER THAN THIS RULE, though no longer by much.
+# default-deny-ingress in
 # kubernetes/infrastructure/network-policies/paperless-ns.yaml uses
-# `endpointSelector: {}`, so it selects this pod as well, and Cilium unions
-# the allow rules of every CNP selecting an endpoint (the same mechanism
-# described below). The effective ingress set is this outpost PLUS, on
-# every port:
-#   - the paperless namespace   (paperless-ns.yaml:9-11)
-#   - the database namespace    (paperless-ns.yaml:31-33)
-#   - the monitoring namespace  (paperless-ns.yaml:34-36)
-#   - host / remote-node        (paperless-ns.yaml:37-39)
-# So any pod in database or monitoring can reach this HTTP listener
-# directly and set X-Authentik-Username to any value. Do not read this
-# rule as the whole boundary. Narrowing is tracked in REN-28.
+# `endpointSelector: {}`, so it selects this pod as well, and Cilium unions the
+# allow rules of every CNP selecting an endpoint (the same mechanism described
+# below). The effective ingress set for THIS pod is the paperless-gate outpost
+# below, plus:
+#   - the paperless namespace, every port
+#     ../../infrastructure/network-policies/paperless-ns.yaml#paperless-admit
+#   - host / remote-node, every port
+#     ../../infrastructure/network-policies/paperless-ns.yaml#host-admit
+#     — the kubelet sources the HTTP probes from the host netns, so this one
+#     can never be removed while the pod has HTTP probes.
+#   - blackbox-exporter on :8000
+#     ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox
+# The `database` and whole-`monitoring` admits that used to be in this list
+# were removed in REN-28. Any pod in the paperless namespace, and anything on a
+# node, can still reach this listener and set X-Authentik-Username to any
+# value. Do not read the gate rule as the whole boundary.
 #
+# anchor: cnp-union
 # This is a CiliumNetworkPolicy (not a standard NetworkPolicy) so it
 # composes additively with the cluster's baseline CNP
 # `default-deny-ingress` for the paperless namespace — Cilium's allow
@@ -24,6 +24,7 @@ data:
  PAPERLESS_ACCOUNT_DEFAULT_GROUPS: "Familie"
  PAPERLESS_URL: "https://paperless.renner.uno"
  PAPERLESS_CSRF_TRUSTED_ORIGINS: "https://paperless.renner.uno,https://docs.renner.uno"
+  # anchor: allowed-hosts
  PAPERLESS_ALLOWED_HOSTS: "*"
  PAPERLESS_TIME_ZONE: "Europe/Berlin"
  PAPERLESS_OCR_LANGUAGE: "deu+eng"
@@ -43,6 +44,7 @@ data:
  PAPERLESS_CONSUMER_POLLING_INTERVAL: "10"
  PAPERLESS_REDIS: "redis://redis:6379"
  PAPERLESS_DBENGINE: "postgresql"
+  # anchor: db-host
  PAPERLESS_DBHOST: "pprl-pg-rw"
  PAPERLESS_DBNAME: "paperless"
  PAPERLESS_TIKA_ENABLED: "1"
@@ -249,6 +251,7 @@ spec:
          # answers, giving that boot 30 * 20s = 10 minutes. Without it the
          # 120s liveness delay plus three 30s periods kills the pod 210s in,
          # part-way through a migration, and the next boot starts over.
+          # anchor: http-probes
          startupProbe:
            httpGet:
              path: /accounts/login/
@@ -282,6 +285,7 @@ spec:
            capabilities:
              drop: ["ALL"]
              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
+        # anchor: bridge-tls-proxy
        - name: bridge-tls-proxy
          image: alpine/socat:1.8.0.3
          args:
@@ -72,6 +72,7 @@ metadata:
 spec:
  selector:
    app.kubernetes.io/name: redis
+  # anchor: service-port
  ports:
    - port: 6379
      targetPort: redis
@@ -53,6 +53,7 @@ metadata:
 spec:
  selector:
    app.kubernetes.io/name: tika
+  # anchor: service-port
  ports:
    - port: 9998
      targetPort: http
@@ -0,0 +1,38 @@
+---
+# Renovate bot (global) configuration (REN-216).
+#
+# This is the configuration of the RUNNER only: where Gitea is, and which
+# repositories to process. What Renovate does INSIDE a repository — schedule,
+# grouping, commit titles, digest pinning — lives in the shared preset repository
+# `k.renner/renovate-config`, which each repository's renovate.json extends. Change
+# update policy there, not here.
+#
+#   autodiscover + requireConfig=required + onboarding=false
+#       Process every repository renovate-bot can see that already has a
+#       renovate.json, and nothing else. No onboarding pull requests: a
+#       repository opts in when its maintainer commits renovate.json.
+#   persistRepoData=false
+#       The pod is ephemeral; every run clones fresh into an emptyDir.
+#   binarySource=install
+#       The slim image installs a tool (for example helm) only when a manager
+#       needs it. The managers homelab uses (flux, kubernetes, woodpecker) need
+#       none; gomod (go.sum) needs golang. The install writes into the image
+#       root file system, see the security context in cronjob.yaml.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: renovate-config
+  namespace: renovate
+data:
+  config.json: |
+    {
+      "platform": "gitea",
+      "endpoint": "https://git.renner.uno/api/v1",
+      "gitAuthor": "Renovate Bot <renovate-bot@renner.uno>",
+      "autodiscover": true,
+      "onboarding": false,
+      "requireConfig": "required",
+      "persistRepoData": false,
+      "binarySource": "install",
+      "baseDir": "/tmp/renovate"
+    }
@@ -0,0 +1,104 @@
+---
+# Renovate runner (REN-216).
+#
+# RUNS DAILY, BUT OPENS ROUTINE PULL REQUESTS ON MONDAY ONLY. The weekly rhythm
+# is enforced by the `schedule` in the shared preset (k.renner/renovate-config),
+# not by this cron line. The daily run exists for one reason: a security update
+# (`vulnerabilityAlerts`, which the preset leaves unscheduled) must not wait up
+# to six days. On Tuesday to Sunday a run finds nothing in schedule, opens and
+# rebases nothing (`updateNotScheduled: false` in the preset), and exits.
+#
+# Blast radius: this pod has no Kubernetes API access (no token mounted, no
+# Role) and writes only to Gitea, as renovate-bot, on branches `renovate/*`. It
+# cannot merge. If it misbehaves, the worst result is unwanted pull requests;
+# suspend it with `spec.suspend: true` in git, or revert this directory.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: renovate
+  namespace: renovate
+  labels:
+    app.kubernetes.io/name: renovate
+spec:
+  schedule: "0 4 * * *"
+  timeZone: Europe/Berlin
+  # Two runs against the same repositories would race on the same branches.
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 2
+  failedJobsHistoryLimit: 3
+  startingDeadlineSeconds: 3600
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 3600
+      ttlSecondsAfterFinished: 604800
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: renovate
+        spec:
+          restartPolicy: Never
+          enableServiceLinks: false
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            # `ubuntu` in the upstream image: uid 12021, primary group 0.
+            runAsUser: 12021
+            # Group 0, not 12021 (REN-315). binarySource=install makes
+            # containerbase write a tool (golang for go.sum, helm, ...) and its
+            # state files into /opt/containerbase, which the image ships as
+            # root:root 0775. Only group 0 can write there. With gid 12021 every
+            # tool install failed ("Install tool golang failed"). Group 0 gives
+            # no privilege beyond those image directories: the user is still
+            # non-root, all capabilities are dropped, no API token is mounted.
+            runAsGroup: 0
+            fsGroup: 12021
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: renovate
+              # Pinned by tag and digest. Renovate updates itself through
+              # homelab's renovate.json, like any other image here.
+              image: docker.io/renovate/renovate:44.115.10@sha256:e262ed52b23dd8c545c80faf5b1ad5c95a07b3f198888de8d7b8928f45c307c7
+              env:
+                - name: RENOVATE_CONFIG_FILE
+                  value: /config/config.json
+                - name: LOG_LEVEL
+                  value: info
+                - name: LOG_FORMAT
+                  value: json
+                # Keep scratch data off the image layer; git, npm caches and
+                # containerbase all write below $HOME.
+                - name: HOME
+                  value: /tmp/home
+              # RENOVATE_TOKEN and RENOVATE_HOST_RULES.
+              envFrom:
+                - secretRef:
+                    name: renovate-token
+              volumeMounts:
+                - name: config
+                  mountPath: /config
+                  readOnly: true
+                - name: tmp
+                  mountPath: /tmp
+              resources:
+                requests:
+                  cpu: 250m
+                  memory: 512Mi
+                limits:
+                  memory: 2Gi
+              securityContext:
+                allowPrivilegeEscalation: false
+                # false (REN-315): containerbase installs tools into
+                # /opt/containerbase of the image. An emptyDir there would hide
+                # the node runtime and state files the image ships there.
+                readOnlyRootFilesystem: false
+                capabilities:
+                  drop: ["ALL"]
+          volumes:
+            - name: config
+              configMap:
+                name: renovate-config
+            - name: tmp
+              emptyDir:
+                sizeLimit: 4Gi
@@ -0,0 +1,53 @@
+---
+# Gitea token for the `renovate-bot` account (REN-216).
+#
+# OpenBao key infrastructure/gitea/renovate-bot, field `token`. Hermes owns the
+# account and the token. The token needs these scopes, and no others:
+#   write:repository   push update branches, open pull requests
+#   write:issue        the Dependency Dashboard issue, labels
+#   read:user          Renovate reads its own user at startup
+#   read:organization  autodiscover lists repositories through org membership
+#   read:package       look up tags of CI images on gitea.renner.uno
+#
+# Which repositories Renovate touches is decided by Gitea, not by this file:
+# autodiscover finds only the repositories renovate-bot is a collaborator on,
+# and `requireConfig: required` (configmap.yaml) skips any of those that has no
+# renovate.json. So a repository opts in twice: Hermes grants access, and the
+# maintainer commits renovate.json. Branch protection on `main`/`master` keeps
+# the merge with Argus; the bot cannot merge.
+#
+# `data` + `property`, not `dataFrom: extract`, for the reason written in
+# apps/uptime-monitor/externalsecret.yaml: name the one field, so a field added
+# at the KV path later does not land here unseen.
+#
+# The template renders two env vars from the one token:
+#   RENOVATE_TOKEN       the platform token.
+#   RENOVATE_HOST_RULES  the same token as registry credentials for
+#                        gitea.renner.uno, so Renovate can list the tags of the
+#                        in-house CI images (.woodpecker/*.yml).
+#
+# refreshInterval 1h, as for the other un-rotated keys. The CronJob reads the
+# Secret at pod start, so a new value is used from the next run. No Reloader
+# annotation is needed.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: renovate-token
+  namespace: renovate
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: renovate-token
+    template:
+      engineVersion: v2
+      data:
+        RENOVATE_TOKEN: "{{ .token }}"
+        RENOVATE_HOST_RULES: '[{"matchHost":"gitea.renner.uno","hostType":"docker","username":"renovate-bot","password":"{{ .token }}"}]'
+  data:
+    - secretKey: token
+      remoteRef:
+        key: infrastructure/gitea/renovate-bot
+        property: token
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - externalsecret.yaml
+  - configmap.yaml
+  - cronjob.yaml
@@ -239,7 +239,7 @@ spec:
        # the panel after one lands. ARCH-SUFFIXED image: upstream publishes
        # no multi-arch manifest.
        - name: renderer
-          image: ghcr.io/usetrmnl/trmnl-ha-amd64:0.10.3 # {"$imagepolicy": "flux-system:trmnl-renderer"}
+          image: ghcr.io/usetrmnl/trmnl-ha-amd64:0.11.0 # {"$imagepolicy": "flux-system:trmnl-renderer"}
          ports:
            - name: http
              containerPort: 10000
@@ -36,7 +36,7 @@
 # raised — no further PR needed. Until then this object logs one refused POST
 # every 300s and nothing else: nothing pages, nothing degrades, and
 # `audiobooks.renner.uno` stays covered by its 60s Gatus check
-# (apps/gatus/configmap.yaml:163) throughout.
+# (Web Service 8 in ../gatus/configmap.yaml#gatus-endpoints) throughout.
 #
 # REVERT: revert this commit. Flux prunes the object again and, because the
 # monitor does not exist to begin with, nothing in UptimeRobot changes.
@@ -72,7 +72,8 @@
 # namespace, and the reason is structural rather than a permissions gap:
 #
 #   * `IngressURLSource` has exactly one field, `name` — there is no namespace
-#     field on it (crd-endpointmonitors.yaml:443-459).
+#     field on it (crd-endpointmonitors.yaml, schema path
+#     `spec.urlFrom.ingressRef`).
 #   * `discoverURLFromRefs` resolves that name against
 #     `ingressMonitor.Namespace` — the EndpointMonitor's OWN namespace
 #     (v2.2.13 pkg/kube/util/url.go:71 → :38-40).
@@ -111,19 +112,31 @@
 # template.
 #
 # ---
-# NO `alertContacts` YET — AND THIS IS A GAP, NOT AN OVERSIGHT
+# `alertContacts` — WIRED TO THE OWNER'S EXISTING CHANNEL (REN-7, REN-138)
 #
 # `processProviderConfig` ALWAYS appends `&alert_contacts=`, falling back to the
 # provider-level value from config.yaml when the CR sets none
-# (uptime-monitor.go:240-244). Neither is set today, so these two monitors are
-# created with an EMPTY alert-contact list: UptimeRobot will record the outage
-# and show it on the page, and will notify nobody.
+# (uptime-monitor.go:240-244). Neither was set before this change, so the pilot
+# monitors were created notifying nobody.
 #
-# That is the safe direction to be wrong in for a pilot, but it means REN-7's
-# "an alert reaches the owner's existing alert channel" is NOT satisfied by this
-# file. Closing it needs the numeric alert-contact id from `getAlertContacts`,
-# which needs the API key. Tracked separately; do not read a green pilot as
-# proof the alert path works.
+# The value below is copied from the account, not chosen: every one of the nine
+# hand-made monitors carries exactly these two contacts, at threshold 0 and
+# recurrence 0 (read with `getMonitors&alert_contacts=1`, 2026-09-26). So an IMC
+# monitor now alerts wherever the owner's existing monitors already alert — no
+# new channel, no new recipient. Contact ids from `getAlertContacts`:
+#
+#   7880965   type 2  (e-mail)
+#   8026823   type 13
+#
+# ORDER MATTERS. `Equal` compares the rendered request strings, and the read-back
+# side builds its string from the API's `alert_contacts` array in the order the
+# API returns it (uptime-mappers.go:22-29). The API returns 7880965 first, so the
+# value is written in that order; reversed, IMC would see a "change" and re-send
+# `editMonitor` every 300s forever.
+#
+# ALERT DELIVERY IS OWNER-VERIFIED, NOT PROVEN HERE. By the owner's ruling on
+# REN-138 no test alert is fired from this repo: no broken monitor, no sentinel.
+# This file proves the wiring only.
 #
 # ---
 # NO `statusPages` — AND THE EXPERIMENT THAT JUSTIFIED LEAVING IT UNSET IS NOW
@@ -240,7 +253,7 @@ metadata:
 spec:
  # `https://www.renner.uno` and not `/wp-admin` or `/wp-login.php`: those two
  # paths are served by a SECOND Ingress that sits behind the Authentik outpost
-  # (apps/wordpress/ingress.yaml:50,57). `/` is the ungated one.
+  # (../wordpress/ingress.yaml#gated-paths). `/` is the ungated one.
  url: https://www.renner.uno/
  uptimeRobotConfig:
    # `http`, not `keyword`. The survey recommends a keyword check here
@@ -254,6 +267,8 @@ spec:
    # matches the interval of the nine monitors already in the account. Written
    # out rather than left implicit so the value is reviewable here.
    interval: 300
+    # See "alertContacts" in the header: the owner's existing two contacts.
+    alertContacts: 7880965_0_0-8026823_0_0
 ---
 # Restored by this commit. This object already proved criterion 3 end to end:
 # created as id 804092841 at 03:26:27Z, removed as `Monitor Removed` at
@@ -273,8 +288,9 @@ spec:
  # explicitly ignored when `spec.url` is set (pkg/kube/util/url.go:31-33).
  # `/healthcheck` is ungated, so the check reaches application health rather
  # than mere reachability — it is the same endpoint the Gatus check uses
-  # (apps/gatus/configmap.yaml:163).
+  # (Web Service 8 in ../gatus/configmap.yaml#gatus-endpoints).
  url: https://audiobooks.renner.uno/healthcheck
  uptimeRobotConfig:
    monitorType: http
    interval: 300
+    alertContacts: 7880965_0_0-8026823_0_0
@@ -75,6 +75,7 @@ spec:
                name: ak-outpost-wordpress-gate
                port:
                  number: 9000
+          # anchor: gated-paths
          # Covers admin-ajax.php too. That is only safe because the gift
          # registry plugin -- the sole thing on this site that ever emitted
          # an ajaxurl -- is being dropped in the migration. Verified against
@@ -0,0 +1,37 @@
+# Renovate (REN-216): a CronJob that opens dependency-update pull requests on
+# git.renner.uno. It holds the renovate-bot token ExternalSecret, the runner
+# config and the CronJob. The namespace and its CiliumNetworkPolicy live in
+# infrastructure/ (see the `renovate` entry in namespaces.yaml).
+#
+# A dedicated Kustomization, and the same ExternalSecret health check as
+# uptime-monitor.yaml, for the same reasons written out there at length: a
+# missing or failed key must hold THIS Kustomization NotReady, and nothing
+# depends on this one, so the blast radius of that is Renovate alone.
+# `wait: true` without `healthCheckExprs` would go Ready with no Secret behind
+# it (REN-100).
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: renovate
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 2m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/apps/renovate
+  prune: true
+  wait: true
+  healthCheckExprs:
+    - apiVersion: external-secrets.io/v1
+      kind: ExternalSecret
+      failed: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'False' && (c.reason == 'SecretOwnedByOther' || c.reason == 'SecretImmutable'))"
+      current: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')"
+  dependsOn:
+    # ESO itself, and the `renovate` namespace.
+    - name: infrastructure
+    # The `openbao` ClusterSecretStore, including the `renovate` entry in its
+    # conditions.namespaces allowlist.
+    - name: infrastructure-crds
@@ -0,0 +1,137 @@
+# kubernetes-sigs/agent-sandbox controller (REN-285, plan REN-284). Provides the
+# `sandboxes.agents.x-k8s.io` CRD and reconciles each Sandbox into one pod. The
+# Paperclip Kubernetes plugin (backend `sandbox-cr`) uses it to run each agent
+# run in its own pod in namespace `paperclip-renner`.
+#
+# Upstream publishes the Helm chart only inside the git repo (./helm, chart
+# version frozen at 0.1.0), not in a chart repository. So the source is a
+# GitRepository pinned to a release tag, and `reconcileStrategy: Revision`
+# makes a tag bump re-render the chart even though its version does not move.
+#
+# TWO PINS, ONE RELEASE: the git tag below and `image.tag` in the values. The
+# chart requires the tag and does not derive it from the git ref. Renovate
+# tracks both (flux manager: GitRepository tag, HelmRelease values image) and
+# the `agent-sandbox` packageRule in renovate.json groups them into one PR.
+#
+# HELD AT v0.5.x (REN-298): the Paperclip plugin (0.1.0-alpha.1) hardcodes
+# `agents.x-k8s.io/v1alpha1`; v1.0.0 dropped it. v0.5.6 serves v1alpha1
+# (converted by the controller's webhook on :9443) and stores v1beta1, like
+# v1.0.4. renovate.json caps the version below 1.0.0 until the plugin moves
+# to v1beta1.
+#
+# Not a healthCheck of `infrastructure`: nothing in the cluster depends on this
+# at startup, so a failing controller must not hold up GitOps.
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: agent-sandbox
+  namespace: agent-sandbox-system
+spec:
+  interval: 24h
+  url: https://github.com/kubernetes-sigs/agent-sandbox
+  ref:
+    tag: v0.5.6
+  # Only the chart; the rest of the repo is not needed in the artifact.
+  ignore: |
+    /*
+    !/helm/
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: agent-sandbox
+  namespace: agent-sandbox-system
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ./helm
+      reconcileStrategy: Revision
+      sourceRef:
+        kind: GitRepository
+        name: agent-sandbox
+        namespace: agent-sandbox-system
+  install:
+    # The chart ships its CRDs in crds/, which plain Helm never upgrades.
+    # CreateReplace keeps them in step with the controller on every tag bump.
+    # It also installs the three extensions CRDs (sandboxclaims,
+    # sandboxtemplates, sandboxwarmpools); they stay unused while
+    # `controller.extensions` is false.
+    crds: CreateReplace
+    remediation:
+      retries: 3
+  upgrade:
+    crds: CreateReplace
+    remediation:
+      retries: 3
+  # The chart's common labels render two invalid label values here, and the
+  # API server rejects the install:
+  # - `app.kubernetes.io/version` is `.Values.image.tag`, which carries the
+  #   digest (`@sha256:…`, over 63 chars);
+  # - `helm.sh/chart` gets a `+<rev>` suffix from `reconcileStrategy: Revision`.
+  # Drop both labels from every object that has them. Removing them (instead of
+  # setting a fixed value) adds no third pin for Renovate to keep in step.
+  # Pod template and selector labels do not use them and are not touched.
+  postRenderers:
+    - kustomize:
+        patches:
+          - target:
+              labelSelector: app.kubernetes.io/version
+            patch: |
+              - op: remove
+                path: /metadata/labels/app.kubernetes.io~1version
+          - target:
+              labelSelector: helm.sh/chart
+            patch: |
+              - op: remove
+                path: /metadata/labels/helm.sh~1chart
+          # v0.5.x (REN-298): the controller writes its self-signed conversion
+          # webhook cert to /tmp/k8s-webhook-server/serving-certs at startup;
+          # the root filesystem is read-only and the chart mounts nothing
+          # there. Drop this patch when the hold lifts (v1.x has no webhook).
+          - target:
+              kind: Deployment
+              name: agent-sandbox-controller
+            patch: |
+              - op: add
+                path: /spec/template/spec/volumes/-
+                value:
+                  name: webhook-certs
+                  emptyDir:
+                    medium: Memory
+                    sizeLimit: 1Mi
+              - op: add
+                path: /spec/template/spec/containers/0/volumeMounts/-
+                value:
+                  name: webhook-certs
+                  mountPath: /tmp/k8s-webhook-server/serving-certs
+  values:
+    namespace:
+      # Declared in infrastructure/namespaces with the PSS labels.
+      create: false
+      name: agent-sandbox-system
+    image:
+      repository: registry.k8s.io/agent-sandbox/agent-sandbox-controller
+      # Keep equal to the GitRepository tag above.
+      tag: v0.5.6@sha256:dc23fb0d5624c306ca2f8ef0d41848dba670ebaf62beb500f870175aec529ffd
+    controller:
+      leaderElect: true
+      extensions: false
+    resources:
+      requests:
+        cpu: 10m
+        memory: 64Mi
+      limits:
+        memory: 256Mi
+    podSecurityContext:
+      runAsNonRoot: true
+      runAsUser: 65532
+      runAsGroup: 65532
+      seccompProfile:
+        type: RuntimeDefault
+    containerSecurityContext:
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+          - ALL
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helmrelease.yaml
@@ -90,7 +90,7 @@ for image-policy to sort against:
 - `openthread/otbr` — only Ubuntu codenames (`focal`, `jammy`,
  `hirsute`) and `:latest`/`:test`/`:reference-device`; no semver
  release stream.
- `ghcr.io/konairius/ical-proxy` — only `main`, `latest`, and
+- `gitea.renner.uno/konairius/ical-proxy` — only `main`, `latest`, and
  `main-<sha>` commit builds; no release tags.

 The security-scan agent will continue to surface vuln findings on
@@ -31,6 +31,7 @@ resources:
  - external-secrets/
  - reloader/
  - admission-policies/
+  - agent-sandbox/
 # The OpenBao ClusterSecretStore lives in post-crds/ (needs ESO's CRDs and
 # an initialized OpenBao; see kubernetes/MIGRATION-sops-openbao.md).
 # cnpg-barman-plugin/ lives in its own Flux Kustomization
@@ -260,3 +260,52 @@ apiVersion: v1
 kind: Namespace
 metadata:
  name: uptime-monitor
+---
+# Renovate (REN-216): a weekly CronJob that opens dependency-update pull
+# requests on git.renner.uno. Declared here, not in apps/renovate, for the same
+# ordering reason as `uptime-monitor` above: this namespace has a
+# CiliumNetworkPolicy (../network-policies/renovate-ns.yaml), and
+# infrastructure-crds applies that policy after `infrastructure` creates the
+# namespace.
+#
+# No pod-security labels: the job needs no privilege, so the cluster's default
+# `baseline` enforcement is enough.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: renovate
+---
+# kubernetes-sigs/agent-sandbox controller (REN-285, plan REN-284). Declared
+# here for the same ordering reason as `uptime-monitor` above: it has a
+# CiliumNetworkPolicy (../network-policies/agent-sandbox-system-ns.yaml). The
+# chart's own namespace template is switched off (`namespace.create: false`).
+#
+# `restricted`: the controller is a plain API client; helmrelease.yaml sets the
+# securityContext that level requires.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: agent-sandbox-system
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted
+---
+# One pod per Paperclip agent run (REN-284). The Paperclip Kubernetes plugin
+# creates Sandbox CRs, NetworkPolicies and Secrets here; the rights it has to do
+# that are in ../../apps/paperclip/sandbox-rbac.yaml. Declared here because that
+# file puts a RoleBinding in this namespace, and the paperclip Kustomization
+# runs after `infrastructure`.
+#
+# `restricted` is load-bearing, not cosmetic: the paperclip ServiceAccount can
+# create pods here but has no right to patch this Namespace, so these labels are
+# what keeps it from starting a privileged pod. Do not grant `patch`/`update` on
+# namespaces to that account.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: paperclip-renner
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted
@@ -0,0 +1,25 @@
+# agent-sandbox controller (REN-285). The controller only talks to the
+# kube-apiserver: it watches Sandbox CRs and creates pods, services and PVCs.
+# The kube-apiserver calls it on :9443 for CRD conversion (v1alpha1 <->
+# v1beta1, v0.5.x, REN-298).
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: default-deny-ingress
+  namespace: agent-sandbox-system
+spec:
+  endpointSelector: {}
+  ingress:
+    # kubelet liveness/readiness probes on :8081; kube-apiserver conversion
+    # webhook calls on :9443.
+    - fromEntities:
+        - host
+        - remote-node
+        - kube-apiserver
+  egress:
+    # host/remote-node: kube-apiserver — watches, pod create, leader-election
+    # Lease. The client uses KUBERNETES_SERVICE_HOST (an IP), so no DNS rule.
+    - toEntities:
+        - host
+        - remote-node
+        - kube-apiserver
@@ -30,4 +30,6 @@ resources:
  - ai-ns.yaml
  - navidrome-ns.yaml
  - uptime-monitor-ns.yaml
+  - renovate-ns.yaml
+  - agent-sandbox-system-ns.yaml
  - allow-cilium-health.yaml
@@ -113,3 +113,27 @@ spec:
            - { type: 135, family: IPv6 }
            - { type: 136, family: IPv6 }
            - { type: 143, family: IPv6 }
+---
+# Paperclip agents -> Lidarr API only (REN-350, owner-approved): the Lidarr pod
+# on 8686, nothing else in the namespace. Lidarr runs AUTH__METHOD External, so
+# this admit gives every agent in the paperclip pod full Lidarr access (the API
+# key is readable from the UI bootstrap). Revert this doc and the matching
+# egress rule in apps/paperclip/network-policy.yaml to close it again.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: lidarr-from-paperclip
+  namespace: navidrome
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: lidarr
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+            app.kubernetes.io/name: paperclip
+      toPorts:
+        - ports:
+            - port: "8686"
+              protocol: TCP
@@ -1,3 +1,14 @@
+# Baseline ingress/egress policy for the `paperless` namespace.
+#
+# CAUTION when adding rules here: this policy has `endpointSelector: {}`, so
+# every rule below applies to EVERY pod in the namespace — nine workloads, one
+# of which (paperless) grants identity from the X-Authentik-Username header and
+# will believe anything that can open a TCP connection to it. A `toPorts` block
+# narrows the PORT, not the destination workload; the destination is governed
+# solely by the enclosing policy's endpointSelector. Anything meant for one
+# workload belongs in its own policy with its own selector, as
+# pprl-pg-allow-cnpg-operator, pprl-pg-allow-prometheus and
+# paperless-allow-blackbox below do.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
@@ -6,6 +17,7 @@ metadata:
 spec:
  endpointSelector: {}
  ingress:
+    # anchor: paperless-admit
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": paperless
@@ -19,21 +31,15 @@ spec:
    # kubernetes/apps/paperless/network-policy.yaml for the
    # gate-specific ingress rule.
    #
-    # Note: the database and monitoring allows below DO broaden the trust
-    # boundary in the way this note warns about, because endpointSelector
-    # is {} and therefore covers the paperless pod too. They are kept
-    # because both are load-bearing — CNPG reaches the postgres cluster
-    # from `database`, and the blackbox probe of
-    # http://paperless.paperless.svc.cluster.local:80/accounts/login/
-    # (kubernetes/apps/monitoring-rules/probes-blackbox.yaml:250) runs
-    # from `monitoring`.
-    # Scoping them to specific pods and ports is tracked in REN-28.
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": database
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": monitoring
+    # `database` and `monitoring` are intentionally NOT in this list either,
+    # and for the same reason: on endpointSelector:{} they reached all nine
+    # workloads on every port, including paperless's HTTP listener. Both are
+    # load-bearing, so both are preserved as per-workload policies at the
+    # bottom of this file (REN-28):
+    #   pprl-pg-allow-cnpg-operator  database/cloudnative-pg -> pprl-pg   :8000
+    #   pprl-pg-allow-prometheus     monitoring/prometheus   -> pprl-pg   :9187
+    #   paperless-allow-blackbox     monitoring/blackbox     -> paperless :8000
+    # anchor: host-admit
    - fromEntities:
        - host
        - remote-node
@@ -168,3 +174,101 @@ spec:
              protocol: UDP
            - port: "21027"
              protocol: UDP
+---
+# anchor: pprl-pg-allow-cnpg-operator
+# The CNPG operator polls each instance's status endpoint over HTTPS
+# (https://<pod-ip>:8000/pg/status). Without it the Cluster reports "Instance
+# Status Extraction Error" while Postgres serves normally, and the operator
+# loses failover and switchover decisions (#433).
+#
+# Shape and port lifted from wp-pg-allow-cnpg-operator (woodpecker-ns.yaml),
+# which runs in production against a wp-pg that registers the same barman-cloud
+# WAL archiver as pprl-pg (cluster.yaml plugins[].isWALArchiver) and takes daily
+# plugin-method base backups, with no `database` rule on its baseline at all.
+#
+# barman-cloud needs no ingress here: the archiver is a container INSIDE this
+# instance pod — see the debug recipe in
+# monitoring-rules/prometheusrule-cnpg-backup.yaml
+# (`kubectl logs <pod> -c plugin-barman-cloud`) — and this namespace's egress
+# does not permit `database` at all, so every flow on that path is
+# database-initiated. The three mariadb-operator pods in `database` have no
+# MariaDB resource in this namespace to act on.
+#
+# No ICMP rule here, deliberately: the baseline in this file already allows all
+# standard ICMP from any cluster pod on endpointSelector:{}, and admits
+# host/remote-node L3-only, which wildcards every protocol from those entities.
+# Cilium unions allow rules, so attaching toPorts cannot subtract either grant.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: pprl-pg-allow-cnpg-operator
+  namespace: paperless
+spec:
+  endpointSelector:
+    matchLabels:
+      cnpg.io/cluster: pprl-pg
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": database
+            app.kubernetes.io/name: cloudnative-pg
+      toPorts:
+        - ports:
+            - port: "8000"
+              protocol: TCP
+---
+# anchor: pprl-pg-allow-prometheus
+# Prometheus -> the pprl-pg CNPG built-in exporter. PodMonitor cnpg-pprl-pg
+# (monitoring-rules/cnpg-podmonitors.yaml) selects cnpg.io/cluster: pprl-pg and
+# is live and up on both instances. Dropping this silently blinds
+# monitoring-rules/prometheusrule-cnpg-backup.yaml.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: pprl-pg-allow-prometheus
+  namespace: paperless
+spec:
+  endpointSelector:
+    matchLabels:
+      cnpg.io/cluster: pprl-pg
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": monitoring
+            app.kubernetes.io/name: prometheus
+      toPorts:
+        - ports:
+            - port: "9187"
+              protocol: TCP
+---
+# anchor: paperless-allow-blackbox
+# blackbox-exporter -> the paperless web listener only.
+#
+# NOTE: Cilium matches the POD port after service translation. The probe in
+# kubernetes/apps/monitoring-rules/probes-blackbox.yaml#paperless-app-health
+# targets :80 on the Service, which resolves to containerPort 8000
+# (paperless.yaml). `port: "80"` here would read clean in review and break the
+# probe.
+#
+# Scoped to the paperless pod so this does NOT also open pprl-pg's
+# instance-manager API, which serves on :8000 as well — the hazard refused in
+# woodpecker-ns.yaml for woodpecker-ci. Measured safe besides: pprl-pg's pods
+# carry app.kubernetes.io/name: postgresql, not paperless.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: paperless-allow-blackbox
+  namespace: paperless
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: paperless
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": monitoring
+            app.kubernetes.io/name: prometheus-blackbox-exporter
+      toPorts:
+        - ports:
+            - port: "8000"
+              protocol: TCP
@@ -0,0 +1,51 @@
+---
+# Baseline policy for the `renovate` namespace (REN-216).
+#
+# The namespace holds one CronJob pod that serves nothing and only calls out.
+# It has no probes and no port, so it admits no ingress at all. `ingress: [{}]`
+# is the Cilium idiom for that: an empty rule selects no peer, and its presence
+# puts the endpoint into ingress default-deny. An absent `ingress` key would not.
+#
+# Egress, and why each rule is there:
+#   * kube-dns — name resolution.
+#   * Traefik in `ingress` — git.renner.uno (Gitea API, git over HTTPS) and the
+#     gitea.renner.uno container registry, when the name resolves to the
+#     in-cluster load balancer. Same rule as paperclip-ns.yaml.
+#   * world:443 — the upstream registries and release-note sources Renovate
+#     looks up (Docker Hub, ghcr.io, quay.io, Helm repositories, GitHub), and
+#     git.renner.uno when it resolves to the public address.
+#
+# Deliberately no kube-apiserver egress: Renovate never talks to the Kubernetes
+# API. Its ServiceAccount has no token mounted and no Role. ESO, in the
+# external-secrets namespace, writes the token Secret; the pod reads it as an
+# env var. Deliberately no OpenBao egress either, for the same reason.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: default-deny
+  namespace: renovate
+spec:
+  endpointSelector: {}
+  ingress:
+    - {}
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+            - port: "53"
+              protocol: TCP
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ingress
+            app.kubernetes.io/name: traefik
+    - toEntities:
+        - world
+      toPorts:
+        - ports:
+            - port: "443"
+              protocol: TCP
@@ -0,0 +1,21 @@
+# OpenBao policy: secret-exists-check
+#
+# Lets the Paperclip agents Argus and Atlas check whether a secret exists in the
+# `homelab` KV v2 mount (paths, versions, timestamps) without reading any value.
+# Metadata only: no access to homelab/data/*, no writes.
+#
+# Identity: userpass user `agent-secret-check`. Its password is a Paperclip
+# agent secret bound to Argus and Atlas.
+#
+# NOT applied by Flux. Apply ONCE with an admin token:
+#
+#   bao policy write secret-exists-check kubernetes/infrastructure/openbao/policies/secret-exists-check.hcl
+#   bao write auth/userpass/users/agent-secret-check \
+#     password=<generated> token_policies=secret-exists-check \
+#     token_ttl=1h token_max_ttl=4h
+#
+# Check: `bao kv metadata get homelab/<path>` succeeds, `bao kv get homelab/<path>` is denied.
+
+path "homelab/metadata/*" {
+  capabilities = ["read", "list"]
+}
@@ -29,6 +29,7 @@ spec:
        - monitoring
        - paperclip
        - paperless
+        - renovate # renovate-bot token (apps/renovate/externalsecret.yaml)
        - trmnl
        - uptime-monitor # uptime-robot-api-key (apps/uptime-monitor/externalsecret.yaml)
        - velero
@@ -196,9 +196,10 @@ spec:
  # several apps under apps/ set `namespace: home-assistant` in their
  # Kustomization, so their PVCs land here too.
  #
-  # home-assistant-config has its own restore path: the HA pod's `seed`
-  # initContainer pulls the native HAOS backup. Layering a Velero PVC restore
-  # on top of that for the same PVC needs its own review.
+  # This is the ONLY restore path for home-assistant-config (.storage,
+  # secrets.yaml, the config git checkout). The HAOS-backup `seed`
+  # initContainer that used to repopulate it was removed in 21a0a43f.
+  # Restore procedure and test: REN-249 runbook section C.2.
  schedule: "45 6 * * *"
  useOwnerReferencesInBackup: false
  template:
@@ -88,12 +88,21 @@ spec:
    nodeAgent:
      privileged: false
      # Reference the node-agent BackupPVC config (node-agent-config.yaml).
-      # Sets `readOnly: true` for ceph-block and ceph-filesystem source
-      # storage classes so the data-mover skips the CSI clone and reads
-      # snapshot contents directly. Saves ~2.5 h per multi-hundred-GiB
+      # Sets `readOnly: true` for the CephFS source storage classes
+      # (ceph-filesystem, ceph-filesystem-bulk) so the data-mover skips
+      # the CSI clone and reads snapshot contents directly. Saves ~2.5 h per multi-hundred-GiB
      # CephFS Backup. Requires Velero >= 1.15 and ceph-csi >= 3.10.
      extraArgs:
        - --node-agent-configmap=node-agent-config
+      # node-agent reads node-agent-config only at startup and the chart
+      # does not hash it into the pod template. Bump this label whenever
+      # node-agent-config.yaml changes so Flux rolls the node-agent pods
+      # (and only them — top-level podAnnotations would also roll the
+      # server). Rolling kills in-flight DataUploads: merge changes here
+      # outside the backup window (schedules 03:15–09:15Z, immich upload
+      # runs ~4 h).
+      podLabels:
+        renner.uno/node-agent-config: ren-177
    metrics:
      # Service stays (chart-rendered) so Prometheus has a target;
      # ServiceMonitor + nodeAgent PodMonitor are declared standalone under
@@ -11,6 +11,16 @@
 # skip the clone entirely. Requires ceph-csi >= 3.10 (cluster runs
 # 3.16.2) and Velero >= 1.15 (cluster runs 1.18.0).
 #
+# backupPVC is keyed by the SOURCE PVC's storage class name, so every
+# CephFS class needs its own entry. ceph-filesystem-bulk (added for the
+# immich / audiobookshelf bulk libraries) was missing and fell back to
+# the full clone: its data-mover pods sat Pending on "unbound immediate
+# PersistentVolumeClaims" for 4 and 22 min on their first nightly run
+# (REN-177), growing with the data. Its backup PVC uses
+# ceph-filesystem-bulk-snapshot-readonly (snapshot-readonly-
+# storageclass.yaml): explicit backingSnapshot, and reclaimPolicy Delete
+# instead of the source class's Retain.
+#
 # ceph-block (RBD) volumes intentionally use the default clone path
 # (no readOnly here). The previous approach used readOnly: true with
 # storageClass: ceph-block-snapshot-readonly (noload) to skip ext4
@@ -38,6 +48,10 @@ data:
      "backupPVC": {
        "ceph-filesystem": {
          "readOnly": true
+        },
+        "ceph-filesystem-bulk": {
+          "readOnly": true,
+          "storageClass": "ceph-filesystem-bulk-snapshot-readonly"
        }
      }
    }
@@ -37,3 +37,39 @@ mountOptions:
  - noload
 reclaimPolicy: Delete
 allowVolumeExpansion: true
+---
+# Storage class used only by Velero's data-mover exposer pod for
+# ceph-filesystem-bulk source PVCs (node-agent-config.yaml,
+# backupPVC."ceph-filesystem-bulk".storageClass). Same cluster / fs /
+# secrets as ceph-filesystem-bulk, plus:
+#
+# - backingSnapshot: "true" — ceph-csi provisions a shallow ReadOnlyMany
+#   volume that points at the CephFS snapshot instead of cloning the
+#   subvolume. Binds in seconds regardless of data size (REN-177: the
+#   clone path left the immich data mover Pending 22 min, and it grows
+#   with the library). Stated explicitly rather than relying on the
+#   ceph-csi default.
+# - reclaimPolicy: Delete — the backup PVC is thrown away after each
+#   DataUpload; the source class's Retain would strand a PV per run.
+# - no volume expansion — shallow volumes are read-only snapshots.
+#
+# Workload PVCs keep using ceph-filesystem-bulk; nothing else may use
+# this class.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ceph-filesystem-bulk-snapshot-readonly
+provisioner: rook-ceph.cephfs.csi.ceph.com
+parameters:
+  clusterID: rook-ceph
+  fsName: cephfs
+  pool: cephfs-bulk
+  backingSnapshot: "true"
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+reclaimPolicy: Delete
+allowVolumeExpansion: false
@@ -0,0 +1,238 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "local>k.renner/renovate-config"
+  ],
+  "enabledManagers": [
+    "flux",
+    "kubernetes",
+    "woodpecker",
+    "dockerfile"
+  ],
+  "ignorePaths": [
+    "kubernetes/clusters/homelab/flux-system/**"
+  ],
+  "flux": {
+    "managerFilePatterns": [
+      "/^kubernetes/.+\\.ya?ml$/"
+    ]
+  },
+  "kubernetes": {
+    "managerFilePatterns": [
+      "/^kubernetes/.+\\.ya?ml$/"
+    ]
+  },
+  "packageRules": [
+    {
+      "description": "Flux image automation (kubernetes/infrastructure/image-automation) already bumps these images. Renovate must not open a second, competing update. Keep this list equal to the ImageRepository images there.",
+      "matchDatasources": [
+        "docker"
+      ],
+      "matchPackageNames": [
+        "apache/tika",
+        "deluan/navidrome",
+        "eclipse-mosquitto",
+        "everythingsmarthome/everything-presence-mmwave-configurator",
+        "ghcr.io/advplyr/audiobookshelf",
+        "ghcr.io/cloudnative-pg/plugin-barman-cloud",
+        "ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar",
+        "ghcr.io/home-assistant/home-assistant",
+        "ghcr.io/immich-app/immich-machine-learning",
+        "ghcr.io/immich-app/immich-server",
+        "ghcr.io/matter-js/python-matter-server",
+        "ghcr.io/music-assistant/server",
+        "ghcr.io/paperclipai/paperclip",
+        "ghcr.io/paperless-ngx/paperless-ngx",
+        "ghcr.io/usetrmnl/terminus",
+        "ghcr.io/usetrmnl/trmnl-ha-amd64",
+        "gitea.renner.uno/k.renner/shelly-manager",
+        "gitea.renner.uno/k.renner/vsftpd",
+        "gitea.renner.uno/konairius/wiki",
+        "gotenberg/gotenberg",
+        "lscr.io/linuxserver/lidarr",
+        "netbirdio/netbird",
+        "quay.io/ceph/ceph",
+        "redis",
+        "rhasspy/wyoming-openwakeword",
+        "rhasspy/wyoming-speech-to-phrase",
+        "rhasspy/wyoming-whisper",
+        "shenxn/protonmail-bridge",
+        "twinproduction/gatus",
+        "valkey/valkey",
+        "velero/velero-plugin-for-aws",
+        "woodpeckerci/woodpecker-agent",
+        "woodpeckerci/woodpecker-server"
+      ],
+      "enabled": false
+    },
+    {
+      "description": "homelab only: pinning a digest changes the pod spec and restarts the workload. Keep these out of the version-update PR, so a reviewer can merge them in a window, separately.",
+      "matchUpdateTypes": [
+        "pinDigest"
+      ],
+      "groupName": "pin image digests",
+      "groupSlug": "pin-image-digests"
+    },
+    {
+      "description": "Cilium and Rook-Ceph: a minor version is a planned change with a window (maintainer duties). One PR per update, never in the weekly group.",
+      "matchPackageNames": [
+        "cilium",
+        "rook-ceph",
+        "rook-ceph-cluster"
+      ],
+      "matchUpdateTypes": [
+        "minor",
+        "major"
+      ],
+      "groupName": null
+    },
+    {
+      "description": "agent-sandbox (REN-285): the chart comes from the git tag and the controller image is pinned separately in the HelmRelease values. Both must move together, in one PR.",
+      "matchPackageNames": [
+        "kubernetes-sigs/agent-sandbox",
+        "registry.k8s.io/agent-sandbox/agent-sandbox-controller"
+      ],
+      "groupName": "agent-sandbox",
+      "groupSlug": "agent-sandbox"
+    },
+    {
+      "description": "agent-sandbox held below 1.0.0 (REN-298): the Paperclip Kubernetes sandbox plugin (0.1.0-alpha.1) uses Sandbox v1alpha1, which 1.0.0 removed. Lift this rule when the plugin moves to v1beta1.",
+      "matchPackageNames": [
+        "kubernetes-sigs/agent-sandbox",
+        "registry.k8s.io/agent-sandbox/agent-sandbox-controller"
+      ],
+      "allowedVersions": "<1.0.0"
+    },
+    {
+      "description": "Our own images (gitea.renner.uno) are pinned by DATED TAG, never by digest (REN-315, ci-manifests-test no-digest-pins). Registry GC deletes their old digests, and a pod that restarts on a deleted digest cannot pull. Existing digest pins still get digest updates.",
+      "matchDatasources": [
+        "docker"
+      ],
+      "matchPackageNames": [
+        "gitea.renner.uno/**"
+      ],
+      "pinDigests": false
+    },
+    {
+      "description": "Home Assistant namespace: one PR of its own, so Hestia can do the HA-impact review and it merges 02:00-05:00 (REN-240, REN-315).",
+      "matchFileNames": [
+        "kubernetes/apps/home-assistant/**"
+      ],
+      "matchUpdateTypes": [
+        "patch",
+        "minor",
+        "digest",
+        "pin"
+      ],
+      "groupName": "home-assistant non-major",
+      "groupSlug": "home-assistant-minor-patch"
+    },
+    {
+      "description": "CI only (pipelines and CI images): nothing here reaches the cluster, so it merges without a window (REN-315).",
+      "matchFileNames": [
+        ".woodpecker/**",
+        "ci/images/**"
+      ],
+      "matchUpdateTypes": [
+        "patch",
+        "minor",
+        "digest",
+        "pin"
+      ],
+      "groupName": "CI non-major",
+      "groupSlug": "ci-minor-patch"
+    },
+    {
+      "description": "Stateful apps and the CNI: one PR each, never in a group (REN-315). A git revert does not undo a DB migration or a CNI change, so each needs its own review and, where it can disrupt the cluster, a window.",
+      "matchUpdateTypes": [
+        "patch",
+        "minor",
+        "digest",
+        "pin"
+      ],
+      "matchPackageNames": [
+        "gitea/gitea"
+      ],
+      "groupName": "gitea",
+      "groupSlug": "gitea"
+    },
+    {
+      "description": "authentik: chart and images together, one PR (REN-315, DB migrations).",
+      "matchUpdateTypes": [
+        "patch",
+        "minor",
+        "digest",
+        "pin"
+      ],
+      "matchPackageNames": [
+        "authentik",
+        "ghcr.io/goauthentik/**"
+      ],
+      "groupName": "authentik",
+      "groupSlug": "authentik"
+    },
+    {
+      "description": "InfluxDB: chart and image together, one PR (REN-315, data migrations).",
+      "matchUpdateTypes": [
+        "patch",
+        "minor",
+        "digest",
+        "pin"
+      ],
+      "matchPackageNames": [
+        "influxdb",
+        "influxdb2"
+      ],
+      "groupName": "influxdb",
+      "groupSlug": "influxdb"
+    },
+    {
+      "description": "OpenBao: chart and image together, one PR (REN-315, raft storage, the secrets backend). rangeStrategy bump: the default (replace) rewrote the bounded chart range \">=0.28.4 <0.29.0\" to \"<0.30.0\" (homelab #1054) and dropped the lower bound. bump keeps the shape, e.g. \">=0.29.6 <0.30.0\".",
+      "matchUpdateTypes": [
+        "patch",
+        "minor",
+        "digest",
+        "pin"
+      ],
+      "matchPackageNames": [
+        "openbao",
+        "quay.io/openbao/openbao"
+      ],
+      "groupName": "openbao",
+      "groupSlug": "openbao"
+    },
+    {
+      "matchDatasources": [
+        "helm"
+      ],
+      "matchPackageNames": [
+        "openbao"
+      ],
+      "rangeStrategy": "bump"
+    },
+    {
+      "description": "Multus, whereabouts and the CNI plugins: one PR for the directory (REN-315). A CNI change can cut pod networking on every node.",
+      "matchFileNames": [
+        "kubernetes/infrastructure/multus/**"
+      ],
+      "matchUpdateTypes": [
+        "patch",
+        "minor",
+        "digest",
+        "pin"
+      ],
+      "groupName": "multus CNI",
+      "groupSlug": "multus"
+    },
+    {
+      "description": "LinuxServer tags are <semver>-ls<n>; the default docker versioning reads -ls<n> as a compatibility suffix and never proposes a new one. Beets (REN-350) is Renovate-tracked, not Flux image automation.",
+      "matchDatasources": [
+        "docker"
+      ],
+      "matchPackageNames": [
+        "lscr.io/linuxserver/beets"
+      ],
+      "versioning": "regex:^(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)-ls(?<build>\\d+)$"
+    }
+  ]
+}
@@ -0,0 +1,62 @@
+#!/bin/sh
+# Offline self-test for yaml-citations.sh: each case is a small fixture tree
+# that must FAIL (or, for happy-path, PASS). Run: sh scripts/ci/test-yaml-citations.sh
+set -eu
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+SUT="$HERE/yaml-citations.sh"
+[ -f "$SUT" ] || { echo "FAIL: $SUT not found"; exit 1; }
+
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT INT TERM
+fails=0
+
+# new_tree NAME — a fixture with one anchored target the cases can cite.
+new_tree() {
+  d="$TMP/$1"; mkdir -p "$d/k/a" "$d/k/b"
+  printf 'kind: X\n# anchor: target-block\nspec: {}\n' > "$d/k/a/target.yaml"
+  echo "$d"
+}
+
+# expect CASE want(pass|fail) DIR
+expect() {
+  if YAML_CITATIONS_FLOOR=1 sh "$SUT" "$3" > "$TMP/$1.out" 2>&1; then got=pass; else got=fail; fi
+  if [ "$got" = "$2" ]; then echo "ok    $1 ($got)"
+  else echo "FAIL  $1: wanted $2, got $got"; sed 's/^/        /' "$TMP/$1.out"; fails=1; fi
+}
+
+d=$(new_tree happy-path)
+printf '# see target.yaml#target-block and ../a/target.yaml#target-block\nx: 1\n' > "$d/k/a/same-dir.yaml"
+printf '# root-relative: k/a/target.yaml#target-block\nx: 1\n' > "$d/k/b/root.yaml"
+printf 'url: "http://h/p.yaml#frag"  # a # inside a value is not a comment\n' > "$d/k/b/url.yaml"
+expect happy-path pass "$d"
+
+d=$(new_tree line-number)
+printf '# see ../a/target.yaml:2\nx: 1\n' > "$d/k/b/c.yaml"
+expect line-number fail "$d"
+
+d=$(new_tree line-range-yml)
+printf '# see ../a/target.yml:2-3\nx: 1\n' > "$d/k/b/c.yml"
+expect line-range-yml fail "$d"
+
+d=$(new_tree missing-file)
+printf '# see ../a/gone.yaml#target-block\nx: 1\n' > "$d/k/b/c.yaml"
+expect missing-file fail "$d"
+
+d=$(new_tree missing-anchor)
+printf '# see ../a/target.yaml#no-such-block\nx: 1\n' > "$d/k/b/c.yaml"
+expect missing-anchor fail "$d"
+
+d=$(new_tree duplicate-anchor)
+printf '# anchor: target-block\n' >> "$d/k/a/target.yaml"
+expect duplicate-anchor fail "$d"
+
+d=$(new_tree malformed-anchor)
+printf '# anchor: Not_Kebab\nx: 1\n' > "$d/k/b/c.yaml"
+expect malformed-anchor fail "$d"
+
+d="$TMP/empty-discovery"; mkdir -p "$d"
+if sh "$SUT" "$d" > "$TMP/empty.out" 2>&1; then echo "FAIL  empty-discovery: passed with no files"; fails=1
+else echo "ok    empty-discovery (fail)"; fi
+
+exit $fails
@@ -0,0 +1,110 @@
+#!/bin/sh
+# Guard the citation convention in YAML comments (REN-183, see CONTRIBUTING.md).
+#
+# Run:  sh scripts/ci/yaml-citations.sh [repo-root]
+# Needs: sh, find, awk, grep (BusyBox is fine). Offline.
+# Tests: scripts/ci/test-yaml-citations.sh
+#
+# A comment cites another block as `path.yaml#name`, and the cited block
+# carries a marker line `# anchor: name`. `path` is relative to the citing
+# file's directory or to the repo root. This fails when:
+#
+#   * a comment cites by line number (`foo.yaml:12`). Line numbers go stale on
+#     the first unrelated edit above them, silently; three drift fixes were
+#     merged before this guard existed (REN-181).
+#   * a `path#name` citation names a file or an anchor that does not exist.
+#   * an anchor is not kebab-case, or is declared twice in one file.
+#
+# Existence only. It does not check that the anchored block still says what the
+# citing comment claims — that stays a reviewer's job, and it is why this guard
+# has no false positives from ordinary edits.
+set -eu
+
+ROOT="${1:-.}"
+cd "$ROOT"
+
+# Same loose-floor reasoning as ci-manifests-test.yml: catch discovery that
+# silently returns nothing, not normal churn. ~650 YAML files exist today.
+FLOOR="${YAML_CITATIONS_FLOOR:-50}"
+
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT INT TERM
+
+# Not `find -printf` (BusyBox lacks it).
+find . -path ./.git -prune -o -type f \( -name '*.yaml' -o -name '*.yml' \) -print \
+  | sed 's|^\./||' | sort > "$TMP/files"
+count=$(grep -c . "$TMP/files" || true)
+if [ "$count" -lt "$FLOOR" ]; then
+  echo "FAIL: found only $count YAML files; floor is $FLOOR. Discovery is broken."
+  exit 1
+fi
+
+# One pass over every file. A YAML comment starts at a `#` that begins the line
+# or follows whitespace, so a `#` inside a URL or a quoted value is not one.
+# Emits tab-separated records:
+#   L  file line text        a line-number citation
+#   C  file line path name   a path#name citation
+#   A  file line name        an anchor declaration
+#   B  file line text        a malformed anchor declaration
+while IFS= read -r f; do
+  awk -v f="$f" '
+    {
+      if (!match($0, /(^|[ \t])#/)) next
+      c = substr($0, RSTART)
+      if (c ~ /^[ \t]*# anchor:/) {
+        if (c ~ /^[ \t]*# anchor: [a-z0-9]+(-[a-z0-9]+)*[ \t]*$/) {
+          n = c; sub(/^[ \t]*# anchor: /, "", n); sub(/[ \t]*$/, "", n)
+          print "A\t" f "\t" NR "\t" n
+        } else {
+          print "B\t" f "\t" NR "\t" c
+        }
+        next
+      }
+      if (c ~ /\.ya?ml:[0-9]/) print "L\t" f "\t" NR "\t" c
+      while (match(c, /[A-Za-z0-9_.\/-]+\.ya?ml#[A-Za-z0-9_-]+/)) {
+        m = substr(c, RSTART, RLENGTH)
+        p = m; sub(/#.*/, "", p)
+        n = m; sub(/.*#/, "", n)
+        print "C\t" f "\t" NR "\t" p "\t" n
+        c = substr(c, RSTART + RLENGTH)
+      }
+    }' "$f"
+done < "$TMP/files" > "$TMP/records"
+
+fail=0
+while IFS="$(printf '\t')" read -r kind f line a b; do
+  case "$kind" in
+    L)
+      echo "FAIL  $f:$line cites by line number; use path.yaml#anchor instead"
+      echo "        $a"
+      fail=1 ;;
+    B)
+      echo "FAIL  $f:$line malformed anchor; want '# anchor: kebab-case-name'"
+      fail=1 ;;
+    C)
+      dir=$(dirname "$f")
+      if [ -f "$dir/$a" ]; then target="$dir/$a"
+      elif [ -f "$a" ]; then target="$a"
+      else
+        echo "FAIL  $f:$line cites $a#$b, but no such file (tried $dir/$a and ./$a)"
+        fail=1; continue
+      fi
+      if ! grep -qE "^[[:space:]]*# anchor: $b[[:space:]]*\$" "$target"; then
+        echo "FAIL  $f:$line cites $a#$b, but $target has no '# anchor: $b'"
+        fail=1
+      fi ;;
+  esac
+done < "$TMP/records"
+
+# Duplicate anchors in one file make a citation ambiguous.
+awk -F '\t' '$1 == "A" { k = $2 "\t" $4; if (k in seen) print $2 ":" $3 " duplicate anchor " $4 " (first at line " seen[k] ")"; else seen[k] = $3 }' \
+  "$TMP/records" > "$TMP/dups"
+if [ -s "$TMP/dups" ]; then
+  sed 's/^/FAIL  /' "$TMP/dups"
+  fail=1
+fi
+
+anchors=$(grep -c '^A' "$TMP/records" || true)
+cites=$(grep -c '^C' "$TMP/records" || true)
+[ "$fail" -eq 0 ] && echo "ok    $count YAML files, $anchors anchors, $cites path#anchor citations, 0 line-number citations"
+exit $fail