feat(mqtt): deploy mqtt-account-controller with a Dynamic Security test broker (REN-888)

REN-802 step 1. Adds: - infrastructure/post-crds/mqtt-ca.yaml: private MQTT CA (self-signed root in cert-manager ns) and ClusterIssuer mqtt-ca. - apps/home-assistant/mosquitto-dynsec-test.yaml: throwaway mosquitto 2.0.22 with the dynsec plugin, mTLS listener (require_certificate, use_identity_as_username), initContainer that loads Secret mqtt-dynsec-seed or a static base file. ClusterIP only; not labelled mosquitto, so prod Services and the world-ingress policy do not select it. - apps/home-assistant/mqtt-account-controller.yaml: ServiceAccount, RBAC (cluster-wide list certificates; certificates in home-assistant; secrets by resourceName only), controller client cert CN mqtt-account-controller, Deployment main-47213ef pointed at the test broker. - apps/home-assistant/mqtt-accounts.yaml: register with two cert test accounts, delivered as ConfigMap mqtt-accounts. The production broker is not changed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
chore(image-automation): bump gitea.renner.uno/konairius/paperclip:build.1791297080@sha256:0764e882521410d8ce8ed1359f8a1aa64eaa7746aa909653ef70050c239ca195
2026-10-06 15:23:36 +00:00 · 2026-10-06 14:47:22 +00:00 · 2026-10-06 14:38:49 +00:00 · 2026-10-06 14:32:13 +00:00 · 2026-10-06 14:30:54 +00:00 · 2026-10-06 14:28:17 +00:00
155 changed files with 6018 additions and 506 deletions
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,8 +30,10 @@ when:

 steps:
  - name: build-agent-runtime
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images
+    # Context is ci/images (not the variant dir) so both Containerfiles can
+    # COPY the shared agent-runtime-common/ files.
    # One pod, both images one after the other (kaniko --cleanup between
    # builds), at 1 CPU so it still fits if it overlaps another pipeline.
    backend_options:
@@ -56,7 +58,7 @@ steps:
        for VARIANT in go ops; do
          /kaniko/executor \
            --cleanup \
-            --context "agent-runtime-$VARIANT" \
+            --context . \
            --dockerfile "agent-runtime-$VARIANT/Containerfile" \
            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:$TAG" \
            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:v1" \
@@ -43,7 +43,7 @@ when:

 steps:
  - name: build-ansible
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/ansible
    depends_on: []
    # Only rebuild when THIS image's source (or the pipeline file) changed, or
@@ -71,7 +71,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest

  - name: build-k8s
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/k8s
    depends_on: []
    when:
@@ -95,7 +95,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-vmtest
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/vmtest
    depends_on: []
    when:
@@ -119,7 +119,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-vmtest:latest

  - name: build-tofu
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/tofu
    depends_on: []
    when:
@@ -150,7 +150,7 @@ steps:
  # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
  # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
  - name: build-secret-rotator
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: secret-rotator
    depends_on: []
    when:
@@ -201,7 +201,7 @@ steps:
  #
  # The gitea_admin/ copy stays until nothing references it any more.
  - name: build-alpine-jq
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/alpine-jq
    depends_on: []
    when:
@@ -254,7 +254,7 @@ steps:
  # note on build-alpine-jq. Anything an app needs while the cluster is still
  # being built cannot live only in that cluster's own registry.
  - name: build-cni-plugins
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/cni-plugins
    depends_on: []
    when:
@@ -296,7 +296,7 @@ steps:
  # gitea.renner.uno is not there to serve it. A cold-rebuild rehearsal left
  # paperless in Init:ErrImagePull on exactly this image.
  - name: build-alpine-openssl
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/alpine-openssl
    depends_on: []
    when:
@@ -333,7 +333,7 @@ steps:
  # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
  # builds the same way and lands in the same registry so the audit sees it.
  - name: build-heating-can
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/heating-can
    depends_on: []
    when:
@@ -360,7 +360,7 @@ steps:
  # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
  # Runtime artifact like heating-can; same registry so the audit tracks it.
  - name: build-netbird-service-sync
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/netbird-service-sync
    depends_on: []
    when:
@@ -386,7 +386,7 @@ steps:
  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
  # artifact like heating-can; same registry so the audit tracks it.
  - name: build-bird
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/bird
    depends_on: []
    when:
@@ -0,0 +1,69 @@
+# Paperclip server image (REN-401): upstream paperclipai/paperclip at a pinned
+# master commit plus the patches in ci/images/paperclip/. Needed while the k8s
+# sandbox fix (upstream PR #13733) is not merged; see the README there for how
+# to go back to upstream nightly.
+#
+# Own workflow for the reason given in build-agent-runtime.yml: a step in
+# build-ci-images.yml runs on every change to that file and competes for the
+# woodpecker-ci-quota. Re-run alone with a deployment to target
+# `build-paperclip`.
+#
+# Tag `build.<unix-ts>` is what the Flux ImagePolicy `paperclip` sorts on
+# (image-automation/paperclip.yaml); image automation then rolls it out.
+# konairius/ and ci_images_token as for the agent-runtime images.
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/paperclip/**"
+      - ".woodpecker/build-paperclip.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-paperclip"'
+
+steps:
+  - name: prepare-source
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh ci/images/paperclip/prepare.sh build/paperclip-src
+
+  - name: build-paperclip
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    directory: build/paperclip-src
+    depends_on: [prepare-source]
+    # Rust runner + UI + server build in one pod. The server build alone sets
+    # a 4 GiB Node heap, so the memory limit is the LimitRange max (8Gi).
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: "1"
+            memory: 2Gi
+          limits:
+            cpu: "4"
+            memory: 8Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - . ../../ci/images/paperclip/upstream.env
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=build.$(date +%s)
+      # Same target and build args as upstream's docker.yml for nightly.
+      # --compressed-caching=false: pipeline 11717 was killed at the 8Gi limit
+      # (exit 137) in the server build; kaniko otherwise holds compressed
+      # layer tarballs in memory next to the 4 GiB Node heap.
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Dockerfile.kaniko \
+          --target production \
+          --skip-unused-stages \
+          --snapshot-mode=redo \
+          --compressed-caching=false \
+          --build-arg PAPERCLIP_BUILD_COMMIT=$PAPERCLIP_COMMIT \
+          --build-arg PAPERCLIP_BUILD_VERSION=$PAPERCLIP_VERSION \
+          --build-arg CLI_TOOLS_CACHE_EPOCH=$(date -u +%G-W%V) \
+          --destination gitea.renner.uno/konairius/paperclip:$TAG
@@ -131,7 +131,7 @@ steps:
  #    --build-arg to the resolved version (not "latest") guarantees the tag and
  #    the image content agree even if a new release lands mid-run.
  - name: build
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/claude
    depends_on: [decide]
    environment:
@@ -172,7 +172,7 @@ steps:
  # whenever it trails the deployed server. If you bump the server, bump here.
  - name: woodpecker-lint
    depends_on: [] # see note above `steps:`
-    image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
+    image: woodpeckerci/woodpecker-cli:v3.18.1-alpine@sha256:7eac3c4254befc1ba1e65269c3304c370b6032530b7386fc0c7648378fdd0516
    commands:
      - |
        set -eu
@@ -35,18 +35,18 @@ when:
 steps:
  - name: model-selection-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh

  - name: promtool-rules-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-promtool-rules.sh

  - name: yaml-citations-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-yaml-citations.sh
@@ -13,7 +13,7 @@ when:

 steps:
  - name: validate-secrets
-    image: alpine:3.24.1
+    image: alpine:3.24.2@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
    environment:
      TF_VAR_proxmox_api_url:
        from_secret: proxmox_api_url
@@ -26,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -70,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -17,7 +17,7 @@ when:

 steps:
  - name: checkout-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -54,7 +54,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: distill
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -89,14 +89,14 @@ steps:
          -p

  - name: audit
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    commands:
      # Findings are report lines in output/, not a pipeline failure.
      - python3 -B notes/bin/wiki_audit.py --vault "$$PWD/notes"

  - name: consolidate
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -145,7 +145,7 @@ steps:
        exit $$rc

  - name: push-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -54,11 +54,15 @@ Pod images for the Paperclip Kubernetes sandbox (REN-284). Each is the stock
 set. Base user (1000, non-root), entrypoint and cmd are unchanged; the
 images run with a read-only root FS. Source in `ci/images/agent-runtime-*`,
 built by `.woodpecker/build-agent-runtime.yml` (re-run alone: deployment to
-target `build-agent-runtime`), versions bumped by Renovate.
+target `build-agent-runtime`), versions bumped by Renovate. Both also
+carry `claude-agent-acp` for the ACP engine, pinned and patched like
+Paperclip ships it, SDK override included
+(`ci/images/agent-runtime-common/`); bump it by hand
+together with Paperclip. All tools are on the default `PATH`.

 | Image | `imageRegistry` | Adds |
 |---|---|---|
-| `gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-go` | Go toolchain, golangci-lint |
+| `gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-go` | Go toolchain, golangci-lint, govulncheck |
 | `gitea.renner.uno/konairius/agent-runtime-ops/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-ops` | kubectl, flux, kustomize, helm |

 The plugin keeps the name `agent-runtime-claude:v1` and swaps only host and
@@ -0,0 +1,66 @@
+diff --git a/dist/acp-agent.js b/dist/acp-agent.js
+index 0214a509eb5e7b777a69e4e34637941c275d27b1..403be02df66e25eee29e7d5bef14627011e02b30 100644
+--- a/dist/acp-agent.js
+++ b/dist/acp-agent.js
+@@ -2922,9 +2922,19 @@ export class ClaudeAcpAgent {
+                                             amount: message.total_cost_usd,
+                                             currency: "USD",
+                                         },
+-                                        ...(message.origin && {
+-                                            _meta: { "_claude/origin": message.origin },
+-                                        }),
+                                        _meta: {
+                                            ...(message.origin && { "_claude/origin": message.origin }),
+                                            // ACP's core Usage shape exposes context occupancy and
+                                            // cost, but not the billable input/output split. Keep the
+                                            // SDK's bounded aggregate in extension metadata so ACPX can
+                                            // normalize it without persisting a raw provider message.
+                                            usage: {
+                                                input_tokens: message.usage.input_tokens,
+                                                output_tokens: message.usage.output_tokens,
+                                                cache_read_input_tokens: message.usage.cache_read_input_tokens,
+                                                cache_creation_input_tokens: message.usage.cache_creation_input_tokens,
+                                            },
+                                        },
+                                     },
+                                 });
+                             }
+@@ -5311,6 +5321,10 @@ export class ClaudeAcpAgent {
+             settingSources: ["user", "project", "local"],
+             ...(thinking !== undefined && { thinking }),
+             ...userProvidedOptions,
+            // Paperclip Runner owns the complete session context. Its isolated
+            // user root contains only assigned skills; project/local settings
+            // would reintroduce host prompts, plugins, skills, and MCP servers.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && { settingSources: ["user"] }),
+             ...(settings && { settings }),
+             env,
+             // Override certain fields that must be controlled by ACP
+@@ -5318,7 +5332,9 @@ export class ClaudeAcpAgent {
+             includePartialMessages: true,
+             forwardSubagentText,
+             mcpServers: {
+-                ...(userProvidedOptions?.mcpServers || {}),
+                ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1"
+                    ? {}
+                    : (userProvidedOptions?.mcpServers || {})),
+                 ...mcpServers,
+                 ...(fileChangeAuditSupport
+                     ? { [FILE_CHANGE_AUDIT_SERVER_NAME]: fileChangeAuditSupport.mcpServer }
+@@ -5329,6 +5345,16 @@ export class ClaudeAcpAgent {
+             allowDangerouslySkipPermissions: ALLOW_BYPASS,
+             permissionMode: initialPermissionMode,
+             canUseTool: this.canUseTool(sessionId),
+            // The runner pins this actual MCP endpoint before provider launch.
+            // Exempt only task delivery/control tools at the SDK dispatch boundary,
+            // not arbitrary provider permission metadata or MCP name prefixes.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && {
+                allowedTools: mcpServers.paperclip?.type === "http"
+                    && mcpServers.paperclip.url === process.env.PAPERCLIP_ACPX_TASK_TOOL_BRIDGE_URL
+                    ? ["paperclip_finish", "paperclip_block", "read_current_wake_comments", "request_human_input"]
+                        .map((tool) => `mcp__paperclip__${tool}`)
+                    : [],
+            }),
+             // Forward MCP elicitation requests onto ACP elicitation. Only attached
+             // when the client advertised support, so non-supporting clients keep the
+             // SDK's default (auto-decline) behavior. (AskUserQuestion is handled in
@@ -0,0 +1,9 @@
+{
+  "private": true,
+  "dependencies": {
+    "@agentclientprotocol/claude-agent-acp": "0.73.0"
+  },
+  "overrides": {
+    "@anthropic-ai/claude-agent-sdk": "0.3.280"
+  }
+}
@@ -7,6 +7,13 @@
 #
 # Both binaries are static; they run on the Ubuntu 22.04 base unchanged.
 FROM golang:1.27.1-bookworm@sha256:69a7b9788769bec032d238959b61854e9ae87f57be9029ec04e9885fabf99195 AS go
+
+# govulncheck (REN-488) has no upstream image: build it in the go stage, static.
+# The version pins the module; go verifies it against sum.golang.org. Renovate
+# does not track it (no regex manager here): bump by hand.
+FROM go AS govulncheck
+RUN CGO_ENABLED=0 GOBIN=/out go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
+
 FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS golangci-lint

 # upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
@@ -15,6 +22,34 @@ FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518

 COPY --from=go /usr/local/go /usr/local/go
 COPY --from=golangci-lint /usr/bin/golangci-lint /usr/local/bin/golangci-lint
+COPY --from=govulncheck /out/govulncheck /usr/local/bin/govulncheck
+# go and gofmt also in /usr/local/bin: a process started with the default PATH
+# (not the image ENV) still finds them.
+USER root
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/go \
+    && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
+USER 1000:1000
+
+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000

 # The sandbox pod has a read-only root FS. Writable mounts: $HOME
 # (/home/paperclip, 1Gi), $HOME/.cache (1Gi), /tmp (2Gi), /workspace.
@@ -27,4 +62,7 @@ ENV PATH=/usr/local/go/bin:$PATH \
    GOTOOLCHAIN=local

 # Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
-RUN go version && golangci-lint version
+# The PATH check uses the base image's default PATH, not the ENV above.
+RUN go version && golangci-lint version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in go gofmt golangci-lint govulncheck claude-agent-acp; do command -v "$t"; done'
@@ -7,8 +7,8 @@
 #
 # kubectl and flux match the ci-k8s image. All four binaries are static; they
 # run on the Ubuntu 22.04 base unchanged.
-FROM registry.k8s.io/kubectl:v1.36.5@sha256:0667e3f3e6ef435d7cd4c16684244703ebd67314c17511c2a9f87ea44f6cdf6b AS kubectl
-FROM ghcr.io/fluxcd/flux-cli:v2.9.5@sha256:704d5529535570a495249b643159c93a9aa88ffca07813c974a54b7ba6edbd86 AS flux
+FROM registry.k8s.io/kubectl:v1.37.1@sha256:b7cab618e281b1ee7484e7b706a96e2135fbb6e072c2a573a7dab4e87d7f2385 AS kubectl
+FROM ghcr.io/fluxcd/flux-cli:v2.9.6@sha256:b1ac18156f227af9a524b842a96f2c20986c7a779ef721df3ba4e4540f449d76 AS flux
 FROM registry.k8s.io/kustomize/kustomize:v5.8.1@sha256:899fcd3bc898160e62bcaf82932b0cb29ba38d16272353db2e7acbba82129429 AS kustomize
 FROM alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2 AS helm

@@ -21,8 +21,31 @@ COPY --from=flux /usr/local/bin/flux /usr/local/bin/flux
 COPY --from=kustomize /app/kustomize /usr/local/bin/kustomize
 COPY --from=helm /usr/bin/helm /usr/local/bin/helm

+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000
+
 # Read-only root FS: kubectl (~/.kube/cache) and helm (~/.cache, ~/.config)
 # write only under $HOME, which the sandbox pod mounts writable. No ENV needed.

 # Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
-RUN kubectl version --client && flux version --client && kustomize version && helm version
+RUN kubectl version --client && flux version --client && kustomize version && helm version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in kubectl flux kustomize helm claude-agent-acp; do command -v "$t"; done'
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine plus jq. Nothing application-specific — a general-purpose base for any
 # workload that needs to read JSON from a shell script.
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine plus the openssl CLI. Nothing application-specific — named for what it
 # is, so anything needing `openssl s_client` / `x509` from a shell script can use
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 RUN apk add --no-cache \
      ansible \
@@ -1,4 +1,4 @@
-FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
 # step — the image is the binary plus its runtime.
@@ -1,4 +1,4 @@
-FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2
+FROM node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1

 # Default to the latest Claude Code. Pinning a fixed version is the wrong
 # default here: old releases eventually stop working against the API, so a
@@ -28,10 +28,10 @@
 # amd64 only, matching the ARCH the installer already hardcoded and the cluster's
 # node architecture. Add a second stage if a different arch is ever needed.

-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d AS fetch
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS fetch

-ARG CNI_PLUGINS_VERSION=v1.5.1
-ARG CNI_PLUGINS_SHA256=77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b
+ARG CNI_PLUGINS_VERSION=v1.9.1
+ARG CNI_PLUGINS_SHA256=b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303

 # busybox wget and tar are already in the base; nothing is installed here
 # either. sha256sum -c fails the BUILD if the artifact does not match, which is
@@ -61,7 +61,7 @@ RUN set -eu; \
      [ -x "/plugins/$p" ] || { echo "FATAL: /plugins/$p missing or not executable" >&2; exit 1; }; \
    done

-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine rather than scratch: the installer needs a shell to copy the binaries
 # onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
@@ -23,7 +23,7 @@
 # namespace, so can0 is only reachable with network_mode: host. That is also
 # how the local MQTT broker on 127.0.0.1:1883 is reached. CAN_RAW sockets need
 # no capabilities, so both run unprivileged.
-FROM python:3.13-slim AS build
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151 AS build

 # Pins live here, in the artifact, rather than in Ansible variables.
 #
@@ -56,7 +56,7 @@ RUN git clone --quiet https://github.com/MyHomeMyData/E3onCAN.git /opt/e3oncan \
 && rm -rf /opt/e3oncan/.git


-FROM python:3.13-slim
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151

 ARG OPEN3E_REF=6553c10453a6bae1210908f38a5af2481a46a3e5
 ARG E3ONCAN_REF=461dcbdf97e8d72d2edb9929c63b39382af6c3a1
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 ARG TALOS_VERSION=v1.13.10
 ARG VELERO_VERSION=v1.16.0
@@ -2,7 +2,7 @@
 # domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
 # public label. Standard library only, so the image is the interpreter plus one
 # file; runs unprivileged as its own user.
-FROM python:3.13-slim
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151

 RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
 COPY controller.py /opt/controller.py
@@ -0,0 +1,87 @@
+# Paperclip server image (REN-401)
+
+Upstream `paperclipai/paperclip` at the commit in `upstream.env`, plus every
+`*.patch` in this directory. Built by `.woodpecker/build-paperclip.yml`,
+pushed as `gitea.renner.uno/konairius/paperclip:build.<unix-ts>`, rolled out
+by Flux image automation (`kubernetes/infrastructure/image-automation/paperclip.yaml`).
+
+## Why
+
+k8s sandbox runs fail: the run path never sets the lease `remoteCwd`, so file
+sync into the pod fails (paperclipai/paperclip#13864). The fix is
+paperclipai/paperclip#13733, open and not merged.
+`pr-13733-lease-remote-cwd.patch` is that PR's diff (head `5f320bf6`).
+
+#13733 covers only the run path. Adapter environment tests (binding an agent
+to a k8s environment) sync managed config before any workspace is realized,
+so the lease still has no `remoteCwd` and the Claude test fails with
+`claude_managed_config_dir_failed` (REN-409). The fix is
+paperclipai/paperclip#14769, also open: the plugin sets `remoteCwd:
+/workspace` when it acquires or resumes a lease.
+`pr-14769-probe-remote-cwd.patch` is that PR's diff (head `10dc7371`).
+
+Claude auto memory is lost after every run (REN-541): it lives under
+`CLAUDE_CONFIG_DIR`, which Paperclip makes fresh per run and deletes.
+`ren-541-claude-auto-memory.patch` makes the ACPX engine write
+`autoMemoryDirectory: <AGENT_HOME>/memory` into the per-run
+`.claude/settings.local.json`. `AGENT_HOME` is restored before and saved after
+every run, local and in k8s pods. The value is not part of the session
+fingerprint, so resume is unchanged. No upstream PR yet; rename the patch to
+`pr-<n>-…` once there is one.
+
+Claude sessions never resume with a managed AI connection (REN-563): the
+server gives every run a fresh temp home (`HOME`, `XDG_*`,
+`CLAUDE_CONFIG_DIR`, …) and deletes it after the run.
+`ren-563-acpx-claude-session-resume.patch` fixes both halves for local runs:
+the ACPX engine hashes a fixed placeholder for those paths in the session
+fingerprint (`adapterEnvHash`), as the server's own fingerprint already does,
+and links `CLAUDE_CONFIG_DIR/projects` (the Claude transcripts) to
+`<stateDir>/claude-projects`, which persists. Both must ship together: a
+stable fingerprint without the transcript makes the resumed turn fail with
+"Resource not found" (homelab#1111, reverted in #1113). Remote runs are
+unchanged. No upstream PR yet.
+
+A cancelled or failed k8s pod run blocks its card for good (REN-841). The
+owner's next comment waits on `remote_cleanup`, because the server continues
+only when every remote lease carries a provider termination receipt
+(`explicit-native-continuation.ts`), and the Kubernetes plugin's
+`onEnvironmentReleaseLease` / `onEnvironmentDestroyLease` returned `void`.
+`ren-841-k8s-termination-receipt.patch` makes both return
+`{ providerLeaseId, state: "destroyed" }`, but only after the Sandbox CR or
+Job (and, on destroy, the pod) reads 404, polling for up to 20 s, below the
+server's 30 s plugin RPC timeout. If that check times out or fails, the hook
+returns no receipt, as before. No upstream PR yet.
+
+## Files
+
+- `upstream.env` — pinned commit and the version string stamped into the
+  image. Bump both together.
+- `prepare.sh` — fetches the commit, applies the patches, writes
+  `Dockerfile.kaniko` (upstream's `COPY --parents` line split into plain
+  `COPY` lines; kaniko does not support `--parents`), and adds the build of
+  the Kubernetes sandbox provider plugin to the `build` stage (REN-405):
+  upstream's `production` target ships only its source, so the plugin fails
+  with "no longer exposes a Paperclip manifest". The build fails if
+  `dist/manifest.js` is missing.
+- `*.patch` — applied in name order with `git apply`.
+
+## Moving to a newer upstream commit
+
+Set `PAPERCLIP_COMMIT` to the new master commit and `PAPERCLIP_VERSION` to
+`git describe --tags --match 'v*' --long <commit>`, then run
+`sh prepare.sh /tmp/src`: it fails if a patch no longer applies.
+
+## Exit: when every patch here is merged upstream
+
+Renovate cannot watch an upstream PR, so this is a manual step. Once
+upstream nightly contains all of them:
+
+1. `kubernetes/infrastructure/image-automation/paperclip.yaml`: back to
+   `ghcr.io/paperclipai/paperclip` with `filterTags.pattern: '^nightly$'`
+   (the version before REN-401 in git history).
+2. `renovate.json`: swap `gitea.renner.uno/konairius/paperclip` back to
+   `ghcr.io/paperclipai/paperclip` in the image-automation exclusion list.
+3. Delete this directory and `.woodpecker/build-paperclip.yml`.
+
+Image automation then rewrites `kubernetes/apps/paperclip/paperclip.yaml`
+to the nightly digest by itself.
@@ -0,0 +1,93 @@
+diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts
+index f3d1fc659..812cae357 100644
+--- a/server/src/__tests__/environment-run-orchestrator.test.ts
+++ b/server/src/__tests__/environment-run-orchestrator.test.ts
+@@ -430,6 +430,51 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
+     expect(result.persistedExecutionWorkspace).toEqual(updatedEw);
+   });
+ 
+  it("persists the realized cwd as remoteCwd when realization returns no workspaceRealization payload", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({ environment: makeEnvironment("sandbox") }),
+    );
+
+    expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(
+      "lease-1",
+      expect.objectContaining({ remoteCwd: "/workspace" }),
+    );
+    // Nothing to record about the realization shape, so the execution workspace
+    // row is left untouched.
+    expect(mockUpdateExecutionWorkspace).not.toHaveBeenCalled();
+  });
+
+  it("does not rewrite lease metadata when the realized cwd already matches the lease", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({
+        environment: makeEnvironment("sandbox"),
+        lease: makeLease({ metadata: { remoteCwd: "/workspace" } }),
+      }),
+    );
+
+    expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled();
+  });
+
+   it("runs a remote provision command after workspace realization when configured", async () => {
+     mockBuildWorkspaceRealizationRequest.mockReturnValue({
+       version: 1,
+diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts
+index 97cca667e..e76f64913 100644
+--- a/server/src/services/environment-run-orchestrator.ts
+++ b/server/src/services/environment-run-orchestrator.ts
+@@ -485,16 +485,29 @@ export function environmentRunOrchestrator(
+     }
+ 
+     // Step 3: Persist realization metadata on lease and execution workspace
+-    if (Object.keys(workspaceRealization).length > 0) {
+    const hasWorkspaceRealization = Object.keys(workspaceRealization).length > 0;
+    // A driver that realizes the workspace reports the directory it prepared
+    // through the realization `cwd`. Persist it on the lease as `remoteCwd`, so
+    // every later step that receives only the lease — native file sync, a
+    // resumed lease, a second run against the same lease — confines itself to
+    // the same root. Without it a driver whose realization returns no
+    // `workspaceRealization` payload leaves the lease without a root at all,
+    // and the sync step fails even though realization succeeded.
+    const currentRemoteCwd =
+      typeof lease.metadata?.remoteCwd === "string" ? lease.metadata.remoteCwd.trim() : "";
+    const realizedRemoteCwd =
+      realizedWorkspaceCwd && realizedWorkspaceCwd !== currentRemoteCwd ? realizedWorkspaceCwd : null;
+    if (hasWorkspaceRealization || realizedRemoteCwd) {
+       const nextLeaseMetadata = {
+         ...(lease.metadata ?? {}),
+-        workspaceRealization,
+        ...(hasWorkspaceRealization ? { workspaceRealization } : {}),
+        ...(realizedRemoteCwd ? { remoteCwd: realizedRemoteCwd } : {}),
+       };
+       const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, nextLeaseMetadata);
+       if (updatedLease) {
+         lease = updatedLease;
+       }
+-      if (persistedExecutionWorkspace) {
+      if (hasWorkspaceRealization && persistedExecutionWorkspace) {
+         const updatedEw = await executionWorkspacesSvc.update(persistedExecutionWorkspace.id, {
+           metadata: {
+             ...(persistedExecutionWorkspace.metadata ?? {}),
@@ -0,0 +1,86 @@
+diff --git a/packages/plugins/sandbox-providers/kubernetes/README.md b/packages/plugins/sandbox-providers/kubernetes/README.md
+index a51c6ecee60e..72618cb11296 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/README.md
+++ b/packages/plugins/sandbox-providers/kubernetes/README.md
+@@ -140,6 +140,8 @@ Every agent pod is:
+ 
+ Plus per-namespace `pod-security.kubernetes.io/enforce: restricted` and a deny-all NetworkPolicy baseline with explicit egress allow-list (DNS, paperclip-server, configured FQDNs/CIDRs).
+ 
+The lease starts with `/workspace` as the native file-sync confinement root. Adapter environment tests can upload managed config before the server realizes a workspace. When resuming a lease, the provider keeps a previously recorded `remoteCwd` so a custom realized workspace is not replaced with the default.
+
+ The per-run Secret carrying the bootstrap token and adapter API keys has `ownerReferences` pointing at the owning Job, so a single `kubectl delete job …` cascades cleanly to the Pod and Secret.
+ 
+ ## Optional Kata-FC microVM isolation
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+index 8ae47de8ecac..98016385da05 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+@@ -454,6 +454,9 @@ const plugin = definePlugin({
+       secretName,
+       phase: "Pending",
+       backend: config.backend,
+      // Adapter environment probes sync managed assets before workspace
+      // realization. Give native sync the pod's mounted confinement root now.
+      remoteCwd: "/workspace",
+       scopedNetworkPolicyName,
+       scopedNetworkEgress,
+       // Native file sync streams over a pod exec; only the sandbox-cr backend
+@@ -529,6 +532,10 @@ const plugin = definePlugin({
+       secretName,
+       phase: check.phase,
+       backend: leaseBackend,
+      remoteCwd:
+        typeof params.leaseMetadata?.remoteCwd === "string" && params.leaseMetadata.remoteCwd.trim()
+          ? params.leaseMetadata.remoteCwd.trim()
+          : "/workspace",
+       scopedNetworkPolicyName:
+         typeof params.leaseMetadata?.scopedNetworkPolicyName === "string"
+           ? params.leaseMetadata.scopedNetworkPolicyName
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/types.ts b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+index 5626daa93846..ef3d0d889a4c 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+@@ -90,6 +90,8 @@ export interface KubernetesLeaseMetadata {
+   phase: "Pending" | "Running" | "Succeeded" | "Failed";
+   /** Which backend provisioned this lease. */
+   backend: "sandbox-cr" | "job";
+  /** Native sync confinement root; present before workspace realization for probes. */
+  remoteCwd: string;
+   scopedNetworkPolicyName: string | null;
+   scopedNetworkEgress: {
+     allowFqdns: string[];
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+index 7ad0ca4807e5..4c23410d4d15 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+@@ -80,6 +80,7 @@ describe("onEnvironmentResumeLease", () => {
+         secretName: "pc-abc-env",
+         phase: "Running",
+         backend: "sandbox-cr",
+        remoteCwd: "/workspace",
+         resumedLease: true,
+         // sandbox-cr has a pod-exec channel, so native file sync stays enabled.
+         nativeFileSyncUnsupported: false,
+@@ -87,6 +88,22 @@ describe("onEnvironmentResumeLease", () => {
+     );
+   });
+ 
+  it("keeps a custom realized cwd when resuming a sandbox lease", async () => {
+    h.clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockResolvedValue(readySandboxCr("pc-abc-pod")) },
+      core: { readNamespacedPod: vi.fn().mockResolvedValue({ status: { phase: "Running" } }) },
+    };
+    const lease = await plugin.definition.onEnvironmentResumeLease!({
+      driverKey: "kubernetes",
+      companyId: "acme",
+      environmentId: "env-1",
+      config: CONFIG,
+      providerLeaseId: "pc-abc",
+      leaseMetadata: leaseMetadata({ remoteCwd: "/workspace/custom" }),
+    });
+    expect(lease.metadata?.remoteCwd).toBe("/workspace/custom");
+  });
+
+   it("flags a resumed job-backend lease as native-sync-unsupported so the server keeps the base64 fallback", async () => {
+     h.clients = {
+       batch: {
@@ -0,0 +1,59 @@
+#!/bin/sh
+# Fetch upstream Paperclip at the pinned commit, apply our patches and make
+# the Dockerfile buildable with kaniko. Usage: prepare.sh <target-dir>
+set -eu
+HERE=$(cd "$(dirname "$0")" && pwd)
+. "$HERE/upstream.env"
+OUT=$1
+
+rm -rf "$OUT"
+git init -q "$OUT"
+cd "$OUT"
+git fetch -q --depth 1 https://github.com/paperclipai/paperclip.git "$PAPERCLIP_COMMIT"
+git -c advice.detachedHead=false checkout -q FETCH_HEAD
+test "$(git rev-parse HEAD)" = "$PAPERCLIP_COMMIT"
+
+for p in "$HERE"/*.patch; do
+  echo "applying $(basename "$p")"
+  git apply "$p"
+done
+
+# kaniko does not know `COPY --parents` (BuildKit only). Upstream uses it for
+# one line: every sandbox provider's package.json. Replace it with one plain
+# COPY per provider directory, which copies the same files to the same place.
+grep -c -- '^COPY --parents packages/plugins/sandbox-providers/\./\*/package.json packages/plugins/sandbox-providers/$' Dockerfile | grep -qx 1
+awk '
+  /^COPY --parents packages\/plugins\/sandbox-providers\// {
+    while (("ls -d packages/plugins/sandbox-providers/*/package.json" | getline f) > 0) {
+      d = f; sub(/package.json$/, "", d)
+      print "COPY " f " " d
+    }
+    next
+  }
+  { print }
+' Dockerfile > Dockerfile.kaniko
+if grep -q -- '--parents' Dockerfile.kaniko; then
+  echo "Dockerfile.kaniko still uses COPY --parents" >&2
+  exit 1
+fi
+
+# Build the Kubernetes sandbox provider into the image (REN-405). Upstream's
+# `production` target ships only its source; the plugin loader needs
+# dist/manifest.js. Same commands as upstream's `cloud-plugins` stage, run in
+# the `build` stage right after the server build, so production's
+# `COPY --from=build /app /app` carries dist/ and node_modules along. No extra
+# stage: kaniko would snapshot the whole build stage again for it.
+export ANCHOR='RUN test -f server/dist/index.js || (echo "ERROR: server build output missing" && exit 1)'
+grep -c -F -x -- "$ANCHOR" Dockerfile.kaniko | grep -qx 1
+K=packages/plugins/sandbox-providers/kubernetes
+awk -v k="$K" '
+  { print }
+  $0 == ENVIRON["ANCHOR"] {
+    print "RUN pnpm -C " k " install --ignore-workspace --no-lockfile \\"
+    print "  && pnpm -C " k " build \\"
+    print "  && test -f " k "/dist/manifest.js"
+  }
+' Dockerfile.kaniko > Dockerfile.kaniko.new
+mv Dockerfile.kaniko.new Dockerfile.kaniko
+grep -q -- "test -f $K/dist/manifest.js" Dockerfile.kaniko
+rm -rf .git
@@ -0,0 +1,118 @@
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+index a26f980..112e2eb 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.test.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+@@ -2544,6 +2544,37 @@ describe("shared ACPX engine runtime behavior", () => {
+     expect(note).toBeTruthy();
+   });
+ 
+  it("points Claude auto memory at the run's AGENT_HOME without changing the session fingerprint", async () => {
+    const root = await makeTempRoot();
+    const stateDir = path.join(root, "state");
+    const cwd = path.join(root, "worktree");
+    await fs.mkdir(cwd, { recursive: true });
+    const stableHome = path.join(root, "agent-home");
+    const runAgentHome = (runId: string) => path.join(root, "file-sync", "runs", runId, "live");
+    const readSettings = async () =>
+      JSON.parse(await fs.readFile(path.join(cwd, ".claude", "settings.local.json"), "utf8")) as {
+        autoMemoryDirectory?: string;
+        permissions?: { additionalDirectories?: string[] };
+      };
+
+    const first = await runExecutor(
+      { agent: "claude", stateDir, cwd },
+      { context: { paperclipWorkspace: { cwd, agentHome: runAgentHome("a"), agentHomeForPermissions: stableHome } } },
+    );
+    expect((await readSettings()).autoMemoryDirectory).toBe(path.join(runAgentHome("a"), "memory"));
+
+    const second = await runExecutor(
+      { agent: "claude", stateDir, cwd },
+      { context: { paperclipWorkspace: { cwd, agentHome: runAgentHome("b"), agentHomeForPermissions: stableHome } } },
+    );
+    const written = await readSettings();
+    expect(written.autoMemoryDirectory).toBe(path.join(runAgentHome("b"), "memory"));
+    expect(written.permissions?.additionalDirectories).toContain(stableHome);
+    expect(written.permissions?.additionalDirectories).not.toContain(runAgentHome("b"));
+    expect(second.result.sessionParams?.configFingerprint).toBeTypeOf("string");
+    expect(second.result.sessionParams?.configFingerprint).toBe(first.result.sessionParams?.configFingerprint);
+  });
+
+   it("merges Paperclip allowlist into an existing .claude/settings.local.json without losing user entries", async () => {
+     const root = await makeTempRoot();
+     const stateDir = path.join(root, "state");
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts
+index b1dc569..e5681b8 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.ts
+@@ -1558,6 +1558,7 @@ interface PaperclipClaudeSettingsResult {
+   additionalDirectories: string[];
+   defaultMode: string;
+   overrodeDontAsk: boolean;
+  autoMemoryDirectory: string | null;
+ }
+ 
+ function uniqueSorted(values: Array<string | null | undefined>): string[] {
+@@ -1571,10 +1572,19 @@ function uniqueSorted(values: Array<string | null | undefined>): string[] {
+ // denies every non-allowlisted tool and never reaches `canUseTool`), and we
+ // widen the SDK's Read sandbox to include the Paperclip state dirs the agent
+ // needs to talk to its own control plane.
+//
+// It also points Claude's auto memory at `<AGENT_HOME>/memory`. The default
+// location lives under CLAUDE_CONFIG_DIR, which is a per-run temp home, so
+// memories were lost after every run. AGENT_HOME is restored before and saved
+// after each run, locally and in remote sandboxes. Claude ignores
+// `autoMemoryDirectory` only in checked-in project settings, not local ones.
+ async function writePaperclipClaudeSettings(input: {
+   cwd: string;
+   stateDir: string;
+   agentHome: string;
+  // The run's actual AGENT_HOME (the in-sandbox path for remote runs). It may
+  // differ from `agentHome`, which stays stable for session identity.
+  runAgentHome: string;
+   companyId: string;
+ }): Promise<PaperclipClaudeSettingsResult> {
+   const filePath = path.join(input.cwd, ".claude", "settings.local.json");
+@@ -1630,7 +1640,14 @@ async function writePaperclipClaudeSettings(input: {
+     additionalDirectories: mergedAdditionalDirectories,
+     defaultMode,
+   };
+-  const next: Record<string, unknown> = { ...existing, permissions: nextPermissions };
+  const autoMemoryDirectory = input.runAgentHome
+    ? path.posix.join(input.runAgentHome, "memory")
+    : null;
+  const next: Record<string, unknown> = {
+    ...existing,
+    permissions: nextPermissions,
+    ...(autoMemoryDirectory ? { autoMemoryDirectory } : {}),
+  };
+   await writeFileAtomically({
+     target: filePath,
+     contents: `${JSON.stringify(next, null, 2)}\n`,
+@@ -1642,6 +1659,7 @@ async function writePaperclipClaudeSettings(input: {
+     additionalDirectories: mergedAdditionalDirectories,
+     defaultMode,
+     overrodeDontAsk,
+    autoMemoryDirectory,
+   };
+ }
+ 
+@@ -2064,12 +2082,18 @@ async function buildRuntime(input: {
+       // by the company root already granted in these settings; the process env
+       // still receives this run's actual AGENT_HOME.
+       agentHome: asString(workspaceContext.agentHomeForPermissions, agentHome),
+      // Auto memory follows the actual AGENT_HOME so it is saved with the run.
+      // It is not part of the session fingerprint: a moved memory directory
+      // must not invalidate a resumable conversation.
+      runAgentHome: agentHome,
+       companyId: agent.companyId,
+     });
+     skillCommandNotes.push(
+       `Wrote Paperclip-managed Claude settings to ${paperclipClaudeSettings.filePath} (defaultMode=${paperclipClaudeSettings.defaultMode}${
+         paperclipClaudeSettings.overrodeDontAsk ? "; overrode user dontAsk" : ""
+-      }, +${paperclipClaudeSettings.additionalDirectories.length} read root(s), +${paperclipClaudeSettings.allow.length} allow rule(s)).`,
+      }, +${paperclipClaudeSettings.additionalDirectories.length} read root(s), +${paperclipClaudeSettings.allow.length} allow rule(s)${
+        paperclipClaudeSettings.autoMemoryDirectory ? `, autoMemoryDirectory=${paperclipClaudeSettings.autoMemoryDirectory}` : ""
+      }).`,
+     );
+   } else if (acpxAgent === "codex") {
+     // Step 2 — codex-home.seed: the codex managed-home + skills preparation.
@@ -0,0 +1,128 @@
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+index 112e2eb..004baf0 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.test.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+@@ -1857,6 +1857,45 @@ describe("shared ACPX engine runtime behavior", () => {
+     expect(changed.result.sessionParams?.configFingerprint).not.toBe(first.result.sessionParams?.configFingerprint);
+   });
+ 
+  it("keeps the per-run managed AI home out of session identity and keeps Claude transcripts", async () => {
+    const root = await makeTempRoot();
+    const stateDir = path.join(root, "state");
+    const config = { agentCommand: "node ./fake-acp.js", stateDir };
+    async function withManagedHome(home: string, claudeConfigDir = path.join(home, "provider")) {
+      return runExecutor({
+        ...config,
+        managedAiConnection: { identity: "grant-1:shared:abc" },
+        env: {
+          HOME: home,
+          XDG_CONFIG_HOME: path.join(home, "config"),
+          XDG_DATA_HOME: path.join(home, "data"),
+          CODEX_HOME: path.join(home, "provider"),
+          GROK_HOME: path.join(home, "provider"),
+          CLAUDE_CONFIG_DIR: claudeConfigDir,
+        },
+      }, { context: { taskId: "issue-1" } });
+    }
+    const fp = (run: Awaited<ReturnType<typeof runExecutor>>) => run.result.sessionParams?.configFingerprint;
+    const first = await withManagedHome(path.join(root, "paperclip-ai-a"));
+    const second = await withManagedHome(path.join(root, "paperclip-ai-b"));
+    const overridden = await withManagedHome(path.join(root, "paperclip-ai-c"), path.join(root, "custom-claude"));
+    expect(fp(first)).toBeTypeOf("string");
+    expect(fp(second)).toBe(fp(first));
+    expect(fp(overridden)).not.toBe(fp(first));
+
+    // The transcript dir of each managed home points at one dir in the state
+    // dir, so a resumed session finds its transcript after the home is deleted.
+    const transcripts = path.join(stateDir, "claude-projects");
+    expect(await fs.readlink(path.join(root, "paperclip-ai-a", "provider", "projects"))).toBe(transcripts);
+    expect(await fs.readlink(path.join(root, "paperclip-ai-b", "provider", "projects"))).toBe(transcripts);
+    await expect(fs.lstat(path.join(root, "custom-claude", "projects"))).rejects.toThrow();
+
+    // Without a managed AI connection the same paths are user config and still count.
+    const plainA = await runExecutor({ ...config, env: { HOME: path.join(root, "home-a") } }, { context: { taskId: "issue-1" } });
+    const plainB = await runExecutor({ ...config, env: { HOME: path.join(root, "home-b") } }, { context: { taskId: "issue-1" } });
+    expect(fp(plainB)).not.toBe(fp(plainA));
+  });
+
+   it("busts the session fingerprint when a stable configured PAPERCLIP_* value rotates", async () => {
+     const root = await makeTempRoot();
+     const stateDir = path.join(root, "state");
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts
+index e5681b8..0fd486c 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.ts
+@@ -1663,6 +1663,26 @@ async function writePaperclipClaudeSettings(input: {
+   };
+ }
+ 
+// Claude keeps conversation transcripts in CLAUDE_CONFIG_DIR/projects. A
+// managed AI home is a per-run temp dir that the server deletes after the run,
+// so the next run could not resume ("Resource not found"). Point `projects` at
+// a directory in the agent's state dir, like ~/.claude/projects on a normal
+// install. The server's recursive cleanup removes the link, not its target.
+async function linkManagedClaudeTranscripts(input: {
+  claudeConfigDir: string;
+  stateDir: string;
+}): Promise<string> {
+  const target = path.join(input.stateDir, "claude-projects");
+  await fs.mkdir(input.claudeConfigDir, { recursive: true, mode: 0o700 });
+  await fs.mkdir(target, { recursive: true, mode: 0o700 });
+  const link = path.join(input.claudeConfigDir, "projects");
+  const current = await fs.readlink(link).catch(() => null);
+  if (current === target) return target;
+  await fs.rm(link, { recursive: true, force: true });
+  await fs.symlink(target, link, "dir");
+  return target;
+}
+
+ // Cross the CLI's staging seam for a runner-backed remote sandbox: ship the
+ // workspace (and, in PR 2, the per-adapter managed-home `assets`) into the
+ // sandbox and obtain the in-sandbox `workspaceRemoteDir` plus the non-null
+@@ -2027,6 +2047,30 @@ async function buildRuntime(input: {
+     // are absent from tempKeysApplied and keep their compatibility protection.
+     if (!scratchKeys.has(key) || value !== scratch.dir) resolvedAdapterEnv[key] = value;
+   }
+  // A managed AI connection gets a fresh server-made temp home on every run
+  // (HOME, XDG_*, and the provider home under it). Hash a fixed placeholder for
+  // those paths so the session can resume; the server's own fingerprint does
+  // the same (managedAiSessionFingerprintConfig). Values that differ from the
+  // managed layout are real overrides and stay in the hash. A Claude session
+  // can only resume if its transcript survives the run, so this pairs with
+  // linkManagedClaudeTranscripts below. Remote runs keep their transcript in
+  // the sandbox and are left as they are.
+  const managedHome = config.managedAiConnection && !executionTargetIsRemote ? resolvedAdapterEnv.HOME : undefined;
+  if (managedHome) {
+    const managedLayout: Record<string, string> = {
+      HOME: "",
+      XDG_CONFIG_HOME: "config",
+      XDG_DATA_HOME: "data",
+      CODEX_HOME: "provider",
+      GROK_HOME: "provider",
+      CLAUDE_CONFIG_DIR: "provider",
+    };
+    for (const [key, sub] of Object.entries(managedLayout)) {
+      if (resolvedAdapterEnv[key] === (sub ? path.join(managedHome, sub) : managedHome)) {
+        resolvedAdapterEnv[key] = sub ? `<managed-ai-home>/${sub}` : "<managed-ai-home>";
+      }
+    }
+  }
+   if (authToken) env.PAPERCLIP_API_KEY = authToken;
+   // For the claude agent, set model via ANTHROPIC_MODEL at startup rather than
+   // via session/set_config_option — the ACP server's set_config_option handler
+@@ -2088,6 +2132,15 @@ async function buildRuntime(input: {
+       runAgentHome: agentHome,
+       companyId: agent.companyId,
+     });
+    // Same condition as the fingerprint placeholder above: a resumable session
+    // must find its transcript in the next run's managed home.
+    if (resolvedAdapterEnv.CLAUDE_CONFIG_DIR === "<managed-ai-home>/provider" && env.CLAUDE_CONFIG_DIR) {
+      const transcriptsDir = await linkManagedClaudeTranscripts({
+        claudeConfigDir: env.CLAUDE_CONFIG_DIR,
+        stateDir,
+      });
+      skillCommandNotes.push(`Linked Claude transcripts to ${transcriptsDir}.`);
+    }
+     skillCommandNotes.push(
+       `Wrote Paperclip-managed Claude settings to ${paperclipClaudeSettings.filePath} (defaultMode=${paperclipClaudeSettings.defaultMode}${
+         paperclipClaudeSettings.overrodeDontAsk ? "; overrode user dontAsk" : ""
@@ -0,0 +1,384 @@
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts b/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+index 87e18ae..2224e60 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+@@ -21,6 +21,7 @@ import { deleteJob, findPodForJob, getJobStatus } from "./job-orchestrator.js";
+ import {
+   deleteSandboxCr,
+   findPodForSandbox,
+  getSandboxCrStatus,
+   waitForSandboxReady,
+ } from "./sandbox-cr-orchestrator.js";
+ 
+@@ -191,3 +192,60 @@ export async function destroyLeaseResources(
+     );
+   }
+ }
+
+export interface WorkloadGoneInput {
+  namespace: string;
+  /** Workload resource name (Sandbox CR name or Job name) == providerLeaseId. */
+  name: string;
+  backend: "sandbox-cr" | "job";
+  podName?: string | null;
+  /** Stays below the server's 30 s default plugin RPC timeout. */
+  timeoutMs?: number;
+  pollMs?: number;
+}
+
+/**
+ * Poll until the workload (and the named pod, if any) returns 404. Deletes use
+ * Foreground propagation, so the workload disappears only after its pod is
+ * gone. True is the evidence behind a `destroyed` termination receipt; false
+ * (still present at the deadline, or a read failed) means the caller must not
+ * claim one. Never throws: the delete itself already succeeded.
+ */
+export async function waitForWorkloadGone(
+  clients: KubeClients,
+  input: WorkloadGoneInput,
+): Promise<boolean> {
+  const deadline = Date.now() + (input.timeoutMs ?? 20_000);
+  const pollMs = input.pollMs ?? 500;
+  const gone = async (read: () => Promise<unknown>): Promise<boolean> => {
+    try {
+      await read();
+      return false;
+    } catch (err) {
+      if (isKubeNotFoundError(err)) return true;
+      throw err;
+    }
+  };
+  try {
+    return await pollUntilGone();
+  } catch {
+    return false;
+  }
+
+  async function pollUntilGone(): Promise<boolean> {
+    for (;;) {
+      const workloadGone = await gone(() =>
+        input.backend === "sandbox-cr"
+          ? getSandboxCrStatus(clients, input.namespace, input.name)
+          : getJobStatus(clients, input.namespace, input.name),
+      );
+      const podName = input.podName;
+      const podGone = !podName || await gone(() =>
+        clients.core.readNamespacedPod({ namespace: input.namespace, name: podName }),
+      );
+      if (workloadGone && podGone) return true;
+      if (Date.now() >= deadline) return false;
+      await new Promise((resolve) => setTimeout(resolve, pollMs));
+    }
+  }
+}
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+index 9801638..bf5ce0a 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+@@ -15,6 +15,7 @@ import type {
+   PluginEnvironmentSyncInParams,
+   PluginEnvironmentSyncOutParams,
+   PluginEnvironmentSyncResult,
+  PluginEnvironmentTerminationReceipt,
+   PluginEnvironmentValidateConfigParams,
+   PluginEnvironmentValidationResult,
+ } from "@paperclipai/plugin-sdk";
+@@ -38,7 +39,11 @@ import {
+ } from "./sandbox-cr-orchestrator.js";
+ import { execInPod, execInPodStreaming, wrapCommandWithEnv } from "./pod-exec.js";
+ import { performSyncIn, performSyncOut, type PodStreamExec } from "./file-sync.js";
+-import { checkLeaseResumable, destroyLeaseResources } from "./lease-lifecycle.js";
+import {
+  checkLeaseResumable,
+  destroyLeaseResources,
+  waitForWorkloadGone,
+} from "./lease-lifecycle.js";
+ import {
+   appendNetworkEgressDenyHint,
+   createScopedNetworkEgressPolicyOrReleaseWorkload,
+@@ -578,7 +583,7 @@ const plugin = definePlugin({
+ 
+   async onEnvironmentReleaseLease(
+     params: PluginEnvironmentReleaseLeaseParams,
+-  ): Promise<void> {
+  ): Promise<PluginEnvironmentTerminationReceipt | void> {
+     if (!params.providerLeaseId) return;
+     const config = kubernetesProviderConfigSchema.parse(params.config);
+     const namespace =
+@@ -613,11 +618,22 @@ const plugin = definePlugin({
+         ?? (err as { code?: number; statusCode?: number }).statusCode;
+       if (code !== 404) throw err;
+     }
+
+    // The server grants continuation after a stopped run only with a
+    // termination receipt; without one a cancelled run's queued comment
+    // waits on `remote_cleanup` forever (REN-841). Claim `destroyed` only
+    // once the workload is confirmed gone.
+    const gone = await waitForWorkloadGone(clients, {
+      namespace,
+      name: params.providerLeaseId,
+      backend: leaseBackend,
+    });
+    if (gone) return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
+   },
+ 
+   async onEnvironmentDestroyLease(
+     params: PluginEnvironmentDestroyLeaseParams,
+-  ): Promise<void> {
+  ): Promise<PluginEnvironmentTerminationReceipt | void> {
+     if (!params.providerLeaseId) return;
+     const config = kubernetesProviderConfigSchema.parse(params.config);
+     const namespace =
+@@ -658,6 +674,13 @@ const plugin = definePlugin({
+       podName,
+       secretName,
+     });
+    const gone = await waitForWorkloadGone(clients, {
+      namespace,
+      name: params.providerLeaseId,
+      backend: leaseBackend,
+      podName,
+    });
+    if (gone) return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
+   },
+ 
+   async onEnvironmentExecute(
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+index af0875f..d775799 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+@@ -3,6 +3,7 @@ import {
+   checkLeaseResumable,
+   destroyLeaseResources,
+   isKubeNotFoundError,
+  waitForWorkloadGone,
+ } from "../../src/lease-lifecycle.js";
+ 
+ const SANDBOX_GROUP = "agents.x-k8s.io";
+@@ -346,3 +347,64 @@ describe("destroyLeaseResources", () => {
+     ).rejects.toThrow("forbidden");
+   });
+ });
+
+describe("waitForWorkloadGone", () => {
+  const input = { namespace: "ns", name: "pc-abc", backend: "sandbox-cr" as const, pollMs: 1 };
+
+  it("is true once the Sandbox CR and the pod return 404", async () => {
+    const getCr = vi.fn()
+      .mockResolvedValueOnce(readySandboxCr("pc-abc-pod"))
+      .mockRejectedValue(notFound());
+    const readPod = vi.fn().mockRejectedValue(notFound());
+    const clients = { custom: { getNamespacedCustomObject: getCr }, core: { readNamespacedPod: readPod } };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, podName: "pc-abc-pod" }),
+    ).resolves.toBe(true);
+    expect(getCr).toHaveBeenCalledTimes(2);
+    expect(getCr).toHaveBeenCalledWith(expect.objectContaining({
+      group: SANDBOX_GROUP, version: SANDBOX_VERSION, plural: SANDBOX_PLURAL, namespace: "ns", name: "pc-abc",
+    }));
+  });
+
+  it("is false while the workload still exists at the deadline", async () => {
+    const clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockResolvedValue(readySandboxCr()) },
+    };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, timeoutMs: 5 }),
+    ).resolves.toBe(false);
+  });
+
+  it("is false while the pod still exists even if the workload is gone", async () => {
+    const clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()) },
+      core: { readNamespacedPod: vi.fn().mockResolvedValue({ status: { phase: "Running" } }) },
+    };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, podName: "pc-abc-pod", timeoutMs: 5 }),
+    ).resolves.toBe(false);
+  });
+
+  it("checks the Job for job-backend leases", async () => {
+    const readJob = vi.fn().mockRejectedValue(notFound());
+    const clients = { batch: { readNamespacedJobStatus: readJob } };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, backend: "job" }),
+    ).resolves.toBe(true);
+    expect(readJob).toHaveBeenCalledWith({ namespace: "ns", name: "pc-abc" });
+  });
+
+  it("is false (never throws) when a read fails with another error", async () => {
+    const clients = {
+      custom: {
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(Object.assign(new Error("x"), { code: 403 })),
+      },
+    };
+
+    await expect(waitForWorkloadGone(clients as never, input)).resolves.toBe(false);
+  });
+});
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+index 4c23410..f4fd073 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+@@ -184,12 +184,19 @@ describe("onEnvironmentDestroyLease", () => {
+     const deletePod = vi.fn().mockResolvedValue({});
+     const deleteSecret = vi.fn().mockResolvedValue({});
+     h.clients = {
+-      custom: { deleteNamespacedCustomObject: deleteCr },
+-      core: { deleteNamespacedPod: deletePod, deleteNamespacedSecret: deleteSecret },
+      custom: {
+        deleteNamespacedCustomObject: deleteCr,
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+      core: {
+        deleteNamespacedPod: deletePod,
+        deleteNamespacedSecret: deleteSecret,
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+       batch: { deleteNamespacedJob: vi.fn() },
+     };
+ 
+-    await plugin.definition.onEnvironmentDestroyLease!({
+    const receipt = await plugin.definition.onEnvironmentDestroyLease!({
+       driverKey: "kubernetes",
+       companyId: "acme",
+       environmentId: "env-1",
+@@ -209,14 +216,47 @@ describe("onEnvironmentDestroyLease", () => {
+       namespace: "paperclip-acme",
+       name: "pc-abc-env",
+     });
+    expect(receipt).toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+   });
+ 
+   it("is idempotent: resolves cleanly when every resource is already gone (404)", async () => {
+     h.clients = {
+-      custom: { deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()) },
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+       core: {
+         deleteNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+         deleteNamespacedSecret: vi.fn().mockRejectedValue(notFound()),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+      batch: { deleteNamespacedJob: vi.fn() },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentDestroyLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns no receipt when the gone check cannot read the cluster", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(
+          Object.assign(new Error("forbidden"), { code: 403 }),
+        ),
+      },
+      core: {
+        deleteNamespacedPod: vi.fn().mockResolvedValue({}),
+        deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+       },
+       batch: { deleteNamespacedJob: vi.fn() },
+     };
+@@ -261,8 +301,12 @@ describe("onEnvironmentDestroyLease", () => {
+       core: {
+         deleteNamespacedPod: vi.fn().mockResolvedValue({}),
+         deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+      batch: {
+        deleteNamespacedJob: deleteJob,
+        readNamespacedJobStatus: vi.fn().mockRejectedValue(notFound()),
+       },
+-      batch: { deleteNamespacedJob: deleteJob },
+     };
+ 
+     await plugin.definition.onEnvironmentDestroyLease!({
+@@ -280,3 +324,71 @@ describe("onEnvironmentDestroyLease", () => {
+     expect(deleteCr).not.toHaveBeenCalled();
+   });
+ });
+
+describe("onEnvironmentReleaseLease", () => {
+  it("returns a destroyed receipt once the Sandbox CR is gone (REN-841)", async () => {
+    const deleteCr = vi.fn().mockResolvedValue({});
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: deleteCr,
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+    };
+
+    const receipt = await plugin.definition.onEnvironmentReleaseLease!({
+      driverKey: "kubernetes",
+      companyId: "acme",
+      environmentId: "env-1",
+      config: CONFIG,
+      providerLeaseId: "pc-abc",
+      leaseMetadata: leaseMetadata(),
+    });
+
+    expect(deleteCr).toHaveBeenCalledWith(
+      expect.objectContaining({ namespace: "paperclip-acme", name: "pc-abc" }),
+    );
+    expect(receipt).toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns a destroyed receipt when the Sandbox CR was already deleted", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns no receipt when the gone check fails", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(
+          Object.assign(new Error("server error"), { code: 500 }),
+        ),
+      },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toBeUndefined();
+  });
+});
@@ -0,0 +1,7 @@
+# Upstream paperclipai/paperclip commit this image is built from (REN-401).
+# a027f76 is the master commit the nightly image ran when this was pinned, so
+# the only change against that image is the patch next to this file.
+# PAPERCLIP_VERSION is `git describe --tags --match 'v*' --long` of that
+# commit, the value upstream stamps into its own images. Bump both together.
+PAPERCLIP_COMMIT=a027f76a726e8a556674eded807c2de0e55f5cc0
+PAPERCLIP_VERSION=v2026.916.0-219-ga027f76a7
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 ARG TOFU_VERSION=1.11.0

@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Everything `bootstrap.sh vmtest` needs, so the rebuild test can run as a Job in
 # the cluster instead of on a workstation. The workstation has 31G and the test
@@ -16,7 +16,7 @@ credentials are out of scope (see "Deferred").
 |---|---|
 | Execution model | **B — server-local runs** (upstream default): agent CLIs are child processes of the server, in the server container. Trial only. |
 | Upgrade path | If Paperclip is kept: **C** — `@paperclipai/plugin-kubernetes` sandbox provider + kubernetes-sigs agent-sandbox controller (`agents.x-k8s.io`), per-run Pods and Secrets. Separate spec. |
-| Claude auth | **One shared Claude subscription AI connection**, created once via Paperclip's local sign-in (`claude auth login` into `ai-local-logins/<id>`); Paperclip stores and refreshes it. Server-env `CLAUDE_CODE_OAUTH_TOKEN` is not used: `stripAiAuthBindings` blanks it for managed runs, and upstream accepts no pasted setup-token (the device-code flow needs a Daytona sandbox). |
+| Claude auth | **One shared Claude subscription AI connection**, created once via Paperclip's local sign-in (`claude auth login` into `ai-local-logins/<id>`), then its value replaced with a long-lived `claude setup-token` token. Paperclip stores the value as company secret "My Claude subscription" and injects it per run as `CLAUDE_CODE_OAUTH_TOKEN`; it never refreshes Anthropic credentials, so the login's ~8h access token expires (runs then fail with "ACP agent reported a terminal access failure", 0 tokens). Renew by rotating that secret with a new setup-token (~yearly). Server-env `CLAUDE_CODE_OAUTH_TOKEN` is not used: `stripAiAuthBindings` blanks it for managed runs, and upstream accepts no pasted setup-token (the device-code flow needs a Daytona sandbox). |
 | Bot credentials | Out of scope. Future "AIR-Manager" Paperclip role handles permission requests. |
 | Exposure | LAN-only, behind an Authentik forward-auth gate, then Paperclip's own login. |

@@ -189,7 +189,8 @@ available since v0.35.0). First use of digest reflection in this repo.
   **as the node user** (the entrypoint makes exec sessions root, and root-owned 0600
   credentials are unreadable to the server):
   `kubectl exec -it -n paperclip deploy/paperclip -- gosu node sh -c 'export CLAUDE_CONFIG_DIR=<shown path> && mkdir -p "$CLAUDE_CONFIG_DIR" && claude auth login'`
-   Repeat after loss of the PVC or database.
+   Then Settings → Secrets → "My Claude subscription": replace the value with the
+   output of `claude setup-token`. Repeat both after loss of the database.

 ## Verification
 - Flux Kustomization `paperclip` Ready; CNPG `pclp-pg` healthy; first ScheduledBackup
@@ -165,6 +165,21 @@ permanent recovery.

 ---

+## Velero restore-test: 80% OSD capacity gate
+
+The monthly Velero restore-test restores into the production Ceph pools. Before
+each schedule's restore it projects the restore's raw size (DataUpload
+`totalBytes` x the pool's replication/EC factor, spread over all OSDs) onto
+current OSD usage from Prometheus. If the fullest OSD would go above **80%**
+(`OSD_MAX_PCT` in `restore-test-cronjob.yaml`), that schedule is not restored:
+the Job log shows it as **SKIP** with the projected %, and the summary lists it
+under "Skipped (backup NOT restore-tested)". A SKIP does not fail the Job, so
+read the summary each month. If the projection cannot be made (Prometheus
+down, unsized DataUpload), the schedule is not restored and counts as FAIL.
+Background: REN-441, REN-442.
+
+---
+
 ## CNPG restore-test (verify backups are restorable — non-destructive)

 The monthly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
@@ -48,7 +48,7 @@ spec:
        # cert when doing server-side OIDC token exchange. Bundle both
        # roots into one PEM since NODE_EXTRA_CA_CERTS accepts one file.
        - name: fetch-le-staging-roots
-          image: curlimages/curl:8.10.1
+          image: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777
          command:
            - sh
            - -c
@@ -70,7 +70,7 @@ spec:
              mountPath: /ca
      containers:
        - name: audiobookshelf
-          image: ghcr.io/advplyr/audiobookshelf:2.36.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
+          image: ghcr.io/advplyr/audiobookshelf:2.37.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
          ports:
            - name: http
              containerPort: 80
@@ -2121,6 +2121,74 @@ data:
            - url: https://dash.renner.uno/login/generic_oauth
              matching_mode: strict

+      # Matrix (REN-567): upstream login for MAS (account.matrix.renner.uno).
+      # Git owns this provider, its application and the member list.
+      # Public client + PKCE, so there is no client secret to share.
+      # RS256 with the built-in self-signed keypair: a public client has no
+      # secret for HS256, and MAS verifies the ID token signature.
+      - id: matrix-users-group
+        model: authentik_core.group
+        identifiers:
+          name: Matrix Users
+        state: present
+        attrs:
+          is_superuser: false
+          # Every member can sign in to Matrix and gets an account named
+          # after their Authentik username. Add/remove members HERE.
+          users:
+            - !Find [authentik_core.user, [username, k.renner]]
+
+      - id: matrix-mas-provider
+        model: authentik_providers_oauth2.oauth2provider
+        identifiers:
+          name: Matrix (MAS)
+        state: present
+        attrs:
+          authentication_flow: !KeyOf passkey-passwordless-flow
+          authorization_flow: !KeyOf passkey-stepup-flow
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          client_type: public
+          client_id: matrix-mas
+          # Authentik 2026.5 checks the grant type against this list, and a
+          # provider created by a blueprint starts with it empty, so every
+          # authorize fails with invalid_request ("Invalid grant_type for
+          # provider"). Providers made in the UI get the defaults.
+          grant_types:
+            - authorization_code
+            - refresh_token
+          signing_key: !Find [authentik_crypto.certificatekeypair, [name, authentik Self-signed Certificate]]
+          property_mappings:
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-profile]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]]
+          redirect_uris:
+            # The path ends in the provider ULID from apps/matrix/mas.yaml.
+            - url: https://account.matrix.renner.uno/upstream/callback/01M4480KWZGN9HBNB5SSNGT2M1
+              matching_mode: strict
+
+      - model: authentik_core.application
+        identifiers:
+          slug: matrix
+        state: present
+        attrs:
+          name: Matrix
+          slug: matrix
+          provider: !KeyOf matrix-mas-provider
+          meta_launch_url: https://element.renner.uno
+          meta_description: "Matrix chat (Element)"
+
+      # Only the Matrix Users group passes.
+      - model: authentik_policies.policybinding
+        identifiers:
+          target: !Find [authentik_core.application, [slug, matrix]]
+          group: !KeyOf matrix-users-group
+          order: 0
+        state: present
+        attrs:
+          enabled: true
+          negate: false
+          failure_result: false
+
      # Superseded by the single passkey-webauthn-validation stage above.
      # Their flowstagebindings are FK-CASCADEd away with them; the
      # identification stage's webauthn_stage FK is SET_NULL and is
@@ -23,7 +23,7 @@ spec:
      enableServiceLinks: false
      containers:
        - name: evcc
-          image: evcc/evcc:0.310.1
+          image: evcc/evcc:0.316.2@sha256:070c8a8912c5107e8424cef60bf8c3d3cac0eb32aa28d1b021d226545f0aac7b
          # evcc auto-discovers /etc/evcc.yaml on its default search path;
          # the file is delivered (with the MQTT password) via the
          # sops-encrypted evcc-config Secret mounted below.
@@ -30,6 +30,9 @@ metadata:
 #     Web Service 9 = Music Assistant
 #     Web Service 10 = TRMNL
 #     Web Service 11 = Paperless (gate-level only, see its entry)
+#     Web Service 12 = Element Web
+#     Web Service 13 = Matrix (Synapse client API)
+#     Web Service 14 = Matrix Authentication Service
 #
 # CORRECTION (REN-21, 2026-09-25) to the Web Service 11 comment further down,
 # which claims that an in-cluster probe of the paperless ClusterIP "needs a
@@ -44,14 +47,15 @@ metadata:
 # production restart for a comment. This header is outside `data:`, so it is
 # inert; the served config is byte-identical.
 #
-#   * ../../infrastructure/network-policies/paperless-ns.yaml#monitoring-admit
-#     ALREADY admits the whole monitoring namespace on every port, and
-#     monitoring is where
-#     blackbox-exporter runs. Cilium unions the allow rules of every CNP
-#     selecting an endpoint, so no extra policy is required.
-#   * The in-cluster paperless probe therefore exists, added with no change
-#     to the paperless namespace:
-#     monitoring-rules/probes-blackbox.yaml, target
+#   * When written, the paperless baseline admitted the whole monitoring
+#     namespace on every port, and monitoring is where blackbox-exporter runs.
+#     That admit is gone (REN-28); the probe is now admitted by the dedicated
+#     ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
+#     scoped to blackbox-exporter on the paperless pod's :8000. Cilium unions
+#     the allow rules of every CNP selecting an endpoint, so nothing else is
+#     required.
+#   * The in-cluster paperless probe therefore exists:
+#     ../monitoring-rules/probes-blackbox.yaml#paperless-app-health, target
 #     http://paperless.paperless.svc.cluster.local:80/accounts/login/
 #   * The claim does NOT generalise to gatus itself. Gatus runs in `apps`,
 #     which is a different source namespace; a gatus-sourced in-cluster check
@@ -227,6 +231,46 @@ data:
        conditions:
          - "[STATUS] == any(200, 301, 302)"

+      # Matrix (REN-663): Element Web, the Synapse client API and MAS.
+      - name: Web Service 12
+        group: services
+        url: https://element.renner.uno/config.json
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
+      - name: Web Service 13
+        group: services
+        url: https://matrix.renner.uno/_matrix/client/versions
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
+      - name: Web Service 14
+        group: services
+        url: https://account.matrix.renner.uno/.well-known/openid-configuration
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
      # Removed 2026-05-21: Jellyfin (media.renner.uno), Wiki (wiki.renner.uno),
      # InfluxDB (influx.renner.uno) — hostnames NXDOMAIN, no Ingress; services
      # no longer deployed. Re-add when/if they come back.
@@ -6,6 +6,8 @@ kind: Kustomization
 # and verified. See MIGRATION-gitea-woodpecker.md.
 resources:
  - statefulset.yaml
+  # The live size of the data volume; see the file (REN-583).
+  - pvc.yaml
  - service.yaml
  - ingressroute-ssh.yaml
  - ingress.yaml
@@ -0,0 +1,39 @@
+# The live data volume of gitea-0, owned here so its size is set in git
+# (REN-583). The StatefulSet controller created this claim from the
+# volumeClaimTemplates in statefulset.yaml, but that template is immutable on
+# an existing StatefulSet, so changing the size there cannot be applied. A
+# PVC's resources.requests.storage CAN grow in place (ceph-block has
+# allowVolumeExpansion: true; ceph-csi grows the RBD image and the filesystem
+# online, no pod restart). Flux applies this over the existing claim with
+# server-side apply and the resize follows.
+#
+# - Name, namespace, accessModes, storageClassName and volumeMode must match
+#   the live claim exactly: they are immutable, a mismatch fails the whole
+#   gitea Kustomization (no change to the live claim).
+# - volumeName must stay: kustomize-controller has owned spec.volumeName
+#   since the namespace move pre-created this claim (eb23d3ac, #878).
+#   Without it, server-side apply tries to clear the field and the API server
+#   rejects the whole claim as an immutable-spec change.
+# - Grow only. A smaller value is rejected by the API server.
+# - prune: disabled, so removing this file from git never deletes the claim
+#   and its data. To hand ownership back, delete the file; the claim stays.
+#
+# Size history: 20Gi -> 40Gi 2026-07-22 (#336, live resize by hand);
+# 40Gi -> 60Gi 2026-10-03 (REN-583, after the volume filled again on REN-580).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: data-gitea-0
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes: ["ReadWriteOnce"]
+  storageClassName: ceph-block
+  volumeMode: Filesystem
+  volumeName: pvc-e7c56db2-481d-40cb-b94a-b264b7b4b5be
+  resources:
+    requests:
+      storage: 60Gi
@@ -147,14 +147,10 @@ spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: ceph-block
        resources:
-          # Expanded 20Gi -> 40Gi on 2026-07-22 (ceph-block online resize) to
-          # clear a 100%-full outage (#336). The live PVC was resized directly.
-          # NOTE: volumeClaimTemplates is immutable on an existing StatefulSet,
-          # so bumping this value cannot be `kubectl apply`-ed in place — Flux
-          # will reject the whole object. Landing this size change requires a
-          # one-time `kubectl delete statefulset gitea -n gitea --cascade=orphan`
-          # (pod + PVC keep running) so Flux re-creates the STS from this
-          # template and re-adopts them. After the cleanup_packages GC reclaims
-          # the ~10G of orphaned blobs, real usage is ~9G, so 40Gi is headroom.
+          # This template only applies when the claim does not exist yet. The
+          # live size of data-gitea-0 is set in pvc.yaml (REN-583), not here:
+          # volumeClaimTemplates is immutable on an existing StatefulSet, so a
+          # change to this value cannot be applied and Flux would reject the
+          # whole object. Leave it at 40Gi.
          requests:
            storage: 40Gi
@@ -1,15 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: home-assistant-backup-password
-  namespace: home-assistant
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: home-assistant-backup-password # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: apps/home-assistant/backup-password
@@ -60,7 +60,7 @@ spec:
        # repo's configuration.yaml reads this key, so HA does not start
        # without it. Removing this init container takes the recorder down.
        - name: render-db-url
-          image: busybox:1.36
+          image: busybox:1.38@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e
          command:
            - sh
            - -c
@@ -159,7 +159,7 @@ spec:
        # substitution to POD_IP so nginx-runtime variables like $host
        # / $http_x_forwarded_for in the template stay intact.
        - name: proxy
-          image: nginxinc/nginx-unprivileged:1.27-alpine
+          image: nginxinc/nginx-unprivileged:1.31-alpine@sha256:26b0bf6fbf07297983cb341998d79c831508787de26627dd2a112321b9c3a4af
          env:
            - name: POD_IP
              valueFrom:
@@ -193,7 +193,7 @@ spec:
        # any LAN client can edit, matching the user's trust-the-network
        # model. Runs as root so its writes match HA's root-owned files.
        - name: code-server
-          image: codercom/code-server:4.118.0
+          image: codercom/code-server:4.140.0@sha256:1c9ad972a78450f430155144110da4c02d3627c72a598e9197e2781fb9f92e28
          args:
            - "--auth=none"
            - "--bind-addr=0.0.0.0:8080"
@@ -36,7 +36,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: esphome
-          image: ghcr.io/esphome/esphome:2026.6.2
+          image: ghcr.io/esphome/esphome:2026.9.1@sha256:776a845d508909bf426d2ee7bdd81796361ebe6a7f091807ce1d6d0b48985413
          imagePullPolicy: IfNotPresent
          # Default entrypoint runs the dashboard (Device Builder) against the
          # mounted /config — no command override needed.
@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - backup-password-externalsecret.yaml
  - pvcs.yaml
  - deployment.yaml
  - service.yaml
@@ -69,6 +68,11 @@ resources:
  - ha-pg-objectstore.yaml
  - ha-pg-cluster.yaml
  - ha-pg-scheduled-backup.yaml
+  # MQTT credential management (REN-802): the account controller and, for the
+  # test phase, its own Dynamic Security broker. Neither touches the
+  # production broker. Register: mqtt-accounts.yaml (configMapGenerator).
+  - mosquitto-dynsec-test.yaml
+  - mqtt-account-controller.yaml
 configMapGenerator:
  # nginx config for the reverse-proxy sidecar. Hash suffix disabled so
  # the cm name stays stable; editing proxy.conf does not auto-roll the
@@ -104,5 +108,20 @@ configMapGenerator:
    namespace: home-assistant
    files:
      - guard.sh=zigbee2mqtt-discovery-guard.sh
+  # Z2M external converters, copied into /app/data/external_converters by the
+  # seed-data init container on every start. Hash suffix disabled (name stays
+  # stable), so editing a converter does not auto-roll the pod — restart
+  # manually with `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`.
+  - name: zigbee2mqtt-external-converters
+    namespace: home-assistant
+    files:
+      - wetterstation-ws90.js=zigbee2mqtt-ws90-converter.js
+  # MQTT account register, read by mqtt-account-controller on every loop.
+  # Hash suffix disabled (name stays stable); the volume is not a subPath
+  # mount, so an edit reaches the running controller without a restart.
+  - name: mqtt-accounts
+    namespace: home-assistant
+    files:
+      - mqtt-accounts.yaml
 generatorOptions:
  disableNameSuffixHash: true
@@ -42,6 +42,9 @@ spec:
      labels:
        app.kubernetes.io/name: mdns-relay
      annotations:
+        # Bump to restart the pod and re-attach net1 (DHCP via multus).
+        # 2026-10-06: first attach on containernetworking/plugins v1.9.1.
+        homelab.renner.uno/restarted-for: "2026-10-06 cni-plugins v1.9.1 attach check"
        k8s.v1.cni.cncf.io/networks: mdns-relay-iot
    spec:
      enableServiceLinks: false
@@ -82,7 +85,7 @@ spec:
        # deliver, and the ClusterIP has no backend while HA restarts. The
        # trap forwards SIGTERM so the pod still terminates promptly.
        - name: socat-ipv4
-          image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
+          image: alpine/socat:1.8.1.3@sha256:82ad20f6f6e29b91ff33b6662d24063522b1378f08fd8569cd3ab412cac13f50
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
@@ -124,7 +127,7 @@ spec:
        # depends on the kernel's bindv6only default rather than on anything
        # stated here, so the roles are made explicit instead of inherited.
        - name: socat-ipv6
-          image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
+          image: alpine/socat:1.8.1.3@sha256:82ad20f6f6e29b91ff33b6662d24063522b1378f08fd8569cd3ab412cac13f50
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
@@ -44,6 +44,10 @@ kind: Job
 metadata:
  name: mosquitto-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -55,7 +59,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -112,6 +116,10 @@ kind: Job
 metadata:
  name: mosquitto-admin-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -123,7 +131,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -179,6 +187,10 @@ kind: Job
 metadata:
  name: mosquitto-shellies-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -190,7 +202,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -245,6 +257,10 @@ kind: Job
 metadata:
  name: mosquitto-sml2mqtt-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -256,7 +272,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -313,6 +329,10 @@ kind: Job
 metadata:
  name: mosquitto-open3e-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -324,7 +344,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -383,6 +403,10 @@ kind: Job
 metadata:
  name: mosquitto-zigbee2mqtt-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -394,7 +418,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -0,0 +1,233 @@
+# Test broker for mqtt-account-controller (REN-802 step 1, REN-888).
+#
+# A second, throwaway mosquitto with the Dynamic Security plugin and a mutual-
+# TLS listener, so the controller can be proven (cert login, reconcile, seed
+# after restart, revocation) without touching the production broker. Nothing
+# in production connects here.
+#
+# Deliberately NOT labelled app.kubernetes.io/name: mosquitto: that label is
+# the selector of the production Services and of mosquitto-world-ingress, so
+# reusing it would put this pod behind mqtt.renner.uno and the IoT VLAN.
+# ClusterIP only; reachable from inside the home-assistant namespace.
+#
+# Removal: delete this file and its kustomization.yaml entry once production
+# runs Dynamic Security (REN-802 step 2).
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mosquitto-dynsec-test-tls
+  namespace: home-assistant
+spec:
+  secretName: mosquitto-dynsec-test-tls
+  dnsNames:
+    - mosquitto-dynsec-test
+    - mosquitto-dynsec-test.home-assistant.svc
+    - mosquitto-dynsec-test.home-assistant.svc.cluster.local
+  usages:
+    - digital signature
+    - server auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mosquitto-dynsec-test-config
+  namespace: home-assistant
+data:
+  mosquitto.conf: |
+    persistence false
+    log_dest stdout
+    log_type error
+    log_type warning
+    log_type notice
+    log_type information
+
+    # Dynamic Security owns authentication and ACLs. The plugin writes its
+    # state back to this file; it is an emptyDir, refilled by the
+    # initContainer on every start.
+    plugin /usr/lib/mosquitto_dynamic_security.so
+    plugin_opt_config_file /mosquitto/dynsec/dynamic-security.json
+
+    # Mutual TLS only. The client certificate's CN is the username; the
+    # Dynamic Security client of that name supplies the roles.
+    listener 8883
+    cafile /mosquitto/tls/ca.crt
+    certfile /mosquitto/tls/tls.crt
+    keyfile /mosquitto/tls/tls.key
+    require_certificate true
+    use_identity_as_username true
+
+    allow_anonymous false
+  # Start file used until the controller has written the Secret
+  # mqtt-dynsec-seed. No secrets: only the controller's certificate client
+  # and a role that reaches the Dynamic Security API and nothing else. Same
+  # shape as the controller's seed renderer (internal/dynsec/config.go).
+  dynamic-security.json: |
+    {
+      "defaultACLAccess": {
+        "publishClientSend": false,
+        "publishClientReceive": true,
+        "subscribe": false,
+        "unsubscribe": true
+      },
+      "clients": [
+        {
+          "username": "mqtt-account-controller",
+          "roles": [{"rolename": "mqtt-account-controller-admin"}]
+        }
+      ],
+      "groups": [],
+      "roles": [
+        {
+          "rolename": "mqtt-account-controller-admin",
+          "acls": [
+            {"acltype": "publishClientReceive", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "publishClientSend", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "subscribePattern", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true}
+          ]
+        }
+      ]
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mosquitto-dynsec-test
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mosquitto-dynsec-test
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mosquitto-dynsec-test
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 10
+      # Runs as the mosquitto UID from the start instead of starting as root
+      # and dropping, so the dynsec file the initContainer writes is the
+      # broker's own.
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1883
+        runAsGroup: 1883
+        fsGroup: 1883
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # Every start: the controller's seed if it exists, else the static
+        # base file. The seed carries every account and password hash, so the
+        # broker is in the desired state before the first client connects.
+        - name: dynsec-seed
+          image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
+          command:
+            - sh
+            - -c
+            - |
+              set -eu
+              if [ -s /seed/dynamic-security.json ]; then
+                cp /seed/dynamic-security.json /mosquitto/dynsec/dynamic-security.json
+                echo "dynsec: loaded seed from Secret mqtt-dynsec-seed"
+              else
+                cp /mosquitto/config/dynamic-security.json /mosquitto/dynsec/dynamic-security.json
+                echo "dynsec: no seed yet, loaded base file"
+              fi
+              chmod 0600 /mosquitto/dynsec/dynamic-security.json
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /mosquitto/config
+              readOnly: true
+            - name: seed
+              mountPath: /seed
+              readOnly: true
+            - name: dynsec
+              mountPath: /mosquitto/dynsec
+      containers:
+        - name: mosquitto
+          # Same tag and image policy as the production broker, so the test
+          # runs the version that production will run.
+          image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
+          command: ["mosquitto", "-c", "/mosquitto/config/mosquitto.conf"]
+          ports:
+            - name: mqtts
+              containerPort: 8883
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /mosquitto/config
+              readOnly: true
+            - name: tls
+              mountPath: /mosquitto/tls
+              readOnly: true
+            - name: dynsec
+              mountPath: /mosquitto/dynsec
+          readinessProbe:
+            tcpSocket:
+              port: mqtts
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: mqtts
+            periodSeconds: 30
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 5m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+      volumes:
+        - name: config
+          configMap:
+            name: mosquitto-dynsec-test-config
+        - name: tls
+          secret:
+            secretName: mosquitto-dynsec-test-tls
+        # Written by mqtt-account-controller. Optional: absent on the very
+        # first start, when the base file is used instead.
+        - name: seed
+          secret:
+            secretName: mqtt-dynsec-seed
+            optional: true
+        - name: dynsec
+          emptyDir:
+            sizeLimit: 4Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mosquitto-dynsec-test
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+  ports:
+    - name: mqtts
+      port: 8883
+      targetPort: mqtts
+      protocol: TCP
@@ -0,0 +1,222 @@
+# mqtt-account-controller (REN-802, code REN-837, deploy REN-888).
+# Source: konairius/mqtt-account-controller (maintainer Hephaestus). Reconciles
+# the broker's Dynamic Security clients/roles/ACLs, the cert-manager
+# Certificates of `auth: cert` accounts and device passwords to the register
+# mqtt-accounts.yaml, every INTERVAL (2m).
+#
+# TEST PHASE: BROKER_URL points at mosquitto-dynsec-test, not the production
+# broker. Switching it to `mosquitto` is REN-802 step 2 (HA window).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+---
+# Find the Certificates it manages (label app.kubernetes.io/managed-by=
+# mqtt-account-controller) in every namespace. List only: no get, watch or
+# write cluster-wide.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: mqtt-account-controller-list-certificates
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: mqtt-account-controller-list-certificates
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: mqtt-account-controller-list-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# Per deliver.namespace: manage the account Certificates there (server-side
+# apply = patch, which creates through `create`). One Role + RoleBinding per
+# namespace that appears as `deliver.namespace` in the register; today only
+# home-assistant.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "create", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# Secrets in home-assistant, by name only. This namespace holds every broker
+# and HA credential, so an unnamed `delete secrets` grant would let the
+# controller wipe all of them.
+#   mqtt-account-hashes, mqtt-dynsec-seed: its own state (server-side apply
+#     creates them, hence create + patch; resourceNames hold for an apply
+#     that creates, because the name is in the request path).
+#   mqtt-<name>-tls: delete on revocation, one entry per cert account
+#     delivered to this namespace. Add the name with the register entry.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-secrets-home-assistant
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["mqtt-account-hashes", "mqtt-dynsec-seed"]
+    verbs: ["get", "create", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames:
+      - mqtt-test-cert-tls
+      - mqtt-test-revoke-tls
+    verbs: ["delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-secrets-home-assistant
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-secrets-home-assistant
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# The controller's own MQTT identity: CN = Dynamic Security admin client
+# (ADMIN_USER). Not labelled managed-by, so the controller never treats it
+# as an account certificate and never revokes it. Re-read on every TLS
+# handshake, so a renewal needs no restart.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+spec:
+  secretName: mqtt-account-controller-tls
+  commonName: mqtt-account-controller
+  usages:
+    - digital signature
+    - client auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+    rotationPolicy: Always
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+spec:
+  replicas: 1
+  # One writer at a time: two controllers would race on rotations and the
+  # seed Secret.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mqtt-account-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mqtt-account-controller
+    spec:
+      serviceAccountName: mqtt-account-controller
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: controller
+          # Woodpecker builds main-<sha> on every merge to main; no sortable
+          # tag, so bumps are manual (same as ical-proxy).
+          image: gitea.renner.uno/konairius/mqtt-account-controller:main-47213ef
+          env:
+            - name: BROKER_URL
+              value: ssl://mosquitto-dynsec-test.home-assistant.svc.cluster.local:8883
+            # Not :8080: several :8080 ingress grants in the home-assistant
+            # default-deny apply to every pod in the namespace.
+            - name: LISTEN
+              value: ":8081"
+          ports:
+            - name: http
+              containerPort: 8081
+              protocol: TCP
+          # /healthz is 200 while a reconcile succeeded within 3 x INTERVAL.
+          # Readiness only: a restart does not fix an unreachable broker.
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: register
+              mountPath: /etc/mqtt-accounts
+              readOnly: true
+            - name: tls
+              mountPath: /etc/mqtt-tls
+              readOnly: true
+          resources:
+            requests:
+              cpu: 5m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+      volumes:
+        # Not a subPath mount, so a register change reaches the running pod
+        # (kubelet sync, ~1 min) and the next loop reads it without a restart.
+        - name: register
+          configMap:
+            name: mqtt-accounts
+        - name: tls
+          secret:
+            secretName: mqtt-account-controller-tls
@@ -0,0 +1,26 @@
+# MQTT account register (REN-802). The only definition of who may use the
+# broker: mqtt-account-controller reconciles certificates, device passwords
+# and Dynamic Security clients/roles/ACLs to this list. Format: README of
+# konairius/mqtt-account-controller.
+#
+# Delivered to the controller as the ConfigMap `mqtt-accounts` (see the
+# configMapGenerator in kustomization.yaml). An empty list is refused, so it
+# never revokes everyone.
+#
+# TEST PHASE (REN-888): the controller points at the test broker
+# `mosquitto-dynsec-test`, not the production `mosquitto`. Both entries are
+# certificate test accounts. `test-revoke` exists to be deleted again, which
+# checks revocation (Certificate and Secret gone, client and role gone).
+#
+# A cert account needs its Secret `mqtt-<name>-tls` added to the
+# `mqtt-account-controller-secrets-<namespace>` Role in
+# mqtt-account-controller.yaml, or revoking it fails with 403.
+- name: test-cert
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite test/cert/#"]
+
+- name: test-revoke
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["read test/revoke/#"]
@@ -61,7 +61,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: shelly-manager
-          image: gitea.renner.uno/k.renner/shelly-manager:build.1784957186 # {"$imagepolicy": "flux-system:shelly-manager"}
+          image: gitea.renner.uno/k.renner/shelly-manager:build.1791295339 # {"$imagepolicy": "flux-system:shelly-manager"}
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
@@ -132,6 +132,13 @@ spec:
              else
                echo "existing Z2M state present; NOT touching coordinator_backup.json"
              fi
+              # External converters: overwritten from git on EVERY start (not
+              # seed-only). Copied rather than mounted because Z2M writes a temp
+              # copy of each converter into this directory while loading it
+              # (dist/extension/externalJS.js), which a read-only mount refuses.
+              mkdir -p /app/data/external_converters
+              cp /external-converters/*.js /app/data/external_converters/
+              echo "copied external converters: $(ls /external-converters)"
          env:
            - name: MQTT_PASSWORD
              valueFrom:
@@ -147,6 +154,9 @@ spec:
            - name: coordinator-backup
              mountPath: /backup
              readOnly: true
+            - name: external-converters
+              mountPath: /external-converters
+              readOnly: true
          resources:
            requests:
              cpu: 10m
@@ -158,7 +168,7 @@ spec:
          # Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
          # add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
          # follow-up if desired; pinned-and-manual is fine for the initial cutover.
-          image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
+          image: ghcr.io/koenkk/zigbee2mqtt:2.14.2@sha256:addbfb08fcef2e4477de1535636118241e10d13e898fbe4d9faba8a4d7396311
          imagePullPolicy: IfNotPresent
          env:
            - name: TZ
@@ -194,12 +204,15 @@ spec:
            periodSeconds: 30
            timeoutSeconds: 5
            failureThreshold: 4
+          # Node sizes the V8 heap from this limit: 512Mi gave a 259 MB heap,
+          # and Z2M died on "JavaScript heap out of memory" (exit 134) at
+          # 2026-09-28 17:56Z and 2026-09-29 07:36Z.
          resources:
            requests:
              cpu: 50m
-              memory: 128Mi
+              memory: 256Mi
            limits:
-              memory: 512Mi
+              memory: 1Gi
        # Fires a forced interview whenever a still-un-interviewed device
        # transmits — i.e. at the only moment a sleepy battery device can
        # actually complete one. Z2M itself only retries interviews on a ZDO
@@ -337,6 +350,9 @@ spec:
        - name: config-template
          configMap:
            name: zigbee2mqtt-config
+        - name: external-converters
+          configMap:
+            name: zigbee2mqtt-external-converters
        - name: coordinator-backup
          secret:
            secretName: zigbee2mqtt-coordinator-backup
@@ -0,0 +1,48 @@
+// Wetterstation (WS90) = the built-in zigbee-herdsman-converters definition
+// plus one extra configure step that applies the reporting configuration from
+// zigbee2mqtt-ws90-reporting.md. Z2M runs configure after every interview and
+// on bridge/request/device/configure, so those no longer revert the device to
+// the converter defaults (as happened 2026-09-24..28).
+//
+// The extra step runs after the built-in ones (extend order), so its values
+// win. `version` makes Z2M reconfigure once when this file is first loaded:
+// it re-runs configure when definition.version differs from the device's
+// meta.configured. Bump it whenever REPORTING changes.
+const {definitions} = require("zigbee-herdsman-converters/devices/shelly");
+
+const base = definitions.find((d) => d.model === "WS90");
+if (!base) throw new Error("WS90 definition not found in zigbee-herdsman-converters/devices/shelly");
+
+const SHELLY = 5264;
+
+// [cluster, attribute, reportable_change, manufacturerCode]
+const REPORTING = [
+    ["msTemperatureMeasurement", "measuredValue", 10],
+    ["msRelativeHumidity", "measuredValue", 50],
+    ["msPressureMeasurement", "measuredValue", 1],
+    ["msIlluminanceMeasurement", "measuredValue", 5],
+    ["shellyWS90Wind", "windSpeed", 1, SHELLY],
+    ["shellyWS90Wind", "windDirection", 1, SHELLY],
+    ["shellyWS90Wind", "gustSpeed", 1, SHELLY],
+    ["shellyWS90UV", "uvIndex", 1, SHELLY],
+    ["shellyWS90Rain", "rainStatus", 1, SHELLY],
+    ["shellyWS90Rain", "precipitation", 1, SHELLY],
+];
+
+const reporting = {
+    isModernExtend: true,
+    configure: [
+        async (device) => {
+            const endpoint = device.getEndpoint(1);
+            for (const [cluster, attribute, change, manufacturerCode] of REPORTING) {
+                await endpoint.configureReporting(
+                    cluster,
+                    [{attribute, minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: change}],
+                    manufacturerCode ? {manufacturerCode} : {},
+                );
+            }
+        },
+    ],
+};
+
+module.exports = {...base, version: "ws90-reporting-1", extend: [...base.extend, reporting]};
@@ -2,10 +2,15 @@

 Records the ZCL attribute-reporting configuration applied to the Wetterstation
 (Ecowitt WS90 powered by Shelly, IEEE `0xc02cedfffe958f40`) and how to restore
-it, because **this configuration lives on the device and in Z2M's
-`database.db`, not in git**. Zigbee2MQTT has no settings-schema field for
-reporting configuration, so there is nothing declarative to reconcile — Flux
-cannot restore it and a fresh PVC will not have it.
+it. The configuration itself lives on the device and in Z2M's `database.db`.
+
+**Since 2026-10-01 it is applied from git:** the external converter
+`zigbee2mqtt-ws90-converter.js` extends the built-in WS90 definition with a
+configure step that sets the values below, so every interview and every
+`bridge/request/device/configure` applies them instead of the converter
+defaults. The manual commands under "Re-applying" are only needed if the
+converter is not loaded. Keep the table and `REPORTING` in the converter in
+sync, and bump the converter's `version` when they change.

 ## Applied configuration

@@ -71,14 +76,16 @@ the modernExtend value is the outlier.

 ## When this is lost

-Anything that makes Z2M re-run `configure` on the device overwrites all of it
-with the converter defaults:
+Without the external converter, anything that makes Z2M re-run `configure` on
+the device overwrites all of it with the converter defaults:

- re-pairing or re-joining the device
+- re-pairing, re-joining or re-interviewing the device
 - an explicit `zigbee2mqtt/bridge/request/device/configure`
 - a converter update that changes the definition and triggers reconfiguration

-It also will not exist on a fresh PVC. Re-apply with the commands below.
+With the converter loaded, those same events apply the values above. Check
+that it is loaded: Z2M logs `Successfully configured 'Wetterstation'
+(definition vws90-reporting-1)` after a configure.

 ## Re-applying

@@ -46,12 +46,15 @@ spec:
          # Retry-After). main-73b5d97 then raises the server WriteTimeout
          # from 10s to 40s (the 30s upstream fetch timeout + 10s headroom),
          # so a slow-but-legitimate upstream is no longer cut off before
-          # /proxy can answer. The summary_match/summary_replace parameters
-          # this deployment needs are unchanged.
+          # /proxy can answer. main-926ead0 caps the upstream response body
+          # at 2 MiB (larger feeds get 413 instead of being buffered whole,
+          # REN-230) and logs upstream fetch failures with the host,
+          # answering 502 instead of 500. The summary_match/summary_replace
+          # parameters this deployment needs are unchanged.
          #
          # Bumping this is a manual, reviewed edit. The tag format is
          # main-<short-sha>, the first 7 characters of the main commit.
-          image: gitea.renner.uno/konairius/ical-proxy:main-73b5d97
+          image: gitea.renner.uno/konairius/ical-proxy:main-926ead0
          ports:
            - containerPort: 8080
              name: http
@@ -32,7 +32,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.2.4 # {"$imagepolicy": "flux-system:immich-machine-learning"}
          ports:
            - name: http
              containerPort: 3003
@@ -8,26 +8,11 @@
 # back into `kubernetes/apps/immich/` at retirement is the cleanup.
 #
 # Sizing:
-# - library: 300Gi — LXC 225's /mnt/media/immich is currently ~250 GiB.
-#   StorageClass ceph-filesystem has allowVolumeExpansion=true; grow with
-#   `kubectl patch pvc immich-library -p '{"spec":{"resources":{"requests":{"storage":"<new>"}}}}'`.
+# - library-bulk: 300Gi — the media library is ~250 GiB.
+#   StorageClass ceph-filesystem-bulk has allowVolumeExpansion=true; grow with
+#   `kubectl patch pvc immich-library-bulk -p '{"spec":{"resources":{"requests":{"storage":"<new>"}}}}'`.
 # - ml-cache: 20Gi — face-recognition + CLIP model cache. Survives pod
 #   restarts so ML doesn't re-download on every rollout.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: immich-library
-  namespace: immich
-  annotations:
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 300Gi
---
 # Media root on the EC 2+1 bulk pool; thumbs mount separately below.
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -66,7 +66,7 @@ spec:
        # The library PVC is populated by the seed phase before this
        # Deployment reconciles, so no rsync init is needed here.
        - name: fetch-le-staging-roots
-          image: curlimages/curl:8.10.1
+          image: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777
          command:
            - sh
            - -c
@@ -87,7 +87,7 @@ spec:
              mountPath: /ca
      containers:
        - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.2.4 # {"$imagepolicy": "flux-system:immich-server"}
          ports:
            - name: http
              containerPort: 3001
@@ -57,7 +57,7 @@ spec:
        runAsNonRoot: false
      initContainers:
        - name: wait-for-influxd
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -80,7 +80,7 @@ spec:
              echo "[$(date -u +%H:%M:%S)] $INFLUX_HOST ready and authenticated"
      containers:
        - name: create
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -88,7 +88,7 @@ spec:
        # state). Two-stage so we don't try to create an auth before
        # the chart's `influx setup` has finished.
        - name: wait-for-influxd
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -111,7 +111,7 @@ spec:
              echo "[$(date -u +%H:%M:%S)] $INFLUX_HOST ready and authenticated"
      containers:
        - name: create
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -133,7 +133,7 @@ spec:
              set -euo pipefail
              log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }

-              # Need jq + kubectl. influxdb:2.7.4 has neither baked in.
+              # Need jq + kubectl. The influxdb image has neither baked in.
              apt-get update -qq
              apt-get install -y --no-install-recommends curl jq ca-certificates >/dev/null
              KV=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
@@ -53,8 +53,9 @@ spec:
  values:
    image:
      repository: influxdb
-      # Match VM 108 for format compatibility with `influx restore`.
-      tag: "2.7.4"
+      # Was pinned to VM 108's 2.7.4 for the one-time `influx restore`; that
+      # migration is done. No downgrade below 2.9 without `influxd downgrade`.
+      tag: "2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e"
    # Single replica — InfluxDB OSS doesn't cluster. RWO Ceph-block + Velero
    # backups (pending) is the durability story.
    persistence:
@@ -93,6 +94,12 @@ spec:
    env:
      - name: INFLUXD_STORAGE_WRITE_TIMEOUT
        value: "60s"
+      # 2.9 hashes API tokens on startup by default. Hashed tokens cannot be
+      # read back, so the token Jobs' `influx auth list` reuse would break,
+      # and a downgrade below 2.8 would erase every token (HA, Grafana).
+      # Keep tokens unhashed; enable hashing later as its own change.
+      - name: INFLUXD_USE_HASHED_TOKENS
+        value: "false"
    # The chart defaults to liveness/readiness with initialDelaySeconds=0,
    # so after a crash restart the kubelet kills the pod before shards
    # finish opening. Give it 10 minutes to come up cleanly.
@@ -5,3 +5,4 @@ resources:
  - helmrelease.yaml
  - ha-token-job.yaml
  - grafana-token-job.yaml
+  - volumesnapshot-pre-2.9.yaml
@@ -0,0 +1,18 @@
+---
+# Point-in-time copy of the InfluxDB data PVC, taken before the 2.7.4 -> 2.9.1
+# upgrade (homelab #1130, REN-723). Merge this first and check readyToUse=true
+# before #1130: 2.9 migrates the metadata store, so the way back is
+# `influxd downgrade 2.4` or a restore from this snapshot.
+#
+# Crash-consistent (taken from the live RBD volume), like the velero CSI
+# snapshots. Delete this file once 2.9 has run cleanly; the snapshot goes
+# with it (prune).
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: influxdb-influxdb2-pre-2-9
+  namespace: influxdb
+spec:
+  volumeSnapshotClassName: ceph-block-snapclass
+  source:
+    persistentVolumeClaimName: influxdb-influxdb2
@@ -48,7 +48,7 @@ spec:
            # this sentinel: the entrypoint copies AccountsSettings.json
            # into ephemeral /config-internal rather than symlinking it.
            - name: seed-config
-              image: busybox:1.37.0
+              image: busybox:1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e
              command: ["/bin/sh", "-eu", "-c"]
              args:
                - |
@@ -94,7 +94,7 @@ spec:
            - name: libation
              # Pinned by digest: Spegel caches mutable tag resolutions, so
              # :latest stayed on 13.7.11 despite imagePullPolicy Always.
-              image: docker.io/rmcrackan/libation:14.2.1@sha256:6f9d8ba19dc3803dd6ba295ab5a7abfb6ff3bc06286eebf4ecd8ccbe43245888
+              image: docker.io/rmcrackan/libation:14.2.2@sha256:fe188ac3c4ce63cc305971c76e648748341427e16bcf884011da8a384fd3908c
              # The entrypoint stages config into ephemeral /config-internal
              # by copy (only the DB is symlinked to the PVC), so Audible
              # tokens refreshed during a run are lost at container exit.
@@ -0,0 +1,222 @@
+# matrix-bridge (REN-569, REN-587): Matrix appservice between Element and the
+# Paperclip agents. One room per agent with the owner; owner messages become
+# comments on the agent's chat card, the agent's comments come back to the
+# room. Source: gitea.renner.uno/konairius/matrix-bridge.
+#
+# No volume: the bridge keeps its state in the bot's account data on Synapse.
+# No ingress: only Synapse talks to it (appservice transactions on :8080).
+#
+# Secrets:
+#   matrix-bridge-tokens  as_token (bridge -> Synapse) and hs_token (Synapse
+#                         -> bridge), generated once in-cluster by ESO, plus
+#                         the appservice registration Synapse loads from
+#                         app_service_config_files (synapse.yaml). Both sides
+#                         read the same Secret, so the tokens cannot drift.
+#                         Deleting the Secret regenerates both; Reloader
+#                         restarts Synapse and the bridge.
+#   matrix-bridge         paperclip_api_key from OpenBao
+#                         homelab/apps/matrix-bridge: the Paperclip board key
+#                         "matrix-bridge (board)" (REN-582).
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: matrix-bridge-tokens
+  namespace: matrix
+spec:
+  length: 64
+  digits: 16
+  symbols: 0
+  allowRepeat: true
+  noUpper: false
+  secretKeys:
+    - as_token
+    - hs_token
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: matrix-bridge-tokens
+  namespace: matrix
+spec:
+  # Generate once; Periodic would rotate the tokens every refresh.
+  refreshPolicy: CreatedOnce
+  target:
+    name: matrix-bridge-tokens
+    template:
+      engineVersion: v2
+      data:
+        as_token: "{{ .as_token }}"
+        hs_token: "{{ .hs_token }}"
+        # Agent users are @agent_<urlKey>:matrix.renner.uno; the bot is
+        # @paperclip:matrix.renner.uno.
+        registration.yaml: |
+          id: matrix-bridge
+          url: http://matrix-bridge:8080
+          as_token: "{{ .as_token }}"
+          hs_token: "{{ .hs_token }}"
+          sender_localpart: paperclip
+          rate_limited: false
+          namespaces:
+            users:
+              - exclusive: true
+                regex: "@agent_.*:matrix\\.renner\\.uno"
+            aliases: []
+            rooms: []
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: matrix-bridge-tokens
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: matrix-bridge
+  data:
+    - secretKey: paperclip_api_key
+      remoteRef:
+        key: apps/matrix-bridge
+        property: paperclip_api_key
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+  annotations:
+    secret.reloader.stakater.com/reload: matrix-bridge,matrix-bridge-tokens
+  labels:
+    app.kubernetes.io/name: matrix-bridge
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  # One poller only: two pods would bridge every comment twice.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: matrix-bridge
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: matrix-bridge
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        # `node` in node:24-alpine.
+        runAsUser: 1000
+        runAsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: matrix-bridge
+          image: gitea.renner.uno/konairius/matrix-bridge:build.1791195981
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: MATRIX_HOMESERVER_URL
+              value: http://synapse:8008
+            - name: MATRIX_SERVER_NAME
+              value: matrix.renner.uno
+            - name: MATRIX_OWNER
+              value: "@k.renner:matrix.renner.uno"
+            - name: MATRIX_AS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge-tokens
+                  key: as_token
+            - name: MATRIX_HS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge-tokens
+                  key: hs_token
+            - name: PAPERCLIP_API_URL
+              value: http://paperclip.paperclip.svc.cluster.local:3100
+            - name: PAPERCLIP_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge
+                  key: paperclip_api_key
+            - name: PAPERCLIP_COMPANY_ID
+              value: 72f7b75a-10b0-4084-ad5e-a0cdf31c7caf
+            - name: PAPERCLIP_PROJECT_ID
+              value: 67143c0b-e9a1-44c1-b012-0f8fa25625aa
+            # Card link in the topic of each per-card room (REN-668).
+            - name: PAPERCLIP_PUBLIC_URL
+              value: https://paperclip.renner.uno
+          ports:
+            - name: http
+              containerPort: 8080
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          # /health is 503 when the Paperclip comment poll fails, so it gates
+          # readiness only; a Paperclip outage must not restart the bridge.
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: http
+            periodSeconds: 30
+            failureThreshold: 4
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: matrix-bridge
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# The bridge calls the Paperclip API in-cluster. Synapse <-> bridge and DNS
+# are covered by matrix-app-egress (part-of: matrix); the Paperclip side of
+# this flow is in apps/paperclip/network-policy.yaml.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: matrix-bridge-paperclip-egress
+  namespace: matrix
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: matrix-bridge
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+            app.kubernetes.io/name: paperclip
+      toPorts:
+        - ports:
+            - port: "3100"
+              protocol: TCP
@@ -0,0 +1,45 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: mx-pg
+  namespace: matrix
+spec:
+  instances: 2
+  smartShutdownTimeout: 10
+  stopDelay: 60
+  imageName: ghcr.io/cloudnative-pg/postgresql:16
+  enableSuperuserAccess: true
+  storage:
+    storageClass: ceph-block
+    size: 5Gi
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: mx-pg-store
+  bootstrap:
+    initdb:
+      database: synapse
+      owner: synapse
+      # Synapse refuses to start on a database whose collation is not C
+      # (its text indexes depend on byte order). Fixed at initdb time; it
+      # cannot be changed on an existing database.
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  postgresql:
+    parameters:
+      shared_buffers: "128MB"
+      max_connections: "50"
+      # Same write-reduction set as the other CNPG clusters; rationale in
+      # apps/trmnl/cluster.yaml.
+      wal_compression: "zstd"
+      archive_timeout: "30min"
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      memory: 512Mi
@@ -0,0 +1,137 @@
+# Element Web at element.renner.uno, locked to matrix.renner.uno: no custom
+# homeserver, no guests, no registration, no integration manager, no
+# analytics (no posthog block), no bug-report endpoint.
+#
+# feature_login_with_qr: "Sign in with QR code" on the welcome page, so a new
+# browser can be linked from a signed-in device. Linking a phone from the
+# desktop (Settings → Sessions → Link new device) needs no flag; it shows
+# when the server offers MSC4108 and the session has cross-signing.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: element-config
+  namespace: matrix
+data:
+  config.json: |
+    {
+      "default_server_config": {
+        "m.homeserver": {
+          "base_url": "https://matrix.renner.uno",
+          "server_name": "matrix.renner.uno"
+        }
+      },
+      "disable_custom_urls": true,
+      "disable_guests": true,
+      "disable_3pid_login": true,
+      "default_country_code": "DE",
+      "room_directory": { "servers": ["matrix.renner.uno"] },
+      "integrations_ui_url": "",
+      "integrations_rest_url": "",
+      "integrations_widgets_urls": [],
+      "bug_report_endpoint_url": "",
+      "show_labs_settings": false,
+      "features": {
+        "feature_login_with_qr": true
+      },
+      "setting_defaults": {
+        "UIFeature.registration": false,
+        "UIFeature.passwordReset": false,
+        "UIFeature.thirdPartyId": false,
+        "UIFeature.identityServer": false,
+        "UIFeature.deactivate": false
+      }
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: element-web
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: element-config
+  labels:
+    app.kubernetes.io/name: element-web
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: element-web
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: element-web
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 101
+        runAsGroup: 101
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: element-web
+          image: ghcr.io/element-hq/element-web:v1.12.30
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # Unprivileged port: the image runs as nginx, not root.
+            - name: ELEMENT_WEB_PORT
+              value: "8080"
+          ports:
+            - name: http
+              containerPort: 8080
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              memory: 128Mi
+          readinessProbe:
+            httpGet:
+              path: /config.json
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /config.json
+              port: http
+            periodSeconds: 30
+          volumeMounts:
+            # The entrypoint copies /app/config*.json to /tmp and serves it
+            # from there, so the ConfigMap lands at the source path.
+            - name: config
+              mountPath: /app/config.json
+              subPath: config.json
+              readOnly: true
+            # Writable paths for a read-only root: the entrypoint renders the
+            # nginx template into conf.d; pid and temp files go to /tmp.
+            - name: conf-d
+              mountPath: /etc/nginx/conf.d
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: element-config
+        - name: conf-d
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: element-web
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: element-web
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
@@ -0,0 +1,26 @@
+# OpenBao homelab/apps/matrix (seeded by Hermes):
+#   owner_user                 localpart of the owner account (k.renner)
+#   owner_password             the owner's password (MAS password login)
+#   registration_shared_secret random; Synapse's macaroon key fallback
+#   mas_matrix_secret          random; MAS <-> Synapse shared secret
+#   mas_encryption             32 bytes hex; MAS encrypts tokens at rest
+#   mas_rsa_key, mas_ec_key    PEM private keys; MAS signs its tokens
+#
+# The owner reads the login from OpenBao; it never appears in a comment, a
+# manifest or a log. owner_password is the password the account was created
+# with; changing it in OpenBao does not change the account password.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: synapse-secrets
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: synapse-secrets
+  dataFrom:
+    - extract:
+        key: apps/matrix
@@ -0,0 +1,15 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hetzner-s3-creds
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hetzner-s3-creds
+  dataFrom:
+    - extract:
+        key: infrastructure/hetzner-s3
@@ -0,0 +1,161 @@
+# Public (REN-663): each Ingress carries dns.renner.uno/public, so
+# netbird-service-sync publishes its host through the NetBird proxy
+# (Cloudflare wildcard -> proxy.renner.uno -> TLS passthrough to Traefik) and
+# external-dns-unifi skips it; LAN clients follow the public answer and the UDM
+# hairpins back in. Same path as audiobooks/immich. Federation stays off.
+#
+# Synapse: only the client-facing paths. /_synapse/admin stays off the
+# Ingress (in-cluster only); /_matrix/federation and /_matrix/key are not
+# served by the listener anyway.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: synapse
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - matrix.renner.uno
+      secretName: matrix-tls
+  rules:
+    - host: matrix.renner.uno
+      http:
+        paths:
+          # Legacy login/logout/refresh go to MAS (its compat layer). Longer
+          # paths win in Traefik, so these override /_matrix/client below.
+          - path: /_matrix/client/v3/login
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/v3/logout
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/v3/refresh
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/login
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/logout
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/refresh
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /_matrix/media
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /_synapse/client
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /.well-known/matrix/client
+            pathType: Exact
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: element-web
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - element.renner.uno
+      secretName: element-tls
+  rules:
+    - host: element.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: element-web
+                port:
+                  number: 80
+---
+# MAS: login pages, OAuth endpoints and account management.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: mas
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - account.matrix.renner.uno
+      secretName: mas-tls
+  rules:
+    - host: account.matrix.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - externalsecret.yaml
+  - synapse.yaml
+  - mas.yaml
+  - element.yaml
+  - bridge.yaml
+  - ingress.yaml
+  # mx-pg CNPG Postgres + offsite Barman backup. The media store and signing
+  # key are on synapse-data, backed up by Velero (matrix-daily).
+  - cluster.yaml
+  - objectstore.yaml
+  - scheduled-backup.yaml
+  - hetzner-creds-externalsecret.yaml
@@ -0,0 +1,229 @@
+# Matrix Authentication Service (MAS) for matrix.renner.uno (REN-567).
+#
+# Synapse delegates all logins to MAS (matrix_authentication_service in
+# synapse.yaml). Why: Element X only signs in with a password or through
+# MAS. MAS gives it a one-tap passkey login through Authentik
+# (upstream_oauth2 below) and makes QR sign-in possible.
+#
+# Accounts: the owner account was moved from Synapse into MAS by syn2mas
+# (the migrate-to-mas init container in synapse.yaml), with its password
+# hash and sessions. The first Authentik login links to it by localpart
+# (on_conflict: add). The Authentik application admits only the Matrix
+# Users group, so no one else can create an account through it. Password
+# login stays on as a fallback; self-registration and recovery are off.
+#
+# Secrets: OpenBao homelab/apps/matrix (synapse-secrets). The database is
+# `mas` in mx-pg (Barman backup); MAS keeps no other state.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mas-config
+  namespace: matrix
+data:
+  config.yaml: |
+    http:
+      public_base: https://account.matrix.renner.uno/
+      listeners:
+        - name: web
+          resources:
+            - name: discovery
+            - name: human
+            - name: oauth
+            - name: compat
+            - name: graphql
+            - name: assets
+          binds:
+            - address: "[::]:8080"
+        - name: internal
+          resources:
+            - name: health
+          binds:
+            - address: "[::]:8081"
+
+    # Password from the mounted mx-pg-app Secret; nothing in this file.
+    database:
+      host: mx-pg-rw
+      port: 5432
+      username: synapse
+      password_file: /pg/password
+      database: mas
+
+    matrix:
+      kind: synapse
+      homeserver: matrix.renner.uno
+      endpoint: http://synapse:8008/
+      secret_file: /secrets/mas_matrix_secret
+
+    secrets:
+      encryption_file: /secrets/mas_encryption
+      keys:
+        - key_file: /secrets/mas_rsa_key
+        - key_file: /secrets/mas_ec_key
+
+    passwords:
+      enabled: true
+      # Version 1 must be bcrypt with no pepper: syn2mas imports Synapse's
+      # bcrypt hashes under it. New passwords are hashed with argon2id.
+      schemes:
+        - version: 1
+          algorithm: bcrypt
+        - version: 2
+          algorithm: argon2id
+      minimum_complexity: 3
+
+    account:
+      password_registration_enabled: false
+      password_recovery_enabled: false
+      email_change_allowed: false
+      login_with_email_allowed: false
+      account_deactivation_allowed: false
+
+    # No mail server; MAS sends nothing.
+    email:
+      transport: blackhole
+
+    upstream_oauth2:
+      providers:
+        - id: 01M4480KWZGN9HBNB5SSNGT2M1
+          human_name: Authentik (passkey)
+          issuer: https://auth.renner.uno/application/o/matrix/
+          client_id: matrix-mas
+          # Public client + PKCE: no shared client secret to keep in sync.
+          token_endpoint_auth_method: none
+          pkce_method: always
+          scope: "openid profile email"
+          claims_imports:
+            # Authentik username = Matrix localpart. `add` links the first
+            # login to the migrated account of the same name.
+            localpart:
+              action: require
+              template: "{{ user.preferred_username }}"
+              on_conflict: add
+            displayname:
+              action: suggest
+              template: "{{ user.name }}"
+            email:
+              action: ignore
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mas
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: mas-config
+    secret.reloader.stakater.com/reload: synapse-secrets
+  labels:
+    app.kubernetes.io/name: mas
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mas
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mas
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: mas
+          image: ghcr.io/element-hq/matrix-authentication-service:1.26.0
+          # The server applies database migrations and syncs the upstream
+          # providers from the config on start.
+          args: ["server", "--config", "/config/config.yaml"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          ports:
+            - name: http
+              containerPort: 8080
+            - name: internal
+              containerPort: 8081
+          resources:
+            requests:
+              cpu: 20m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: internal
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: internal
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 4
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: pg
+              mountPath: /pg
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: mas-config
+        - name: secrets
+          secret:
+            secretName: synapse-secrets
+            items:
+              - key: mas_matrix_secret
+                path: mas_matrix_secret
+              - key: mas_encryption
+                path: mas_encryption
+              - key: mas_rsa_key
+                path: mas_rsa_key
+              - key: mas_ec_key
+                path: mas_ec_key
+        - name: pg
+          secret:
+            secretName: mx-pg-app
+            items:
+              - key: password
+                path: password
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mas
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: mas
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# `mas` database in mx-pg, owned by the synapse role (one app, one
+# namespace), so MAS reuses mx-pg-app. Barman backs it up with synapse.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: mx-pg-mas
+  namespace: matrix
+spec:
+  cluster:
+    name: mx-pg
+  name: mas
+  owner: synapse
@@ -0,0 +1,30 @@
+---
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: mx-pg-store
+  namespace: matrix
+spec:
+  configuration:
+    # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG
+    # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud
+    # (plugin #646), so at-rest encryption is unavailable; transit is TLS.
+    destinationPath: s3://renner-uno-k8s-cnpg/mx-pg
+    endpointURL: https://fsn1.your-objectstorage.com
+    s3Credentials:
+      accessKeyId:
+        name: hetzner-s3-creds
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: hetzner-s3-creds
+        key: ACCESS_SECRET_KEY
+      # Region is required (REGION=fsn1) — without it boto3's HeadBucket
+      # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s.
+      region:
+        name: hetzner-s3-creds
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  retentionPolicy: "30d"
@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: mx-pg-daily
+  namespace: matrix
+spec:
+  # 03:00 local, same slot as the other CNPG clusters (the Velero PVC
+  # schedule for this namespace runs later, at 09:45).
+  schedule: "0 0 3 * * *"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: mx-pg
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
@@ -0,0 +1,324 @@
+# Synapse homeserver for matrix.renner.uno (REN-567).
+#
+# Private: federation is off at three layers — federation_domain_whitelist
+# is empty, no listener serves the federation/keys resources, and the
+# namespace egress policy lets Synapse reach nothing outside the cluster
+# except the push gateway. Logins and accounts are delegated to MAS
+# (mas.yaml); registration is closed.
+#
+# server_name is permanent: it is part of every user and room ID, and
+# cannot be changed without a new, empty server. matrix.renner.uno needs no
+# .well-known delegation from the apex, which is the WordPress site.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: synapse-config
+  namespace: matrix
+data:
+  homeserver.yaml: |
+    server_name: matrix.renner.uno
+    public_baseurl: https://matrix.renner.uno/
+    # Lets Element X and other clients discover the homeserver from the
+    # server name. No server well-known: there is no federation to point at.
+    serve_client_wellknown: true
+    pid_file: /tmp/homeserver.pid
+    report_stats: false
+
+    listeners:
+      # Client API only (includes media and /.well-known). Leaving out the
+      # `federation` and `keys` resources means this server does not answer
+      # federation at all.
+      - port: 8008
+        type: http
+        tls: false
+        bind_addresses: ["::"]
+        x_forwarded: true
+        resources:
+          - names: [client]
+            compress: false
+
+    # The password comes from PGPASSWORD (libpq reads it), so no credential
+    # is written into this file.
+    database:
+      name: psycopg2
+      args:
+        host: mx-pg-rw
+        port: 5432
+        dbname: synapse
+        user: synapse
+        cp_min: 2
+        cp_max: 10
+
+    # Signing key is generated on first start if missing; it lives on the
+    # data PVC next to the media store, so Velero backs up both.
+    signing_key_path: /data/signing.key
+    media_store_path: /data/media_store
+    max_upload_size: 100M
+    # Also the macaroon key: Synapse falls back to it when
+    # macaroon_secret_key is unset, so rotating it logs every session out.
+    registration_shared_secret_path: /secrets/registration_shared_secret
+
+    # MAS handles login, sessions and accounts. Synapse asks it to check
+    # every access token; password login here must be off.
+    matrix_authentication_service:
+      enabled: true
+      endpoint: http://mas:8080/
+      secret_path: /secrets/mas_matrix_secret
+    password_config:
+      enabled: false
+
+    # Sign in with QR code (Element X): the rendezvous channel between the
+    # new and the signed-in device. Both require MAS. `open` lets the new,
+    # not-yet-signed-in device create the channel; the server is LAN-only.
+    experimental_features:
+      msc4108_enabled: true
+      msc4388_mode: open
+
+    # Federation off.
+    federation_domain_whitelist: []
+    trusted_key_servers: []
+    allow_public_rooms_over_federation: false
+
+    enable_registration: false
+    allow_guest_access: false
+    url_preview_enabled: false
+    user_directory:
+      search_all_users: true
+
+    # Phones get pushes through the Element push gateway on matrix.org.
+    # Only event and room IDs leave the cluster, never message text.
+    push:
+      include_content: false
+
+    # Appservice registrations (bridges), one file each, mounted from a
+    # Secret. matrix-bridge: bridge.yaml (REN-587).
+    app_service_config_files:
+      - /appservices/matrix-bridge/registration.yaml
+---
+# Media store and the server signing key. Not reconstructable: lost media
+# is gone, and a lost signing key invalidates the server's past signatures.
+# Backed up by Velero (matrix-daily); the database by Barman (mx-pg).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: synapse-data
+  namespace: matrix
+  labels:
+    app.kubernetes.io/name: synapse
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ceph-block
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: synapse
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: synapse-config,mas-config
+    secret.reloader.stakater.com/reload: synapse-secrets,matrix-bridge-tokens
+  labels:
+    app.kubernetes.io/name: synapse
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: synapse
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: synapse
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 991
+        runAsGroup: 991
+        fsGroup: 991
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # One-time move of the Synapse accounts (owner, password hash,
+        # sessions) into the MAS database. syn2mas needs Synapse offline;
+        # this runs before Synapse starts, and Recreate stops the old pod
+        # first. The marker on the data PVC makes later starts skip it.
+        # Debug image: the plain one has no shell for the marker check.
+        - name: migrate-to-mas
+          image: ghcr.io/element-hq/matrix-authentication-service:1.26.0-debug
+          command:
+            - /busybox/sh
+            - -ec
+            - |
+              if [ -f /data/.syn2mas-done ]; then
+                echo "syn2mas already done"; exit 0
+              fi
+              mas-cli syn2mas migrate \
+                --config /mas-config/config.yaml \
+                --synapse-config /config/homeserver.yaml \
+                --synapse-database-uri postgresql:
+              touch /data/.syn2mas-done
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # libpq-style variables for the Synapse database; the URI above
+            # is empty on purpose.
+            - name: PGHOST
+              value: mx-pg-rw
+            - name: PGUSER
+              value: synapse
+            - name: PGDATABASE
+              value: synapse
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mx-pg-app
+                  key: password
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: mas-config
+              mountPath: /mas-config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: pg
+              mountPath: /pg
+              readOnly: true
+            - name: data
+              mountPath: /data
+      containers:
+        - name: synapse
+          image: ghcr.io/element-hq/synapse:v1.162.0
+          # Bypass /start.py: it expects to run as root and template a config
+          # from env. Default mode is generate-missing-and-run, which creates
+          # the signing key on first start.
+          command:
+            - python
+            - -m
+            - synapse.app.homeserver
+            - --config-path
+            - /config/homeserver.yaml
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mx-pg-app
+                  key: password
+          ports:
+            - name: http
+              containerPort: 8008
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+          # First start runs every schema migration against an empty
+          # database; give it up to 10 min.
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+            failureThreshold: 60
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 4
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: data
+              mountPath: /data
+            - name: tmp
+              mountPath: /tmp
+            - name: appservice-matrix-bridge
+              mountPath: /appservices/matrix-bridge
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: synapse-config
+        - name: secrets
+          secret:
+            secretName: synapse-secrets
+            items:
+              - key: registration_shared_secret
+                path: registration_shared_secret
+              - key: mas_matrix_secret
+                path: mas_matrix_secret
+              - key: mas_encryption
+                path: mas_encryption
+              - key: mas_rsa_key
+                path: mas_rsa_key
+              - key: mas_ec_key
+                path: mas_ec_key
+        - name: mas-config
+          configMap:
+            name: mas-config
+        - name: pg
+          secret:
+            secretName: mx-pg-app
+            items:
+              - key: password
+                path: password
+        - name: data
+          persistentVolumeClaim:
+            claimName: synapse-data
+        - name: tmp
+          emptyDir: {}
+        - name: appservice-matrix-bridge
+          secret:
+            secretName: matrix-bridge-tokens
+            items:
+              - key: registration.yaml
+                path: registration.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: synapse
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: synapse
+  ports:
+    - name: http
+      port: 8008
+      targetPort: http
@@ -1,71 +1,4 @@
 ---
-# The agent metrics Service, declared here because the chart does not render
-# one under our values (REN-147).
-#
-# cilium's templates/cilium-agent/service.yaml is gated twice:
-#
-#   {{- if and .Values.agent (not .Values.preflight.enabled) .Values.prometheus.enabled }}
-#   {{- if (or .Values.prometheus.serviceMonitor.enabled .Values.prometheus.metricsService) }}
-#
-# infrastructure/cilium/helmrelease.yaml sets `prometheus.enabled: true` and
-# `prometheus.serviceMonitor.enabled: false`, and leaves `metricsService` at its
-# default of false. First gate passes, second does not. The chart's fallback
-# branch is envoy-only and needs envoy in the agent pod, but envoy runs as its
-# own DaemonSet here — so no `cilium-agent` Service was rendered at all.
-#
-# The consequence was silent: agents exported metrics on :9962 all along, the
-# ServiceMonitor below selected a Service that did not exist, and the scrape
-# pool sat at 0 active / 46 dropped targets. `cilium_*` had zero series, so
-# CiliumBPFMapPressureHigh and CiliumBPFMapFull in prometheusrule-cilium.yaml
-# evaluated an empty vector forever and could never fire.
-#
-# Why a Service here and not `prometheus.metricsService: true` in the
-# HelmRelease: that flag is the chart-native fix and its rendered diff is
-# exactly this one object (it appears in no other template — not the
-# ConfigMap, not the DaemonSet). But setting it changes the HelmRelease values,
-# which makes helm-controller run `helm upgrade` on the CNI release, and that
-# three-way-merges against live objects. PROPOSAL-cilium-single-owner.md
-# records that `cilium-config` has carried a `kubectl-patch` field manager; if
-# any of that drift is still live, the upgrade reconciles it, and the
-# `configmap.reloader.stakater.com/reload: cilium-config` annotation on the
-# HelmRelease then rolls every agent node by node. A Service applied by the
-# apps Kustomization cannot do that. If the drift is ever proven gone, delete
-# this object and set `prometheus.metricsService: true` instead.
-#
-# Name and selector follow authentik-outpost-servicemonitors.yaml: a distinct
-# name plus `app.kubernetes.io/managed-by: homelab`, so flipping either chart
-# flag later renders the chart's own `cilium-agent` Service without an
-# ownership conflict and without double-scraping the same endpoints.
-apiVersion: v1
-kind: Service
-metadata:
-  name: cilium-agent-metrics
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/managed-by: homelab
-    app.kubernetes.io/name: cilium-agent
-    app.kubernetes.io/part-of: cilium
-    # Copied onto every series by the ServiceMonitor's targetLabels below.
-    # The dashboard the chart ships queries
-    # cilium_bpf_map_pressure{k8s_app="cilium"}, so this label is load-bearing.
-    k8s-app: cilium
-spec:
-  type: ClusterIP
-  # Headless on purpose: agent metrics are per-node and every endpoint has to
-  # be scraped. A ClusterIP would hand Prometheus one random agent per scrape.
-  clusterIP: None
-  # Agent pods only. cilium-envoy carries k8s-app: cilium-envoy and exposes
-  # 9964, so it cannot match.
-  selector:
-    k8s-app: cilium
-  ports:
-    # Named and numbered exactly as the chart's own Service, so the
-    # ServiceMonitor below stays a field-for-field copy of the chart's.
-    - name: metrics
-      port: 9962
-      targetPort: prometheus
-      protocol: TCP
---
 # Cilium's agent ServiceMonitor, declared here rather than by the chart.
 #
 # Reproduces what cilium renders with prometheus.serviceMonitor.enabled: same
@@ -89,11 +22,13 @@ metadata:
    app.kubernetes.io/part-of: cilium
 spec:
  selector:
-    # Pinned to the Service above. Without managed-by this would also match a
-    # chart-rendered `cilium-agent` Service and scrape the same endpoints twice.
+    # The headless `cilium-agent` Service that the HelmRelease renders with
+    # prometheus.metricsService (REN-166). It carries k8s-app: cilium, which
+    # targetLabels below copies onto every series; the dashboard the chart
+    # ships queries cilium_bpf_map_pressure{k8s_app="cilium"}.
    matchLabels:
-      app.kubernetes.io/managed-by: homelab
      app.kubernetes.io/name: cilium-agent
+      app.kubernetes.io/part-of: cilium
  namespaceSelector:
    matchNames:
      - kube-system
@@ -29,6 +29,17 @@
 # been dropped and the metric would have stayed absent. The matching ingress rule
 # is added to infrastructure/network-policies/external-secrets-ns.yaml in the
 # same pull request. Without it this file is decoration.
+#
+# NO TLS AND NO AUTH ON THIS SCRAPE, and that is the chart's posture rather than
+# an omission here. In 2.11.0 `metrics.listen.secure.enabled` and
+# `metrics.listen.auth.enabled` are both false in the chart's values.yaml and
+# infrastructure/external-secrets/helmrelease.yaml overrides neither, so the
+# controller serves /metrics over plain HTTP to anything that can reach :8080.
+# Hence no `scheme`, no `tlsConfig` and no `bearerTokenSecret` below: adding
+# them would break the scrape, not harden it. What limits exposure is the
+# network policy — a single port from a single namespace — so turning on
+# `secure.enabled` here would mean a Helm upgrade of ESO to close a hole the
+# CiliumNetworkPolicy already closes.
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -15,6 +15,11 @@ resources:
  - prometheusrule-netbird-availability.yaml
  - prometheusrule-netbird-ingress.yaml
  - prometheusrule-openbao.yaml
+  # Prometheus's own retention. Until REN-107 `retention: 90d` and
+  # `retentionSize: 30GB` were both set, the size cap bound first, and the
+  # effective window was a 20.5-27.2 day saw-tooth with nothing reporting it.
+  # Fires if the size cap binds before the time retention again.
+  - prometheusrule-prometheus-retention.yaml
  - prometheusrule-velero.yaml
  - prometheusrule-woodpecker-rotation.yaml
  # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
@@ -72,8 +77,7 @@ resources:
  # customResourceState block in apps/monitoring/helmrelease-prometheus.yaml —
  # these two files must land and be reverted together.
  - prometheusrule-flux.yaml
+  # Gitea's data volume. It filled twice (#336, REN-580) and the chart's
+  # KubePersistentVolumeFillingUp warned 33 minutes before the second (REN-583).
+  - prometheusrule-gitea.yaml
  - probes-blackbox.yaml
-  # TEMPORARY, REN-140. Negative-test probes for the REN-28 §7 bar, taking the
-  # "reachable before" reading that stops being takeable once the paperless PR
-  # merges. Delete this line and the file once the "after" reading is recorded.
-  - probes-ren28-negative.yaml
@@ -222,16 +222,16 @@ spec:
        # this is the other half of the pair.
        #
        # That gatus comment says the in-cluster probe "needs a
-        # CiliumNetworkPolicy first". That sentence is simply WRONG, not
-        # merely inapplicable:
-        # ../../infrastructure/network-policies/paperless-ns.yaml#monitoring-admit
-        # already admits the whole monitoring namespace, which is where
-        # blackbox-exporter runs,
-        # and Cilium unions the allow rules of every CNP selecting an
+        # CiliumNetworkPolicy first". That sentence was WRONG when written,
+        # and still is, on different grounds. The probe is admitted by the
+        # dedicated policy
+        # ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
+        # scoped to blackbox-exporter on the paperless pod's :8000 (REN-28);
+        # the namespace-wide `monitoring` admit this probe used to rely on is
+        # gone. Cilium unions the allow rules of every CNP selecting an
        # endpoint (the mechanism ../paperless/network-policy.yaml#cnp-union
-        # relies on). The EGRESS paragraph in the block above applies here
-        # identically. Nothing in the paperless namespace is edited by this
-        # change.
+        # relies on), so that policy is all the probe needs. The EGRESS
+        # paragraph in the block above applies here identically.
        #
        # The stale sentence is deliberately left in place in
        # apps/gatus/configmap.yaml: it sits INSIDE the `config.yaml: |`
@@ -261,6 +261,32 @@ spec:
        # at 2b71dda1, and the tested one-line deletion for this target alone
        # is the second command listed there (20 -> 19 targets).
        - http://paperless.paperless.svc.cluster.local:80/accounts/login/
+        # anchor: shelly-manager-app-health
+        # APP-HEALTH probe for shelly-manager (REN-531/REN-533). Same rule as
+        # the blocks above (monitoring-vantage-point-rule): shelly.renner.uno
+        # sits behind the shelly-manager-gate outpost, so only the in-cluster
+        # Service can tell a dead app from a live gate.
+        #
+        # /health is unauthenticated and answers a plain 200 — not a 3xx —
+        # per the app's own test (shelly-manager repo,
+        # AuthenticationTests.Health_IsUnauthenticated); it is also the pod's
+        # kubelet liveness/readiness path
+        # (../home-assistant/shelly-manager-deployment.yaml). AllowedHosts is
+        # "*", so the cluster-local Host header is accepted. Service :80 ->
+        # container :8080 (../home-assistant/shelly-manager-service.yaml).
+        #
+        # NO CiliumNetworkPolicy change: the home-assistant default-deny
+        # admits the whole monitoring namespace on every port
+        # (../../infrastructure/network-policies/home-assistant-ns.yaml,
+        # "Prometheus scrape" rule). If that rule is ever narrowed to
+        # Prometheus alone, add a blackbox-exporter grant on :8080 in a
+        # DEDICATED policy selecting shelly-manager — not on the {} selector,
+        # where :8080 is shared by three apps. The EGRESS paragraph above
+        # applies too: a future monitoring egress block must list
+        # home-assistant.
+        #
+        # REVERT: delete the target line below.
+        - http://shelly-manager.home-assistant.svc.cluster.local:80/health
 # WAN-path probes of the public edge BY PROXY IP. The name-based
 # home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway)
 # and never leaves the LAN — the vantage-point blind spot that let the
@@ -324,12 +350,13 @@ spec:
        - proxy-1.renner.uno:33080
        - proxy-2.renner.uno:33080
 ---
+# anchor: negative-control-paperless
 # NEGATIVE CONTROLS for the paperless ingress scoping (REN-28 PR 2a).
 #
 # These four targets are here TO BE REFUSED. They are not availability
 # monitoring: every one of them is a workload in the `paperless` namespace that
 # blackbox-exporter has no business reaching, and each must read
-# `probe_success == 1` today and `0` once REN-28's PR 2b lands. That flip is the
+# `probe_success == 1` before REN-28's PR 2b and `0` after it. That flip is the
 # test. Without them, "the whole-namespace admit is gone" is an assertion about
 # a CNP diff rather than a measured property of the cluster, and a later
 # re-widening of the grant would be silent.
@@ -353,10 +380,31 @@ spec:
 # these to blackbox-http instead would
 # page the on-call forever.
 #
-# The inverted alert that gives these targets teeth — CNPNegativeControlReachable,
-# `probe_success{job="blackbox-negative-control"} == 1` — ships in PR 2b, NOT
-# here. Between 2a and 2b these controls are SUPPOSED to read 1, so the alert
-# would fire on the correct state. Do not add it before the policy change.
+# The inverted alert that gives these targets teeth — CNPNegativeControlReachable
+# (prometheusrule-blackbox.yaml#cnp-negative-control-reachable),
+# `probe_success{job="blackbox-negative-control"} == 1` — landed with PR 2b,
+# together with the policy change, not before: until then these controls were
+# SUPPOSED to read 1, so the alert would have fired on the correct state.
+#
+# TWO WAYS A BROKEN HARNESS READS AS A PASSING TEST. The inverted alert fires
+# only on `1`, so anything that forces `0` for a reason other than the policy
+# disarms it silently. Both of these are load-bearing:
+#
+#   * The IPv4 fallback. The shared tcp_connect module
+#     (monitoring/helmrelease-blackbox-exporter.yaml) prefers ip6 with
+#     ip_protocol_fallback: true. All four Services are SingleStack IPv4, so
+#     ip6 never resolves and every `1` these controls ever read came through
+#     the fallback (probe_ip_protocol == 4). Set the fallback to false and all
+#     four drop to `0` with the policy wide open. Do not change the module for
+#     these targets — blackbox-tcp-relays pins ip6-first on purpose; give this
+#     job its own module instead.
+#   * scrapeTimeout: 10s below. Cilium drops the SYN, so a refused connect runs
+#     to the module's own 5s timeout. 5s < 10s is what keeps the SCRAPE
+#     succeeding, so `up` stays 1 and probe_success reads 0 instead of going
+#     absent. Do not trim scrapeTimeout toward 5s: a scrape that times out
+#     leaves no probe_success series for the inverted alert to read, and sets
+#     `up` to 0, which the chart's job-agnostic TargetDown (defaultRules are on
+#     in monitoring/helmrelease-prometheus.yaml) can page on.
 #
 # A bare TCP connect is the right probe: it completes or it times out, with no
 # application-layer ambiguity about whether a 403 meant the policy or the app.
@@ -386,9 +434,10 @@ spec:
        # Proves BOTH halves of the pprl-pg scoping PR 2b will introduce: the
        # source (blackbox is not Prometheus, and the pprl-pg-allow-prometheus
        # rule 2b adds admits only Prometheus) and the port (5432 is not the
-        # exporter's 9187). Neither that rule nor paperless-allow-blackbox
-        # exists yet at this commit — that is the point, and why this target
-        # must read 1 now. Service is CNPG-generated from Cluster `pprl-pg`
+        # exporter's 9187). Both that rule and paperless-allow-blackbox
+        # (../../infrastructure/network-policies/paperless-ns.yaml#pprl-pg-allow-prometheus,
+        # ...#paperless-allow-blackbox) now exist, which is why this target
+        # must read 0. Service is CNPG-generated from Cluster `pprl-pg`
        # (../paperless/cluster.yaml#pprl-pg) and is the name paperless itself
        # dials (../paperless/paperless.yaml#db-host).
        - pprl-pg-rw.paperless.svc.cluster.local:5432
@@ -1,100 +0,0 @@
-# TEMPORARY — negative-test probes for REN-28 §7. DELETE AFTER THE "AFTER"
-# READING IS TAKEN. This file exists to make two of the four §7 negative tests
-# measurable without granting `pods/exec` in `monitoring`, which the owner
-# declined on REN-140 as a wider grant than the change it would verify.
-#
-# WHAT IT IS. blackbox-exporter is already an HTTP/TCP prober running in the
-# `monitoring` namespace. Pointing it at targets that the REN-28 paperless PR
-# is supposed to CLOSE turns "is this path still open?" into a number in
-# Prometheus, readable through the existing read-only REN-36 grant. Every
-# target below answers today and MUST stop answering after the PR lands. A
-# `probe_success` that stays 1 after the merge means the rule does not do what
-# §5 claims.
-#
-# WHY IT DOES NOT PAGE ANYONE. jobName is `blackbox-ren28-negative`, not
-# `blackbox-http`. Every rule in prometheusrule-blackbox.yaml is scoped to
-# job="blackbox-http" (BlackboxProbeFailing) or absent(...{job="blackbox-http"})
-# (BlackboxProbesAbsent), so neither can match these series — before or after.
-# TargetDown (general.rules) cannot match either: the scrape target of a Probe
-# is blackbox-exporter itself, which answers 200 with probe_success 0 on a
-# failed probe, so `up` stays 1 regardless of the result. Verified against the
-# live rule set on 2026-09-26. This is deliberate: a monitor that pages for an
-# expected failure is worse than no monitor.
-#
-# NO POLICY CHANGE IS NEEDED TO MAKE THESE WORK TODAY. Ingress: the paperless
-# baseline still admits the whole `monitoring` namespace on every port
-# (../../infrastructure/network-policies/paperless-ns.yaml#monitoring-admit) —
-# that admission is
-# exactly what REN-28 removes. Egress: the `monitoring` namespace is in no
-# egress enforcement at all; its only two Cilium policies, monitoring-ns.yaml
-# and monitoring-alloy-syslog.yaml, are ingress-only. Same dependency the three
-# cross-namespace targets in probes-blackbox.yaml already rest on.
-#
-# TWO POD-IP TARGETS. pprl-pg's :9187 metrics port is on the POD, not on any
-# Service — that is why it is scraped by a PodMonitor and not a ServiceMonitor
-# (cnpg-podmonitors.yaml). There is no stable name for it, so the literal pod
-# IPs are used. They were read live on 2026-09-26; if either pod is rescheduled
-# the probe goes red for the wrong reason. BEFORE TRUSTING AN "AFTER" READING,
-# re-confirm the IP still belongs to the same pod:
-#
-#   kube_pod_info{namespace="paperless", pod=~"pprl-pg-.*"}   -> pod_ip
-#
-# The pprl-pg-r:5432 target below is the stable-name version of the same
-# statement and survives a reschedule; it is on the database port rather than
-# the metrics port, so the two are kept together on purpose.
-#
-# REVERT. Delete this file and its line in kustomization.yaml. Nothing else
-# references it; no workload is touched; Prometheus drops the job on the next
-# operator reconcile.
-apiVersion: monitoring.coreos.com/v1
-kind: Probe
-metadata:
-  name: blackbox-ren28-negative
-  namespace: monitoring
-  labels:
-    # probeSelector is NOT loosened on this cluster (helmrelease-prometheus.yaml
-    # keeps ruleSelector/probeSelector/scrapeConfigSelector strict), so this
-    # label is load-bearing, not decorative. Without it the Probe is created and
-    # never discovered — and a test that silently does not run reads exactly
-    # like a test that passed.
-    release: kube-prometheus-stack
-spec:
-  jobName: blackbox-ren28-negative
-  prober:
-    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
-  module: tcp_connect
-  interval: 60s
-  # 10s matches blackbox-tcp-relays. It is also the discriminator: a Cilium
-  # deny DROPS the SYN, so a refused path completes in ~1ms and a denied path
-  # burns the full timeout. probe_duration_seconds distinguishes "nothing is
-  # listening" from "policy stopped it" — probe_success alone does not.
-  scrapeTimeout: 10s
-  targets:
-    staticConfig:
-      static:
-        # REN-28 §7 test 4 — "from a monitoring pod that is not Prometheus, a
-        # connection to pprl-pg :9187 must time out". blackbox-exporter is
-        # precisely that pod. Both CNPG instances, because the PR's
-        # pprl-pg-allow-prometheus rule selects on a label both carry: if the
-        # selector were wrong in a way that left one instance open, one target
-        # would tell us and the other would not.
-        - 10.244.4.166:9187 # pprl-pg-2, node k8s-2
-        - 10.244.2.220:9187 # pprl-pg-3, node k8s-3
-        # Same statement on a stable name and the port that actually matters:
-        # after the PR the only monitoring→pprl-pg allow is Prometheus on
-        # :9187, so a non-Prometheus monitoring pod must not reach Postgres.
-        - pprl-pg-r.paperless.svc.cluster.local:5432
-        # REN-28 §7 test 2, workload dimension — "from blackbox-exporter, a
-        # paperless target other than the one it is scoped to must time out".
-        # The test as written in §7 says "a paperless POD port other than
-        # 8000"; there is no such port. The paperless pod's only other listener
-        # is the bridge-tls-proxy socat, bound to 127.0.0.1:11143
-        # (../paperless/paperless.yaml#bridge-tls-proxy), so every other port
-        # on that pod
-        # is closed today and answers RST — there is no "reachable before" to
-        # record and the port half of paperless-allow-blackbox (:8000) cannot
-        # be tested at all. gotenberg is the half that CAN be: it is one of the
-        # other eight workloads the endpointSelector:{} baseline currently
-        # exposes to all of `monitoring`, and the PR's per-workload selector is
-        # what closes it.
-        - gotenberg.paperless.svc.cluster.local:3000
@@ -59,3 +59,42 @@ spec:
              they are all blind until it returns.
          labels:
            severity: critical
+    # The inverted alert for the paperless negative controls (REN-28 §5.6).
+    # A separate group so nothing above can grow a matcher for it by accident.
+    - name: blackbox-negative-control.rules
+      rules:
+        # The four targets in probes-blackbox.yaml#negative-control-paperless are
+        # paperless-namespace workloads that blackbox-exporter must NOT reach
+        # since REN-28 PR 2b scoped the paperless ingress to per-workload
+        # policies. `0` is healthy here and `1` is the failure: a control that
+        # connects means the grant was re-widened — a namespace-wide
+        # `monitoring` admit back on the paperless baseline, or a per-workload
+        # policy whose selector grew — and blackbox, a pod with no business
+        # there, can open TCP into the namespace whose paperless pod trusts
+        # X-Authentik-Username from anything that connects.
+        #
+        # Scoped to job="blackbox-negative-control" ONLY. Never blackbox-http,
+        # whose targets are meant to read 1.
+        #
+        # This rule reads 0 as silence, so anything that forces 0 for another
+        # reason disarms it without a sound. The two known ways are commented
+        # on the Probe: the IPv4 fallback of the shared tcp_connect module, and
+        # scrapeTimeout staying above the module's 5s timeout.
+        #
+        # for: 15m rides out a short policy churn during a Cilium restart or a
+        # CNP re-apply; a real re-widening is permanent and fires regardless.
+        # anchor: cnp-negative-control-reachable
+        - alert: CNPNegativeControlReachable
+          expr: probe_success{job="blackbox-negative-control"} == 1
+          for: 15m
+          annotations:
+            summary: "Negative control {{ $labels.instance }} is reachable from blackbox-exporter"
+            description: >-
+              blackbox-exporter in `monitoring` completed a TCP connect to
+              {{ $labels.instance }}, which the paperless ingress policy must
+              refuse (REN-28). The paperless ingress grant has been re-widened.
+              Check infrastructure/network-policies/paperless-ns.yaml for a
+              namespace-wide `monitoring` admit on default-deny-ingress, or a
+              per-workload policy whose selector now matches this workload.
+          labels:
+            severity: warning
@@ -11,6 +11,31 @@ spec:
  groups:
    - name: disk-wear.rules
      rules:
+        # The drives carrying Talos's EPHEMERAL volume (/var), as (node, device)
+        # in smartctl-exporter's labels. EPHEMERAL sits on retired Ceph drives
+        # whose contents rebuild, and the owner's call is that they run to
+        # failure rather than be replaced on wear (REN-419, REN-420). So the two
+        # wear alerts below skip them. The failure alerts — spare, critical
+        # warning, media errors — still cover them: run to failure still means
+        # knowing when it failed.
+        #
+        # Resolved from node-exporter's /var mount, not a list of serials, so it
+        # follows a drive swap without an edit. If the join stops matching (a
+        # relabel, an encrypted /var on a dm device) the series is empty, and
+        # the wear alerts return for EPHEMERAL drives: it fails loud, not
+        # silent.
+        - record: node_device:nvme_ephemeral_volume:info
+          expr: >-
+            count by (node, device) (
+              label_replace(
+                label_replace(
+                  node_filesystem_size_bytes{mountpoint="/var"}
+                    * on (instance) group_left (nodename) node_uname_info,
+                  "node", "$1", "nodename", "(.+)"
+                ),
+                "device", "$1", "device", "/dev/(nvme[0-9]+)n[0-9]+(p[0-9]+)?"
+              )
+            )
        # Wear percentages are self-reported: 100 means the rated endurance is
        # spent, not that the drive stops.
        #
@@ -45,6 +70,7 @@ spec:
              )
            ) < 180
            and smartctl_device_percentage_used >= 80
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
          # Long, because a projection off a 7d rate should not react to a day.
          for: 6h
          labels:
@@ -55,13 +81,17 @@ spec:
              The drive is past 80% of its rated endurance and the write rate
              of the last week would spend the rest within six months. This is
              a lead-time warning, not a fault: decide whether to replace the
-              drive or move what is writing to it. The Disk Health dashboard
+              drive or move what is writing to it. Drives carrying EPHEMERAL
+              are excluded; they run to failure. The Disk Health dashboard
              shows which drives carry which role.
        # Kept as a flat threshold on purpose: past rated endurance the drive
        # runs outside what the manufacturer stands behind regardless of how
-        # fast it got there, and the alert clears by replacing it.
+        # fast it got there, and the alert clears by replacing it. EPHEMERAL
+        # drives are excluded for the same reason as above.
        - alert: NvmeDriveWearCritical
-          expr: smartctl_device_percentage_used >= 95
+          expr: >-
+            smartctl_device_percentage_used >= 95
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
          for: 1h
          labels:
            severity: critical
@@ -0,0 +1,42 @@
+# Gitea's data volume (data-gitea-0) running low (REN-583).
+#
+# It filled twice: 2026-07-22 (#336) and 2026-10-03 (REN-580), both times
+# because the package registry grew faster than cleanup_packages reclaimed it.
+# A full volume fails image pushes (HTTP 500) and Gitea's own SQLite writes.
+#
+# The chart's KubePersistentVolumeFillingUp did not give usable lead time on
+# REN-580: its warning needs under 15% free AND a 4-day linear projection to
+# zero AND one hour of both, so it fired at 20:26Z; the volume was full at
+# 20:59Z. Growth here comes in steps (a run of image builds added 8 GiB in a
+# few hours that day), which a linear projection lags.
+#
+# So this is a plain floor: under 25% free (15 GiB at 60Gi) for 15m. That is
+# about two of the 8 GiB steps seen on REN-580, and time for someone to run the
+# package cleanup or grow pvc.yaml in apps/gitea.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: gitea
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: gitea.rules
+      rules:
+        - alert: GiteaDataVolumeLow
+          expr: >-
+            kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}
+              / kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}
+            < 0.25
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gitea data volume has {{ $value | humanizePercentage }} free"
+            description: >-
+              data-gitea-0 is under 25% free. When it fills, image pushes fail
+              with HTTP 500 and Gitea cannot write its database. Run the
+              cleanup_packages cron in Gitea's site admin and delete old
+              package versions, or grow the claim in apps/gitea/pvc.yaml
+              (ceph-block resizes online).
@@ -0,0 +1,153 @@
+# Catches a Prometheus retention setting that is configured but NOT IN FORCE.
+#
+# kube-prometheus-stack takes both `retention` (time) and `retentionSize`
+# (bytes) and passes both to Prometheus, which honours WHICHEVER BINDS FIRST.
+# When the size cap binds first, the time setting silently becomes decoration:
+# Prometheus drops its oldest blocks, every query still succeeds, and the only
+# symptom is that a range longer than the real window quietly returns less data
+# than asked for. Nothing in the 182 alert rules this cluster already loads
+# reports that — verified against the live rule set when this was written.
+#
+# That is not hypothetical here. Measured through the read-only proxy on
+# 2026-09-26 with `retention: 90d` and `retentionSize: 30GB` in
+# apps/monitoring/helmrelease-prometheus.yaml (REN-107):
+#
+#   storage.tsdb.retention.size = 30GiB    -> 32,212,254,720 bytes
+#   storage.tsdb.retention.time = 90d      -> 7,776,000 s
+#   actual window held                     -> 2,291,239 s  (26.5 days, 29% of 90d)
+#
+# and size eviction has been firing on a ~7-day cycle for at least as long as
+# the TSDB retains history. At 2026-09-19 23:00->23:30 the store went 28.16 GB
+# -> 21.14 GB, `prometheus_tsdb_size_retentions_total` went 0 -> 1, and the
+# window held collapsed 27.21 -> 20.48 days. The effective retention is a
+# 20.5-27.2 day saw-tooth, not 90 days and not any single number.
+#
+# WHY NOT JUST ALERT ON prometheus_tsdb_size_retentions_total > 0
+#
+# Because that is a PROCESS-LIFETIME counter and it lies by omission. It resets
+# to 0 on every Prometheus restart, and this Prometheus restarts every few days
+# (seven distinct pod IPs in the 26 days of history the TSDB holds — the chart
+# is pinned `>=72.0.0`, so Flux upgrades it whenever upstream releases). Each
+# fresh pod reads 0 for the first ~6 days of its life even though the cluster
+# has been evicting data continuously for a month. REN-107 was originally filed
+# as "the cap will bind in ~3 days" on exactly that reading, taken on a pod that
+# had started 5 days earlier. A spot read of this counter cannot distinguish
+# "never evicted" from "evicted, then restarted".
+#
+# So this rule does not read the counter at all. It compares the two things
+# that are true regardless of process age: the window actually on disk, and the
+# limits the running process reports.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-retention
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: prometheus-retention.rules
+      rules:
+        - alert: PrometheusRetentionNotInForce
+          # NO HARDCODED 90d / 30GB. Both limits are read from the running
+          # process via `prometheus_tsdb_retention_limit_seconds` and
+          # `prometheus_tsdb_retention_limit_bytes`, which Prometheus exports
+          # from the flags it actually started with. Edit the HelmRelease and
+          # this rule follows; it cannot drift out of sync with the config the
+          # way a literal threshold would. That matters because a literal here
+          # would be a second copy of the very setting whose accuracy this rule
+          # exists to police.
+          #
+          # TWO CLAUSES, AND THE SECOND IS THE GUARD
+          #
+          # Clause 1 — the window on disk is materially shorter than the
+          # configured time retention (<90% of it).
+          #
+          # Clause 2 — the store has reached 80% of the configured SIZE cap at
+          # some point in the last 7 days.
+          #
+          # Clause 1 alone would fire on a legitimately young TSDB: a freshly
+          # provisioned PVC holds one day of data against a 90d setting and
+          # nothing is wrong. Clause 2 is what distinguishes "not full yet" from
+          # "full, and the size cap is what is actually in force" — the store can
+          # only be near the byte cap AND short of the time window if bytes are
+          # the binding constraint. Dropping either clause breaks the rule in a
+          # different direction; verified both ways against the live TSDB before
+          # merge (see the PR).
+          #
+          # max_over_time(...[7d:1h]) rather than the instantaneous ratio,
+          # because size eviction is a SAW-TOOTH: the store climbs to ~87% of
+          # the cap, sheds a compacted block, and drops back to ~66%. An instant
+          # ratio would resolve and re-fire on that ~7-day cycle and email twice
+          # a week about an unchanging condition. The 7d window spans a full
+          # tooth, so the alert is continuous while the condition is.
+          #
+          # The 7d PEAK IS DIVIDED BY THE CAP IN FORCE NOW, not by the cap in
+          # force when the peak was recorded. The difference is the day the
+          # config is fixed: REN-107 moves 90d/30GiB to 30d/40GiB, the window
+          # on disk is still ~21 days (so clause 1 stays true until it grows
+          # past 27), and last week's 28 GB peak was 87% of the OLD cap. A
+          # per-step ratio would keep this firing for up to 7 days after the
+          # fix that resolves it; against the new cap the same peak is 66% and
+          # it clears at once. tests/prometheus-retention_test.yaml holds both.
+          #
+          # Everything is aggregated (min/sum/max) so the expression carries NO
+          # instance or pod label. That is deliberate and load-bearing: it makes
+          # the 7d lookback survive the pod restarts described in the header,
+          # which would otherwise start a new series and reset the window.
+          # Confirmed on the live TSDB — the 0.874 peak this returns today was
+          # recorded by a pod that no longer exists. There is exactly one
+          # Prometheus here, so losing the per-instance breakdown costs nothing.
+          expr: |
+            (
+              min(time() - prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus"})
+                < 0.9 * min(prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus"})
+            )
+            and
+            (
+                max_over_time(
+                  sum(
+                      prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus"}
+                    + prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus"}
+                    + prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus"}
+                  )[7d:1h]
+                )
+              / max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})
+              > 0.8
+            )
+          # Long, because this is a configuration fact rather than an incident.
+          # Nothing degrades at the moment it becomes true and nothing is fixed
+          # by waiting, so there is no value in a fast signal — only in not
+          # paging during the minutes after a restart while the aggregation
+          # settles. It clears within one evaluation once the config is honest.
+          for: 1h
+          labels:
+            severity: warning
+            # Static, and true: the Prometheus this describes runs in
+            # `monitoring`. The expression aggregates the real label away (see
+            # above), and the chart's inhibit rules plus Alertmanager's
+            # `group_by` both key on `namespace` — without it this alert groups
+            # under the empty namespace, unlike every other alert here.
+            namespace: monitoring
+          annotations:
+            summary: "Prometheus retention setting is not in force — size cap binds first"
+            # $value is clause 1's left-hand side: the window actually on disk,
+            # in seconds. The configured pair is deliberately NOT templated in
+            # via `query` — that runs on every evaluation and can fail the rule
+            # itself. `runtimeinfo.storageRetention` reports both as one string
+            # ("90d or 30GiB"), which is the cheaper and more direct read.
+            description: >-
+              Prometheus is holding only {{ $value | humanizeDuration }} of history while its
+              configured time retention is longer, and the store has been within 20% of its
+              configured size cap in the last 7 days. The size cap is therefore what is actually
+              in force and the `retention` value in
+              kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured —
+              queries over a longer range return less data than requested, silently and
+              successfully. Confirm the running pair with
+              `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise
+              `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention`
+              to a figure the storage budget can actually hold. Do not read
+              `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets
+              on every restart. Background: REN-107.
@@ -30,6 +30,8 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
 | `external-secrets_test.yaml` | Hand-written suite for `prometheusrule-external-secrets.yaml` (REN-103). |
 | `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). |
 | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
+| `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
+| `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |

 ## Adding a suite
@@ -0,0 +1,124 @@
+# promtool test rules for the EPHEMERAL exclusion in prometheusrule-disk-wear.yaml
+# (REN-419).
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh, which strips `spec:` from the manifest into a
+# mirrored tree — hence `../` below, resolved from this file's directory. See
+# README.md here. The negative controls for this suite are the `disk-wear-*`
+# entries in mutations.txt.
+#
+# WHAT IT IS FOR. The two wear alerts skip the drive that carries /var, found
+# by joining node-exporter's /var mount onto smartctl-exporter's (node, device).
+# Each case puts an EPHEMERAL drive and a non-EPHEMERAL drive side by side with
+# identical SMART data, so the only thing that can separate them is the join.
+#
+# Fixture shape: smartctl-exporter labels drives `nvme0`, `nvme1` and carries
+# `node` from the ServiceMonitor relabel; node-exporter carries `instance` and
+# reports /var on the partition, `/dev/nvme1n1p1`.
+rule_files:
+  - ../prometheusrule-disk-wear.yaml
+evaluation_interval: 1m
+tests:
+  # A. Projection. k8s-1 nvme0 and nvme1 wear identically: 89%, 1.5e9 bytes
+  #    per 5m (4.32e11 a day), 9e14 lifetime bytes at 192h. That is 0.04272 % a
+  #    day, 6 % to go, 140 days — under 180. Only nvme0 may fire; nvme1
+  #    carries /var. Neither is at 95%, so the critical alert stays quiet.
+  #
+  #    Sampled at 5m, not 1h: the instant selectors look back 5m, so hourly
+  #    samples leave the series absent for most of each hour and the 6h `for`
+  #    never completes. Not 1m either: a 7d rate over 1m samples exceeds
+  #    promtool's query sample limit.
+  - interval: 5m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '89+0x2320'
+      - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '896544000000000+1500000000x2320'
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '89+0x2320'
+      - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '896544000000000+1500000000x2320'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}'
+        values: '1000000000000+0x2320'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x2320'
+    alert_rule_test:
+      - eval_time: 192h
+        alertname: NvmeDriveWearProjectedExhaustion
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              node: k8s-1
+              device: nvme0
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme0 on 10.0.0.1:9633 projects to 95% wear in 140 days"
+              description: "The drive is past 80% of its rated endurance and the write rate of the last week would spend the rest within six months. This is a lead-time warning, not a fault: decide whether to replace the drive or move what is writing to it. Drives carrying EPHEMERAL are excluded; they run to failure. The Disk Health dashboard shows which drives carry which role."
+      - eval_time: 192h
+        alertname: NvmeDriveWearCritical
+        exp_alerts: []
+
+  # B. Critical, two nodes. Every drive is at 96%. On k8s-1 /var is nvme1; on
+  #    k8s-2 it is nvme2, which reports no SMART here. So k8s-1 nvme1 is the only
+  #    one excluded, and k8s-2 nvme1 must still fire — the exclusion is per
+  #    node, not per device name.
+  - interval: 1m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'smartctl_device_percentage_used{node="k8s-2",device="nvme1",instance="10.0.0.2:9633"}'
+        values: '96+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.2:9100",mountpoint="/var",device="/dev/nvme2n1p1",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.2:9100",nodename="k8s-2"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: NvmeDriveWearCritical
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              node: k8s-1
+              device: nvme0
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme0 on 10.0.0.1:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
+          - exp_labels:
+              severity: critical
+              node: k8s-2
+              device: nvme1
+              instance: 10.0.0.2:9633
+            exp_annotations:
+              summary: "NVMe nvme1 on 10.0.0.2:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
+
+  # C. Fails loud. /var is on a device the join cannot map to an NVMe name (an
+  #    encrypted volume on dm-0). The exclusion then matches nothing and the
+  #    EPHEMERAL drive alerts again, which is the intended direction to fail.
+  - interval: 1m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/dm-0",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: NvmeDriveWearCritical
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              node: k8s-1
+              device: nvme1
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme1 on 10.0.0.1:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
@@ -0,0 +1,46 @@
+# Suite for prometheusrule-gitea.yaml (REN-583). Capacity 60 GiB; free space
+# in GiB. 15 GiB free is exactly 25% and must stay silent.
+rule_files:
+  - ../prometheusrule-gitea.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # 20 GiB free, then 14 GiB free from minute 10 on: pending at 10m, fires
+  # at 25m (15m later), not before.
+  - interval: 1m
+    input_series:
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '21474836480+0x9 15032385536+0x30'
+    alert_rule_test:
+      - eval_time: 24m
+        alertname: GiteaDataVolumeLow
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: GiteaDataVolumeLow
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              namespace: gitea
+              persistentvolumeclaim: data-gitea-0
+            exp_annotations:
+              summary: "Gitea data volume has 23.33% free"
+              description: "data-gitea-0 is under 25% free. When it fills, image pushes fail with HTTP 500 and Gitea cannot write its database. Run the cleanup_packages cron in Gitea's site admin and delete old package versions, or grow the claim in apps/gitea/pvc.yaml (ceph-block resizes online)."
+
+  # Exactly 25% free stays silent; another claim at 1% free is not this alert.
+  - interval: 1m
+    input_series:
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '16106127360+0x40'
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="other",persistentvolumeclaim="data-other-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="other",persistentvolumeclaim="data-other-0"}'
+        values: '644245094+0x40'
+    alert_rule_test:
+      - eval_time: 40m
+        alertname: GiteaDataVolumeLow
+        exp_alerts: []
@@ -141,3 +141,34 @@ ceph-blind-inverted|kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml|s/
 # the critical rule, not only the canary.
 # Detected by: cases C, D and E, which assert the critical rule is silent.
 ceph-critical-any-type|kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml|/^[[:space:]]*absent($/,/^[[:space:]]*)$/!s/{type="replicated"}//
+
+# prometheusrule-prometheus-retention.yaml (REN-107). Detected by the cases in
+# prometheus-retention_test.yaml.
+retention-no-time-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/< 0.9 \* min(prometheus_tsdb_retention_limit_seconds/< 100 * min(prometheus_tsdb_retention_limit_seconds/
+retention-no-size-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0/
+retention-ratio-0.9|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0.9/
+retention-lookback-1d|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/)\[7d:1h\]$/)[1d:1h]/
+retention-cap-at-peak-time|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/\/ max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})$/\/ min_over_time(max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})[7d:1h])/
+
+# prometheusrule-disk-wear.yaml (REN-419): the EPHEMERAL exclusion on the two
+# wear alerts. Detected by the cases in disk-wear_test.yaml.
+#
+# Gate inverted on the projection: the EPHEMERAL drive alone fires.
+# Detected by: case A.
+disk-wear-projection-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearProjectedExhaustion/,/for: 6h/s/unless on (node, device)/and on (node, device)/
+# Gate inverted on the critical rule.
+# Detected by: case B.
+disk-wear-critical-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/and on (node, device)/
+# Matched on device name only: k8s-1's /var on nvme1 then silences nvme1 on
+# every node.
+# Detected by: case B, k8s-2 nvme1.
+disk-wear-critical-device-only|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/unless on (device)/
+# Partition suffix not stripped: /dev/nvme1n1p1 no longer maps to nvme1 and the
+# exclusion matches nothing.
+# Detected by: cases A and B.
+disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|s/n\[0-9\]+(p\[0-9\]+)?"/n[0-9]+"/
+
+# prometheusrule-gitea.yaml (REN-583). Detected by the 24m/25m pair and the
+# exactly-25% case in gitea_test.yaml.
+gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/
+gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/
@@ -0,0 +1,218 @@
+# promtool test rules for prometheusrule-prometheus-retention.yaml (REN-107).
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh — see README.md here. Negative controls are the
+# `retention-*` entries in mutations.txt.
+#
+# WHAT IT IS FOR. PrometheusRetentionNotInForce makes five claims in its
+# comments, and each scenario below asserts one of them:
+#
+#   1. It fires on the state measured on 2026-09-26 (90d/30GiB, 20.5-27.2 day
+#      saw-tooth), across a pod restart and across the post-eviction trough.
+#   2. It goes silent the moment the REN-107 fix (30d/40GiB) is in force,
+#      although the window on disk is still ~21 days and last week's peak was
+#      87% of the OLD cap.
+#   3. It stays silent at 30d steady state, where the store sits above 80% of
+#      the 40GiB cap but the time retention is what binds (clause 1 guards).
+#   4. It stays silent on a freshly provisioned PVC (clause 2 guards).
+#   5. It fires if the size cap binds under the new config.
+#
+# All series are hourly (the rule's subquery step) and evaluated hourly.
+# `lowest_timestamp_seconds` is written as (t - window), so `time() - lowest`
+# reads back the window: `-X+3600xN` holds a constant X-second window while
+# Prometheus deletes old data, a constant value is a window that only grows.
+# Byte figures follow the live pod: WAL ~0.5 GB, head chunks ~0.2 GB.
+rule_files:
+  - ../prometheusrule-prometheus-retention.yaml
+
+evaluation_interval: 1h
+
+tests:
+  # --- 1. Today: 90d / 30GiB, size cap binds ------------------------------
+  # Pod a holds a 26.5-day window at 87.5% of the cap. At t=40h it is
+  # replaced (a chart upgrade) and pod b starts after an eviction: 20.5 GB,
+  # 66% of the cap, same 26.5-day window. The alert must stay up across the
+  # restart and the trough for as long as the peak is inside 7 days, and
+  # clear once it is not — the 7d window is what spans a saw-tooth tooth.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '7776000x39'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '32212254720x39'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-2291239+3600x39'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '27500000000x39'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x39'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x39'
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 7776000x180'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 32212254720x180'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 -2147239+3600x180'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 20500000000x180'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 500000000x180'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x40 200000000x180'
+    alert_rule_test:
+      - eval_time: 0h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: &retention_firing_26d
+          - exp_labels: {severity: warning, namespace: monitoring}
+            exp_annotations:
+              summary: "Prometheus retention setting is not in force — size cap binds first"
+              description: "Prometheus is holding only 26d 12h 27m 19s of history while its configured time retention is longer, and the store has been within 20% of its configured size cap in the last 7 days. The size cap is therefore what is actually in force and the `retention` value in kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured — queries over a longer range return less data than requested, silently and successfully. Confirm the running pair with `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention` to a figure the storage budget can actually hold. Do not read `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets on every restart. Background: REN-107."
+      # After the restart and in the trough: the peak was pod a's.
+      - eval_time: 60h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: *retention_firing_26d
+      - eval_time: 150h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: *retention_firing_26d
+      # Pod a's last sample (39h) is out of the 7d subquery from 208h on.
+      - eval_time: 210h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 2. The fix day: 90d/30GiB -> 30d/40GiB -----------------------------
+  # Pod a is today's state just after an eviction (peak 28.2 GB = 87.5% of
+  # 30GiB until t=79h, then 20.5 GB). The fix rolls out at t=100h; pod b holds
+  # the data it inherited and deletes nothing, so its window grows from 20.8
+  # days and is still under 27 days (90% of 30d) until t~=250h. Clause 1 is
+  # therefore true throughout; the rule must be silent anyway, because the
+  # 28.2 GB peak is 66% of the cap now in force.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '7776000x99'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '32212254720x99'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-1771200+3600x99'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '27500000000x79 20500000000x19'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x99'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x99'
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 2592000x160'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 42949672960x160'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 -1411200x160'
+      # ~1.02 GB/day of ingest, nothing deleted.
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 20500000000+42500000x160'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 500000000x160'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-b"}'
+        values: '_x100 200000000x160'
+    alert_rule_test:
+      # Before the fix it fires (window 20.5 days, peak within 7 days).
+      - eval_time: 99h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts:
+          - exp_labels: {severity: warning, namespace: monitoring}
+            exp_annotations:
+              summary: "Prometheus retention setting is not in force — size cap binds first"
+              description: "Prometheus is holding only 20d 12h 0m 0s of history while its configured time retention is longer, and the store has been within 20% of its configured size cap in the last 7 days. The size cap is therefore what is actually in force and the `retention` value in kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured — queries over a longer range return less data than requested, silently and successfully. Confirm the running pair with `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention` to a figure the storage budget can actually hold. Do not read `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets on every restart. Background: REN-107."
+      - eval_time: 100h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 110h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 170h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 250h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 3. 30d steady state -------------------------------------------------
+  # Time retention binds: the window never drops below 30 days (it runs to
+  # ~31.5 while a block waits for deletion) and the store sits at 38 GB,
+  # 88.5% of 40GiB. Clause 2 is true all the time; clause 1 must keep the
+  # rule silent.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '2592000x200'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '42949672960x200'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-2721600+3600x200'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '37300000000x200'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x200'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x200'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 180h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 4. Fresh PVC --------------------------------------------------------
+  # A new volume holds hours of data against a 30d setting: clause 1 is true
+  # for 27 days. Clause 2 (7.6 GB at t=180h, 18% of the cap) must keep it
+  # silent.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '2592000x200'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '42949672960x200'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '0x200'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '0+42500000x200'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x200'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x200'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+      - eval_time: 180h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts: []
+
+  # --- 5. The size cap binds again under 30d/40GiB ------------------------
+  # Ingest grew: the store is at 38 GB (88.5%) and holds only 25 days.
+  - interval: 1h
+    input_series:
+      - series: 'prometheus_tsdb_retention_limit_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '2592000x10'
+      - series: 'prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '42949672960x10'
+      - series: 'prometheus_tsdb_lowest_timestamp_seconds{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '-2160000+3600x10'
+      - series: 'prometheus_tsdb_storage_blocks_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '37300000000x10'
+      - series: 'prometheus_tsdb_wal_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '500000000x10'
+      - series: 'prometheus_tsdb_head_chunks_storage_size_bytes{job="kube-prometheus-stack-prometheus",pod="prometheus-a"}'
+        values: '200000000x10'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: PrometheusRetentionNotInForce
+        exp_alerts:
+          - exp_labels: {severity: warning, namespace: monitoring}
+            exp_annotations:
+              summary: "Prometheus retention setting is not in force — size cap binds first"
+              description: "Prometheus is holding only 25d 0h 0m 0s of history while its configured time retention is longer, and the store has been within 20% of its configured size cap in the last 7 days. The size cap is therefore what is actually in force and the `retention` value in kubernetes/apps/monitoring/helmrelease-prometheus.yaml is not being honoured — queries over a longer range return less data than requested, silently and successfully. Confirm the running pair with `GET /api/v1/status/runtimeinfo` -> `storageRetention`, then either raise `retentionSize` (and the PVC, which must stay ahead of it) or lower `retention` to a figure the storage budget can actually hold. Do not read `prometheus_tsdb_size_retentions_total` to decide whether this is real — it resets on every restart. Background: REN-107."
@@ -17,7 +17,7 @@ spec:
  chart:
    spec:
      chart: kube-prometheus-stack
-      version: "91.7.0"
+      version: "91.9.0"
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
@@ -35,7 +35,7 @@ spec:
    prometheus:
      # prometheus.renner.uno is served by a standalone, Authentik-gated
      # Ingress (prometheus-gate-ingress.yaml), NOT this Helm-generated one.
-      # Prometheus had no auth of its own — its full 90d metrics DB is an
+      # Prometheus had no auth of its own — its full 30d metrics DB is an
      # infra-recon oracle (hostnames, IPs, service inventory). Gated behind
      # the prometheus-gate proxy outpost, same as alertmanager-gate (#146).
      # The chart Ingress is disabled here — two Ingresses for one host would
@@ -76,8 +76,17 @@ spec:
        # Rewrites the in-cluster URL embedded in alert generatorURLs and
        # status pages so click-through from emails works.
        externalUrl: https://prometheus.renner.uno
-        retention: 90d
-        retentionSize: 30GB
+        # Time retention is the one in force; the size cap is a backstop
+        # (REN-107, owner decision: 30 days on the existing disk). This was
+        # 90d/30GB, and at ~1 GB/day of ingest the 30GiB cap bound first: the
+        # real window was a 20.5-27 day saw-tooth while this line said 90d.
+        # 30d peaks near 38 GB (30d plus up to one 3-day block awaiting
+        # deletion, plus WAL), under 40GiB (42.9 GB), under the 50Gi PVC.
+        # Raising either number means re-doing that arithmetic against the
+        # PVC; PrometheusRetentionNotInForce (monitoring-rules) fires if the
+        # size cap starts binding again.
+        retention: 30d
+        retentionSize: 40GiB
        scrapeInterval: 30s
        evaluationInterval: 30s
        storageSpec:
@@ -10,6 +10,7 @@ resources:
  - loki-basic-auth-externalsecret.yaml
  - loki-basic-auth-middleware.yaml
  - loki-ingress.yaml
+  - loki-read-gateway.yaml
  - alertmanager-gate-service.yaml
  - alertmanager-gate-ingress.yaml
  - prometheus-gate-service.yaml
@@ -18,6 +19,13 @@ resources:
  - smartctl-exporter.yaml
  - intel-gpu-exporter.yaml
  - dashboards/
+configMapGenerator:
+  # Hash suffix kept on purpose: an edit to the .conf rolls the gateway pod,
+  # which is stateless and off the ingest path.
+  - name: loki-read-gateway
+    namespace: monitoring
+    files:
+      - loki-read-gateway.conf
 # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
 # separate Flux Kustomization that dependsOn `monitoring` — otherwise the
 # server-side dry-run of the PrometheusRule fails on a fresh cluster
@@ -0,0 +1,60 @@
+# GET-only allow-list in front of Loki (REN-621, owner decision on REN-622).
+#
+# Loki runs SingleBinary with auth_enabled: false, and its HTTP port mixes
+# reads with control endpoints that also answer GET (/ingester/shutdown,
+# /flush). An RBAC `get` on services/proxy for loki:3100 would therefore be a
+# kill switch for log ingest. This server is the read-only boundary instead:
+# the paperclip read ServiceAccount may proxy to this Service only.
+#
+# Each allowed location forwards a URI built here, never the client's raw
+# request URI, so dot-segments or encoded slashes cannot reach another Loki
+# path. Anything not listed gets 403. /loki/api/v1/tail (websocket) is left
+# out on purpose. Adding a path is a new card and a new review.
+
+upstream loki {
+    server loki.monitoring.svc.cluster.local:3100;
+}
+
+server {
+    listen 8080;
+    server_tokens off;
+
+    proxy_http_version 1.1;
+    proxy_set_header Host loki;
+    proxy_read_timeout 60s;
+
+    location / {
+        return 403;
+    }
+
+    location = /healthz {
+        access_log off;
+        default_type text/plain;
+        return 200 "ok\n";
+    }
+
+    location = /loki/api/v1/query {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/query$is_args$args;
+    }
+
+    location = /loki/api/v1/query_range {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/query_range$is_args$args;
+    }
+
+    location = /loki/api/v1/labels {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/labels$is_args$args;
+    }
+
+    location = /loki/api/v1/series {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/series$is_args$args;
+    }
+
+    location ~ ^/loki/api/v1/label/(?<label_name>[A-Za-z_][A-Za-z0-9_]*)/values$ {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/label/$label_name/values$is_args$args;
+    }
+}
@@ -0,0 +1,97 @@
+---
+# Read-only Loki gateway for the agents in the paperclip pod (REN-621,
+# owner decision on REN-622). The allow-list and the reason it exists are in
+# loki-read-gateway.conf. Callers reach it through the apiserver proxy only:
+#   /api/v1/namespaces/monitoring/services/loki-read-gateway:8080/proxy/loki/api/v1/query_range?...
+# granted by ../paperclip/prometheus-read-rbac.yaml.
+#
+# Stateless and off the ingest path: Alloy pushes straight to the `loki`
+# Service, so restarting or deleting this pod never touches log ingest.
+# Revert: delete this file, the .conf and their kustomization entries, and the
+# resourceName in the paperclip Role.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki-read-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: loki-read-gateway
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: loki-read-gateway
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: loki-read-gateway
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 101
+        runAsGroup: 101
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: nginx
+          image: nginxinc/nginx-unprivileged:1.31-alpine@sha256:26b0bf6fbf07297983cb341998d79c831508787de26627dd2a112321b9c3a4af
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            periodSeconds: 30
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+          volumeMounts:
+            # Replaces the image's default.conf; the entrypoint's ipv6 and
+            # envsubst steps skip a read-only conf.d and the server starts.
+            - name: config
+              mountPath: /etc/nginx/conf.d
+              readOnly: true
+            # pid file and proxy temp paths of the unprivileged image.
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: loki-read-gateway
+        - name: tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: 32Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki-read-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: loki-read-gateway
+spec:
+  selector:
+    app.kubernetes.io/name: loki-read-gateway
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+      protocol: TCP
@@ -55,7 +55,7 @@ spec:
        # stdlib only; Flux keeps the tag in lockstep via the shared image
        # policy) — no extra dependency.
        - name: reconcile-publish-ip
-          image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.5 # {"$imagepolicy": "flux-system:music-assistant"}
          command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
          volumeMounts:
            - name: data
@@ -65,7 +65,7 @@ spec:
              readOnly: true
      containers:
        - name: music-assistant
-          image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.5 # {"$imagepolicy": "flux-system:music-assistant"}
          env:
            - name: LOG_LEVEL
              value: info
--- a/Show More
+++ b/Show More