fix: [alert] KubePersistentVolumeFillingUp (paperclip) (#1184 )

chore(image-automation): bump gitea.renner.uno/k.renner/shelly-manager:build.1791385485
Updates managed by image-automation-controller.
2026-10-07 15:42:50 +00:00 · 2026-10-07 15:21:31 +00:00 · 2026-10-07 15:19:42 +00:00 · 2026-10-07 15:04:18 +00:00 · 2026-10-07 14:51:31 +00:00 · 2026-10-07 13:21:31 +00:00
157 changed files with 5654 additions and 436 deletions
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: build-agent-runtime
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images
    # Context is ci/images (not the variant dir) so both Containerfiles can
    # COPY the shared agent-runtime-common/ files.
@@ -43,7 +43,7 @@ when:

 steps:
  - name: build-ansible
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/ansible
    depends_on: []
    # Only rebuild when THIS image's source (or the pipeline file) changed, or
@@ -71,7 +71,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest

  - name: build-k8s
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/k8s
    depends_on: []
    when:
@@ -95,7 +95,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-vmtest
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/vmtest
    depends_on: []
    when:
@@ -119,7 +119,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-vmtest:latest

  - name: build-tofu
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/tofu
    depends_on: []
    when:
@@ -150,7 +150,7 @@ steps:
  # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
  # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
  - name: build-secret-rotator
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: secret-rotator
    depends_on: []
    when:
@@ -201,7 +201,7 @@ steps:
  #
  # The gitea_admin/ copy stays until nothing references it any more.
  - name: build-alpine-jq
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/alpine-jq
    depends_on: []
    when:
@@ -254,7 +254,7 @@ steps:
  # note on build-alpine-jq. Anything an app needs while the cluster is still
  # being built cannot live only in that cluster's own registry.
  - name: build-cni-plugins
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/cni-plugins
    depends_on: []
    when:
@@ -296,7 +296,7 @@ steps:
  # gitea.renner.uno is not there to serve it. A cold-rebuild rehearsal left
  # paperless in Init:ErrImagePull on exactly this image.
  - name: build-alpine-openssl
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/alpine-openssl
    depends_on: []
    when:
@@ -333,7 +333,7 @@ steps:
  # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
  # builds the same way and lands in the same registry so the audit sees it.
  - name: build-heating-can
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/heating-can
    depends_on: []
    when:
@@ -360,7 +360,7 @@ steps:
  # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
  # Runtime artifact like heating-can; same registry so the audit tracks it.
  - name: build-netbird-service-sync
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/netbird-service-sync
    depends_on: []
    when:
@@ -386,7 +386,7 @@ steps:
  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
  # artifact like heating-can; same registry so the audit tracks it.
  - name: build-bird
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/bird
    depends_on: []
    when:
@@ -0,0 +1,69 @@
+# Paperclip server image (REN-401): upstream paperclipai/paperclip at a pinned
+# master commit plus the patches in ci/images/paperclip/. Needed while the k8s
+# sandbox fix (upstream PR #13733) is not merged; see the README there for how
+# to go back to upstream nightly.
+#
+# Own workflow for the reason given in build-agent-runtime.yml: a step in
+# build-ci-images.yml runs on every change to that file and competes for the
+# woodpecker-ci-quota. Re-run alone with a deployment to target
+# `build-paperclip`.
+#
+# Tag `build.<unix-ts>` is what the Flux ImagePolicy `paperclip` sorts on
+# (image-automation/paperclip.yaml); image automation then rolls it out.
+# konairius/ and ci_images_token as for the agent-runtime images.
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/paperclip/**"
+      - ".woodpecker/build-paperclip.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-paperclip"'
+
+steps:
+  - name: prepare-source
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh ci/images/paperclip/prepare.sh build/paperclip-src
+
+  - name: build-paperclip
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    directory: build/paperclip-src
+    depends_on: [prepare-source]
+    # Rust runner + UI + server build in one pod. The server build alone sets
+    # a 4 GiB Node heap, so the memory limit is the LimitRange max (8Gi).
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: "1"
+            memory: 2Gi
+          limits:
+            cpu: "4"
+            memory: 8Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - . ../../ci/images/paperclip/upstream.env
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=build.$(date +%s)
+      # Same target and build args as upstream's docker.yml for nightly.
+      # --compressed-caching=false: pipeline 11717 was killed at the 8Gi limit
+      # (exit 137) in the server build; kaniko otherwise holds compressed
+      # layer tarballs in memory next to the 4 GiB Node heap.
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Dockerfile.kaniko \
+          --target production \
+          --skip-unused-stages \
+          --snapshot-mode=redo \
+          --compressed-caching=false \
+          --build-arg PAPERCLIP_BUILD_COMMIT=$PAPERCLIP_COMMIT \
+          --build-arg PAPERCLIP_BUILD_VERSION=$PAPERCLIP_VERSION \
+          --build-arg CLI_TOOLS_CACHE_EPOCH=$(date -u +%G-W%V) \
+          --destination gitea.renner.uno/konairius/paperclip:$TAG
@@ -131,7 +131,7 @@ steps:
  #    --build-arg to the resolved version (not "latest") guarantees the tag and
  #    the image content agree even if a new release lands mid-run.
  - name: build
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/claude
    depends_on: [decide]
    environment:
@@ -172,7 +172,7 @@ steps:
  # whenever it trails the deployed server. If you bump the server, bump here.
  - name: woodpecker-lint
    depends_on: [] # see note above `steps:`
-    image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
+    image: woodpeckerci/woodpecker-cli:v3.18.1-alpine@sha256:7eac3c4254befc1ba1e65269c3304c370b6032530b7386fc0c7648378fdd0516
    commands:
      - |
        set -eu
@@ -199,7 +199,7 @@ steps:
  # separately rather than one this step quietly implies is covered.
  - name: shellcheck
    depends_on: [] # see note above `steps:`
-    image: koalaman/shellcheck-alpine:v0.11.0
+    image: koalaman/shellcheck-alpine:v0.11.0@sha256:9955be09ea7f0dbf7ae942ac1f2094355bb30d96fffba0ec09f5432207544002
    commands:
      - |
        set -eu
@@ -319,7 +319,7 @@ steps:
  # writes its stripped mirror under mktemp and never into the workspace.
  - name: promtool
    depends_on: [] # see note above `steps:`
-    image: prom/prometheus:v3.15.0
+    image: prom/prometheus:v3.15.0@sha256:efd719c99d83b060d9daefdcf00360461adf279f45ef5391f8d111892118753e
    commands:
      - sh scripts/ci/promtool-rules.sh

@@ -35,18 +35,18 @@ when:
 steps:
  - name: model-selection-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh

  - name: promtool-rules-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-promtool-rules.sh

  - name: yaml-citations-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-yaml-citations.sh
@@ -13,7 +13,7 @@ when:

 steps:
  - name: validate-secrets
-    image: alpine:3.24.1
+    image: alpine:3.24.2@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
    environment:
      TF_VAR_proxmox_api_url:
        from_secret: proxmox_api_url
@@ -26,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -18,12 +18,12 @@ when:

 steps:
  - name: iac-scan
-    image: aquasec/trivy:latest
+    image: aquasec/trivy:latest@sha256:af6acf9a6b85dfe389a1941505c0ce9efef52a4719635e1a962f022a3d855daa
    commands:
      - trivy fs --format json -o trivy-iac.json --severity CRITICAL,HIGH --ignore-unfixed .

  - name: image-scans
-    image: aquasec/trivy:latest
+    image: aquasec/trivy:latest@sha256:af6acf9a6b85dfe389a1941505c0ce9efef52a4719635e1a962f022a3d855daa
    commands:
      - trivy image --download-db-only
      - |
@@ -70,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -17,7 +17,7 @@ when:

 steps:
  - name: checkout-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -54,7 +54,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: distill
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -89,14 +89,14 @@ steps:
          -p

  - name: audit
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    commands:
      # Findings are report lines in output/, not a pipeline failure.
      - python3 -B notes/bin/wiki_audit.py --vault "$$PWD/notes"

  - name: consolidate
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -145,7 +145,7 @@ steps:
        exit $$rc

  - name: push-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.283
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -62,7 +62,7 @@ together with Paperclip. All tools are on the default `PATH`.

 | Image | `imageRegistry` | Adds |
 |---|---|---|
-| `gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-go` | Go toolchain, golangci-lint |
+| `gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-go` | Go toolchain, golangci-lint, govulncheck |
 | `gitea.renner.uno/konairius/agent-runtime-ops/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-ops` | kubectl, flux, kustomize, helm |

 The plugin keeps the name `agent-runtime-claude:v1` and swaps only host and
@@ -7,6 +7,13 @@
 #
 # Both binaries are static; they run on the Ubuntu 22.04 base unchanged.
 FROM golang:1.27.1-bookworm@sha256:69a7b9788769bec032d238959b61854e9ae87f57be9029ec04e9885fabf99195 AS go
+
+# govulncheck (REN-488) has no upstream image: build it in the go stage, static.
+# The version pins the module; go verifies it against sum.golang.org. Renovate
+# does not track it (no regex manager here): bump by hand.
+FROM go AS govulncheck
+RUN CGO_ENABLED=0 GOBIN=/out go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
+
 FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS golangci-lint

 # upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
@@ -15,6 +22,7 @@ FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518

 COPY --from=go /usr/local/go /usr/local/go
 COPY --from=golangci-lint /usr/bin/golangci-lint /usr/local/bin/golangci-lint
+COPY --from=govulncheck /out/govulncheck /usr/local/bin/govulncheck
 # go and gofmt also in /usr/local/bin: a process started with the default PATH
 # (not the image ENV) still finds them.
 USER root
@@ -57,4 +65,4 @@ ENV PATH=/usr/local/go/bin:$PATH \
 # The PATH check uses the base image's default PATH, not the ENV above.
 RUN go version && golangci-lint version \
    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
-       sh -ec 'for t in go gofmt golangci-lint claude-agent-acp; do command -v "$t"; done'
+       sh -ec 'for t in go gofmt golangci-lint govulncheck claude-agent-acp; do command -v "$t"; done'
@@ -7,8 +7,8 @@
 #
 # kubectl and flux match the ci-k8s image. All four binaries are static; they
 # run on the Ubuntu 22.04 base unchanged.
-FROM registry.k8s.io/kubectl:v1.36.5@sha256:0667e3f3e6ef435d7cd4c16684244703ebd67314c17511c2a9f87ea44f6cdf6b AS kubectl
-FROM ghcr.io/fluxcd/flux-cli:v2.9.5@sha256:704d5529535570a495249b643159c93a9aa88ffca07813c974a54b7ba6edbd86 AS flux
+FROM registry.k8s.io/kubectl:v1.37.1@sha256:b7cab618e281b1ee7484e7b706a96e2135fbb6e072c2a573a7dab4e87d7f2385 AS kubectl
+FROM ghcr.io/fluxcd/flux-cli:v2.9.6@sha256:b1ac18156f227af9a524b842a96f2c20986c7a779ef721df3ba4e4540f449d76 AS flux
 FROM registry.k8s.io/kustomize/kustomize:v5.8.1@sha256:899fcd3bc898160e62bcaf82932b0cb29ba38d16272353db2e7acbba82129429 AS kustomize
 FROM alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2 AS helm

@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine plus jq. Nothing application-specific — a general-purpose base for any
 # workload that needs to read JSON from a shell script.
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine plus the openssl CLI. Nothing application-specific — named for what it
 # is, so anything needing `openssl s_client` / `x509` from a shell script can use
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 RUN apk add --no-cache \
      ansible \
@@ -1,4 +1,4 @@
-FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
 # step — the image is the binary plus its runtime.
@@ -28,10 +28,10 @@
 # amd64 only, matching the ARCH the installer already hardcoded and the cluster's
 # node architecture. Add a second stage if a different arch is ever needed.

-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d AS fetch
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS fetch

-ARG CNI_PLUGINS_VERSION=v1.5.1
-ARG CNI_PLUGINS_SHA256=77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b
+ARG CNI_PLUGINS_VERSION=v1.9.1
+ARG CNI_PLUGINS_SHA256=b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303

 # busybox wget and tar are already in the base; nothing is installed here
 # either. sha256sum -c fails the BUILD if the artifact does not match, which is
@@ -61,7 +61,7 @@ RUN set -eu; \
      [ -x "/plugins/$p" ] || { echo "FATAL: /plugins/$p missing or not executable" >&2; exit 1; }; \
    done

-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine rather than scratch: the installer needs a shell to copy the binaries
 # onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
@@ -23,7 +23,7 @@
 # namespace, so can0 is only reachable with network_mode: host. That is also
 # how the local MQTT broker on 127.0.0.1:1883 is reached. CAN_RAW sockets need
 # no capabilities, so both run unprivileged.
-FROM python:3.13-slim AS build
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151 AS build

 # Pins live here, in the artifact, rather than in Ansible variables.
 #
@@ -56,7 +56,7 @@ RUN git clone --quiet https://github.com/MyHomeMyData/E3onCAN.git /opt/e3oncan \
 && rm -rf /opt/e3oncan/.git


-FROM python:3.13-slim
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151

 ARG OPEN3E_REF=6553c10453a6bae1210908f38a5af2481a46a3e5
 ARG E3ONCAN_REF=461dcbdf97e8d72d2edb9929c63b39382af6c3a1
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 ARG TALOS_VERSION=v1.13.10
 ARG VELERO_VERSION=v1.16.0
@@ -2,7 +2,7 @@
 # domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
 # public label. Standard library only, so the image is the interpreter plus one
 # file; runs unprivileged as its own user.
-FROM python:3.13-slim
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151

 RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
 COPY controller.py /opt/controller.py
@@ -0,0 +1,87 @@
+# Paperclip server image (REN-401)
+
+Upstream `paperclipai/paperclip` at the commit in `upstream.env`, plus every
+`*.patch` in this directory. Built by `.woodpecker/build-paperclip.yml`,
+pushed as `gitea.renner.uno/konairius/paperclip:build.<unix-ts>`, rolled out
+by Flux image automation (`kubernetes/infrastructure/image-automation/paperclip.yaml`).
+
+## Why
+
+k8s sandbox runs fail: the run path never sets the lease `remoteCwd`, so file
+sync into the pod fails (paperclipai/paperclip#13864). The fix is
+paperclipai/paperclip#13733, open and not merged.
+`pr-13733-lease-remote-cwd.patch` is that PR's diff (head `5f320bf6`).
+
+#13733 covers only the run path. Adapter environment tests (binding an agent
+to a k8s environment) sync managed config before any workspace is realized,
+so the lease still has no `remoteCwd` and the Claude test fails with
+`claude_managed_config_dir_failed` (REN-409). The fix is
+paperclipai/paperclip#14769, also open: the plugin sets `remoteCwd:
+/workspace` when it acquires or resumes a lease.
+`pr-14769-probe-remote-cwd.patch` is that PR's diff (head `10dc7371`).
+
+Claude auto memory is lost after every run (REN-541): it lives under
+`CLAUDE_CONFIG_DIR`, which Paperclip makes fresh per run and deletes.
+`ren-541-claude-auto-memory.patch` makes the ACPX engine write
+`autoMemoryDirectory: <AGENT_HOME>/memory` into the per-run
+`.claude/settings.local.json`. `AGENT_HOME` is restored before and saved after
+every run, local and in k8s pods. The value is not part of the session
+fingerprint, so resume is unchanged. No upstream PR yet; rename the patch to
+`pr-<n>-…` once there is one.
+
+Claude sessions never resume with a managed AI connection (REN-563): the
+server gives every run a fresh temp home (`HOME`, `XDG_*`,
+`CLAUDE_CONFIG_DIR`, …) and deletes it after the run.
+`ren-563-acpx-claude-session-resume.patch` fixes both halves for local runs:
+the ACPX engine hashes a fixed placeholder for those paths in the session
+fingerprint (`adapterEnvHash`), as the server's own fingerprint already does,
+and links `CLAUDE_CONFIG_DIR/projects` (the Claude transcripts) to
+`<stateDir>/claude-projects`, which persists. Both must ship together: a
+stable fingerprint without the transcript makes the resumed turn fail with
+"Resource not found" (homelab#1111, reverted in #1113). Remote runs are
+unchanged. No upstream PR yet.
+
+A cancelled or failed k8s pod run blocks its card for good (REN-841). The
+owner's next comment waits on `remote_cleanup`, because the server continues
+only when every remote lease carries a provider termination receipt
+(`explicit-native-continuation.ts`), and the Kubernetes plugin's
+`onEnvironmentReleaseLease` / `onEnvironmentDestroyLease` returned `void`.
+`ren-841-k8s-termination-receipt.patch` makes both return
+`{ providerLeaseId, state: "destroyed" }`, but only after the Sandbox CR or
+Job (and, on destroy, the pod) reads 404, polling for up to 20 s, below the
+server's 30 s plugin RPC timeout. If that check times out or fails, the hook
+returns no receipt, as before. No upstream PR yet.
+
+## Files
+
+- `upstream.env` — pinned commit and the version string stamped into the
+  image. Bump both together.
+- `prepare.sh` — fetches the commit, applies the patches, writes
+  `Dockerfile.kaniko` (upstream's `COPY --parents` line split into plain
+  `COPY` lines; kaniko does not support `--parents`), and adds the build of
+  the Kubernetes sandbox provider plugin to the `build` stage (REN-405):
+  upstream's `production` target ships only its source, so the plugin fails
+  with "no longer exposes a Paperclip manifest". The build fails if
+  `dist/manifest.js` is missing.
+- `*.patch` — applied in name order with `git apply`.
+
+## Moving to a newer upstream commit
+
+Set `PAPERCLIP_COMMIT` to the new master commit and `PAPERCLIP_VERSION` to
+`git describe --tags --match 'v*' --long <commit>`, then run
+`sh prepare.sh /tmp/src`: it fails if a patch no longer applies.
+
+## Exit: when every patch here is merged upstream
+
+Renovate cannot watch an upstream PR, so this is a manual step. Once
+upstream nightly contains all of them:
+
+1. `kubernetes/infrastructure/image-automation/paperclip.yaml`: back to
+   `ghcr.io/paperclipai/paperclip` with `filterTags.pattern: '^nightly$'`
+   (the version before REN-401 in git history).
+2. `renovate.json`: swap `gitea.renner.uno/konairius/paperclip` back to
+   `ghcr.io/paperclipai/paperclip` in the image-automation exclusion list.
+3. Delete this directory and `.woodpecker/build-paperclip.yml`.
+
+Image automation then rewrites `kubernetes/apps/paperclip/paperclip.yaml`
+to the nightly digest by itself.
@@ -0,0 +1,93 @@
+diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts
+index f3d1fc659..812cae357 100644
+--- a/server/src/__tests__/environment-run-orchestrator.test.ts
+++ b/server/src/__tests__/environment-run-orchestrator.test.ts
+@@ -430,6 +430,51 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
+     expect(result.persistedExecutionWorkspace).toEqual(updatedEw);
+   });
+ 
+  it("persists the realized cwd as remoteCwd when realization returns no workspaceRealization payload", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({ environment: makeEnvironment("sandbox") }),
+    );
+
+    expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(
+      "lease-1",
+      expect.objectContaining({ remoteCwd: "/workspace" }),
+    );
+    // Nothing to record about the realization shape, so the execution workspace
+    // row is left untouched.
+    expect(mockUpdateExecutionWorkspace).not.toHaveBeenCalled();
+  });
+
+  it("does not rewrite lease metadata when the realized cwd already matches the lease", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({
+        environment: makeEnvironment("sandbox"),
+        lease: makeLease({ metadata: { remoteCwd: "/workspace" } }),
+      }),
+    );
+
+    expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled();
+  });
+
+   it("runs a remote provision command after workspace realization when configured", async () => {
+     mockBuildWorkspaceRealizationRequest.mockReturnValue({
+       version: 1,
+diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts
+index 97cca667e..e76f64913 100644
+--- a/server/src/services/environment-run-orchestrator.ts
+++ b/server/src/services/environment-run-orchestrator.ts
+@@ -485,16 +485,29 @@ export function environmentRunOrchestrator(
+     }
+ 
+     // Step 3: Persist realization metadata on lease and execution workspace
+-    if (Object.keys(workspaceRealization).length > 0) {
+    const hasWorkspaceRealization = Object.keys(workspaceRealization).length > 0;
+    // A driver that realizes the workspace reports the directory it prepared
+    // through the realization `cwd`. Persist it on the lease as `remoteCwd`, so
+    // every later step that receives only the lease — native file sync, a
+    // resumed lease, a second run against the same lease — confines itself to
+    // the same root. Without it a driver whose realization returns no
+    // `workspaceRealization` payload leaves the lease without a root at all,
+    // and the sync step fails even though realization succeeded.
+    const currentRemoteCwd =
+      typeof lease.metadata?.remoteCwd === "string" ? lease.metadata.remoteCwd.trim() : "";
+    const realizedRemoteCwd =
+      realizedWorkspaceCwd && realizedWorkspaceCwd !== currentRemoteCwd ? realizedWorkspaceCwd : null;
+    if (hasWorkspaceRealization || realizedRemoteCwd) {
+       const nextLeaseMetadata = {
+         ...(lease.metadata ?? {}),
+-        workspaceRealization,
+        ...(hasWorkspaceRealization ? { workspaceRealization } : {}),
+        ...(realizedRemoteCwd ? { remoteCwd: realizedRemoteCwd } : {}),
+       };
+       const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, nextLeaseMetadata);
+       if (updatedLease) {
+         lease = updatedLease;
+       }
+-      if (persistedExecutionWorkspace) {
+      if (hasWorkspaceRealization && persistedExecutionWorkspace) {
+         const updatedEw = await executionWorkspacesSvc.update(persistedExecutionWorkspace.id, {
+           metadata: {
+             ...(persistedExecutionWorkspace.metadata ?? {}),
@@ -0,0 +1,86 @@
+diff --git a/packages/plugins/sandbox-providers/kubernetes/README.md b/packages/plugins/sandbox-providers/kubernetes/README.md
+index a51c6ecee60e..72618cb11296 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/README.md
+++ b/packages/plugins/sandbox-providers/kubernetes/README.md
+@@ -140,6 +140,8 @@ Every agent pod is:
+ 
+ Plus per-namespace `pod-security.kubernetes.io/enforce: restricted` and a deny-all NetworkPolicy baseline with explicit egress allow-list (DNS, paperclip-server, configured FQDNs/CIDRs).
+ 
+The lease starts with `/workspace` as the native file-sync confinement root. Adapter environment tests can upload managed config before the server realizes a workspace. When resuming a lease, the provider keeps a previously recorded `remoteCwd` so a custom realized workspace is not replaced with the default.
+
+ The per-run Secret carrying the bootstrap token and adapter API keys has `ownerReferences` pointing at the owning Job, so a single `kubectl delete job …` cascades cleanly to the Pod and Secret.
+ 
+ ## Optional Kata-FC microVM isolation
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+index 8ae47de8ecac..98016385da05 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+@@ -454,6 +454,9 @@ const plugin = definePlugin({
+       secretName,
+       phase: "Pending",
+       backend: config.backend,
+      // Adapter environment probes sync managed assets before workspace
+      // realization. Give native sync the pod's mounted confinement root now.
+      remoteCwd: "/workspace",
+       scopedNetworkPolicyName,
+       scopedNetworkEgress,
+       // Native file sync streams over a pod exec; only the sandbox-cr backend
+@@ -529,6 +532,10 @@ const plugin = definePlugin({
+       secretName,
+       phase: check.phase,
+       backend: leaseBackend,
+      remoteCwd:
+        typeof params.leaseMetadata?.remoteCwd === "string" && params.leaseMetadata.remoteCwd.trim()
+          ? params.leaseMetadata.remoteCwd.trim()
+          : "/workspace",
+       scopedNetworkPolicyName:
+         typeof params.leaseMetadata?.scopedNetworkPolicyName === "string"
+           ? params.leaseMetadata.scopedNetworkPolicyName
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/types.ts b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+index 5626daa93846..ef3d0d889a4c 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+@@ -90,6 +90,8 @@ export interface KubernetesLeaseMetadata {
+   phase: "Pending" | "Running" | "Succeeded" | "Failed";
+   /** Which backend provisioned this lease. */
+   backend: "sandbox-cr" | "job";
+  /** Native sync confinement root; present before workspace realization for probes. */
+  remoteCwd: string;
+   scopedNetworkPolicyName: string | null;
+   scopedNetworkEgress: {
+     allowFqdns: string[];
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+index 7ad0ca4807e5..4c23410d4d15 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+@@ -80,6 +80,7 @@ describe("onEnvironmentResumeLease", () => {
+         secretName: "pc-abc-env",
+         phase: "Running",
+         backend: "sandbox-cr",
+        remoteCwd: "/workspace",
+         resumedLease: true,
+         // sandbox-cr has a pod-exec channel, so native file sync stays enabled.
+         nativeFileSyncUnsupported: false,
+@@ -87,6 +88,22 @@ describe("onEnvironmentResumeLease", () => {
+     );
+   });
+ 
+  it("keeps a custom realized cwd when resuming a sandbox lease", async () => {
+    h.clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockResolvedValue(readySandboxCr("pc-abc-pod")) },
+      core: { readNamespacedPod: vi.fn().mockResolvedValue({ status: { phase: "Running" } }) },
+    };
+    const lease = await plugin.definition.onEnvironmentResumeLease!({
+      driverKey: "kubernetes",
+      companyId: "acme",
+      environmentId: "env-1",
+      config: CONFIG,
+      providerLeaseId: "pc-abc",
+      leaseMetadata: leaseMetadata({ remoteCwd: "/workspace/custom" }),
+    });
+    expect(lease.metadata?.remoteCwd).toBe("/workspace/custom");
+  });
+
+   it("flags a resumed job-backend lease as native-sync-unsupported so the server keeps the base64 fallback", async () => {
+     h.clients = {
+       batch: {
@@ -0,0 +1,59 @@
+#!/bin/sh
+# Fetch upstream Paperclip at the pinned commit, apply our patches and make
+# the Dockerfile buildable with kaniko. Usage: prepare.sh <target-dir>
+set -eu
+HERE=$(cd "$(dirname "$0")" && pwd)
+. "$HERE/upstream.env"
+OUT=$1
+
+rm -rf "$OUT"
+git init -q "$OUT"
+cd "$OUT"
+git fetch -q --depth 1 https://github.com/paperclipai/paperclip.git "$PAPERCLIP_COMMIT"
+git -c advice.detachedHead=false checkout -q FETCH_HEAD
+test "$(git rev-parse HEAD)" = "$PAPERCLIP_COMMIT"
+
+for p in "$HERE"/*.patch; do
+  echo "applying $(basename "$p")"
+  git apply "$p"
+done
+
+# kaniko does not know `COPY --parents` (BuildKit only). Upstream uses it for
+# one line: every sandbox provider's package.json. Replace it with one plain
+# COPY per provider directory, which copies the same files to the same place.
+grep -c -- '^COPY --parents packages/plugins/sandbox-providers/\./\*/package.json packages/plugins/sandbox-providers/$' Dockerfile | grep -qx 1
+awk '
+  /^COPY --parents packages\/plugins\/sandbox-providers\// {
+    while (("ls -d packages/plugins/sandbox-providers/*/package.json" | getline f) > 0) {
+      d = f; sub(/package.json$/, "", d)
+      print "COPY " f " " d
+    }
+    next
+  }
+  { print }
+' Dockerfile > Dockerfile.kaniko
+if grep -q -- '--parents' Dockerfile.kaniko; then
+  echo "Dockerfile.kaniko still uses COPY --parents" >&2
+  exit 1
+fi
+
+# Build the Kubernetes sandbox provider into the image (REN-405). Upstream's
+# `production` target ships only its source; the plugin loader needs
+# dist/manifest.js. Same commands as upstream's `cloud-plugins` stage, run in
+# the `build` stage right after the server build, so production's
+# `COPY --from=build /app /app` carries dist/ and node_modules along. No extra
+# stage: kaniko would snapshot the whole build stage again for it.
+export ANCHOR='RUN test -f server/dist/index.js || (echo "ERROR: server build output missing" && exit 1)'
+grep -c -F -x -- "$ANCHOR" Dockerfile.kaniko | grep -qx 1
+K=packages/plugins/sandbox-providers/kubernetes
+awk -v k="$K" '
+  { print }
+  $0 == ENVIRON["ANCHOR"] {
+    print "RUN pnpm -C " k " install --ignore-workspace --no-lockfile \\"
+    print "  && pnpm -C " k " build \\"
+    print "  && test -f " k "/dist/manifest.js"
+  }
+' Dockerfile.kaniko > Dockerfile.kaniko.new
+mv Dockerfile.kaniko.new Dockerfile.kaniko
+grep -q -- "test -f $K/dist/manifest.js" Dockerfile.kaniko
+rm -rf .git
@@ -0,0 +1,118 @@
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+index a26f980..112e2eb 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.test.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+@@ -2544,6 +2544,37 @@ describe("shared ACPX engine runtime behavior", () => {
+     expect(note).toBeTruthy();
+   });
+ 
+  it("points Claude auto memory at the run's AGENT_HOME without changing the session fingerprint", async () => {
+    const root = await makeTempRoot();
+    const stateDir = path.join(root, "state");
+    const cwd = path.join(root, "worktree");
+    await fs.mkdir(cwd, { recursive: true });
+    const stableHome = path.join(root, "agent-home");
+    const runAgentHome = (runId: string) => path.join(root, "file-sync", "runs", runId, "live");
+    const readSettings = async () =>
+      JSON.parse(await fs.readFile(path.join(cwd, ".claude", "settings.local.json"), "utf8")) as {
+        autoMemoryDirectory?: string;
+        permissions?: { additionalDirectories?: string[] };
+      };
+
+    const first = await runExecutor(
+      { agent: "claude", stateDir, cwd },
+      { context: { paperclipWorkspace: { cwd, agentHome: runAgentHome("a"), agentHomeForPermissions: stableHome } } },
+    );
+    expect((await readSettings()).autoMemoryDirectory).toBe(path.join(runAgentHome("a"), "memory"));
+
+    const second = await runExecutor(
+      { agent: "claude", stateDir, cwd },
+      { context: { paperclipWorkspace: { cwd, agentHome: runAgentHome("b"), agentHomeForPermissions: stableHome } } },
+    );
+    const written = await readSettings();
+    expect(written.autoMemoryDirectory).toBe(path.join(runAgentHome("b"), "memory"));
+    expect(written.permissions?.additionalDirectories).toContain(stableHome);
+    expect(written.permissions?.additionalDirectories).not.toContain(runAgentHome("b"));
+    expect(second.result.sessionParams?.configFingerprint).toBeTypeOf("string");
+    expect(second.result.sessionParams?.configFingerprint).toBe(first.result.sessionParams?.configFingerprint);
+  });
+
+   it("merges Paperclip allowlist into an existing .claude/settings.local.json without losing user entries", async () => {
+     const root = await makeTempRoot();
+     const stateDir = path.join(root, "state");
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts
+index b1dc569..e5681b8 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.ts
+@@ -1558,6 +1558,7 @@ interface PaperclipClaudeSettingsResult {
+   additionalDirectories: string[];
+   defaultMode: string;
+   overrodeDontAsk: boolean;
+  autoMemoryDirectory: string | null;
+ }
+ 
+ function uniqueSorted(values: Array<string | null | undefined>): string[] {
+@@ -1571,10 +1572,19 @@ function uniqueSorted(values: Array<string | null | undefined>): string[] {
+ // denies every non-allowlisted tool and never reaches `canUseTool`), and we
+ // widen the SDK's Read sandbox to include the Paperclip state dirs the agent
+ // needs to talk to its own control plane.
+//
+// It also points Claude's auto memory at `<AGENT_HOME>/memory`. The default
+// location lives under CLAUDE_CONFIG_DIR, which is a per-run temp home, so
+// memories were lost after every run. AGENT_HOME is restored before and saved
+// after each run, locally and in remote sandboxes. Claude ignores
+// `autoMemoryDirectory` only in checked-in project settings, not local ones.
+ async function writePaperclipClaudeSettings(input: {
+   cwd: string;
+   stateDir: string;
+   agentHome: string;
+  // The run's actual AGENT_HOME (the in-sandbox path for remote runs). It may
+  // differ from `agentHome`, which stays stable for session identity.
+  runAgentHome: string;
+   companyId: string;
+ }): Promise<PaperclipClaudeSettingsResult> {
+   const filePath = path.join(input.cwd, ".claude", "settings.local.json");
+@@ -1630,7 +1640,14 @@ async function writePaperclipClaudeSettings(input: {
+     additionalDirectories: mergedAdditionalDirectories,
+     defaultMode,
+   };
+-  const next: Record<string, unknown> = { ...existing, permissions: nextPermissions };
+  const autoMemoryDirectory = input.runAgentHome
+    ? path.posix.join(input.runAgentHome, "memory")
+    : null;
+  const next: Record<string, unknown> = {
+    ...existing,
+    permissions: nextPermissions,
+    ...(autoMemoryDirectory ? { autoMemoryDirectory } : {}),
+  };
+   await writeFileAtomically({
+     target: filePath,
+     contents: `${JSON.stringify(next, null, 2)}\n`,
+@@ -1642,6 +1659,7 @@ async function writePaperclipClaudeSettings(input: {
+     additionalDirectories: mergedAdditionalDirectories,
+     defaultMode,
+     overrodeDontAsk,
+    autoMemoryDirectory,
+   };
+ }
+ 
+@@ -2064,12 +2082,18 @@ async function buildRuntime(input: {
+       // by the company root already granted in these settings; the process env
+       // still receives this run's actual AGENT_HOME.
+       agentHome: asString(workspaceContext.agentHomeForPermissions, agentHome),
+      // Auto memory follows the actual AGENT_HOME so it is saved with the run.
+      // It is not part of the session fingerprint: a moved memory directory
+      // must not invalidate a resumable conversation.
+      runAgentHome: agentHome,
+       companyId: agent.companyId,
+     });
+     skillCommandNotes.push(
+       `Wrote Paperclip-managed Claude settings to ${paperclipClaudeSettings.filePath} (defaultMode=${paperclipClaudeSettings.defaultMode}${
+         paperclipClaudeSettings.overrodeDontAsk ? "; overrode user dontAsk" : ""
+-      }, +${paperclipClaudeSettings.additionalDirectories.length} read root(s), +${paperclipClaudeSettings.allow.length} allow rule(s)).`,
+      }, +${paperclipClaudeSettings.additionalDirectories.length} read root(s), +${paperclipClaudeSettings.allow.length} allow rule(s)${
+        paperclipClaudeSettings.autoMemoryDirectory ? `, autoMemoryDirectory=${paperclipClaudeSettings.autoMemoryDirectory}` : ""
+      }).`,
+     );
+   } else if (acpxAgent === "codex") {
+     // Step 2 — codex-home.seed: the codex managed-home + skills preparation.
@@ -0,0 +1,128 @@
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+index 112e2eb..004baf0 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.test.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+@@ -1857,6 +1857,45 @@ describe("shared ACPX engine runtime behavior", () => {
+     expect(changed.result.sessionParams?.configFingerprint).not.toBe(first.result.sessionParams?.configFingerprint);
+   });
+ 
+  it("keeps the per-run managed AI home out of session identity and keeps Claude transcripts", async () => {
+    const root = await makeTempRoot();
+    const stateDir = path.join(root, "state");
+    const config = { agentCommand: "node ./fake-acp.js", stateDir };
+    async function withManagedHome(home: string, claudeConfigDir = path.join(home, "provider")) {
+      return runExecutor({
+        ...config,
+        managedAiConnection: { identity: "grant-1:shared:abc" },
+        env: {
+          HOME: home,
+          XDG_CONFIG_HOME: path.join(home, "config"),
+          XDG_DATA_HOME: path.join(home, "data"),
+          CODEX_HOME: path.join(home, "provider"),
+          GROK_HOME: path.join(home, "provider"),
+          CLAUDE_CONFIG_DIR: claudeConfigDir,
+        },
+      }, { context: { taskId: "issue-1" } });
+    }
+    const fp = (run: Awaited<ReturnType<typeof runExecutor>>) => run.result.sessionParams?.configFingerprint;
+    const first = await withManagedHome(path.join(root, "paperclip-ai-a"));
+    const second = await withManagedHome(path.join(root, "paperclip-ai-b"));
+    const overridden = await withManagedHome(path.join(root, "paperclip-ai-c"), path.join(root, "custom-claude"));
+    expect(fp(first)).toBeTypeOf("string");
+    expect(fp(second)).toBe(fp(first));
+    expect(fp(overridden)).not.toBe(fp(first));
+
+    // The transcript dir of each managed home points at one dir in the state
+    // dir, so a resumed session finds its transcript after the home is deleted.
+    const transcripts = path.join(stateDir, "claude-projects");
+    expect(await fs.readlink(path.join(root, "paperclip-ai-a", "provider", "projects"))).toBe(transcripts);
+    expect(await fs.readlink(path.join(root, "paperclip-ai-b", "provider", "projects"))).toBe(transcripts);
+    await expect(fs.lstat(path.join(root, "custom-claude", "projects"))).rejects.toThrow();
+
+    // Without a managed AI connection the same paths are user config and still count.
+    const plainA = await runExecutor({ ...config, env: { HOME: path.join(root, "home-a") } }, { context: { taskId: "issue-1" } });
+    const plainB = await runExecutor({ ...config, env: { HOME: path.join(root, "home-b") } }, { context: { taskId: "issue-1" } });
+    expect(fp(plainB)).not.toBe(fp(plainA));
+  });
+
+   it("busts the session fingerprint when a stable configured PAPERCLIP_* value rotates", async () => {
+     const root = await makeTempRoot();
+     const stateDir = path.join(root, "state");
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts
+index e5681b8..0fd486c 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.ts
+@@ -1663,6 +1663,26 @@ async function writePaperclipClaudeSettings(input: {
+   };
+ }
+ 
+// Claude keeps conversation transcripts in CLAUDE_CONFIG_DIR/projects. A
+// managed AI home is a per-run temp dir that the server deletes after the run,
+// so the next run could not resume ("Resource not found"). Point `projects` at
+// a directory in the agent's state dir, like ~/.claude/projects on a normal
+// install. The server's recursive cleanup removes the link, not its target.
+async function linkManagedClaudeTranscripts(input: {
+  claudeConfigDir: string;
+  stateDir: string;
+}): Promise<string> {
+  const target = path.join(input.stateDir, "claude-projects");
+  await fs.mkdir(input.claudeConfigDir, { recursive: true, mode: 0o700 });
+  await fs.mkdir(target, { recursive: true, mode: 0o700 });
+  const link = path.join(input.claudeConfigDir, "projects");
+  const current = await fs.readlink(link).catch(() => null);
+  if (current === target) return target;
+  await fs.rm(link, { recursive: true, force: true });
+  await fs.symlink(target, link, "dir");
+  return target;
+}
+
+ // Cross the CLI's staging seam for a runner-backed remote sandbox: ship the
+ // workspace (and, in PR 2, the per-adapter managed-home `assets`) into the
+ // sandbox and obtain the in-sandbox `workspaceRemoteDir` plus the non-null
+@@ -2027,6 +2047,30 @@ async function buildRuntime(input: {
+     // are absent from tempKeysApplied and keep their compatibility protection.
+     if (!scratchKeys.has(key) || value !== scratch.dir) resolvedAdapterEnv[key] = value;
+   }
+  // A managed AI connection gets a fresh server-made temp home on every run
+  // (HOME, XDG_*, and the provider home under it). Hash a fixed placeholder for
+  // those paths so the session can resume; the server's own fingerprint does
+  // the same (managedAiSessionFingerprintConfig). Values that differ from the
+  // managed layout are real overrides and stay in the hash. A Claude session
+  // can only resume if its transcript survives the run, so this pairs with
+  // linkManagedClaudeTranscripts below. Remote runs keep their transcript in
+  // the sandbox and are left as they are.
+  const managedHome = config.managedAiConnection && !executionTargetIsRemote ? resolvedAdapterEnv.HOME : undefined;
+  if (managedHome) {
+    const managedLayout: Record<string, string> = {
+      HOME: "",
+      XDG_CONFIG_HOME: "config",
+      XDG_DATA_HOME: "data",
+      CODEX_HOME: "provider",
+      GROK_HOME: "provider",
+      CLAUDE_CONFIG_DIR: "provider",
+    };
+    for (const [key, sub] of Object.entries(managedLayout)) {
+      if (resolvedAdapterEnv[key] === (sub ? path.join(managedHome, sub) : managedHome)) {
+        resolvedAdapterEnv[key] = sub ? `<managed-ai-home>/${sub}` : "<managed-ai-home>";
+      }
+    }
+  }
+   if (authToken) env.PAPERCLIP_API_KEY = authToken;
+   // For the claude agent, set model via ANTHROPIC_MODEL at startup rather than
+   // via session/set_config_option — the ACP server's set_config_option handler
+@@ -2088,6 +2132,15 @@ async function buildRuntime(input: {
+       runAgentHome: agentHome,
+       companyId: agent.companyId,
+     });
+    // Same condition as the fingerprint placeholder above: a resumable session
+    // must find its transcript in the next run's managed home.
+    if (resolvedAdapterEnv.CLAUDE_CONFIG_DIR === "<managed-ai-home>/provider" && env.CLAUDE_CONFIG_DIR) {
+      const transcriptsDir = await linkManagedClaudeTranscripts({
+        claudeConfigDir: env.CLAUDE_CONFIG_DIR,
+        stateDir,
+      });
+      skillCommandNotes.push(`Linked Claude transcripts to ${transcriptsDir}.`);
+    }
+     skillCommandNotes.push(
+       `Wrote Paperclip-managed Claude settings to ${paperclipClaudeSettings.filePath} (defaultMode=${paperclipClaudeSettings.defaultMode}${
+         paperclipClaudeSettings.overrodeDontAsk ? "; overrode user dontAsk" : ""
@@ -0,0 +1,384 @@
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts b/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+index 87e18ae..2224e60 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+@@ -21,6 +21,7 @@ import { deleteJob, findPodForJob, getJobStatus } from "./job-orchestrator.js";
+ import {
+   deleteSandboxCr,
+   findPodForSandbox,
+  getSandboxCrStatus,
+   waitForSandboxReady,
+ } from "./sandbox-cr-orchestrator.js";
+ 
+@@ -191,3 +192,60 @@ export async function destroyLeaseResources(
+     );
+   }
+ }
+
+export interface WorkloadGoneInput {
+  namespace: string;
+  /** Workload resource name (Sandbox CR name or Job name) == providerLeaseId. */
+  name: string;
+  backend: "sandbox-cr" | "job";
+  podName?: string | null;
+  /** Stays below the server's 30 s default plugin RPC timeout. */
+  timeoutMs?: number;
+  pollMs?: number;
+}
+
+/**
+ * Poll until the workload (and the named pod, if any) returns 404. Deletes use
+ * Foreground propagation, so the workload disappears only after its pod is
+ * gone. True is the evidence behind a `destroyed` termination receipt; false
+ * (still present at the deadline, or a read failed) means the caller must not
+ * claim one. Never throws: the delete itself already succeeded.
+ */
+export async function waitForWorkloadGone(
+  clients: KubeClients,
+  input: WorkloadGoneInput,
+): Promise<boolean> {
+  const deadline = Date.now() + (input.timeoutMs ?? 20_000);
+  const pollMs = input.pollMs ?? 500;
+  const gone = async (read: () => Promise<unknown>): Promise<boolean> => {
+    try {
+      await read();
+      return false;
+    } catch (err) {
+      if (isKubeNotFoundError(err)) return true;
+      throw err;
+    }
+  };
+  try {
+    return await pollUntilGone();
+  } catch {
+    return false;
+  }
+
+  async function pollUntilGone(): Promise<boolean> {
+    for (;;) {
+      const workloadGone = await gone(() =>
+        input.backend === "sandbox-cr"
+          ? getSandboxCrStatus(clients, input.namespace, input.name)
+          : getJobStatus(clients, input.namespace, input.name),
+      );
+      const podName = input.podName;
+      const podGone = !podName || await gone(() =>
+        clients.core.readNamespacedPod({ namespace: input.namespace, name: podName }),
+      );
+      if (workloadGone && podGone) return true;
+      if (Date.now() >= deadline) return false;
+      await new Promise((resolve) => setTimeout(resolve, pollMs));
+    }
+  }
+}
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+index 9801638..bf5ce0a 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+@@ -15,6 +15,7 @@ import type {
+   PluginEnvironmentSyncInParams,
+   PluginEnvironmentSyncOutParams,
+   PluginEnvironmentSyncResult,
+  PluginEnvironmentTerminationReceipt,
+   PluginEnvironmentValidateConfigParams,
+   PluginEnvironmentValidationResult,
+ } from "@paperclipai/plugin-sdk";
+@@ -38,7 +39,11 @@ import {
+ } from "./sandbox-cr-orchestrator.js";
+ import { execInPod, execInPodStreaming, wrapCommandWithEnv } from "./pod-exec.js";
+ import { performSyncIn, performSyncOut, type PodStreamExec } from "./file-sync.js";
+-import { checkLeaseResumable, destroyLeaseResources } from "./lease-lifecycle.js";
+import {
+  checkLeaseResumable,
+  destroyLeaseResources,
+  waitForWorkloadGone,
+} from "./lease-lifecycle.js";
+ import {
+   appendNetworkEgressDenyHint,
+   createScopedNetworkEgressPolicyOrReleaseWorkload,
+@@ -578,7 +583,7 @@ const plugin = definePlugin({
+ 
+   async onEnvironmentReleaseLease(
+     params: PluginEnvironmentReleaseLeaseParams,
+-  ): Promise<void> {
+  ): Promise<PluginEnvironmentTerminationReceipt | void> {
+     if (!params.providerLeaseId) return;
+     const config = kubernetesProviderConfigSchema.parse(params.config);
+     const namespace =
+@@ -613,11 +618,22 @@ const plugin = definePlugin({
+         ?? (err as { code?: number; statusCode?: number }).statusCode;
+       if (code !== 404) throw err;
+     }
+
+    // The server grants continuation after a stopped run only with a
+    // termination receipt; without one a cancelled run's queued comment
+    // waits on `remote_cleanup` forever (REN-841). Claim `destroyed` only
+    // once the workload is confirmed gone.
+    const gone = await waitForWorkloadGone(clients, {
+      namespace,
+      name: params.providerLeaseId,
+      backend: leaseBackend,
+    });
+    if (gone) return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
+   },
+ 
+   async onEnvironmentDestroyLease(
+     params: PluginEnvironmentDestroyLeaseParams,
+-  ): Promise<void> {
+  ): Promise<PluginEnvironmentTerminationReceipt | void> {
+     if (!params.providerLeaseId) return;
+     const config = kubernetesProviderConfigSchema.parse(params.config);
+     const namespace =
+@@ -658,6 +674,13 @@ const plugin = definePlugin({
+       podName,
+       secretName,
+     });
+    const gone = await waitForWorkloadGone(clients, {
+      namespace,
+      name: params.providerLeaseId,
+      backend: leaseBackend,
+      podName,
+    });
+    if (gone) return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
+   },
+ 
+   async onEnvironmentExecute(
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+index af0875f..d775799 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+@@ -3,6 +3,7 @@ import {
+   checkLeaseResumable,
+   destroyLeaseResources,
+   isKubeNotFoundError,
+  waitForWorkloadGone,
+ } from "../../src/lease-lifecycle.js";
+ 
+ const SANDBOX_GROUP = "agents.x-k8s.io";
+@@ -346,3 +347,64 @@ describe("destroyLeaseResources", () => {
+     ).rejects.toThrow("forbidden");
+   });
+ });
+
+describe("waitForWorkloadGone", () => {
+  const input = { namespace: "ns", name: "pc-abc", backend: "sandbox-cr" as const, pollMs: 1 };
+
+  it("is true once the Sandbox CR and the pod return 404", async () => {
+    const getCr = vi.fn()
+      .mockResolvedValueOnce(readySandboxCr("pc-abc-pod"))
+      .mockRejectedValue(notFound());
+    const readPod = vi.fn().mockRejectedValue(notFound());
+    const clients = { custom: { getNamespacedCustomObject: getCr }, core: { readNamespacedPod: readPod } };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, podName: "pc-abc-pod" }),
+    ).resolves.toBe(true);
+    expect(getCr).toHaveBeenCalledTimes(2);
+    expect(getCr).toHaveBeenCalledWith(expect.objectContaining({
+      group: SANDBOX_GROUP, version: SANDBOX_VERSION, plural: SANDBOX_PLURAL, namespace: "ns", name: "pc-abc",
+    }));
+  });
+
+  it("is false while the workload still exists at the deadline", async () => {
+    const clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockResolvedValue(readySandboxCr()) },
+    };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, timeoutMs: 5 }),
+    ).resolves.toBe(false);
+  });
+
+  it("is false while the pod still exists even if the workload is gone", async () => {
+    const clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()) },
+      core: { readNamespacedPod: vi.fn().mockResolvedValue({ status: { phase: "Running" } }) },
+    };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, podName: "pc-abc-pod", timeoutMs: 5 }),
+    ).resolves.toBe(false);
+  });
+
+  it("checks the Job for job-backend leases", async () => {
+    const readJob = vi.fn().mockRejectedValue(notFound());
+    const clients = { batch: { readNamespacedJobStatus: readJob } };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, backend: "job" }),
+    ).resolves.toBe(true);
+    expect(readJob).toHaveBeenCalledWith({ namespace: "ns", name: "pc-abc" });
+  });
+
+  it("is false (never throws) when a read fails with another error", async () => {
+    const clients = {
+      custom: {
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(Object.assign(new Error("x"), { code: 403 })),
+      },
+    };
+
+    await expect(waitForWorkloadGone(clients as never, input)).resolves.toBe(false);
+  });
+});
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+index 4c23410..f4fd073 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+@@ -184,12 +184,19 @@ describe("onEnvironmentDestroyLease", () => {
+     const deletePod = vi.fn().mockResolvedValue({});
+     const deleteSecret = vi.fn().mockResolvedValue({});
+     h.clients = {
+-      custom: { deleteNamespacedCustomObject: deleteCr },
+-      core: { deleteNamespacedPod: deletePod, deleteNamespacedSecret: deleteSecret },
+      custom: {
+        deleteNamespacedCustomObject: deleteCr,
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+      core: {
+        deleteNamespacedPod: deletePod,
+        deleteNamespacedSecret: deleteSecret,
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+       batch: { deleteNamespacedJob: vi.fn() },
+     };
+ 
+-    await plugin.definition.onEnvironmentDestroyLease!({
+    const receipt = await plugin.definition.onEnvironmentDestroyLease!({
+       driverKey: "kubernetes",
+       companyId: "acme",
+       environmentId: "env-1",
+@@ -209,14 +216,47 @@ describe("onEnvironmentDestroyLease", () => {
+       namespace: "paperclip-acme",
+       name: "pc-abc-env",
+     });
+    expect(receipt).toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+   });
+ 
+   it("is idempotent: resolves cleanly when every resource is already gone (404)", async () => {
+     h.clients = {
+-      custom: { deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()) },
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+       core: {
+         deleteNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+         deleteNamespacedSecret: vi.fn().mockRejectedValue(notFound()),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+      batch: { deleteNamespacedJob: vi.fn() },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentDestroyLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns no receipt when the gone check cannot read the cluster", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(
+          Object.assign(new Error("forbidden"), { code: 403 }),
+        ),
+      },
+      core: {
+        deleteNamespacedPod: vi.fn().mockResolvedValue({}),
+        deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+       },
+       batch: { deleteNamespacedJob: vi.fn() },
+     };
+@@ -261,8 +301,12 @@ describe("onEnvironmentDestroyLease", () => {
+       core: {
+         deleteNamespacedPod: vi.fn().mockResolvedValue({}),
+         deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+      batch: {
+        deleteNamespacedJob: deleteJob,
+        readNamespacedJobStatus: vi.fn().mockRejectedValue(notFound()),
+       },
+-      batch: { deleteNamespacedJob: deleteJob },
+     };
+ 
+     await plugin.definition.onEnvironmentDestroyLease!({
+@@ -280,3 +324,71 @@ describe("onEnvironmentDestroyLease", () => {
+     expect(deleteCr).not.toHaveBeenCalled();
+   });
+ });
+
+describe("onEnvironmentReleaseLease", () => {
+  it("returns a destroyed receipt once the Sandbox CR is gone (REN-841)", async () => {
+    const deleteCr = vi.fn().mockResolvedValue({});
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: deleteCr,
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+    };
+
+    const receipt = await plugin.definition.onEnvironmentReleaseLease!({
+      driverKey: "kubernetes",
+      companyId: "acme",
+      environmentId: "env-1",
+      config: CONFIG,
+      providerLeaseId: "pc-abc",
+      leaseMetadata: leaseMetadata(),
+    });
+
+    expect(deleteCr).toHaveBeenCalledWith(
+      expect.objectContaining({ namespace: "paperclip-acme", name: "pc-abc" }),
+    );
+    expect(receipt).toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns a destroyed receipt when the Sandbox CR was already deleted", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns no receipt when the gone check fails", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(
+          Object.assign(new Error("server error"), { code: 500 }),
+        ),
+      },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toBeUndefined();
+  });
+});
@@ -0,0 +1,7 @@
+# Upstream paperclipai/paperclip commit this image is built from (REN-401).
+# a027f76 is the master commit the nightly image ran when this was pinned, so
+# the only change against that image is the patch next to this file.
+# PAPERCLIP_VERSION is `git describe --tags --match 'v*' --long` of that
+# commit, the value upstream stamps into its own images. Bump both together.
+PAPERCLIP_COMMIT=a027f76a726e8a556674eded807c2de0e55f5cc0
+PAPERCLIP_VERSION=v2026.916.0-219-ga027f76a7
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 ARG TOFU_VERSION=1.11.0

@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Everything `bootstrap.sh vmtest` needs, so the rebuild test can run as a Job in
 # the cluster instead of on a workstation. The workstation has 31G and the test
@@ -16,7 +16,7 @@ credentials are out of scope (see "Deferred").
 |---|---|
 | Execution model | **B — server-local runs** (upstream default): agent CLIs are child processes of the server, in the server container. Trial only. |
 | Upgrade path | If Paperclip is kept: **C** — `@paperclipai/plugin-kubernetes` sandbox provider + kubernetes-sigs agent-sandbox controller (`agents.x-k8s.io`), per-run Pods and Secrets. Separate spec. |
-| Claude auth | **One shared Claude subscription AI connection**, created once via Paperclip's local sign-in (`claude auth login` into `ai-local-logins/<id>`); Paperclip stores and refreshes it. Server-env `CLAUDE_CODE_OAUTH_TOKEN` is not used: `stripAiAuthBindings` blanks it for managed runs, and upstream accepts no pasted setup-token (the device-code flow needs a Daytona sandbox). |
+| Claude auth | **One shared Claude subscription AI connection**, created once via Paperclip's local sign-in (`claude auth login` into `ai-local-logins/<id>`), then its value replaced with a long-lived `claude setup-token` token. Paperclip stores the value as company secret "My Claude subscription" and injects it per run as `CLAUDE_CODE_OAUTH_TOKEN`; it never refreshes Anthropic credentials, so the login's ~8h access token expires (runs then fail with "ACP agent reported a terminal access failure", 0 tokens). Renew by rotating that secret with a new setup-token (~yearly). Server-env `CLAUDE_CODE_OAUTH_TOKEN` is not used: `stripAiAuthBindings` blanks it for managed runs, and upstream accepts no pasted setup-token (the device-code flow needs a Daytona sandbox). |
 | Bot credentials | Out of scope. Future "AIR-Manager" Paperclip role handles permission requests. |
 | Exposure | LAN-only, behind an Authentik forward-auth gate, then Paperclip's own login. |

@@ -189,7 +189,8 @@ available since v0.35.0). First use of digest reflection in this repo.
   **as the node user** (the entrypoint makes exec sessions root, and root-owned 0600
   credentials are unreadable to the server):
   `kubectl exec -it -n paperclip deploy/paperclip -- gosu node sh -c 'export CLAUDE_CONFIG_DIR=<shown path> && mkdir -p "$CLAUDE_CONFIG_DIR" && claude auth login'`
-   Repeat after loss of the PVC or database.
+   Then Settings → Secrets → "My Claude subscription": replace the value with the
+   output of `claude setup-token`. Repeat both after loss of the database.

 ## Verification
 - Flux Kustomization `paperclip` Ready; CNPG `pclp-pg` healthy; first ScheduledBackup
@@ -165,6 +165,21 @@ permanent recovery.

 ---

+## Velero restore-test: 80% OSD capacity gate
+
+The monthly Velero restore-test restores into the production Ceph pools. Before
+each schedule's restore it projects the restore's raw size (DataUpload
+`totalBytes` x the pool's replication/EC factor, spread over all OSDs) onto
+current OSD usage from Prometheus. If the fullest OSD would go above **80%**
+(`OSD_MAX_PCT` in `restore-test-cronjob.yaml`), that schedule is not restored:
+the Job log shows it as **SKIP** with the projected %, and the summary lists it
+under "Skipped (backup NOT restore-tested)". A SKIP does not fail the Job, so
+read the summary each month. If the projection cannot be made (Prometheus
+down, unsized DataUpload), the schedule is not restored and counts as FAIL.
+Background: REN-441, REN-442.
+
+---
+
 ## CNPG restore-test (verify backups are restorable — non-destructive)

 The monthly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
@@ -48,7 +48,7 @@ spec:
        # cert when doing server-side OIDC token exchange. Bundle both
        # roots into one PEM since NODE_EXTRA_CA_CERTS accepts one file.
        - name: fetch-le-staging-roots
-          image: curlimages/curl:8.10.1
+          image: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777
          command:
            - sh
            - -c
@@ -2121,6 +2121,74 @@ data:
            - url: https://dash.renner.uno/login/generic_oauth
              matching_mode: strict

+      # Matrix (REN-567): upstream login for MAS (account.matrix.renner.uno).
+      # Git owns this provider, its application and the member list.
+      # Public client + PKCE, so there is no client secret to share.
+      # RS256 with the built-in self-signed keypair: a public client has no
+      # secret for HS256, and MAS verifies the ID token signature.
+      - id: matrix-users-group
+        model: authentik_core.group
+        identifiers:
+          name: Matrix Users
+        state: present
+        attrs:
+          is_superuser: false
+          # Every member can sign in to Matrix and gets an account named
+          # after their Authentik username. Add/remove members HERE.
+          users:
+            - !Find [authentik_core.user, [username, k.renner]]
+
+      - id: matrix-mas-provider
+        model: authentik_providers_oauth2.oauth2provider
+        identifiers:
+          name: Matrix (MAS)
+        state: present
+        attrs:
+          authentication_flow: !KeyOf passkey-passwordless-flow
+          authorization_flow: !KeyOf passkey-stepup-flow
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          client_type: public
+          client_id: matrix-mas
+          # Authentik 2026.5 checks the grant type against this list, and a
+          # provider created by a blueprint starts with it empty, so every
+          # authorize fails with invalid_request ("Invalid grant_type for
+          # provider"). Providers made in the UI get the defaults.
+          grant_types:
+            - authorization_code
+            - refresh_token
+          signing_key: !Find [authentik_crypto.certificatekeypair, [name, authentik Self-signed Certificate]]
+          property_mappings:
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-profile]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]]
+          redirect_uris:
+            # The path ends in the provider ULID from apps/matrix/mas.yaml.
+            - url: https://account.matrix.renner.uno/upstream/callback/01M4480KWZGN9HBNB5SSNGT2M1
+              matching_mode: strict
+
+      - model: authentik_core.application
+        identifiers:
+          slug: matrix
+        state: present
+        attrs:
+          name: Matrix
+          slug: matrix
+          provider: !KeyOf matrix-mas-provider
+          meta_launch_url: https://element.renner.uno
+          meta_description: "Matrix chat (Element)"
+
+      # Only the Matrix Users group passes.
+      - model: authentik_policies.policybinding
+        identifiers:
+          target: !Find [authentik_core.application, [slug, matrix]]
+          group: !KeyOf matrix-users-group
+          order: 0
+        state: present
+        attrs:
+          enabled: true
+          negate: false
+          failure_result: false
+
      # Superseded by the single passkey-webauthn-validation stage above.
      # Their flowstagebindings are FK-CASCADEd away with them; the
      # identification stage's webauthn_stage FK is SET_NULL and is
@@ -5,6 +5,11 @@ metadata:
  namespace: evcc
  labels:
    app.kubernetes.io/name: evcc
+  annotations:
+    # The MQTT client certificate reaches evcc as env (below), which does not
+    # follow a renewal. cert-manager renews it every ~60 days; Reloader then
+    # restarts evcc so it picks up the new PEMs before the old ones expire.
+    secret.reloader.stakater.com/reload: mqtt-evcc-client-tls
 spec:
  replicas: 1
  # RWO PVC (`evcc-data`, ceph-block) can't be mounted twice, so the
@@ -23,13 +28,36 @@ spec:
      enableServiceLinks: false
      containers:
        - name: evcc
-          image: evcc/evcc:0.310.1
+          image: evcc/evcc:0.316.2@sha256:070c8a8912c5107e8424cef60bf8c3d3cac0eb32aa28d1b021d226545f0aac7b
          # evcc auto-discovers /etc/evcc.yaml on its default search path;
          # the file is delivered (with the MQTT password) via the
          # sops-encrypted evcc-config Secret mounted below.
          env:
            - name: TZ
              value: Europe/Berlin
+            # MQTT by client certificate (register account evcc-client,
+            # REN-953) on the mTLS listener. These override the mqtt block of
+            # evcc.yaml (viper AutomaticEnv, checked against 0.316.2); the
+            # file's user/password are ignored there, as the broker takes the
+            # username from the certificate CN. evcc wants the PEM text, not
+            # a path.
+            - name: EVCC_MQTT_BROKER
+              value: tls://mosquitto.home-assistant.svc.cluster.local:8884
+            - name: EVCC_MQTT_CACERT
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-evcc-client-tls
+                  key: ca.crt
+            - name: EVCC_MQTT_CLIENTCERT
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-evcc-client-tls
+                  key: tls.crt
+            - name: EVCC_MQTT_CLIENTKEY
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-evcc-client-tls
+                  key: tls.key
          ports:
            - name: http
              containerPort: 7070
@@ -30,6 +30,9 @@ metadata:
 #     Web Service 9 = Music Assistant
 #     Web Service 10 = TRMNL
 #     Web Service 11 = Paperless (gate-level only, see its entry)
+#     Web Service 12 = Element Web
+#     Web Service 13 = Matrix (Synapse client API)
+#     Web Service 14 = Matrix Authentication Service
 #
 # CORRECTION (REN-21, 2026-09-25) to the Web Service 11 comment further down,
 # which claims that an in-cluster probe of the paperless ClusterIP "needs a
@@ -228,6 +231,46 @@ data:
        conditions:
          - "[STATUS] == any(200, 301, 302)"

+      # Matrix (REN-663): Element Web, the Synapse client API and MAS.
+      - name: Web Service 12
+        group: services
+        url: https://element.renner.uno/config.json
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
+      - name: Web Service 13
+        group: services
+        url: https://matrix.renner.uno/_matrix/client/versions
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
+      - name: Web Service 14
+        group: services
+        url: https://account.matrix.renner.uno/.well-known/openid-configuration
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
      # Removed 2026-05-21: Jellyfin (media.renner.uno), Wiki (wiki.renner.uno),
      # InfluxDB (influx.renner.uno) — hostnames NXDOMAIN, no Ingress; services
      # no longer deployed. Re-add when/if they come back.
@@ -6,6 +6,8 @@ kind: Kustomization
 # and verified. See MIGRATION-gitea-woodpecker.md.
 resources:
  - statefulset.yaml
+  # The live size of the data volume; see the file (REN-583).
+  - pvc.yaml
  - service.yaml
  - ingressroute-ssh.yaml
  - ingress.yaml
@@ -0,0 +1,39 @@
+# The live data volume of gitea-0, owned here so its size is set in git
+# (REN-583). The StatefulSet controller created this claim from the
+# volumeClaimTemplates in statefulset.yaml, but that template is immutable on
+# an existing StatefulSet, so changing the size there cannot be applied. A
+# PVC's resources.requests.storage CAN grow in place (ceph-block has
+# allowVolumeExpansion: true; ceph-csi grows the RBD image and the filesystem
+# online, no pod restart). Flux applies this over the existing claim with
+# server-side apply and the resize follows.
+#
+# - Name, namespace, accessModes, storageClassName and volumeMode must match
+#   the live claim exactly: they are immutable, a mismatch fails the whole
+#   gitea Kustomization (no change to the live claim).
+# - volumeName must stay: kustomize-controller has owned spec.volumeName
+#   since the namespace move pre-created this claim (eb23d3ac, #878).
+#   Without it, server-side apply tries to clear the field and the API server
+#   rejects the whole claim as an immutable-spec change.
+# - Grow only. A smaller value is rejected by the API server.
+# - prune: disabled, so removing this file from git never deletes the claim
+#   and its data. To hand ownership back, delete the file; the claim stays.
+#
+# Size history: 20Gi -> 40Gi 2026-07-22 (#336, live resize by hand);
+# 40Gi -> 60Gi 2026-10-03 (REN-583, after the volume filled again on REN-580).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: data-gitea-0
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes: ["ReadWriteOnce"]
+  storageClassName: ceph-block
+  volumeMode: Filesystem
+  volumeName: pvc-e7c56db2-481d-40cb-b94a-b264b7b4b5be
+  resources:
+    requests:
+      storage: 60Gi
@@ -37,7 +37,7 @@ spec:
          # both handled as explicit steps in MIGRATION-gitea-woodpecker.md. Pinned
          # to match the LXC for a clean SQLite lift; no imagepolicy marker (an
          # auto-bump across versions on live SQLite data must never be applied).
-          image: gitea/gitea:1.21.4-rootless
+          image: gitea/gitea:1.21.4-rootless@sha256:600f1f1e1d090c763866db16882b5eac5c6a80b7664ee53bec0806c656c0033f
          env:
            - name: GITEA_WORK_DIR
              value: /var/lib/gitea
@@ -147,14 +147,10 @@ spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: ceph-block
        resources:
-          # Expanded 20Gi -> 40Gi on 2026-07-22 (ceph-block online resize) to
-          # clear a 100%-full outage (#336). The live PVC was resized directly.
-          # NOTE: volumeClaimTemplates is immutable on an existing StatefulSet,
-          # so bumping this value cannot be `kubectl apply`-ed in place — Flux
-          # will reject the whole object. Landing this size change requires a
-          # one-time `kubectl delete statefulset gitea -n gitea --cascade=orphan`
-          # (pod + PVC keep running) so Flux re-creates the STS from this
-          # template and re-adopts them. After the cleanup_packages GC reclaims
-          # the ~10G of orphaned blobs, real usage is ~9G, so 40Gi is headroom.
+          # This template only applies when the claim does not exist yet. The
+          # live size of data-gitea-0 is set in pvc.yaml (REN-583), not here:
+          # volumeClaimTemplates is immutable on an existing StatefulSet, so a
+          # change to this value cannot be applied and Flux would reject the
+          # whole object. Leave it at 40Gi.
          requests:
            storage: 40Gi
@@ -60,7 +60,7 @@ spec:
        # repo's configuration.yaml reads this key, so HA does not start
        # without it. Removing this init container takes the recorder down.
        - name: render-db-url
-          image: busybox:1.36
+          image: busybox:1.38@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e
          command:
            - sh
            - -c
@@ -159,7 +159,7 @@ spec:
        # substitution to POD_IP so nginx-runtime variables like $host
        # / $http_x_forwarded_for in the template stay intact.
        - name: proxy
-          image: nginxinc/nginx-unprivileged:1.27-alpine
+          image: nginxinc/nginx-unprivileged:1.31-alpine@sha256:26b0bf6fbf07297983cb341998d79c831508787de26627dd2a112321b9c3a4af
          env:
            - name: POD_IP
              valueFrom:
@@ -193,7 +193,7 @@ spec:
        # any LAN client can edit, matching the user's trust-the-network
        # model. Runs as root so its writes match HA's root-owned files.
        - name: code-server
-          image: codercom/code-server:4.118.0
+          image: codercom/code-server:4.140.0@sha256:1c9ad972a78450f430155144110da4c02d3627c72a598e9197e2781fb9f92e28
          args:
            - "--auth=none"
            - "--bind-addr=0.0.0.0:8080"
@@ -253,7 +253,7 @@ spec:
        # in the PVC's remote URL; reload auth uses a dedicated long-lived token.
        # Runs as root to match HA's root-owned /config files. See config-sync.sh.
        - name: config-sync
-          image: alpine/git:2.54.0
+          image: alpine/git:2.54.0@sha256:a4bb51f1a3553df194ce679fc1db721d8bfba2046fa1a88fe9d4ac551ffbce25
          command: ["/bin/sh", "/scripts/sync.sh"]
          env:
            - name: SYNC_BRANCH
@@ -36,7 +36,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: esphome
-          image: ghcr.io/esphome/esphome:2026.6.2
+          image: ghcr.io/esphome/esphome:2026.9.1@sha256:776a845d508909bf426d2ee7bdd81796361ebe6a7f091807ce1d6d0b48985413
          imagePullPolicy: IfNotPresent
          # Default entrypoint runs the dashboard (Device Builder) against the
          # mounted /config — no command override needed.
@@ -12,8 +12,9 @@ resources:
  - prometheus-token-externalsecret.yaml
  - config-sync-token-externalsecret.yaml
  - mosquitto-bootstrap.yaml
-  # health broker credential: OpenBao-backed + Class-A rotated (first of the
-  # eight users to migrate off mosquitto-bootstrap.yaml).
+  # Former `health` probe credential. The broker no longer reads it (the
+  # probes use the cert account mosquitto-probe, REN-953); removed with its
+  # rotator in the REN-802 cleanup step.
  - mosquitto-health-credentials-externalsecret.yaml
  # open3e-int broker credential: open3e-esp32 gateway on the Vitocal internal
  # CAN; OpenBao-backed + Class-A rotated.
@@ -68,6 +69,12 @@ resources:
  - ha-pg-objectstore.yaml
  - ha-pg-cluster.yaml
  - ha-pg-scheduled-backup.yaml
+  # MQTT credential management (REN-802): the account controller manages the
+  # production broker's Dynamic Security (REN-900). mosquitto-dynsec-test is
+  # the step-1 test broker, no longer managed; removal follows separately.
+  # Register: mqtt-accounts.yaml (configMapGenerator).
+  - mosquitto-dynsec-test.yaml
+  - mqtt-account-controller.yaml
 configMapGenerator:
  # nginx config for the reverse-proxy sidecar. Hash suffix disabled so
  # the cm name stays stable; editing proxy.conf does not auto-roll the
@@ -103,5 +110,28 @@ configMapGenerator:
    namespace: home-assistant
    files:
      - guard.sh=zigbee2mqtt-discovery-guard.sh
+  # Z2M external converters, copied into /app/data/external_converters by the
+  # seed-data init container on every start. Hash suffix disabled (name stays
+  # stable), so editing a converter does not auto-roll the pod — restart
+  # manually with `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`.
+  - name: zigbee2mqtt-external-converters
+    namespace: home-assistant
+    files:
+      - wetterstation-ws90.js=zigbee2mqtt-ws90-converter.js
+  # MQTT account register, read by mqtt-account-controller on every loop.
+  # Hash suffix disabled (name stays stable); the volume is not a subPath
+  # mount, so an edit reaches the running controller without a restart.
+  - name: mqtt-accounts
+    namespace: home-assistant
+    files:
+      - mqtt-accounts.yaml
+  # shelly-manager api-proxy sidecar config (REN-935). Hash suffix disabled
+  # (name stays stable), so editing shelly-manager-api-proxy.conf does not
+  # auto-roll the pod — restart manually with
+  # `kubectl rollout restart deploy/shelly-manager -n home-assistant`.
+  - name: shelly-manager-api-proxy
+    namespace: home-assistant
+    files:
+      - api.conf.template=shelly-manager-api-proxy.conf
 generatorOptions:
  disableNameSuffixHash: true
@@ -42,6 +42,9 @@ spec:
      labels:
        app.kubernetes.io/name: mdns-relay
      annotations:
+        # Bump to restart the pod and re-attach net1 (DHCP via multus).
+        # 2026-10-06: first attach on containernetworking/plugins v1.9.1.
+        homelab.renner.uno/restarted-for: "2026-10-06 cni-plugins v1.9.1 attach check"
        k8s.v1.cni.cncf.io/networks: mdns-relay-iot
    spec:
      enableServiceLinks: false
@@ -82,7 +85,7 @@ spec:
        # deliver, and the ClusterIP has no backend while HA restarts. The
        # trap forwards SIGTERM so the pod still terminates promptly.
        - name: socat-ipv4
-          image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
+          image: alpine/socat:1.8.1.3@sha256:82ad20f6f6e29b91ff33b6662d24063522b1378f08fd8569cd3ab412cac13f50
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
@@ -124,7 +127,7 @@ spec:
        # depends on the kernel's bindv6only default rather than on anything
        # stated here, so the roles are made explicit instead of inherited.
        - name: socat-ipv6
-          image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
+          image: alpine/socat:1.8.1.3@sha256:82ad20f6f6e29b91ff33b6662d24063522b1378f08fd8569cd3ab412cac13f50
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
@@ -44,6 +44,10 @@ kind: Job
 metadata:
  name: mosquitto-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -55,7 +59,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -112,6 +116,10 @@ kind: Job
 metadata:
  name: mosquitto-admin-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -123,7 +131,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -179,6 +187,10 @@ kind: Job
 metadata:
  name: mosquitto-shellies-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -190,7 +202,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -245,6 +257,10 @@ kind: Job
 metadata:
  name: mosquitto-sml2mqtt-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -256,7 +272,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -313,6 +329,10 @@ kind: Job
 metadata:
  name: mosquitto-open3e-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -324,7 +344,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -383,6 +403,10 @@ kind: Job
 metadata:
  name: mosquitto-zigbee2mqtt-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -394,7 +418,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -10,3 +10,38 @@ spec:
    kind: ClusterIssuer
  dnsNames:
    - mqtt.renner.uno
+---
+# Server certificate of the mutual-TLS listener :8884 (REN-802). Issued by
+# mqtt-ca because certificate clients and mqtt-account-controller trust only
+# that CA. The Let's Encrypt certificate above stays on :8883 for password
+# clients. In-cluster names only; an outside cert client (edge-1) is REN-802
+# step 3.
+#
+# Long-lived on purpose: mosquitto reads its certificate only at start, and a
+# broker restart is an outage, so nothing restarts it for a renewal. With 2
+# years and renewal after 1, the broker serves a renewed certificate after the
+# next restart of any kind (image bump, node reboot) within that year; the old
+# one stays valid until then.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mosquitto-mtls
+  namespace: home-assistant
+spec:
+  secretName: mosquitto-mtls
+  duration: 17520h
+  renewBefore: 8760h
+  dnsNames:
+    - mosquitto
+    - mosquitto.home-assistant.svc
+    - mosquitto.home-assistant.svc.cluster.local
+  usages:
+    - digital signature
+    - server auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
@@ -64,6 +64,66 @@ data:
    certfile /mosquitto/tls/tls.crt
    keyfile /mosquitto/tls/tls.key

+    # Mutual TLS for certificate clients (REN-802). Only certificates from
+    # mqtt-ca are accepted, and the certificate's CN is the username. Not on
+    # the LoadBalancer: in-cluster only (Service `mosquitto`).
+    listener 8884
+    max_connections 200
+    cafile /mosquitto/mtls/ca.crt
+    certfile /mosquitto/mtls/tls.crt
+    keyfile /mosquitto/mtls/tls.key
+    require_certificate true
+    use_identity_as_username true
+
    allow_anonymous false
+
+    # Two account sources side by side (REN-802 step 2, tested on 2.0.22 in
+    # REN-900, document `coexistence`):
+    #   Dynamic Security: accounts from the register mqtt-accounts.yaml,
+    #     managed live by mqtt-account-controller over $CONTROL.
+    #   password_file/acl_file: the legacy users, unchanged until each client
+    #     moves to the register (step 3).
+    # Dynamic Security is asked first. For a username it does not know, and
+    # for a request none of its roles match, it defers to the files. A
+    # register entry with the same name as a legacy user therefore replaces
+    # that user's password, while its acl_file block still applies to
+    # whatever the entry's roles do not cover. Delete the legacy user when
+    # its client moves.
+    #
+    # The acl_file check of 2.0 allows every SUBSCRIBE and checks only on
+    # delivery, so the Dynamic Security default must deny publishClientReceive.
+    # Otherwise every Dynamic Security client could read every topic.
+    plugin /usr/lib/mosquitto_dynamic_security.so
+    plugin_opt_config_file /mosquitto/auth/dynamic-security.json
    password_file /mosquitto/auth/passwd
    acl_file /mosquitto/auth/acl
+  # Dynamic Security start file, used until mqtt-account-controller has
+  # written the Secret mqtt-dynsec-seed. No secrets: only the controller's
+  # certificate client and a role that reaches the Dynamic Security API and
+  # nothing else.
+  dynamic-security.json: |
+    {
+      "defaultACLAccess": {
+        "publishClientSend": false,
+        "publishClientReceive": false,
+        "subscribe": false,
+        "unsubscribe": true
+      },
+      "clients": [
+        {
+          "username": "mqtt-account-controller",
+          "roles": [{"rolename": "mqtt-account-controller-admin"}]
+        }
+      ],
+      "groups": [],
+      "roles": [
+        {
+          "rolename": "mqtt-account-controller-admin",
+          "acls": [
+            {"acltype": "publishClientReceive", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "publishClientSend", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "subscribePattern", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true}
+          ]
+        }
+      ]
+    }
@@ -12,7 +12,7 @@ metadata:
    #
    # Listed per-Secret rather than with the blanket auto annotation, so that
    # enrolling another broker user is a visible edit to this line.
-    secret.reloader.stakater.com/reload: mosquitto-health-credentials,mosquitto-open3e-int-credentials
+    secret.reloader.stakater.com/reload: mosquitto-open3e-int-credentials
 spec:
  replicas: 1
  strategy:
@@ -54,7 +54,7 @@ spec:
        fsGroup: 1883
      containers:
        - name: mosquitto
-          # Pinned tag, auto-bumped within the 2.x line by Flux image-automation
+          # Pinned tag, auto-bumped within the 2.0 line by Flux image-automation
          # (infrastructure/image-automation/mosquitto.yaml). The marker tells
          # Flux where to write; patch/minor updates land as git commits and
          # roll the broker. HA re-publishes its own discovery afterwards, but
@@ -80,7 +80,6 @@ spec:
              mosquitto_passwd -b /mosquitto/auth/passwd open3e "$OPEN3E_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd open3e-int "$OPEN3E_INT_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd evcc "$EVCC_PASSWORD"
-              mosquitto_passwd -b /mosquitto/auth/passwd health "$HEALTH_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd zigbee2mqtt "$ZIGBEE2MQTT_PASSWORD"
              # ACL: HA keeps full access (subscribes to everything); the
              # shellies user is `readwrite shellies/#` (devices publish state
@@ -136,16 +135,6 @@ spec:
              topic readwrite evcc/#
              topic read energy/#

-              # `health` is the broker's own probe credential (see the
-              # liveness/readiness probes below). Scoped to health/# and
-              # nothing else: the probes publish a random value retained and
-              # read it back to prove the broker actually stores+delivers
-              # (not just PUBACKs) — so the user needs read AND write, but
-              # only under health/#. A leak can only touch the throwaway
-              # probe subtree that nothing else subscribes to.
-              user health
-              topic readwrite health/#
-
              # `zigbee2mqtt` is the Zigbee2MQTT bridge (the ZHA replacement).
              # Owns its own `zigbee2mqtt/#` subtree; may WRITE `homeassistant/#`
              # to publish HA MQTT-discovery configs (write-only there, like the
@@ -165,8 +154,21 @@ spec:
              topic write homeassistant/#
              topic read homeassistant/status
              EOF
-              chown 1883:1883 /mosquitto/auth/passwd /mosquitto/auth/acl
-              chmod 0600 /mosquitto/auth/passwd /mosquitto/auth/acl
+              # Dynamic Security start file (REN-802): the controller's seed
+              # if it exists, else the static base file. The seed carries
+              # every register account and its hash, so the broker is in the
+              # desired state before the first client connects. The plugin
+              # rewrites this file on every $CONTROL change, so the broker
+              # user owns it and the directory.
+              if [ -s /mosquitto/seed/dynamic-security.json ]; then
+                cp /mosquitto/seed/dynamic-security.json /mosquitto/auth/dynamic-security.json
+                echo "dynsec: loaded seed from Secret mqtt-dynsec-seed"
+              else
+                cp /mosquitto/config/dynamic-security.json /mosquitto/auth/dynamic-security.json
+                echo "dynsec: no seed yet, loaded base file"
+              fi
+              chown 1883:1883 /mosquitto/auth /mosquitto/auth/passwd /mosquitto/auth/acl /mosquitto/auth/dynamic-security.json
+              chmod 0600 /mosquitto/auth/passwd /mosquitto/auth/acl /mosquitto/auth/dynamic-security.json
              exec mosquitto -c /mosquitto/config/mosquitto.conf
          env:
            - name: HA_PASSWORD
@@ -206,17 +208,6 @@ spec:
                secretKeyRef:
                  name: mosquitto-evcc-credentials
                  key: password
-            # Probe-only credential. Now OpenBao-backed and Class-A rotated (see
-            # mosquitto-health-credentials-externalsecret.yaml); its bootstrap Job
-            # is gone.
-            # Present here for the same reason as the others — the startup
-            # shim hashes it into the passwd file — and reused by the exec
-            # probes below, which run inside this container.
-            - name: HEALTH_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-health-credentials
-                  key: password
            # Zigbee2MQTT bridge credential (mosquitto-zigbee2mqtt-credentials-
            # bootstrap). Hashed into the passwd file here; Z2M reads the same
            # Secret via its init container to authenticate to the broker.
@@ -232,6 +223,9 @@ spec:
            - name: mqtts
              containerPort: 8883
              protocol: TCP
+            - name: mqtts-cert
+              containerPort: 8884
+              protocol: TCP
          volumeMounts:
            - name: config
              mountPath: /mosquitto/config
@@ -241,6 +235,15 @@ spec:
            - name: tls
              mountPath: /mosquitto/tls
              readOnly: true
+            - name: mtls
+              mountPath: /mosquitto/mtls
+              readOnly: true
+            - name: seed
+              mountPath: /mosquitto/seed
+              readOnly: true
+            - name: probe-tls
+              mountPath: /mosquitto/probe-tls
+              readOnly: true
          # Real MQTT round-trip probes, replacing the old tcpSocket checks.
          # A bare TCP accept proved only that a listener existed. Even a
          # QoS-1 PUBACK proves only that the broker acknowledged the publish
@@ -259,14 +262,14 @@ spec:
          # Liveness and readiness use SEPARATE topics (probe/live vs
          # probe/ready) so their rotating retained values never collide.
          #
-          # Uses the dedicated least-privilege `health` user (ACL: readwrite
-          # health/# only — see the passwd/acl build above and the
-          # mosquitto-health-credentials ExternalSecret). Its password is in
-          # this container's env (HEALTH_PASSWORD) because the startup shim
-          # hashes every user into the passwd file; the exec probes run
-          # inside this container and read it directly. A leak of this
-          # credential can only touch the health/# probe subtree, which
-          # nothing else subscribes to.
+          # Logs in as the register account `mosquitto-probe` (mqtt-accounts.yaml,
+          # REN-953): a client certificate from mqtt-ca on the mTLS listener
+          # :8884, ACL readwrite health/# only. No password exists for it.
+          # The Secret mount is not a subPath, so a cert-manager renewal
+          # reaches the next probe without a restart. --insecure skips only
+          # the hostname check, because the server certificate names the
+          # Service and the probe dials 127.0.0.1; the chain is still checked
+          # against mqtt-ca.
          #
          # mosquitto is a single-replica SPOF (HA, Shelly, sml2mqtt, open3e,
          # evcc). Liveness is deliberately slow to fire — failureThreshold 6
@@ -286,8 +289,9 @@ spec:
                - -c
                - |
                  V="probe-$(head -c8 /dev/urandom | od -An -tx1 | tr -d ' ')"
-                  mosquitto_pub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -q 1 -r -t health/probe/ready -m "$V" &&
-                  [ "$(mosquitto_sub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -t health/probe/ready -C 1 -W 5)" = "$V" ]
+                  T="-h 127.0.0.1 -p 8884 --insecure --cafile /mosquitto/probe-tls/ca.crt --cert /mosquitto/probe-tls/tls.crt --key /mosquitto/probe-tls/tls.key"
+                  mosquitto_pub $T -q 1 -r -t health/probe/ready -m "$V" &&
+                  [ "$(mosquitto_sub $T -t health/probe/ready -C 1 -W 5)" = "$V" ]
            periodSeconds: 15
            timeoutSeconds: 10
            failureThreshold: 3
@@ -298,8 +302,9 @@ spec:
                - -c
                - |
                  V="probe-$(head -c8 /dev/urandom | od -An -tx1 | tr -d ' ')"
-                  mosquitto_pub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -q 1 -r -t health/probe/live -m "$V" &&
-                  [ "$(mosquitto_sub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -t health/probe/live -C 1 -W 5)" = "$V" ]
+                  T="-h 127.0.0.1 -p 8884 --insecure --cafile /mosquitto/probe-tls/ca.crt --cert /mosquitto/probe-tls/tls.crt --key /mosquitto/probe-tls/tls.key"
+                  mosquitto_pub $T -q 1 -r -t health/probe/live -m "$V" &&
+                  [ "$(mosquitto_sub $T -t health/probe/live -C 1 -W 5)" = "$V" ]
            periodSeconds: 30
            timeoutSeconds: 10
            failureThreshold: 6
@@ -320,3 +325,23 @@ spec:
        - name: tls
          secret:
            secretName: mosquitto-tls
+        # mqtt-ca server certificate and CA of the :8884 mTLS listener, read
+        # only at start (renewal timing: mosquitto-certificate.yaml).
+        - name: mtls
+          secret:
+            secretName: mosquitto-mtls
+        # Written by mqtt-account-controller. Optional so the broker still
+        # starts with the base file if the Secret is missing. Not in the
+        # reloader annotation: a register change is applied live over
+        # $CONTROL and must never restart the broker.
+        - name: seed
+          secret:
+            secretName: mqtt-dynsec-seed
+            optional: true
+        # Probe certificate of the register account mosquitto-probe, created
+        # by mqtt-account-controller. Not optional: a pod without it fails its
+        # probes, so it waits for the Secret instead.
+        - name: probe-tls
+          secret:
+            secretName: mqtt-mosquitto-probe-tls
+            defaultMode: 0400
@@ -0,0 +1,233 @@
+# Test broker for mqtt-account-controller (REN-802 step 1, REN-888).
+#
+# A second, throwaway mosquitto with the Dynamic Security plugin and a mutual-
+# TLS listener, so the controller can be proven (cert login, reconcile, seed
+# after restart, revocation) without touching the production broker. Nothing
+# in production connects here.
+#
+# Deliberately NOT labelled app.kubernetes.io/name: mosquitto: that label is
+# the selector of the production Services and of mosquitto-world-ingress, so
+# reusing it would put this pod behind mqtt.renner.uno and the IoT VLAN.
+# ClusterIP only; reachable from inside the home-assistant namespace.
+#
+# Removal: delete this file and its kustomization.yaml entry once production
+# runs Dynamic Security (REN-802 step 2).
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mosquitto-dynsec-test-tls
+  namespace: home-assistant
+spec:
+  secretName: mosquitto-dynsec-test-tls
+  dnsNames:
+    - mosquitto-dynsec-test
+    - mosquitto-dynsec-test.home-assistant.svc
+    - mosquitto-dynsec-test.home-assistant.svc.cluster.local
+  usages:
+    - digital signature
+    - server auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mosquitto-dynsec-test-config
+  namespace: home-assistant
+data:
+  mosquitto.conf: |
+    persistence false
+    log_dest stdout
+    log_type error
+    log_type warning
+    log_type notice
+    log_type information
+
+    # Dynamic Security owns authentication and ACLs. The plugin writes its
+    # state back to this file; it is an emptyDir, refilled by the
+    # initContainer on every start.
+    plugin /usr/lib/mosquitto_dynamic_security.so
+    plugin_opt_config_file /mosquitto/dynsec/dynamic-security.json
+
+    # Mutual TLS only. The client certificate's CN is the username; the
+    # Dynamic Security client of that name supplies the roles.
+    listener 8883
+    cafile /mosquitto/tls/ca.crt
+    certfile /mosquitto/tls/tls.crt
+    keyfile /mosquitto/tls/tls.key
+    require_certificate true
+    use_identity_as_username true
+
+    allow_anonymous false
+  # Start file used until the controller has written the Secret
+  # mqtt-dynsec-seed. No secrets: only the controller's certificate client
+  # and a role that reaches the Dynamic Security API and nothing else. Same
+  # shape as the controller's seed renderer (internal/dynsec/config.go).
+  dynamic-security.json: |
+    {
+      "defaultACLAccess": {
+        "publishClientSend": false,
+        "publishClientReceive": true,
+        "subscribe": false,
+        "unsubscribe": true
+      },
+      "clients": [
+        {
+          "username": "mqtt-account-controller",
+          "roles": [{"rolename": "mqtt-account-controller-admin"}]
+        }
+      ],
+      "groups": [],
+      "roles": [
+        {
+          "rolename": "mqtt-account-controller-admin",
+          "acls": [
+            {"acltype": "publishClientReceive", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "publishClientSend", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "subscribePattern", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true}
+          ]
+        }
+      ]
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mosquitto-dynsec-test
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mosquitto-dynsec-test
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mosquitto-dynsec-test
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 10
+      # Runs as the mosquitto UID from the start instead of starting as root
+      # and dropping, so the dynsec file the initContainer writes is the
+      # broker's own.
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1883
+        runAsGroup: 1883
+        fsGroup: 1883
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # Every start: the controller's seed if it exists, else the static
+        # base file. The seed carries every account and password hash, so the
+        # broker is in the desired state before the first client connects.
+        - name: dynsec-seed
+          image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
+          command:
+            - sh
+            - -c
+            - |
+              set -eu
+              if [ -s /seed/dynamic-security.json ]; then
+                cp /seed/dynamic-security.json /mosquitto/dynsec/dynamic-security.json
+                echo "dynsec: loaded seed from Secret mqtt-dynsec-seed"
+              else
+                cp /mosquitto/config/dynamic-security.json /mosquitto/dynsec/dynamic-security.json
+                echo "dynsec: no seed yet, loaded base file"
+              fi
+              chmod 0600 /mosquitto/dynsec/dynamic-security.json
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /mosquitto/config
+              readOnly: true
+            - name: seed
+              mountPath: /seed
+              readOnly: true
+            - name: dynsec
+              mountPath: /mosquitto/dynsec
+      containers:
+        - name: mosquitto
+          # Same tag and image policy as the production broker, so the test
+          # runs the version that production will run.
+          image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
+          command: ["mosquitto", "-c", "/mosquitto/config/mosquitto.conf"]
+          ports:
+            - name: mqtts
+              containerPort: 8883
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /mosquitto/config
+              readOnly: true
+            - name: tls
+              mountPath: /mosquitto/tls
+              readOnly: true
+            - name: dynsec
+              mountPath: /mosquitto/dynsec
+          readinessProbe:
+            tcpSocket:
+              port: mqtts
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: mqtts
+            periodSeconds: 30
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 5m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+      volumes:
+        - name: config
+          configMap:
+            name: mosquitto-dynsec-test-config
+        - name: tls
+          secret:
+            secretName: mosquitto-dynsec-test-tls
+        # Written by mqtt-account-controller. Optional: absent on the very
+        # first start, when the base file is used instead.
+        - name: seed
+          secret:
+            secretName: mqtt-dynsec-seed
+            optional: true
+        - name: dynsec
+          emptyDir:
+            sizeLimit: 4Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mosquitto-dynsec-test
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+  ports:
+    - name: mqtts
+      port: 8883
+      targetPort: mqtts
+      protocol: TCP
@@ -12,3 +12,7 @@ spec:
    - name: mqtt
      port: 1883
      targetPort: mqtt
+    # Mutual TLS for certificate clients and mqtt-account-controller (REN-802).
+    - name: mqtts-cert
+      port: 8884
+      targetPort: mqtts-cert
@@ -0,0 +1,317 @@
+# mqtt-account-controller (REN-802, code REN-837, deploy REN-888).
+# Source: konairius/mqtt-account-controller (maintainer Hephaestus). Reconciles
+# the broker's Dynamic Security clients/roles/ACLs, the cert-manager
+# Certificates of `auth: cert` accounts and device passwords to the register
+# mqtt-accounts.yaml, every INTERVAL (2m).
+#
+# Manages the production broker `mosquitto` over its mTLS listener :8884
+# (REN-802 step 2, REN-900). The legacy users in that broker's password_file
+# are not Dynamic Security clients, so the controller never sees or touches
+# them.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+---
+# Find the Certificates it manages (label app.kubernetes.io/managed-by=
+# mqtt-account-controller) in every namespace. List only: no get, watch or
+# write cluster-wide.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: mqtt-account-controller-list-certificates
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: mqtt-account-controller-list-certificates
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: mqtt-account-controller-list-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# Per deliver.namespace: manage the account Certificates there (server-side
+# apply = patch, which creates through `create`). One Role + RoleBinding per
+# namespace that appears as `deliver.namespace` in the register: home-assistant,
+# evcc and paperclip.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "create", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# Secrets in home-assistant, by name only. This namespace holds every broker
+# and HA credential, so an unnamed `delete secrets` grant would let the
+# controller wipe all of them.
+#   mqtt-account-hashes, mqtt-dynsec-seed: its own state (server-side apply
+#     creates them, hence create + patch; resourceNames hold for an apply
+#     that creates, because the name is in the request path).
+#   mqtt-<name>-tls: delete on revocation, one entry per cert account
+#     delivered to this namespace. Add the name with the register entry.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-secrets-home-assistant
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["mqtt-account-hashes", "mqtt-dynsec-seed"]
+    verbs: ["get", "create", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames:
+      - mqtt-test-cert-tls
+      - mqtt-mosquitto-probe-tls
+      - mqtt-zigbee2mqtt-bridge-tls
+      - mqtt-home-assistant-tls
+    verbs: ["delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-secrets-home-assistant
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-secrets-home-assistant
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# deliver.namespace evcc (account evcc-client): its Certificate, and its
+# Secret by name for revocation.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: evcc
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "create", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames:
+      - mqtt-evcc-client-tls
+    verbs: ["delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: evcc
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# deliver.namespace paperclip (account agents-read): its Certificate, and its
+# Secret by name for revocation.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: paperclip
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "create", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames:
+      - mqtt-agents-read-tls
+    verbs: ["delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: paperclip
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# The controller's own MQTT identity: CN = Dynamic Security admin client
+# (ADMIN_USER). Not labelled managed-by, so the controller never treats it
+# as an account certificate and never revokes it. Re-read on every TLS
+# handshake, so a renewal needs no restart.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+spec:
+  secretName: mqtt-account-controller-tls
+  commonName: mqtt-account-controller
+  usages:
+    - digital signature
+    - client auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+    rotationPolicy: Always
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+  annotations:
+    # The controller reads the shelly-manager API key once at start; the
+    # quarterly rotation of shelly-manager-auth must restart it too.
+    secret.reloader.stakater.com/reload: shelly-manager-auth
+spec:
+  replicas: 1
+  # One writer at a time: two controllers would race on rotations and the
+  # seed Secret.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mqtt-account-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mqtt-account-controller
+    spec:
+      serviceAccountName: mqtt-account-controller
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: controller
+          # Woodpecker builds main-<sha> on every merge to main; no sortable
+          # tag, so bumps are manual (same as ical-proxy).
+          image: gitea.renner.uno/konairius/mqtt-account-controller:main-982231a@sha256:851dd025a8aea352061ac818f9968e1c4a074c8f33af405efde3424382e77ae1
+          env:
+            - name: BROKER_URL
+              value: ssl://mosquitto.home-assistant.svc.cluster.local:8884
+            # Password push to Shellies (accounts with deliver.shelly-manager).
+            # The API-key-only Service (api-proxy sidecar, :8081), like the
+            # agents; same namespace, so the default-deny allows it.
+            - name: SHELLY_MANAGER_URL
+              value: http://shelly-manager-api.home-assistant.svc.cluster.local
+            - name: SHELLY_MANAGER_API_KEY_FILE
+              value: /etc/shelly-manager/api-key
+            # Not :8080: several :8080 ingress grants in the home-assistant
+            # default-deny apply to every pod in the namespace.
+            - name: LISTEN
+              value: ":8081"
+          ports:
+            - name: http
+              containerPort: 8081
+              protocol: TCP
+          # /healthz is 200 while a reconcile succeeded within 3 x INTERVAL.
+          # Readiness only: a restart does not fix an unreachable broker.
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: register
+              mountPath: /etc/mqtt-accounts
+              readOnly: true
+            - name: tls
+              mountPath: /etc/mqtt-tls
+              readOnly: true
+            - name: shelly-manager-key
+              mountPath: /etc/shelly-manager
+              readOnly: true
+          resources:
+            requests:
+              cpu: 5m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+      volumes:
+        # Not a subPath mount, so a register change reaches the running pod
+        # (kubelet sync, ~1 min) and the next loop reads it without a restart.
+        - name: register
+          configMap:
+            name: mqtt-accounts
+        - name: tls
+          secret:
+            secretName: mqtt-account-controller-tls
+        # Only the api-key item, not the shelly-manager admin password.
+        - name: shelly-manager-key
+          secret:
+            secretName: shelly-manager-auth
+            items:
+              - key: api-key
+                path: api-key
@@ -0,0 +1,206 @@
+# MQTT account register (REN-802). The only definition of who may use the
+# broker: mqtt-account-controller reconciles certificates, device passwords
+# and Dynamic Security clients/roles/ACLs to this list. Format: README of
+# konairius/mqtt-account-controller.
+#
+# Delivered to the controller as the ConfigMap `mqtt-accounts` (see the
+# configMapGenerator in kustomization.yaml). An empty list is refused, so it
+# never revokes everyone.
+#
+# The controller manages the production broker `mosquitto` (REN-900). Its
+# legacy users (password_file in mosquitto-deployment.yaml) are not listed
+# here; each moves here with its client (REN-802 step 3). `test-cert` is the
+# certificate test account from REN-888.
+#
+# A cert account needs its Secret `mqtt-<name>-tls` added to the
+# `mqtt-account-controller-secrets-<namespace>` Role in
+# mqtt-account-controller.yaml, or revoking it fails with 403.
+- name: test-cert
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite test/cert/#"]
+# The broker's own liveness/readiness probe (mosquitto-deployment.yaml).
+# Not named `health`: a Dynamic Security client of that name would shadow the
+# legacy password user while the old broker pod still probes with it.
+- name: mosquitto-probe
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite health/#"]
+# Zigbee2MQTT and its two sidecars (discovery-guard, interview-watch), one
+# certificate for the pod. Same ACL as the legacy user `zigbee2mqtt`; not
+# named like it, so this entry cannot shadow the password login of a running
+# Z2M (REN-953).
+- name: zigbee2mqtt-bridge
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite zigbee2mqtt/#", "write homeassistant/#", "read homeassistant/status"]
+# evcc (evcc namespace). Same ACL as the legacy user `evcc`, different name
+# so it cannot shadow the running password login (REN-953). evcc takes the
+# certificate PEMs by env (EVCC_MQTT_*), see apps/evcc/deployment.yaml.
+- name: evcc-client
+  auth: cert
+  deliver: {namespace: evcc}
+  acl: ["read open3e/#", "readwrite evcc/#", "read energy/#"]
+# Home Assistant, via the ghostunnel sidecar `mqtt-tls` in its pod (HA keeps
+# client certificates only as uploaded PEMs in its config entry, so a renewal
+# could not reach it without a reconfigure). Same ACL as the legacy user
+# `homeassistant`; not named like it, so it cannot shadow that login (REN-953).
+- name: home-assistant
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite #"]
+# Read-only account for agents (REN-953). One account for all of them: the
+# agents share sandboxes, so per-agent separation would be nominal. The
+# Secret lands in `paperclip`; agents fetch it per run with their kubectl
+# identity (Role mqtt-agents-read in apps/paperclip/agent-kubectl-rbac.yaml),
+# so a renewal needs no restart. Network path: agent sandbox pods ->
+# mosquitto :8884 only.
+- name: agents-read
+  auth: cert
+  deliver: {namespace: paperclip}
+  acl: ["read open3e/#", "read zigbee2mqtt/#", "read homeassistant/#"]
+# Shellies (REN-1017): one password account per device, pushed by the
+# controller through shelly-manager's per-device API. Name shelly-<id> =
+# shelly-manager device id. Same ACL as the legacy user `shellies`, which
+# keeps working until every device has moved. Gen2+ devices reboot once
+# when their password is pushed.
+- name: shelly-5               # Plus i4, the API test device of REN-929
+  auth: password
+  deliver: {shelly-manager: 5}
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+# Only devices that log in as `shellies` today. A device with MQTT off never
+# reports connected, so the controller would push (and reboot it) again
+# every ROTATE_GRACE.
+- name: shelly-6
+  auth: password
+  deliver: {shelly-manager: 6}  # Plus i4 90380c3808b8
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-29
+  auth: password
+  deliver: {shelly-manager: 29}  # Plus i4 b8d61a8c7f24
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-3
+  auth: password
+  deliver: {shelly-manager: 3}  # Pro 3EM ac15186d43c0
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-42
+  auth: password
+  deliver: {shelly-manager: 42}  # 3EM-63 G3 d885ac1c5ab4
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-43
+  auth: password
+  deliver: {shelly-manager: 43}  # 3EM-63 G3 d885ac1ab274
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-10
+  auth: password
+  deliver: {shelly-manager: 10}  # Plug S Gen1 4382e5
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-22
+  auth: password
+  deliver: {shelly-manager: 22}  # Plug S Gen1 87e961
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-25
+  auth: password
+  deliver: {shelly-manager: 25}  # Plug S Gen1 1c1a8f
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-15
+  auth: password
+  deliver: {shelly-manager: 15}  # Plug S G3 8cbfea910578
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-16
+  auth: password
+  deliver: {shelly-manager: 16}  # Plug S G3 b08184a63ef0
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-14
+  auth: password
+  deliver: {shelly-manager: 14}  # Plus 1PM Mini 348518e0be40
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-27
+  auth: password
+  deliver: {shelly-manager: 27}  # Plus 1PM Mini 348518e06c04
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-13
+  auth: password
+  deliver: {shelly-manager: 13}  # Plus 2PM 48551997e624
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-19
+  auth: password
+  deliver: {shelly-manager: 19}  # Plus 2PM 48551999a790
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-21
+  auth: password
+  deliver: {shelly-manager: 21}  # Plus 2PM 0cdc7efa8d00
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-34
+  auth: password
+  deliver: {shelly-manager: 34}  # Plus 2PM 3ce90e2f2d7c
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-2
+  auth: password
+  deliver: {shelly-manager: 2}  # Wall Display 00082266e106
+  # Checked but never rotated on "device not connected" (REN-1066).
+  rotate: false
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-26
+  auth: password
+  deliver: {shelly-manager: 26}  # Power Strip G4 206ef102a6cc
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-1
+  auth: password
+  deliver: {shelly-manager: 1}  # Pro 4PM 9454c5b9c954
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-4
+  auth: password
+  deliver: {shelly-manager: 4}  # Pro 4PM ac15186ca578
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-32
+  auth: password
+  deliver: {shelly-manager: 32}  # Pro 4PM 30c6f7834ec4
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-33
+  auth: password
+  deliver: {shelly-manager: 33}  # Pro 3 fce8c0db5f28
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+# HiluX DS8: HA drives them over MQTT only, and their firmware can hang in
+# the MQTT reconnect path until a power cycle (wiki
+# shelly-hilux-ds8-mqtt-reconnect-hang-bug). One unit at a time; 12 is a
+# bathroom mesh receiver, so its wall switch works without MQTT.
+- name: shelly-12
+  auth: password
+  deliver: {shelly-manager: 12}  # HiluX DS8 34b7da890cfc, Bad Spiegel Rechts
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-9
+  auth: password
+  deliver: {shelly-manager: 9}  # HiluX DS8 34b7da8c03d0, Bad Badewanne
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-31
+  auth: password
+  deliver: {shelly-manager: 31}  # HiluX DS8 34b7da888050, Bad Spiegel Links
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-44
+  auth: password
+  deliver: {shelly-manager: 44}  # HiluX DS8 34b7da894604, Toilette (standalone, no mesh)
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+# Sleeping H&Ts (REN-1107): awake only 8-13 s every 1-2 h, so the
+# controller pushes when the device publishes `online` on its wake topic
+# (shelly-manager topic prefix `shellies/<hostname>`), not in the 2-min loop.
+- name: shelly-35
+  auth: password
+  deliver: {shelly-manager: 35}  # Plus H&T 08b61fcea690
+  wake: shellies/shellyplusht-08b61fcea690/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-36
+  auth: password
+  deliver: {shelly-manager: 36}  # Plus H&T 08b61fce90dc
+  wake: shellies/shellyplusht-08b61fce90dc/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-37
+  auth: password
+  deliver: {shelly-manager: 37}  # H&T G3 84fce63ab4d8
+  wake: shellies/shellyhtg3-84fce63ab4d8/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-38
+  auth: password
+  deliver: {shelly-manager: 38}  # H&T G3 ecda3bc0b6a0
+  wake: shellies/shellyhtg3-ecda3bc0b6a0/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
@@ -0,0 +1,30 @@
+# API-key-only listener for shelly-manager (REN-931, REN-935). Rendered by the
+# nginx-unprivileged entrypoint (envsubst, filtered to POD_IP) into
+# /etc/nginx/conf.d/ at start.
+#
+# shelly-manager signs in whoever X-authentik-username names when the peer is
+# in Auth__ReverseProxy__TrustedProxies (the pod CIDR). Only the Authentik
+# outpost may reach :8080 directly. Paperclip agent pods reach this :8081
+# instead, which drops the identity header, so every request here falls through
+# to the API key (or the login cookie) and a forged header gets 401.
+#
+# Bound to ${POD_IP}, not 0.0.0.0, so it stays off the net1 macvlan (see the
+# ASPNETCORE_URLS comment in shelly-manager-deployment.yaml).
+server {
+    listen ${POD_IP}:8081;
+    server_tokens off;
+    client_max_body_size 64m;
+
+    location / {
+        proxy_pass http://${POD_IP}:8080;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        # Empty value = nginx does not pass the header at all, whatever the
+        # client's case or how many copies it sent. X-Forwarded-User is the
+        # app's default HeaderName, dropped too in case the setting changes.
+        # Header names with underscores never pass (ignore_invalid_headers).
+        proxy_set_header X-authentik-username "";
+        proxy_set_header X-Forwarded-User "";
+        proxy_read_timeout 300s;
+    }
+}
@@ -61,7 +61,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: shelly-manager
-          image: gitea.renner.uno/k.renner/shelly-manager:build.1784957186 # {"$imagepolicy": "flux-system:shelly-manager"}
+          image: gitea.renner.uno/k.renner/shelly-manager:build.1791385485 # {"$imagepolicy": "flux-system:shelly-manager"}
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
@@ -121,7 +121,9 @@ spec:
            # outpost pod's address is dynamic, so this trusts the cluster pod
            # CIDR and leans on the namespace CNP to keep other pods off the
            # app port. ADMIN_PASSWORD (above) stays as break-glass, e.g. for
-            # port-forward access when authentik is down.
+            # port-forward access when authentik is down. Never open :8080 to
+            # another client: they get the api-proxy sidecar's :8081 (below),
+            # which drops the header (REN-935).
            - name: Auth__ReverseProxy__Enabled
              value: "true"
            - name: Auth__ReverseProxy__HeaderName
@@ -168,7 +170,62 @@ spec:
              memory: 128Mi
            limits:
              memory: 512Mi
+        # API-key-only listener for clients other than the Authentik outpost
+        # (Paperclip agents, REN-931). nginx on ${POD_IP}:8081 proxies to
+        # :8080 and drops X-authentik-username, so the ProxyHeader sign-in
+        # cannot be reached through it. Config and reasoning:
+        # shelly-manager-api-proxy.conf. Service: shelly-manager-api.
+        # No probes: it only serves the agent path, and a probe failure would
+        # also pull the outpost's endpoint out of the shelly-manager Service.
+        - name: api-proxy
+          image: nginxinc/nginx-unprivileged:1.31-alpine@sha256:26b0bf6fbf07297983cb341998d79c831508787de26627dd2a112321b9c3a4af
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NGINX_ENVSUBST_FILTER
+              value: "^POD_IP$"
+          ports:
+            - name: api
+              containerPort: 8081
+              protocol: TCP
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 101
+            runAsGroup: 101
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+          volumeMounts:
+            - name: api-proxy-config
+              mountPath: /etc/nginx/templates
+              readOnly: true
+            # envsubst output of the template above.
+            - name: api-proxy-conf-d
+              mountPath: /etc/nginx/conf.d
+            # pid file and proxy temp paths of the unprivileged image.
+            - name: api-proxy-tmp
+              mountPath: /tmp
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: shelly-manager-data
+        - name: api-proxy-config
+          configMap:
+            name: shelly-manager-api-proxy
+        - name: api-proxy-conf-d
+          emptyDir:
+            medium: Memory
+            sizeLimit: 1Mi
+        - name: api-proxy-tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: 32Mi
@@ -11,3 +11,21 @@ spec:
      port: 80
      targetPort: http
      protocol: TCP
+---
+# API-key-only entry for Paperclip agent pods (REN-931, REN-935): the
+# api-proxy sidecar, which drops the X-authentik-username header. Agents call
+# http://shelly-manager-api.home-assistant.svc/api/v1/... with the API key.
+# The CNPs admit agents to :8081 only; :8080 stays outpost-only.
+apiVersion: v1
+kind: Service
+metadata:
+  name: shelly-manager-api
+  namespace: home-assistant
+spec:
+  selector:
+    app.kubernetes.io/name: shelly-manager
+  ports:
+    - name: http
+      port: 80
+      targetPort: api
+      protocol: TCP
@@ -11,11 +11,11 @@ data:
  # config once the PVC has been seeded — change the on-PVC file (via the Z2M
  # frontend or an edit) instead. Kept here so a fresh PVC comes up configured.
  #
-  # Secrets are NOT in this file: both the MQTT password and the Zigbee
-  # network_key are quoted parseValueRefs (`"!secret <key>"`) resolved from
-  # /app/data/secret.yaml, which the init container renders — the MQTT password
-  # from the broker-credential Secret, and the network_key extracted from the
-  # OpenBao coordinator backup. The non-secret network params (channel/pan_id/
+  # Secrets are NOT in this file: the Zigbee network_key is a quoted
+  # parseValueRef (`"!secret <key>"`) resolved from /app/data/secret.yaml,
+  # which the init container renders from the OpenBao coordinator backup. MQTT
+  # uses the client certificate mounted at /mqtt-tls (REN-953); the Deployment
+  # also sets the mqtt server/ca/cert/key by env, which wins over this file. The non-secret network params (channel/pan_id/
  # ext_pan_id) are in `advanced` below.
  configuration.yaml: |
    # Home Assistant MQTT discovery — Z2M publishes device configs under the
@@ -32,14 +32,12 @@ data:

    mqtt:
      base_topic: zigbee2mqtt
-      server: mqtt://mosquitto:1883
-      user: zigbee2mqtt
-      # MUST be a QUOTED string: Z2M reads secrets via parseValueRef on the
-      # string value (regex `!<file> <key>`), NOT a YAML `!secret` tag. Unquoted
-      # `!secret ...` is parsed as a tag and throws "unknown scalar tag
-      # !<!secret>". The init container renders /app/data/secret.yaml with
-      # `mqtt_password` from the broker-credential Secret.
-      password: "!secret mqtt_password"
+      # Mutual TLS: the broker takes the username from the certificate CN
+      # (zigbee2mqtt-bridge), so no user or password here.
+      server: mqtts://mosquitto:8884
+      ca: /mqtt-tls/ca.crt
+      cert: /mqtt-tls/tls.crt
+      key: /mqtt-tls/tls.key

    # SMLIGHT SLZB-06 (Texas Instruments CC2652, zigpy-znp under ZHA) reached
    # over TCP on the IoT VLAN — the SAME coordinator ZHA uses today
@@ -32,9 +32,11 @@ spec:
        # backup also handles the ZHA<->Z2M ext_pan_id byte-order reversal —
        # which is why we restore a backup rather than hand-set advanced.ext_pan_id).
        # Renders /app/data/secret.yaml (referenced by the config as quoted
-        # "!secret" parseValueRefs) with the MQTT password AND the Zigbee
-        # network_key, the latter extracted from the OpenBao coordinator backup
-        # as a natural-byte-order int array.
+        # "!secret" parseValueRefs) with the Zigbee network_key, extracted from
+        # the OpenBao coordinator backup as a natural-byte-order int array. No
+        # MQTT password since REN-953: `mqtt_password` is absent, so Z2M's
+        # "!secret mqtt_password" resolves to nothing and it logs in by
+        # certificate alone.
        - name: seed-data
          # Our own alpine + jq image, built by build-ci-images.yml. DATED tag,
          # never :latest — rebuilding this base for an unrelated consumer
@@ -114,7 +116,6 @@ spec:
              # hex key "fe0b..2d" -> "0xfe,0x0b,..,0x2d" (natural order)
              NK=$(printf '%s' "$KEY" | sed 's/../0x&,/g; s/,$//')
              cat > /app/data/secret.yaml <<EOF
-              mqtt_password: "${MQTT_PASSWORD}"
              network_key: [${NK}]
              EOF
              if [ ! -f /app/data/configuration.yaml ]; then
@@ -132,12 +133,13 @@ spec:
              else
                echo "existing Z2M state present; NOT touching coordinator_backup.json"
              fi
-          env:
-            - name: MQTT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-zigbee2mqtt-credentials
-                  key: password
+              # External converters: overwritten from git on EVERY start (not
+              # seed-only). Copied rather than mounted because Z2M writes a temp
+              # copy of each converter into this directory while loading it
+              # (dist/extension/externalJS.js), which a read-only mount refuses.
+              mkdir -p /app/data/external_converters
+              cp /external-converters/*.js /app/data/external_converters/
+              echo "copied external converters: $(ls /external-converters)"
          volumeMounts:
            - name: data
              mountPath: /app/data
@@ -147,6 +149,9 @@ spec:
            - name: coordinator-backup
              mountPath: /backup
              readOnly: true
+            - name: external-converters
+              mountPath: /external-converters
+              readOnly: true
          resources:
            requests:
              cpu: 10m
@@ -158,19 +163,36 @@ spec:
          # Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
          # add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
          # follow-up if desired; pinned-and-manual is fine for the initial cutover.
-          image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
+          image: ghcr.io/koenkk/zigbee2mqtt:2.14.2@sha256:addbfb08fcef2e4477de1535636118241e10d13e898fbe4d9faba8a4d7396311
          imagePullPolicy: IfNotPresent
          env:
            - name: TZ
              value: Europe/Berlin
            - name: ZIGBEE2MQTT_DATA
              value: /app/data
+            # MQTT by client certificate (register account zigbee2mqtt-bridge,
+            # REN-953). Env, not the seed template: configuration.yaml on the
+            # PVC is Z2M's own and is never re-seeded. Z2M copies env values
+            # into that file on its next settings write, so a rollback must
+            # set these to the old values, not just drop them. The file keeps
+            # `user: zigbee2mqtt`; the broker ignores it and uses the CN.
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_SERVER
+              value: mqtts://mosquitto:8884
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_CA
+              value: /mqtt-tls/ca.crt
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_CERT
+              value: /mqtt-tls/tls.crt
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_KEY
+              value: /mqtt-tls/tls.key
          ports:
            - name: http
              containerPort: 8080
          volumeMounts:
            - name: data
              mountPath: /app/data
+            - name: mqtt-tls
+              mountPath: /mqtt-tls
+              readOnly: true
          # The frontend only serves once Z2M has started its adapter+MQTT, so
          # an HTTP 200 on / is a decent liveness/readiness signal. Generous
          # startup budget covers coordinator connect + herdsman network adopt.
@@ -229,19 +251,16 @@ spec:
          imagePullPolicy: IfNotPresent
          command: [sh, /script/watch.sh]
          env:
-            - name: MQTT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-zigbee2mqtt-credentials
-                  key: password
-            # These three must track zigbee2mqtt-config.yaml: MQTT_HOST the
-            # mosquitto Service name, MQTT_USER and BASE_TOPIC the mqtt.user and
-            # mqtt.base_topic it connects with. Set explicitly rather than left
-            # to the script's defaults so the coupling is visible here.
+            # Must track the Z2M container: MQTT_HOST/MQTT_PORT the broker it
+            # connects to, BASE_TOPIC its mqtt.base_topic. Set explicitly
+            # rather than left to the script's defaults so the coupling is
+            # visible here. Logs in with the pod's certificate (/mqtt-tls).
            - name: MQTT_HOST
              value: mosquitto
-            - name: MQTT_USER
-              value: zigbee2mqtt
+            - name: MQTT_PORT
+              value: "8884"
+            - name: MQTT_TLS_DIR
+              value: /mqtt-tls
            - name: BASE_TOPIC
              value: zigbee2mqtt
            - name: STATEDIR
@@ -264,6 +283,9 @@ spec:
            - name: interview-watch-script
              mountPath: /script
              readOnly: true
+            - name: mqtt-tls
+              mountPath: /mqtt-tls
+              readOnly: true
            # Cooldown stamps. An explicit emptyDir rather than the image's
            # /tmp, so it stays writable as non-root without depending on the
            # container filesystem.
@@ -295,19 +317,16 @@ spec:
          imagePullPolicy: IfNotPresent
          command: [sh, /script/guard.sh]
          env:
-            # Reuses the Z2M broker credential: its ACL is `readwrite
-            # zigbee2mqtt/#`, which covers reading bridge/state/bridge/groups and
-            # writing bridge/request/restart. No new broker user needed.
-            - name: MQTT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-zigbee2mqtt-credentials
-                  key: password
-            # Must track zigbee2mqtt-config.yaml, same as interview-watch.
+            # Shares the Z2M certificate: its ACL is `readwrite zigbee2mqtt/#`,
+            # which covers reading bridge/state/bridge/groups and writing
+            # bridge/request/restart. Must track the Z2M container, same as
+            # interview-watch.
            - name: MQTT_HOST
              value: mosquitto
-            - name: MQTT_USER
-              value: zigbee2mqtt
+            - name: MQTT_PORT
+              value: "8884"
+            - name: MQTT_TLS_DIR
+              value: /mqtt-tls
            - name: BASE_TOPIC
              value: zigbee2mqtt
            - name: STATEDIR
@@ -323,6 +342,9 @@ spec:
            - name: discovery-guard-script
              mountPath: /script
              readOnly: true
+            - name: mqtt-tls
+              mountPath: /mqtt-tls
+              readOnly: true
            # Restart cooldown stamp. Its own emptyDir rather than the image's
            # /tmp, so it stays writable as non-root.
            - name: discovery-guard-state
@@ -340,12 +362,24 @@ spec:
        - name: config-template
          configMap:
            name: zigbee2mqtt-config
+        - name: external-converters
+          configMap:
+            name: zigbee2mqtt-external-converters
        - name: coordinator-backup
          secret:
            secretName: zigbee2mqtt-coordinator-backup
        - name: interview-watch-script
          configMap:
            name: zigbee2mqtt-interview-watch-script
+        # Client certificate of the register account zigbee2mqtt-bridge,
+        # issued by mqtt-account-controller. Not subPath, so a cert-manager
+        # renewal reaches the files in place; Z2M reads them at connect, the
+        # sidecars on every call. 0444 because the sidecars run as UID 1883
+        # and Z2M as root; only this pod mounts it.
+        - name: mqtt-tls
+          secret:
+            secretName: mqtt-zigbee2mqtt-bridge-tls
+            defaultMode: 0444
        - name: interview-watch-state
          emptyDir:
            medium: Memory
@@ -61,16 +61,19 @@
 # herdsman-only restart. discover() republishes only configs that are missing or
 # whose payload changed, which after a wipe is all of them.
 #
-# Credentials: reuses the existing zigbee2mqtt broker user, whose ACL is
+# Credentials: the pod's client certificate (register account
+# zigbee2mqtt-bridge, REN-953) on the mTLS listener :8884. Its ACL is
 # `readwrite zigbee2mqtt/#` — enough to read bridge/state and bridge/groups and
-# to write bridge/request/restart. No new broker user, and no read grant on
-# homeassistant/# (checking for the discovery configs directly would have needed
-# one, which the ACL denies on purpose).
+# to write bridge/request/restart. No read grant on homeassistant/# (checking
+# for the discovery configs directly would have needed one, which the ACL
+# denies on purpose).
 set -eu

 MQTT_HOST="${MQTT_HOST:-mosquitto}"
-MQTT_PORT="${MQTT_PORT:-1883}"
-MQTT_USER="${MQTT_USER:-zigbee2mqtt}"
+MQTT_PORT="${MQTT_PORT:-8884}"
+# ca.crt, tls.crt, tls.key of the client certificate. The broker takes the
+# username from the certificate CN.
+MQTT_TLS_DIR="${MQTT_TLS_DIR:-/mqtt-tls}"
 BASE_TOPIC="${BASE_TOPIC:-zigbee2mqtt}"
 # Poll cadence. Worst-case time-to-repair is CHECK_INTERVAL + CONFIRM_DELAY.
 CHECK_INTERVAL="${CHECK_INTERVAL:-300}"
@@ -105,7 +108,7 @@ log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] discovery-guard: $*"; }
 # ever paid when the topic is genuinely empty.
 retained() {
  timeout "$((READ_TIMEOUT + 5))" mosquitto_sub -h "$MQTT_HOST" -p "$MQTT_PORT" \
-    -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+    --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" \
    -t "$1" -C 1 -W "$READ_TIMEOUT" 2>/dev/null || true
 }

@@ -152,7 +155,7 @@ while true; do

  log "confirmed — requesting a Zigbee2MQTT restart to re-run discovery"
  if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" \
-       -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+       --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" \
       -t "$BASE_TOPIC/bridge/request/restart" -m '' 2>/dev/null; then
    date -u +%s > "$STAMP"
    log "restart requested"
@@ -53,7 +53,10 @@ set -u
 DATA=/app/data
 DB="$DATA/database.db"
 MQTT_HOST=${MQTT_HOST:-mosquitto}
-MQTT_USER=${MQTT_USER:-zigbee2mqtt}
+MQTT_PORT=${MQTT_PORT:-8884}
+# ca.crt, tls.crt, tls.key of the pod's client certificate (register account
+# zigbee2mqtt-bridge, REN-953). The broker takes the username from its CN.
+MQTT_TLS_DIR=${MQTT_TLS_DIR:-/mqtt-tls}
 # Must track mqtt.base_topic in zigbee2mqtt-config.yaml. Overridable via env so
 # the two can be kept in step from the Deployment rather than by editing this
 # script; the MQTT ACL for this user is scoped to `zigbee2mqtt/#`, so a change
@@ -219,10 +222,6 @@ while :; do
    fi

    log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview"
-    # NOTE: mosquitto_pub can only take a password via argv (-P, or an -L URL);
-    # it has no --pw-file or env option, so this cannot be avoided with this
-    # client. Not a cross-container exposure: the pod does not set
-    # shareProcessNamespace, so /proc is not shared with the Z2M container.
    # Bounded with `timeout`, which also covers a TCP connect that hangs rather
    # than being refused. The read loop is single-threaded, so an unbounded
    # publish would stall it and miss other devices' wake windows.
@@ -230,7 +229,8 @@ while :; do
    # NOT `mosquitto_pub -W`: that flag does not exist for mosquitto_pub (it is
    # a mosquitto_sub option), and passing it makes every publish fail on a
    # usage error — verified against the client in this image.
-    if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+    if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" \
+         --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" \
         -t "$BASE_TOPIC/bridge/request/device/interview" \
         -m "{\"id\":\"$ieee\"}" 2>/dev/null; then
      # Stamp only on a SUCCESSFUL publish. Stamping unconditionally would burn
@@ -0,0 +1,48 @@
+// Wetterstation (WS90) = the built-in zigbee-herdsman-converters definition
+// plus one extra configure step that applies the reporting configuration from
+// zigbee2mqtt-ws90-reporting.md. Z2M runs configure after every interview and
+// on bridge/request/device/configure, so those no longer revert the device to
+// the converter defaults (as happened 2026-09-24..28).
+//
+// The extra step runs after the built-in ones (extend order), so its values
+// win. `version` makes Z2M reconfigure once when this file is first loaded:
+// it re-runs configure when definition.version differs from the device's
+// meta.configured. Bump it whenever REPORTING changes.
+const {definitions} = require("zigbee-herdsman-converters/devices/shelly");
+
+const base = definitions.find((d) => d.model === "WS90");
+if (!base) throw new Error("WS90 definition not found in zigbee-herdsman-converters/devices/shelly");
+
+const SHELLY = 5264;
+
+// [cluster, attribute, reportable_change, manufacturerCode]
+const REPORTING = [
+    ["msTemperatureMeasurement", "measuredValue", 10],
+    ["msRelativeHumidity", "measuredValue", 50],
+    ["msPressureMeasurement", "measuredValue", 1],
+    ["msIlluminanceMeasurement", "measuredValue", 5],
+    ["shellyWS90Wind", "windSpeed", 1, SHELLY],
+    ["shellyWS90Wind", "windDirection", 1, SHELLY],
+    ["shellyWS90Wind", "gustSpeed", 1, SHELLY],
+    ["shellyWS90UV", "uvIndex", 1, SHELLY],
+    ["shellyWS90Rain", "rainStatus", 1, SHELLY],
+    ["shellyWS90Rain", "precipitation", 1, SHELLY],
+];
+
+const reporting = {
+    isModernExtend: true,
+    configure: [
+        async (device) => {
+            const endpoint = device.getEndpoint(1);
+            for (const [cluster, attribute, change, manufacturerCode] of REPORTING) {
+                await endpoint.configureReporting(
+                    cluster,
+                    [{attribute, minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: change}],
+                    manufacturerCode ? {manufacturerCode} : {},
+                );
+            }
+        },
+    ],
+};
+
+module.exports = {...base, version: "ws90-reporting-1", extend: [...base.extend, reporting]};
@@ -2,10 +2,15 @@

 Records the ZCL attribute-reporting configuration applied to the Wetterstation
 (Ecowitt WS90 powered by Shelly, IEEE `0xc02cedfffe958f40`) and how to restore
-it, because **this configuration lives on the device and in Z2M's
-`database.db`, not in git**. Zigbee2MQTT has no settings-schema field for
-reporting configuration, so there is nothing declarative to reconcile — Flux
-cannot restore it and a fresh PVC will not have it.
+it. The configuration itself lives on the device and in Z2M's `database.db`.
+
+**Since 2026-10-01 it is applied from git:** the external converter
+`zigbee2mqtt-ws90-converter.js` extends the built-in WS90 definition with a
+configure step that sets the values below, so every interview and every
+`bridge/request/device/configure` applies them instead of the converter
+defaults. The manual commands under "Re-applying" are only needed if the
+converter is not loaded. Keep the table and `REPORTING` in the converter in
+sync, and bump the converter's `version` when they change.

 ## Applied configuration

@@ -71,14 +76,16 @@ the modernExtend value is the outlier.

 ## When this is lost

-Anything that makes Z2M re-run `configure` on the device overwrites all of it
-with the converter defaults:
+Without the external converter, anything that makes Z2M re-run `configure` on
+the device overwrites all of it with the converter defaults:

- re-pairing or re-joining the device
+- re-pairing, re-joining or re-interviewing the device
 - an explicit `zigbee2mqtt/bridge/request/device/configure`
 - a converter update that changes the definition and triggers reconfiguration

-It also will not exist on a fresh PVC. Re-apply with the commands below.
+With the converter loaded, those same events apply the values above. Check
+that it is loaded: Z2M logs `Successfully configured 'Wetterstation'
+(definition vws90-reporting-1)` after a configure.

 ## Re-applying

@@ -96,25 +103,25 @@ POD=$(kubectl get pod -n home-assistant -l app.kubernetes.io/name=zigbee2mqtt -o
 ```

 ```sh
-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msTemperatureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":10}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msTemperatureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":10}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msRelativeHumidity\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":50}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msRelativeHumidity\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":50}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msPressureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msPressureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msIlluminanceMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":5}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msIlluminanceMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":5}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windDirection\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windDirection\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"gustSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"gustSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90UV\",\"attribute\":\"uvIndex\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90UV\",\"attribute\":\"uvIndex\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"rainStatus\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"rainStatus\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"precipitation\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"precipitation\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
 ```

 These are idempotent — re-sending a request that matches the current
@@ -8,26 +8,11 @@
 # back into `kubernetes/apps/immich/` at retirement is the cleanup.
 #
 # Sizing:
-# - library: 300Gi — LXC 225's /mnt/media/immich is currently ~250 GiB.
-#   StorageClass ceph-filesystem has allowVolumeExpansion=true; grow with
-#   `kubectl patch pvc immich-library -p '{"spec":{"resources":{"requests":{"storage":"<new>"}}}}'`.
+# - library-bulk: 300Gi — the media library is ~250 GiB.
+#   StorageClass ceph-filesystem-bulk has allowVolumeExpansion=true; grow with
+#   `kubectl patch pvc immich-library-bulk -p '{"spec":{"resources":{"requests":{"storage":"<new>"}}}}'`.
 # - ml-cache: 20Gi — face-recognition + CLIP model cache. Survives pod
 #   restarts so ML doesn't re-download on every rollout.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: immich-library
-  namespace: immich
-  annotations:
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 300Gi
---
 # Media root on the EC 2+1 bulk pool; thumbs mount separately below.
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -66,7 +66,7 @@ spec:
        # The library PVC is populated by the seed phase before this
        # Deployment reconciles, so no rsync init is needed here.
        - name: fetch-le-staging-roots
-          image: curlimages/curl:8.10.1
+          image: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777
          command:
            - sh
            - -c
@@ -57,7 +57,7 @@ spec:
        runAsNonRoot: false
      initContainers:
        - name: wait-for-influxd
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -80,7 +80,7 @@ spec:
              echo "[$(date -u +%H:%M:%S)] $INFLUX_HOST ready and authenticated"
      containers:
        - name: create
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -88,7 +88,7 @@ spec:
        # state). Two-stage so we don't try to create an auth before
        # the chart's `influx setup` has finished.
        - name: wait-for-influxd
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -111,7 +111,7 @@ spec:
              echo "[$(date -u +%H:%M:%S)] $INFLUX_HOST ready and authenticated"
      containers:
        - name: create
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -133,7 +133,7 @@ spec:
              set -euo pipefail
              log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }

-              # Need jq + kubectl. influxdb:2.7.4 has neither baked in.
+              # Need jq + kubectl. The influxdb image has neither baked in.
              apt-get update -qq
              apt-get install -y --no-install-recommends curl jq ca-certificates >/dev/null
              KV=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
@@ -53,8 +53,9 @@ spec:
  values:
    image:
      repository: influxdb
-      # Match VM 108 for format compatibility with `influx restore`.
-      tag: "2.7.4"
+      # Was pinned to VM 108's 2.7.4 for the one-time `influx restore`; that
+      # migration is done. No downgrade below 2.9 without `influxd downgrade`.
+      tag: "2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e"
    # Single replica — InfluxDB OSS doesn't cluster. RWO Ceph-block + Velero
    # backups (pending) is the durability story.
    persistence:
@@ -93,6 +94,12 @@ spec:
    env:
      - name: INFLUXD_STORAGE_WRITE_TIMEOUT
        value: "60s"
+      # 2.9 hashes API tokens on startup by default. Hashed tokens cannot be
+      # read back, so the token Jobs' `influx auth list` reuse would break,
+      # and a downgrade below 2.8 would erase every token (HA, Grafana).
+      # Keep tokens unhashed; enable hashing later as its own change.
+      - name: INFLUXD_USE_HASHED_TOKENS
+        value: "false"
    # The chart defaults to liveness/readiness with initialDelaySeconds=0,
    # so after a crash restart the kubelet kills the pod before shards
    # finish opening. Give it 10 minutes to come up cleanly.
@@ -5,3 +5,4 @@ resources:
  - helmrelease.yaml
  - ha-token-job.yaml
  - grafana-token-job.yaml
+  - volumesnapshot-pre-2.9.yaml
@@ -0,0 +1,18 @@
+---
+# Point-in-time copy of the InfluxDB data PVC, taken before the 2.7.4 -> 2.9.1
+# upgrade (homelab #1130, REN-723). Merge this first and check readyToUse=true
+# before #1130: 2.9 migrates the metadata store, so the way back is
+# `influxd downgrade 2.4` or a restore from this snapshot.
+#
+# Crash-consistent (taken from the live RBD volume), like the velero CSI
+# snapshots. Delete this file once 2.9 has run cleanly; the snapshot goes
+# with it (prune).
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: influxdb-influxdb2-pre-2-9
+  namespace: influxdb
+spec:
+  volumeSnapshotClassName: ceph-block-snapclass
+  source:
+    persistentVolumeClaimName: influxdb-influxdb2
@@ -48,7 +48,7 @@ spec:
            # this sentinel: the entrypoint copies AccountsSettings.json
            # into ephemeral /config-internal rather than symlinking it.
            - name: seed-config
-              image: busybox:1.37.0
+              image: busybox:1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e
              command: ["/bin/sh", "-eu", "-c"]
              args:
                - |
@@ -94,7 +94,7 @@ spec:
            - name: libation
              # Pinned by digest: Spegel caches mutable tag resolutions, so
              # :latest stayed on 13.7.11 despite imagePullPolicy Always.
-              image: docker.io/rmcrackan/libation:14.2.1@sha256:6f9d8ba19dc3803dd6ba295ab5a7abfb6ff3bc06286eebf4ecd8ccbe43245888
+              image: docker.io/rmcrackan/libation:14.2.2@sha256:fe188ac3c4ce63cc305971c76e648748341427e16bcf884011da8a384fd3908c
              # The entrypoint stages config into ephemeral /config-internal
              # by copy (only the DB is symlinked to the PVC), so Audible
              # tokens refreshed during a run are lost at container exit.
@@ -54,7 +54,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: llama-server
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
          args:
            # The one place this model is named; llama-server resolves the repo
            # and downloads it itself, so there is nothing to keep in sync.
@@ -43,7 +43,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: llama-embed
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
          args:
            - --hf-repo
            - gpustack/bge-m3-GGUF:Q8_0
@@ -32,7 +32,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: wiki-embed
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
          args:
            - --hf-repo
            - gpustack/bge-m3-GGUF:Q8_0
@@ -0,0 +1,222 @@
+# matrix-bridge (REN-569, REN-587): Matrix appservice between Element and the
+# Paperclip agents. One room per agent with the owner; owner messages become
+# comments on the agent's chat card, the agent's comments come back to the
+# room. Source: gitea.renner.uno/konairius/matrix-bridge.
+#
+# No volume: the bridge keeps its state in the bot's account data on Synapse.
+# No ingress: only Synapse talks to it (appservice transactions on :8080).
+#
+# Secrets:
+#   matrix-bridge-tokens  as_token (bridge -> Synapse) and hs_token (Synapse
+#                         -> bridge), generated once in-cluster by ESO, plus
+#                         the appservice registration Synapse loads from
+#                         app_service_config_files (synapse.yaml). Both sides
+#                         read the same Secret, so the tokens cannot drift.
+#                         Deleting the Secret regenerates both; Reloader
+#                         restarts Synapse and the bridge.
+#   matrix-bridge         paperclip_api_key from OpenBao
+#                         homelab/apps/matrix-bridge: the Paperclip board key
+#                         "matrix-bridge (board)" (REN-582).
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: matrix-bridge-tokens
+  namespace: matrix
+spec:
+  length: 64
+  digits: 16
+  symbols: 0
+  allowRepeat: true
+  noUpper: false
+  secretKeys:
+    - as_token
+    - hs_token
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: matrix-bridge-tokens
+  namespace: matrix
+spec:
+  # Generate once; Periodic would rotate the tokens every refresh.
+  refreshPolicy: CreatedOnce
+  target:
+    name: matrix-bridge-tokens
+    template:
+      engineVersion: v2
+      data:
+        as_token: "{{ .as_token }}"
+        hs_token: "{{ .hs_token }}"
+        # Agent users are @agent_<urlKey>:matrix.renner.uno; the bot is
+        # @paperclip:matrix.renner.uno.
+        registration.yaml: |
+          id: matrix-bridge
+          url: http://matrix-bridge:8080
+          as_token: "{{ .as_token }}"
+          hs_token: "{{ .hs_token }}"
+          sender_localpart: paperclip
+          rate_limited: false
+          namespaces:
+            users:
+              - exclusive: true
+                regex: "@agent_.*:matrix\\.renner\\.uno"
+            aliases: []
+            rooms: []
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: matrix-bridge-tokens
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: matrix-bridge
+  data:
+    - secretKey: paperclip_api_key
+      remoteRef:
+        key: apps/matrix-bridge
+        property: paperclip_api_key
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+  annotations:
+    secret.reloader.stakater.com/reload: matrix-bridge,matrix-bridge-tokens
+  labels:
+    app.kubernetes.io/name: matrix-bridge
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  # One poller only: two pods would bridge every comment twice.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: matrix-bridge
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: matrix-bridge
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        # `node` in node:24-alpine.
+        runAsUser: 1000
+        runAsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: matrix-bridge
+          image: gitea.renner.uno/konairius/matrix-bridge:build.1791195981
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: MATRIX_HOMESERVER_URL
+              value: http://synapse:8008
+            - name: MATRIX_SERVER_NAME
+              value: matrix.renner.uno
+            - name: MATRIX_OWNER
+              value: "@k.renner:matrix.renner.uno"
+            - name: MATRIX_AS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge-tokens
+                  key: as_token
+            - name: MATRIX_HS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge-tokens
+                  key: hs_token
+            - name: PAPERCLIP_API_URL
+              value: http://paperclip.paperclip.svc.cluster.local:3100
+            - name: PAPERCLIP_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge
+                  key: paperclip_api_key
+            - name: PAPERCLIP_COMPANY_ID
+              value: 72f7b75a-10b0-4084-ad5e-a0cdf31c7caf
+            - name: PAPERCLIP_PROJECT_ID
+              value: 67143c0b-e9a1-44c1-b012-0f8fa25625aa
+            # Card link in the topic of each per-card room (REN-668).
+            - name: PAPERCLIP_PUBLIC_URL
+              value: https://paperclip.renner.uno
+          ports:
+            - name: http
+              containerPort: 8080
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          # /health is 503 when the Paperclip comment poll fails, so it gates
+          # readiness only; a Paperclip outage must not restart the bridge.
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: http
+            periodSeconds: 30
+            failureThreshold: 4
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: matrix-bridge
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# The bridge calls the Paperclip API in-cluster. Synapse <-> bridge and DNS
+# are covered by matrix-app-egress (part-of: matrix); the Paperclip side of
+# this flow is in apps/paperclip/network-policy.yaml.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: matrix-bridge-paperclip-egress
+  namespace: matrix
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: matrix-bridge
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+            app.kubernetes.io/name: paperclip
+      toPorts:
+        - ports:
+            - port: "3100"
+              protocol: TCP
@@ -0,0 +1,45 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: mx-pg
+  namespace: matrix
+spec:
+  instances: 2
+  smartShutdownTimeout: 10
+  stopDelay: 60
+  imageName: ghcr.io/cloudnative-pg/postgresql:16
+  enableSuperuserAccess: true
+  storage:
+    storageClass: ceph-block
+    size: 5Gi
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: mx-pg-store
+  bootstrap:
+    initdb:
+      database: synapse
+      owner: synapse
+      # Synapse refuses to start on a database whose collation is not C
+      # (its text indexes depend on byte order). Fixed at initdb time; it
+      # cannot be changed on an existing database.
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  postgresql:
+    parameters:
+      shared_buffers: "128MB"
+      max_connections: "50"
+      # Same write-reduction set as the other CNPG clusters; rationale in
+      # apps/trmnl/cluster.yaml.
+      wal_compression: "zstd"
+      archive_timeout: "30min"
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      memory: 512Mi
@@ -0,0 +1,137 @@
+# Element Web at element.renner.uno, locked to matrix.renner.uno: no custom
+# homeserver, no guests, no registration, no integration manager, no
+# analytics (no posthog block), no bug-report endpoint.
+#
+# feature_login_with_qr: "Sign in with QR code" on the welcome page, so a new
+# browser can be linked from a signed-in device. Linking a phone from the
+# desktop (Settings → Sessions → Link new device) needs no flag; it shows
+# when the server offers MSC4108 and the session has cross-signing.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: element-config
+  namespace: matrix
+data:
+  config.json: |
+    {
+      "default_server_config": {
+        "m.homeserver": {
+          "base_url": "https://matrix.renner.uno",
+          "server_name": "matrix.renner.uno"
+        }
+      },
+      "disable_custom_urls": true,
+      "disable_guests": true,
+      "disable_3pid_login": true,
+      "default_country_code": "DE",
+      "room_directory": { "servers": ["matrix.renner.uno"] },
+      "integrations_ui_url": "",
+      "integrations_rest_url": "",
+      "integrations_widgets_urls": [],
+      "bug_report_endpoint_url": "",
+      "show_labs_settings": false,
+      "features": {
+        "feature_login_with_qr": true
+      },
+      "setting_defaults": {
+        "UIFeature.registration": false,
+        "UIFeature.passwordReset": false,
+        "UIFeature.thirdPartyId": false,
+        "UIFeature.identityServer": false,
+        "UIFeature.deactivate": false
+      }
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: element-web
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: element-config
+  labels:
+    app.kubernetes.io/name: element-web
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: element-web
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: element-web
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 101
+        runAsGroup: 101
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: element-web
+          image: ghcr.io/element-hq/element-web:v1.12.30@sha256:3e7dbd4424de9e11c812bc740acf2725b2abe66ea0d0802023b56fd85d3fa09a
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # Unprivileged port: the image runs as nginx, not root.
+            - name: ELEMENT_WEB_PORT
+              value: "8080"
+          ports:
+            - name: http
+              containerPort: 8080
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              memory: 128Mi
+          readinessProbe:
+            httpGet:
+              path: /config.json
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /config.json
+              port: http
+            periodSeconds: 30
+          volumeMounts:
+            # The entrypoint copies /app/config*.json to /tmp and serves it
+            # from there, so the ConfigMap lands at the source path.
+            - name: config
+              mountPath: /app/config.json
+              subPath: config.json
+              readOnly: true
+            # Writable paths for a read-only root: the entrypoint renders the
+            # nginx template into conf.d; pid and temp files go to /tmp.
+            - name: conf-d
+              mountPath: /etc/nginx/conf.d
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: element-config
+        - name: conf-d
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: element-web
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: element-web
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
@@ -0,0 +1,26 @@
+# OpenBao homelab/apps/matrix (seeded by Hermes):
+#   owner_user                 localpart of the owner account (k.renner)
+#   owner_password             the owner's password (MAS password login)
+#   registration_shared_secret random; Synapse's macaroon key fallback
+#   mas_matrix_secret          random; MAS <-> Synapse shared secret
+#   mas_encryption             32 bytes hex; MAS encrypts tokens at rest
+#   mas_rsa_key, mas_ec_key    PEM private keys; MAS signs its tokens
+#
+# The owner reads the login from OpenBao; it never appears in a comment, a
+# manifest or a log. owner_password is the password the account was created
+# with; changing it in OpenBao does not change the account password.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: synapse-secrets
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: synapse-secrets
+  dataFrom:
+    - extract:
+        key: apps/matrix
@@ -0,0 +1,15 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hetzner-s3-creds
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hetzner-s3-creds
+  dataFrom:
+    - extract:
+        key: infrastructure/hetzner-s3
@@ -0,0 +1,161 @@
+# Public (REN-663): each Ingress carries dns.renner.uno/public, so
+# netbird-service-sync publishes its host through the NetBird proxy
+# (Cloudflare wildcard -> proxy.renner.uno -> TLS passthrough to Traefik) and
+# external-dns-unifi skips it; LAN clients follow the public answer and the UDM
+# hairpins back in. Same path as audiobooks/immich. Federation stays off.
+#
+# Synapse: only the client-facing paths. /_synapse/admin stays off the
+# Ingress (in-cluster only); /_matrix/federation and /_matrix/key are not
+# served by the listener anyway.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: synapse
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - matrix.renner.uno
+      secretName: matrix-tls
+  rules:
+    - host: matrix.renner.uno
+      http:
+        paths:
+          # Legacy login/logout/refresh go to MAS (its compat layer). Longer
+          # paths win in Traefik, so these override /_matrix/client below.
+          - path: /_matrix/client/v3/login
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/v3/logout
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/v3/refresh
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/login
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/logout
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/refresh
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /_matrix/media
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /_synapse/client
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /.well-known/matrix/client
+            pathType: Exact
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: element-web
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - element.renner.uno
+      secretName: element-tls
+  rules:
+    - host: element.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: element-web
+                port:
+                  number: 80
+---
+# MAS: login pages, OAuth endpoints and account management.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: mas
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - account.matrix.renner.uno
+      secretName: mas-tls
+  rules:
+    - host: account.matrix.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - externalsecret.yaml
+  - synapse.yaml
+  - mas.yaml
+  - element.yaml
+  - bridge.yaml
+  - ingress.yaml
+  # mx-pg CNPG Postgres + offsite Barman backup. The media store and signing
+  # key are on synapse-data, backed up by Velero (matrix-daily).
+  - cluster.yaml
+  - objectstore.yaml
+  - scheduled-backup.yaml
+  - hetzner-creds-externalsecret.yaml
@@ -0,0 +1,229 @@
+# Matrix Authentication Service (MAS) for matrix.renner.uno (REN-567).
+#
+# Synapse delegates all logins to MAS (matrix_authentication_service in
+# synapse.yaml). Why: Element X only signs in with a password or through
+# MAS. MAS gives it a one-tap passkey login through Authentik
+# (upstream_oauth2 below) and makes QR sign-in possible.
+#
+# Accounts: the owner account was moved from Synapse into MAS by syn2mas
+# (the migrate-to-mas init container in synapse.yaml), with its password
+# hash and sessions. The first Authentik login links to it by localpart
+# (on_conflict: add). The Authentik application admits only the Matrix
+# Users group, so no one else can create an account through it. Password
+# login stays on as a fallback; self-registration and recovery are off.
+#
+# Secrets: OpenBao homelab/apps/matrix (synapse-secrets). The database is
+# `mas` in mx-pg (Barman backup); MAS keeps no other state.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mas-config
+  namespace: matrix
+data:
+  config.yaml: |
+    http:
+      public_base: https://account.matrix.renner.uno/
+      listeners:
+        - name: web
+          resources:
+            - name: discovery
+            - name: human
+            - name: oauth
+            - name: compat
+            - name: graphql
+            - name: assets
+          binds:
+            - address: "[::]:8080"
+        - name: internal
+          resources:
+            - name: health
+          binds:
+            - address: "[::]:8081"
+
+    # Password from the mounted mx-pg-app Secret; nothing in this file.
+    database:
+      host: mx-pg-rw
+      port: 5432
+      username: synapse
+      password_file: /pg/password
+      database: mas
+
+    matrix:
+      kind: synapse
+      homeserver: matrix.renner.uno
+      endpoint: http://synapse:8008/
+      secret_file: /secrets/mas_matrix_secret
+
+    secrets:
+      encryption_file: /secrets/mas_encryption
+      keys:
+        - key_file: /secrets/mas_rsa_key
+        - key_file: /secrets/mas_ec_key
+
+    passwords:
+      enabled: true
+      # Version 1 must be bcrypt with no pepper: syn2mas imports Synapse's
+      # bcrypt hashes under it. New passwords are hashed with argon2id.
+      schemes:
+        - version: 1
+          algorithm: bcrypt
+        - version: 2
+          algorithm: argon2id
+      minimum_complexity: 3
+
+    account:
+      password_registration_enabled: false
+      password_recovery_enabled: false
+      email_change_allowed: false
+      login_with_email_allowed: false
+      account_deactivation_allowed: false
+
+    # No mail server; MAS sends nothing.
+    email:
+      transport: blackhole
+
+    upstream_oauth2:
+      providers:
+        - id: 01M4480KWZGN9HBNB5SSNGT2M1
+          human_name: Authentik (passkey)
+          issuer: https://auth.renner.uno/application/o/matrix/
+          client_id: matrix-mas
+          # Public client + PKCE: no shared client secret to keep in sync.
+          token_endpoint_auth_method: none
+          pkce_method: always
+          scope: "openid profile email"
+          claims_imports:
+            # Authentik username = Matrix localpart. `add` links the first
+            # login to the migrated account of the same name.
+            localpart:
+              action: require
+              template: "{{ user.preferred_username }}"
+              on_conflict: add
+            displayname:
+              action: suggest
+              template: "{{ user.name }}"
+            email:
+              action: ignore
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mas
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: mas-config
+    secret.reloader.stakater.com/reload: synapse-secrets
+  labels:
+    app.kubernetes.io/name: mas
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mas
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mas
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: mas
+          image: ghcr.io/element-hq/matrix-authentication-service:1.26.0@sha256:e089f1048a1d4a9a492ed17b9fe759100f1bd619407b001f5927928d88b780c4
+          # The server applies database migrations and syncs the upstream
+          # providers from the config on start.
+          args: ["server", "--config", "/config/config.yaml"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          ports:
+            - name: http
+              containerPort: 8080
+            - name: internal
+              containerPort: 8081
+          resources:
+            requests:
+              cpu: 20m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: internal
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: internal
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 4
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: pg
+              mountPath: /pg
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: mas-config
+        - name: secrets
+          secret:
+            secretName: synapse-secrets
+            items:
+              - key: mas_matrix_secret
+                path: mas_matrix_secret
+              - key: mas_encryption
+                path: mas_encryption
+              - key: mas_rsa_key
+                path: mas_rsa_key
+              - key: mas_ec_key
+                path: mas_ec_key
+        - name: pg
+          secret:
+            secretName: mx-pg-app
+            items:
+              - key: password
+                path: password
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mas
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: mas
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# `mas` database in mx-pg, owned by the synapse role (one app, one
+# namespace), so MAS reuses mx-pg-app. Barman backs it up with synapse.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: mx-pg-mas
+  namespace: matrix
+spec:
+  cluster:
+    name: mx-pg
+  name: mas
+  owner: synapse
@@ -0,0 +1,30 @@
+---
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: mx-pg-store
+  namespace: matrix
+spec:
+  configuration:
+    # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG
+    # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud
+    # (plugin #646), so at-rest encryption is unavailable; transit is TLS.
+    destinationPath: s3://renner-uno-k8s-cnpg/mx-pg
+    endpointURL: https://fsn1.your-objectstorage.com
+    s3Credentials:
+      accessKeyId:
+        name: hetzner-s3-creds
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: hetzner-s3-creds
+        key: ACCESS_SECRET_KEY
+      # Region is required (REGION=fsn1) — without it boto3's HeadBucket
+      # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s.
+      region:
+        name: hetzner-s3-creds
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  retentionPolicy: "30d"
@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: mx-pg-daily
+  namespace: matrix
+spec:
+  # 03:00 local, same slot as the other CNPG clusters (the Velero PVC
+  # schedule for this namespace runs later, at 09:45).
+  schedule: "0 0 3 * * *"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: mx-pg
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
@@ -0,0 +1,324 @@
+# Synapse homeserver for matrix.renner.uno (REN-567).
+#
+# Private: federation is off at three layers — federation_domain_whitelist
+# is empty, no listener serves the federation/keys resources, and the
+# namespace egress policy lets Synapse reach nothing outside the cluster
+# except the push gateway. Logins and accounts are delegated to MAS
+# (mas.yaml); registration is closed.
+#
+# server_name is permanent: it is part of every user and room ID, and
+# cannot be changed without a new, empty server. matrix.renner.uno needs no
+# .well-known delegation from the apex, which is the WordPress site.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: synapse-config
+  namespace: matrix
+data:
+  homeserver.yaml: |
+    server_name: matrix.renner.uno
+    public_baseurl: https://matrix.renner.uno/
+    # Lets Element X and other clients discover the homeserver from the
+    # server name. No server well-known: there is no federation to point at.
+    serve_client_wellknown: true
+    pid_file: /tmp/homeserver.pid
+    report_stats: false
+
+    listeners:
+      # Client API only (includes media and /.well-known). Leaving out the
+      # `federation` and `keys` resources means this server does not answer
+      # federation at all.
+      - port: 8008
+        type: http
+        tls: false
+        bind_addresses: ["::"]
+        x_forwarded: true
+        resources:
+          - names: [client]
+            compress: false
+
+    # The password comes from PGPASSWORD (libpq reads it), so no credential
+    # is written into this file.
+    database:
+      name: psycopg2
+      args:
+        host: mx-pg-rw
+        port: 5432
+        dbname: synapse
+        user: synapse
+        cp_min: 2
+        cp_max: 10
+
+    # Signing key is generated on first start if missing; it lives on the
+    # data PVC next to the media store, so Velero backs up both.
+    signing_key_path: /data/signing.key
+    media_store_path: /data/media_store
+    max_upload_size: 100M
+    # Also the macaroon key: Synapse falls back to it when
+    # macaroon_secret_key is unset, so rotating it logs every session out.
+    registration_shared_secret_path: /secrets/registration_shared_secret
+
+    # MAS handles login, sessions and accounts. Synapse asks it to check
+    # every access token; password login here must be off.
+    matrix_authentication_service:
+      enabled: true
+      endpoint: http://mas:8080/
+      secret_path: /secrets/mas_matrix_secret
+    password_config:
+      enabled: false
+
+    # Sign in with QR code (Element X): the rendezvous channel between the
+    # new and the signed-in device. Both require MAS. `open` lets the new,
+    # not-yet-signed-in device create the channel; the server is LAN-only.
+    experimental_features:
+      msc4108_enabled: true
+      msc4388_mode: open
+
+    # Federation off.
+    federation_domain_whitelist: []
+    trusted_key_servers: []
+    allow_public_rooms_over_federation: false
+
+    enable_registration: false
+    allow_guest_access: false
+    url_preview_enabled: false
+    user_directory:
+      search_all_users: true
+
+    # Phones get pushes through the Element push gateway on matrix.org.
+    # Only event and room IDs leave the cluster, never message text.
+    push:
+      include_content: false
+
+    # Appservice registrations (bridges), one file each, mounted from a
+    # Secret. matrix-bridge: bridge.yaml (REN-587).
+    app_service_config_files:
+      - /appservices/matrix-bridge/registration.yaml
+---
+# Media store and the server signing key. Not reconstructable: lost media
+# is gone, and a lost signing key invalidates the server's past signatures.
+# Backed up by Velero (matrix-daily); the database by Barman (mx-pg).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: synapse-data
+  namespace: matrix
+  labels:
+    app.kubernetes.io/name: synapse
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ceph-block
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: synapse
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: synapse-config,mas-config
+    secret.reloader.stakater.com/reload: synapse-secrets,matrix-bridge-tokens
+  labels:
+    app.kubernetes.io/name: synapse
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: synapse
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: synapse
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 991
+        runAsGroup: 991
+        fsGroup: 991
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # One-time move of the Synapse accounts (owner, password hash,
+        # sessions) into the MAS database. syn2mas needs Synapse offline;
+        # this runs before Synapse starts, and Recreate stops the old pod
+        # first. The marker on the data PVC makes later starts skip it.
+        # Debug image: the plain one has no shell for the marker check.
+        - name: migrate-to-mas
+          image: ghcr.io/element-hq/matrix-authentication-service:1.26.0-debug@sha256:46f7480f5ea4243ccb6fd2c447ea859e0230c8b3c0e676ee285d4ae6ca344ee3
+          command:
+            - /busybox/sh
+            - -ec
+            - |
+              if [ -f /data/.syn2mas-done ]; then
+                echo "syn2mas already done"; exit 0
+              fi
+              mas-cli syn2mas migrate \
+                --config /mas-config/config.yaml \
+                --synapse-config /config/homeserver.yaml \
+                --synapse-database-uri postgresql:
+              touch /data/.syn2mas-done
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # libpq-style variables for the Synapse database; the URI above
+            # is empty on purpose.
+            - name: PGHOST
+              value: mx-pg-rw
+            - name: PGUSER
+              value: synapse
+            - name: PGDATABASE
+              value: synapse
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mx-pg-app
+                  key: password
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: mas-config
+              mountPath: /mas-config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: pg
+              mountPath: /pg
+              readOnly: true
+            - name: data
+              mountPath: /data
+      containers:
+        - name: synapse
+          image: ghcr.io/element-hq/synapse:v1.162.0@sha256:6b84a7bbac36f080b2d2e51e0289cf1b08b349598ea44a558df38d558f2c2311
+          # Bypass /start.py: it expects to run as root and template a config
+          # from env. Default mode is generate-missing-and-run, which creates
+          # the signing key on first start.
+          command:
+            - python
+            - -m
+            - synapse.app.homeserver
+            - --config-path
+            - /config/homeserver.yaml
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mx-pg-app
+                  key: password
+          ports:
+            - name: http
+              containerPort: 8008
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+          # First start runs every schema migration against an empty
+          # database; give it up to 10 min.
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+            failureThreshold: 60
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 4
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: data
+              mountPath: /data
+            - name: tmp
+              mountPath: /tmp
+            - name: appservice-matrix-bridge
+              mountPath: /appservices/matrix-bridge
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: synapse-config
+        - name: secrets
+          secret:
+            secretName: synapse-secrets
+            items:
+              - key: registration_shared_secret
+                path: registration_shared_secret
+              - key: mas_matrix_secret
+                path: mas_matrix_secret
+              - key: mas_encryption
+                path: mas_encryption
+              - key: mas_rsa_key
+                path: mas_rsa_key
+              - key: mas_ec_key
+                path: mas_ec_key
+        - name: mas-config
+          configMap:
+            name: mas-config
+        - name: pg
+          secret:
+            secretName: mx-pg-app
+            items:
+              - key: password
+                path: password
+        - name: data
+          persistentVolumeClaim:
+            claimName: synapse-data
+        - name: tmp
+          emptyDir: {}
+        - name: appservice-matrix-bridge
+          secret:
+            secretName: matrix-bridge-tokens
+            items:
+              - key: registration.yaml
+                path: registration.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: synapse
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: synapse
+  ports:
+    - name: http
+      port: 8008
+      targetPort: http
@@ -1,71 +1,4 @@
 ---
-# The agent metrics Service, declared here because the chart does not render
-# one under our values (REN-147).
-#
-# cilium's templates/cilium-agent/service.yaml is gated twice:
-#
-#   {{- if and .Values.agent (not .Values.preflight.enabled) .Values.prometheus.enabled }}
-#   {{- if (or .Values.prometheus.serviceMonitor.enabled .Values.prometheus.metricsService) }}
-#
-# infrastructure/cilium/helmrelease.yaml sets `prometheus.enabled: true` and
-# `prometheus.serviceMonitor.enabled: false`, and leaves `metricsService` at its
-# default of false. First gate passes, second does not. The chart's fallback
-# branch is envoy-only and needs envoy in the agent pod, but envoy runs as its
-# own DaemonSet here — so no `cilium-agent` Service was rendered at all.
-#
-# The consequence was silent: agents exported metrics on :9962 all along, the
-# ServiceMonitor below selected a Service that did not exist, and the scrape
-# pool sat at 0 active / 46 dropped targets. `cilium_*` had zero series, so
-# CiliumBPFMapPressureHigh and CiliumBPFMapFull in prometheusrule-cilium.yaml
-# evaluated an empty vector forever and could never fire.
-#
-# Why a Service here and not `prometheus.metricsService: true` in the
-# HelmRelease: that flag is the chart-native fix and its rendered diff is
-# exactly this one object (it appears in no other template — not the
-# ConfigMap, not the DaemonSet). But setting it changes the HelmRelease values,
-# which makes helm-controller run `helm upgrade` on the CNI release, and that
-# three-way-merges against live objects. PROPOSAL-cilium-single-owner.md
-# records that `cilium-config` has carried a `kubectl-patch` field manager; if
-# any of that drift is still live, the upgrade reconciles it, and the
-# `configmap.reloader.stakater.com/reload: cilium-config` annotation on the
-# HelmRelease then rolls every agent node by node. A Service applied by the
-# apps Kustomization cannot do that. If the drift is ever proven gone, delete
-# this object and set `prometheus.metricsService: true` instead.
-#
-# Name and selector follow authentik-outpost-servicemonitors.yaml: a distinct
-# name plus `app.kubernetes.io/managed-by: homelab`, so flipping either chart
-# flag later renders the chart's own `cilium-agent` Service without an
-# ownership conflict and without double-scraping the same endpoints.
-apiVersion: v1
-kind: Service
-metadata:
-  name: cilium-agent-metrics
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/managed-by: homelab
-    app.kubernetes.io/name: cilium-agent
-    app.kubernetes.io/part-of: cilium
-    # Copied onto every series by the ServiceMonitor's targetLabels below.
-    # The dashboard the chart ships queries
-    # cilium_bpf_map_pressure{k8s_app="cilium"}, so this label is load-bearing.
-    k8s-app: cilium
-spec:
-  type: ClusterIP
-  # Headless on purpose: agent metrics are per-node and every endpoint has to
-  # be scraped. A ClusterIP would hand Prometheus one random agent per scrape.
-  clusterIP: None
-  # Agent pods only. cilium-envoy carries k8s-app: cilium-envoy and exposes
-  # 9964, so it cannot match.
-  selector:
-    k8s-app: cilium
-  ports:
-    # Named and numbered exactly as the chart's own Service, so the
-    # ServiceMonitor below stays a field-for-field copy of the chart's.
-    - name: metrics
-      port: 9962
-      targetPort: prometheus
-      protocol: TCP
---
 # Cilium's agent ServiceMonitor, declared here rather than by the chart.
 #
 # Reproduces what cilium renders with prometheus.serviceMonitor.enabled: same
@@ -89,11 +22,13 @@ metadata:
    app.kubernetes.io/part-of: cilium
 spec:
  selector:
-    # Pinned to the Service above. Without managed-by this would also match a
-    # chart-rendered `cilium-agent` Service and scrape the same endpoints twice.
+    # The headless `cilium-agent` Service that the HelmRelease renders with
+    # prometheus.metricsService (REN-166). It carries k8s-app: cilium, which
+    # targetLabels below copies onto every series; the dashboard the chart
+    # ships queries cilium_bpf_map_pressure{k8s_app="cilium"}.
    matchLabels:
-      app.kubernetes.io/managed-by: homelab
      app.kubernetes.io/name: cilium-agent
+      app.kubernetes.io/part-of: cilium
  namespaceSelector:
    matchNames:
      - kube-system
@@ -29,6 +29,17 @@
 # been dropped and the metric would have stayed absent. The matching ingress rule
 # is added to infrastructure/network-policies/external-secrets-ns.yaml in the
 # same pull request. Without it this file is decoration.
+#
+# NO TLS AND NO AUTH ON THIS SCRAPE, and that is the chart's posture rather than
+# an omission here. In 2.11.0 `metrics.listen.secure.enabled` and
+# `metrics.listen.auth.enabled` are both false in the chart's values.yaml and
+# infrastructure/external-secrets/helmrelease.yaml overrides neither, so the
+# controller serves /metrics over plain HTTP to anything that can reach :8080.
+# Hence no `scheme`, no `tlsConfig` and no `bearerTokenSecret` below: adding
+# them would break the scrape, not harden it. What limits exposure is the
+# network policy — a single port from a single namespace — so turning on
+# `secure.enabled` here would mean a Helm upgrade of ESO to close a hole the
+# CiliumNetworkPolicy already closes.
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -77,4 +77,7 @@ resources:
  # customResourceState block in apps/monitoring/helmrelease-prometheus.yaml —
  # these two files must land and be reverted together.
  - prometheusrule-flux.yaml
+  # Gitea's data volume. It filled twice (#336, REN-580) and the chart's
+  # KubePersistentVolumeFillingUp warned 33 minutes before the second (REN-583).
+  - prometheusrule-gitea.yaml
  - probes-blackbox.yaml
@@ -261,6 +261,32 @@ spec:
        # at 2b71dda1, and the tested one-line deletion for this target alone
        # is the second command listed there (20 -> 19 targets).
        - http://paperless.paperless.svc.cluster.local:80/accounts/login/
+        # anchor: shelly-manager-app-health
+        # APP-HEALTH probe for shelly-manager (REN-531/REN-533). Same rule as
+        # the blocks above (monitoring-vantage-point-rule): shelly.renner.uno
+        # sits behind the shelly-manager-gate outpost, so only the in-cluster
+        # Service can tell a dead app from a live gate.
+        #
+        # /health is unauthenticated and answers a plain 200 — not a 3xx —
+        # per the app's own test (shelly-manager repo,
+        # AuthenticationTests.Health_IsUnauthenticated); it is also the pod's
+        # kubelet liveness/readiness path
+        # (../home-assistant/shelly-manager-deployment.yaml). AllowedHosts is
+        # "*", so the cluster-local Host header is accepted. Service :80 ->
+        # container :8080 (../home-assistant/shelly-manager-service.yaml).
+        #
+        # NO CiliumNetworkPolicy change: the home-assistant default-deny
+        # admits the whole monitoring namespace on every port
+        # (../../infrastructure/network-policies/home-assistant-ns.yaml,
+        # "Prometheus scrape" rule). If that rule is ever narrowed to
+        # Prometheus alone, add a blackbox-exporter grant on :8080 in a
+        # DEDICATED policy selecting shelly-manager — not on the {} selector,
+        # where :8080 is shared by three apps. The EGRESS paragraph above
+        # applies too: a future monitoring egress block must list
+        # home-assistant.
+        #
+        # REVERT: delete the target line below.
+        - http://shelly-manager.home-assistant.svc.cluster.local:80/health
 # WAN-path probes of the public edge BY PROXY IP. The name-based
 # home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway)
 # and never leaves the LAN — the vantage-point blind spot that let the
@@ -11,6 +11,31 @@ spec:
  groups:
    - name: disk-wear.rules
      rules:
+        # The drives carrying Talos's EPHEMERAL volume (/var), as (node, device)
+        # in smartctl-exporter's labels. EPHEMERAL sits on retired Ceph drives
+        # whose contents rebuild, and the owner's call is that they run to
+        # failure rather than be replaced on wear (REN-419, REN-420). So the two
+        # wear alerts below skip them. The failure alerts — spare, critical
+        # warning, media errors — still cover them: run to failure still means
+        # knowing when it failed.
+        #
+        # Resolved from node-exporter's /var mount, not a list of serials, so it
+        # follows a drive swap without an edit. If the join stops matching (a
+        # relabel, an encrypted /var on a dm device) the series is empty, and
+        # the wear alerts return for EPHEMERAL drives: it fails loud, not
+        # silent.
+        - record: node_device:nvme_ephemeral_volume:info
+          expr: >-
+            count by (node, device) (
+              label_replace(
+                label_replace(
+                  node_filesystem_size_bytes{mountpoint="/var"}
+                    * on (instance) group_left (nodename) node_uname_info,
+                  "node", "$1", "nodename", "(.+)"
+                ),
+                "device", "$1", "device", "/dev/(nvme[0-9]+)n[0-9]+(p[0-9]+)?"
+              )
+            )
        # Wear percentages are self-reported: 100 means the rated endurance is
        # spent, not that the drive stops.
        #
@@ -45,6 +70,7 @@ spec:
              )
            ) < 180
            and smartctl_device_percentage_used >= 80
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
          # Long, because a projection off a 7d rate should not react to a day.
          for: 6h
          labels:
@@ -55,13 +81,17 @@ spec:
              The drive is past 80% of its rated endurance and the write rate
              of the last week would spend the rest within six months. This is
              a lead-time warning, not a fault: decide whether to replace the
-              drive or move what is writing to it. The Disk Health dashboard
+              drive or move what is writing to it. Drives carrying EPHEMERAL
+              are excluded; they run to failure. The Disk Health dashboard
              shows which drives carry which role.
        # Kept as a flat threshold on purpose: past rated endurance the drive
        # runs outside what the manufacturer stands behind regardless of how
-        # fast it got there, and the alert clears by replacing it.
+        # fast it got there, and the alert clears by replacing it. EPHEMERAL
+        # drives are excluded for the same reason as above.
        - alert: NvmeDriveWearCritical
-          expr: smartctl_device_percentage_used >= 95
+          expr: >-
+            smartctl_device_percentage_used >= 95
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
          for: 1h
          labels:
            severity: critical
@@ -0,0 +1,42 @@
+# Gitea's data volume (data-gitea-0) running low (REN-583).
+#
+# It filled twice: 2026-07-22 (#336) and 2026-10-03 (REN-580), both times
+# because the package registry grew faster than cleanup_packages reclaimed it.
+# A full volume fails image pushes (HTTP 500) and Gitea's own SQLite writes.
+#
+# The chart's KubePersistentVolumeFillingUp did not give usable lead time on
+# REN-580: its warning needs under 15% free AND a 4-day linear projection to
+# zero AND one hour of both, so it fired at 20:26Z; the volume was full at
+# 20:59Z. Growth here comes in steps (a run of image builds added 8 GiB in a
+# few hours that day), which a linear projection lags.
+#
+# So this is a plain floor: under 25% free (15 GiB at 60Gi) for 15m. That is
+# about two of the 8 GiB steps seen on REN-580, and time for someone to run the
+# package cleanup or grow pvc.yaml in apps/gitea.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: gitea
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: gitea.rules
+      rules:
+        - alert: GiteaDataVolumeLow
+          expr: >-
+            kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}
+              / kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}
+            < 0.25
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gitea data volume has {{ $value | humanizePercentage }} free"
+            description: >-
+              data-gitea-0 is under 25% free. When it fills, image pushes fail
+              with HTTP 500 and Gitea cannot write its database. Run the
+              cleanup_packages cron in Gitea's site admin and delete old
+              package versions, or grow the claim in apps/gitea/pvc.yaml
+              (ceph-block resizes online).
@@ -30,6 +30,8 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
 | `external-secrets_test.yaml` | Hand-written suite for `prometheusrule-external-secrets.yaml` (REN-103). |
 | `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). |
 | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
+| `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
+| `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |

 ## Adding a suite
@@ -0,0 +1,124 @@
+# promtool test rules for the EPHEMERAL exclusion in prometheusrule-disk-wear.yaml
+# (REN-419).
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh, which strips `spec:` from the manifest into a
+# mirrored tree — hence `../` below, resolved from this file's directory. See
+# README.md here. The negative controls for this suite are the `disk-wear-*`
+# entries in mutations.txt.
+#
+# WHAT IT IS FOR. The two wear alerts skip the drive that carries /var, found
+# by joining node-exporter's /var mount onto smartctl-exporter's (node, device).
+# Each case puts an EPHEMERAL drive and a non-EPHEMERAL drive side by side with
+# identical SMART data, so the only thing that can separate them is the join.
+#
+# Fixture shape: smartctl-exporter labels drives `nvme0`, `nvme1` and carries
+# `node` from the ServiceMonitor relabel; node-exporter carries `instance` and
+# reports /var on the partition, `/dev/nvme1n1p1`.
+rule_files:
+  - ../prometheusrule-disk-wear.yaml
+evaluation_interval: 1m
+tests:
+  # A. Projection. k8s-1 nvme0 and nvme1 wear identically: 89%, 1.5e9 bytes
+  #    per 5m (4.32e11 a day), 9e14 lifetime bytes at 192h. That is 0.04272 % a
+  #    day, 6 % to go, 140 days — under 180. Only nvme0 may fire; nvme1
+  #    carries /var. Neither is at 95%, so the critical alert stays quiet.
+  #
+  #    Sampled at 5m, not 1h: the instant selectors look back 5m, so hourly
+  #    samples leave the series absent for most of each hour and the 6h `for`
+  #    never completes. Not 1m either: a 7d rate over 1m samples exceeds
+  #    promtool's query sample limit.
+  - interval: 5m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '89+0x2320'
+      - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '896544000000000+1500000000x2320'
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '89+0x2320'
+      - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '896544000000000+1500000000x2320'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}'
+        values: '1000000000000+0x2320'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x2320'
+    alert_rule_test:
+      - eval_time: 192h
+        alertname: NvmeDriveWearProjectedExhaustion
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              node: k8s-1
+              device: nvme0
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme0 on 10.0.0.1:9633 projects to 95% wear in 140 days"
+              description: "The drive is past 80% of its rated endurance and the write rate of the last week would spend the rest within six months. This is a lead-time warning, not a fault: decide whether to replace the drive or move what is writing to it. Drives carrying EPHEMERAL are excluded; they run to failure. The Disk Health dashboard shows which drives carry which role."
+      - eval_time: 192h
+        alertname: NvmeDriveWearCritical
+        exp_alerts: []
+
+  # B. Critical, two nodes. Every drive is at 96%. On k8s-1 /var is nvme1; on
+  #    k8s-2 it is nvme2, which reports no SMART here. So k8s-1 nvme1 is the only
+  #    one excluded, and k8s-2 nvme1 must still fire — the exclusion is per
+  #    node, not per device name.
+  - interval: 1m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'smartctl_device_percentage_used{node="k8s-2",device="nvme1",instance="10.0.0.2:9633"}'
+        values: '96+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.2:9100",mountpoint="/var",device="/dev/nvme2n1p1",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.2:9100",nodename="k8s-2"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: NvmeDriveWearCritical
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              node: k8s-1
+              device: nvme0
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme0 on 10.0.0.1:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
+          - exp_labels:
+              severity: critical
+              node: k8s-2
+              device: nvme1
+              instance: 10.0.0.2:9633
+            exp_annotations:
+              summary: "NVMe nvme1 on 10.0.0.2:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
+
+  # C. Fails loud. /var is on a device the join cannot map to an NVMe name (an
+  #    encrypted volume on dm-0). The exclusion then matches nothing and the
+  #    EPHEMERAL drive alerts again, which is the intended direction to fail.
+  - interval: 1m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/dm-0",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: NvmeDriveWearCritical
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              node: k8s-1
+              device: nvme1
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme1 on 10.0.0.1:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
@@ -0,0 +1,46 @@
+# Suite for prometheusrule-gitea.yaml (REN-583). Capacity 60 GiB; free space
+# in GiB. 15 GiB free is exactly 25% and must stay silent.
+rule_files:
+  - ../prometheusrule-gitea.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # 20 GiB free, then 14 GiB free from minute 10 on: pending at 10m, fires
+  # at 25m (15m later), not before.
+  - interval: 1m
+    input_series:
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '21474836480+0x9 15032385536+0x30'
+    alert_rule_test:
+      - eval_time: 24m
+        alertname: GiteaDataVolumeLow
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: GiteaDataVolumeLow
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              namespace: gitea
+              persistentvolumeclaim: data-gitea-0
+            exp_annotations:
+              summary: "Gitea data volume has 23.33% free"
+              description: "data-gitea-0 is under 25% free. When it fills, image pushes fail with HTTP 500 and Gitea cannot write its database. Run the cleanup_packages cron in Gitea's site admin and delete old package versions, or grow the claim in apps/gitea/pvc.yaml (ceph-block resizes online)."
+
+  # Exactly 25% free stays silent; another claim at 1% free is not this alert.
+  - interval: 1m
+    input_series:
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '16106127360+0x40'
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="other",persistentvolumeclaim="data-other-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="other",persistentvolumeclaim="data-other-0"}'
+        values: '644245094+0x40'
+    alert_rule_test:
+      - eval_time: 40m
+        alertname: GiteaDataVolumeLow
+        exp_alerts: []
@@ -149,3 +149,26 @@ retention-no-size-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometh
 retention-ratio-0.9|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0.9/
 retention-lookback-1d|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/)\[7d:1h\]$/)[1d:1h]/
 retention-cap-at-peak-time|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/\/ max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})$/\/ min_over_time(max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})[7d:1h])/
+
+# prometheusrule-disk-wear.yaml (REN-419): the EPHEMERAL exclusion on the two
+# wear alerts. Detected by the cases in disk-wear_test.yaml.
+#
+# Gate inverted on the projection: the EPHEMERAL drive alone fires.
+# Detected by: case A.
+disk-wear-projection-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearProjectedExhaustion/,/for: 6h/s/unless on (node, device)/and on (node, device)/
+# Gate inverted on the critical rule.
+# Detected by: case B.
+disk-wear-critical-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/and on (node, device)/
+# Matched on device name only: k8s-1's /var on nvme1 then silences nvme1 on
+# every node.
+# Detected by: case B, k8s-2 nvme1.
+disk-wear-critical-device-only|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/unless on (device)/
+# Partition suffix not stripped: /dev/nvme1n1p1 no longer maps to nvme1 and the
+# exclusion matches nothing.
+# Detected by: cases A and B.
+disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|s/n\[0-9\]+(p\[0-9\]+)?"/n[0-9]+"/
+
+# prometheusrule-gitea.yaml (REN-583). Detected by the 24m/25m pair and the
+# exactly-25% case in gitea_test.yaml.
+gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/
+gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/
@@ -17,7 +17,7 @@ spec:
  chart:
    spec:
      chart: kube-prometheus-stack
-      version: "91.7.0"
+      version: "91.9.0"
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
@@ -10,6 +10,7 @@ resources:
  - loki-basic-auth-externalsecret.yaml
  - loki-basic-auth-middleware.yaml
  - loki-ingress.yaml
+  - loki-read-gateway.yaml
  - alertmanager-gate-service.yaml
  - alertmanager-gate-ingress.yaml
  - prometheus-gate-service.yaml
@@ -18,6 +19,13 @@ resources:
  - smartctl-exporter.yaml
  - intel-gpu-exporter.yaml
  - dashboards/
+configMapGenerator:
+  # Hash suffix kept on purpose: an edit to the .conf rolls the gateway pod,
+  # which is stateless and off the ingest path.
+  - name: loki-read-gateway
+    namespace: monitoring
+    files:
+      - loki-read-gateway.conf
 # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
 # separate Flux Kustomization that dependsOn `monitoring` — otherwise the
 # server-side dry-run of the PrometheusRule fails on a fresh cluster
@@ -0,0 +1,60 @@
+# GET-only allow-list in front of Loki (REN-621, owner decision on REN-622).
+#
+# Loki runs SingleBinary with auth_enabled: false, and its HTTP port mixes
+# reads with control endpoints that also answer GET (/ingester/shutdown,
+# /flush). An RBAC `get` on services/proxy for loki:3100 would therefore be a
+# kill switch for log ingest. This server is the read-only boundary instead:
+# the paperclip read ServiceAccount may proxy to this Service only.
+#
+# Each allowed location forwards a URI built here, never the client's raw
+# request URI, so dot-segments or encoded slashes cannot reach another Loki
+# path. Anything not listed gets 403. /loki/api/v1/tail (websocket) is left
+# out on purpose. Adding a path is a new card and a new review.
+
+upstream loki {
+    server loki.monitoring.svc.cluster.local:3100;
+}
+
+server {
+    listen 8080;
+    server_tokens off;
+
+    proxy_http_version 1.1;
+    proxy_set_header Host loki;
+    proxy_read_timeout 60s;
+
+    location / {
+        return 403;
+    }
+
+    location = /healthz {
+        access_log off;
+        default_type text/plain;
+        return 200 "ok\n";
+    }
+
+    location = /loki/api/v1/query {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/query$is_args$args;
+    }
+
+    location = /loki/api/v1/query_range {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/query_range$is_args$args;
+    }
+
+    location = /loki/api/v1/labels {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/labels$is_args$args;
+    }
+
+    location = /loki/api/v1/series {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/series$is_args$args;
+    }
+
+    location ~ ^/loki/api/v1/label/(?<label_name>[A-Za-z_][A-Za-z0-9_]*)/values$ {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/label/$label_name/values$is_args$args;
+    }
+}
--- a/Show More
+++ b/Show More