fix: [alert] KubePersistentVolumeFillingUp (paperclip) (#1184 )

chore(image-automation): bump gitea.renner.uno/k.renner/shelly-manager:build.1791385485
Updates managed by image-automation-controller.
2026-10-07 15:42:50 +00:00 · 2026-10-07 15:21:31 +00:00 · 2026-10-07 15:19:42 +00:00 · 2026-10-07 15:04:18 +00:00 · 2026-10-07 14:51:31 +00:00 · 2026-10-07 13:21:31 +00:00
228 changed files with 14139 additions and 482 deletions
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -0,0 +1,66 @@
+# Paperclip agent runtime images (REN-284 / REN-286): the stock
+# agent-runtime-claude plus a tool set, one image per sandbox environment.
+# Built with kaniko like build-ci-images.yml and pushed with the same
+# ci_images_token (package read+write only, see the header there).
+#
+# Own workflow, not a step in build-ci-images.yml. A change to that file
+# rebuilds all of its images at once and fills the 24-CPU woodpecker-ci-quota;
+# on the REN-286 merge (pipeline 11306) this step then got no pod
+# ("exceeded quota") and the go image was never built. Here it runs only when
+# its own sources change, and it can be re-run alone with a deployment to
+# target `build-agent-runtime`, without rebuilding the other images.
+#
+# The name looks odd on purpose. The kubernetes sandbox plugin keeps the image
+# name and tag `agent-runtime-claude:v1` and swaps only the registry prefix
+# (`imageRegistry`), replacing host + org. So each variant needs its own
+# prefix with the same name under it:
+#   imageRegistry gitea.renner.uno/konairius/agent-runtime-go
+#   -> gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1
+# `v1` is what the plugin pulls and moves on every build; the date tag is the
+# rollback point. konairius/ because the ci_images_token can push there and
+# it is the target of the gitea_admin/* migration (#330).
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/agent-runtime-*/**"
+      - ".woodpecker/build-agent-runtime.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-agent-runtime"'
+
+steps:
+  - name: build-agent-runtime
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    directory: ci/images
+    # Context is ci/images (not the variant dir) so both Containerfiles can
+    # COPY the shared agent-runtime-common/ files.
+    # One pod, both images one after the other (kaniko --cleanup between
+    # builds), at 1 CPU so it still fits if it overlaps another pipeline.
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: "1"
+            memory: 4Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        for VARIANT in go ops; do
+          /kaniko/executor \
+            --cleanup \
+            --context . \
+            --dockerfile "agent-runtime-$VARIANT/Containerfile" \
+            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:$TAG" \
+            --destination "gitea.renner.uno/konairius/agent-runtime-$VARIANT/agent-runtime-claude:v1" \
+            || exit 1
+        done
@@ -26,8 +26,14 @@
 # 38-reference change tracked separately in #330 rather than bundled here.
 # Note this pipeline runs no LLM agent, so the exfil path that motivated the
 # other cutovers does not apply to it.
+#
+# Push builds run on master only. Without the branch filter a feature branch
+# that touches this file started all steps at once: it pushed unreviewed
+# images and used the whole woodpecker-ci-quota, so the PR checks could not
+# get a pod (REN-286).
 when:
  - event: push
+    branch: master
    path:
      - "ci/images/**"
      - "secret-rotator/**"
@@ -37,7 +43,7 @@ when:

 steps:
  - name: build-ansible
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/ansible
    depends_on: []
    # Only rebuild when THIS image's source (or the pipeline file) changed, or
@@ -65,7 +71,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest

  - name: build-k8s
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/k8s
    depends_on: []
    when:
@@ -89,7 +95,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-vmtest
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/vmtest
    depends_on: []
    when:
@@ -113,7 +119,7 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-vmtest:latest

  - name: build-tofu
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/tofu
    depends_on: []
    when:
@@ -144,7 +150,7 @@ steps:
  # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
  # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
  - name: build-secret-rotator
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: secret-rotator
    depends_on: []
    when:
@@ -195,7 +201,7 @@ steps:
  #
  # The gitea_admin/ copy stays until nothing references it any more.
  - name: build-alpine-jq
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/alpine-jq
    depends_on: []
    when:
@@ -248,7 +254,7 @@ steps:
  # note on build-alpine-jq. Anything an app needs while the cluster is still
  # being built cannot live only in that cluster's own registry.
  - name: build-cni-plugins
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/cni-plugins
    depends_on: []
    when:
@@ -290,7 +296,7 @@ steps:
  # gitea.renner.uno is not there to serve it. A cold-rebuild rehearsal left
  # paperless in Init:ErrImagePull on exactly this image.
  - name: build-alpine-openssl
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/alpine-openssl
    depends_on: []
    when:
@@ -327,7 +333,7 @@ steps:
  # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
  # builds the same way and lands in the same registry so the audit sees it.
  - name: build-heating-can
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/heating-can
    depends_on: []
    when:
@@ -354,7 +360,7 @@ steps:
  # services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
  # Runtime artifact like heating-can; same registry so the audit tracks it.
  - name: build-netbird-service-sync
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/netbird-service-sync
    depends_on: []
    when:
@@ -380,7 +386,7 @@ steps:
  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
  # artifact like heating-can; same registry so the audit tracks it.
  - name: build-bird
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/bird
    depends_on: []
    when:
@@ -0,0 +1,69 @@
+# Paperclip server image (REN-401): upstream paperclipai/paperclip at a pinned
+# master commit plus the patches in ci/images/paperclip/. Needed while the k8s
+# sandbox fix (upstream PR #13733) is not merged; see the README there for how
+# to go back to upstream nightly.
+#
+# Own workflow for the reason given in build-agent-runtime.yml: a step in
+# build-ci-images.yml runs on every change to that file and competes for the
+# woodpecker-ci-quota. Re-run alone with a deployment to target
+# `build-paperclip`.
+#
+# Tag `build.<unix-ts>` is what the Flux ImagePolicy `paperclip` sorts on
+# (image-automation/paperclip.yaml); image automation then rolls it out.
+# konairius/ and ci_images_token as for the agent-runtime images.
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/paperclip/**"
+      - ".woodpecker/build-paperclip.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-paperclip"'
+
+steps:
+  - name: prepare-source
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh ci/images/paperclip/prepare.sh build/paperclip-src
+
+  - name: build-paperclip
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
+    directory: build/paperclip-src
+    depends_on: [prepare-source]
+    # Rust runner + UI + server build in one pod. The server build alone sets
+    # a 4 GiB Node heap, so the memory limit is the LimitRange max (8Gi).
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: "1"
+            memory: 2Gi
+          limits:
+            cpu: "4"
+            memory: 8Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - . ../../ci/images/paperclip/upstream.env
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=build.$(date +%s)
+      # Same target and build args as upstream's docker.yml for nightly.
+      # --compressed-caching=false: pipeline 11717 was killed at the 8Gi limit
+      # (exit 137) in the server build; kaniko otherwise holds compressed
+      # layer tarballs in memory next to the 4 GiB Node heap.
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Dockerfile.kaniko \
+          --target production \
+          --skip-unused-stages \
+          --snapshot-mode=redo \
+          --compressed-caching=false \
+          --build-arg PAPERCLIP_BUILD_COMMIT=$PAPERCLIP_COMMIT \
+          --build-arg PAPERCLIP_BUILD_VERSION=$PAPERCLIP_VERSION \
+          --build-arg CLI_TOOLS_CACHE_EPOCH=$(date -u +%G-W%V) \
+          --destination gitea.renner.uno/konairius/paperclip:$TAG
@@ -131,7 +131,7 @@ steps:
  #    --build-arg to the resolved version (not "latest") guarantees the tag and
  #    the image content agree even if a new release lands mid-run.
  - name: build
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.19-debug@sha256:7795cec509a50a71378cc2c365f55c555ee3a05834d8eb1cd2552df5e66d44d8
    directory: ci/images/claude
    depends_on: [decide]
    environment:
@@ -17,30 +17,44 @@
 # here is a static check that either passes or fails on the checkout alone, so it
 # can gate PRs without becoming a bottleneck or a flake source.
 when:
-  # Path-filtered on purpose, unlike ci-scripts-test.yml. That pipeline is one
-  # step on one already-cached image, so running it everywhere is free; this one
-  # pulls three images and builds every kustomization in the tree, which is real
+  # Path-filtered on purpose, unlike ci-scripts-test.yml. That pipeline is two
+  # steps on already-cached images, so running it everywhere is free; this one
+  # pulls four images and builds every kustomization in the tree, which is real
  # cost to put on a docs-only PR. The filter covers everything the steps actually read:
-  # kubernetes/** for the manifests and scripts, .woodpecker/** for the lint (and
-  # so an edit to this file re-runs its own checks).
+  # kubernetes/** for the manifests, scripts and promtool suites, .woodpecker/**
+  # for the lint (and so an edit to this file re-runs its own checks), and the
+  # promtool driver script by name.
+  #
+  # That last entry is NOT covered by kubernetes/**: promtool-rules.sh lives
+  # under scripts/, so without it a PR that only edits the script would never run
+  # the check it drives. It is listed as one file rather than scripts/ci/**
+  # because the rest of that directory is CI-agent helpers with nothing to do
+  # with this pipeline, and pulling four images for them is the cost this filter
+  # exists to avoid. Their own tests run in ci-scripts-test.yml on every PR.
  - event: pull_request
    path:
      - "kubernetes/**"
      - ".woodpecker/**"
+      - "scripts/ci/promtool-rules.sh"
+      - "scripts/ci/yaml-citations.sh"

-# ADDING A THIRD DISCOVERY CHECK: kustomize-build and shellcheck both hand-roll
-# the same shape — `find`, a floor guard against empty discovery, then a
-# file-fed `while read`. Two copies is deliberate: the two traps being commented
-# around (BusyBox find lacking `-printf`, and a piped loop running in a subshell
-# where `fail` cannot propagate) are different, and both are worth spelling out
-# where they bite. A third instance is the point to stop copying and factor it
-# into a shared script under `scripts/ci/` — which is already covered by
-# ci-scripts-test.yml, so it would gain unit tests in the process.
+# THE THIRD DISCOVERY CHECK WENT TO A SCRIPT. kustomize-build and shellcheck
+# both hand-roll the same shape — `find`, a floor guard against empty discovery,
+# then a file-fed `while read`. Two copies was deliberate: the two traps being
+# commented around (BusyBox find lacking `-printf`, and a piped loop running in
+# a subshell where `fail` cannot propagate) are different, and both are worth
+# spelling out where they bite. An earlier revision of this comment said a third
+# instance would be the point to stop copying and factor it into `scripts/ci/`,
+# where it would gain unit tests — and that is what `promtool` below does. Its
+# discovery, floors and guards live in scripts/ci/promtool-rules.sh and are
+# tested by scripts/ci/test-promtool-rules.sh on every PR via
+# ci-scripts-test.yml. A FOURTH inline copy should go the same way; do not add
+# one here.
 #
-# Every step carries `depends_on: []` so all four run in PARALLEL and every
+# Every step carries `depends_on: []` so all six run in PARALLEL and every
 # failure shows in one run. Woodpecker's default is sequential, which would stop
 # at the first failure and make a contributor fix-and-repush to discover the
-# next — and these four are independent, cheap, offline checks with no ordering
+# next — and these six are independent, cheap, offline checks with no ordering
 # between them. Keep it on every step: `depends_on` is per-step, so dropping it
 # from one silently re-serialises that step behind the others.
 steps:
@@ -158,7 +172,7 @@ steps:
  # whenever it trails the deployed server. If you bump the server, bump here.
  - name: woodpecker-lint
    depends_on: [] # see note above `steps:`
-    image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
+    image: woodpeckerci/woodpecker-cli:v3.18.1-alpine@sha256:7eac3c4254befc1ba1e65269c3304c370b6032530b7386fc0c7648378fdd0516
    commands:
      - |
        set -eu
@@ -185,7 +199,7 @@ steps:
  # separately rather than one this step quietly implies is covered.
  - name: shellcheck
    depends_on: [] # see note above `steps:`
-    image: koalaman/shellcheck-alpine:v0.11.0
+    image: koalaman/shellcheck-alpine:v0.11.0@sha256:9955be09ea7f0dbf7ae942ac1f2094355bb30d96fffba0ec09f5432207544002
    commands:
      - |
        set -eu
@@ -258,3 +272,72 @@ steps:
          exit 1
        fi
        echo "ok    no digest pins under openbao/rotation/"
+
+  # Machine-check the alert rules: parse every PrometheusRule, run the committed
+  # promtool unit suites, then damage the rules on purpose and require the suites
+  # to notice. Everything it does lives in scripts/ci/promtool-rules.sh; read
+  # that file and kubernetes/apps/monitoring-rules/tests/README.md for the why.
+  #
+  # The gap it closes: the other four steps prove a PrometheusRule assembles into
+  # valid YAML and nothing more — not that an expression parses, not that a label
+  # join matches anything, not that a `for` window does what its comment claims.
+  # On REN-129 the CNPG surge rule was written `for: 2m`, which fires on six of
+  # seven measured nights and would have mailed the owner twice a night at 04:16
+  # and 04:22 local for a backlog that clears itself. Nothing here would have
+  # caught it, and the arithmetic that did catch it was hand-written from scratch
+  # twice, in two runs, and thrown away both times because there was nowhere in
+  # the repo to commit it. REN-152 retired that rule; its `for` margin is now
+  # guarded by mutations.txt:backlog-for-12m.
+  #
+  # Offline, like the rest of this pipeline: no cluster, no registry, no secrets,
+  # no network. promtool reads files and exits.
+  #
+  # IMAGE — prom/prometheus, not a registry-local ci-* image, and pinned to an
+  # exact upstream version rather than a dated tag. The dated-tag convention
+  # guarded by no-digest-pins above is about OUR images, whose digests are
+  # garbage-collected out from under a pin; this is upstream and immutable.
+  #
+  # The version must be 3.x, and that is not a preference. The cluster runs
+  # whatever kube-prometheus-stack >= 72.0.0 ships, which is Prometheus 3, and
+  # alert `for` evaluation is version-sensitive — a 2.x promtool would be
+  # checking a different evaluator than production, so the margins the suites
+  # assert would be claims about the wrong program. promtool-rules.sh asserts the
+  # major version it was handed and fails on 2.x, so a careless bump here is a
+  # red step rather than a quietly wrong check. Keep this at the latest stable
+  # 3.x, per the standing rule that any software uses the latest stable version.
+  #
+  # This image is NOT covered by image-automation (same reason as
+  # woodpecker-lint above: a Woodpecker pipeline file has no apiVersion/kind, so
+  # an $imagepolicy marker here is silently ignored — fluxcd/flux2#2826). The
+  # bump is manual.
+  #
+  # `prom/prometheus` is busybox-based and ships /bin/sh, so it can run a
+  # `commands:` block — unlike the plain woodpecker-cli tag, which cannot. Its
+  # find/grep/awk/sed are BusyBox, which is why promtool-rules.sh avoids
+  # `find -printf` and `grep -r --include`; it was verified against BusyBox
+  # 1.35.0 as well as GNU. The image also declares `USER nobody`, so the script
+  # writes its stripped mirror under mktemp and never into the workspace.
+  - name: promtool
+    depends_on: [] # see note above `steps:`
+    image: prom/prometheus:v3.15.0@sha256:efd719c99d83b060d9daefdcf00360461adf279f45ef5391f8d111892118753e
+    commands:
+      - sh scripts/ci/promtool-rules.sh
+
+  # Comments cite other YAML blocks by anchor name, never by line number
+  # (REN-183; the exact syntax is in CONTRIBUTING.md, not repeated here because
+  # an example written in this comment would itself be checked). Line numbers drift on any edit
+  # above them and nothing noticed: three drift fixes were merged by hand
+  # before this step existed. This fails on a new line-number citation and on
+  # a citation whose file or anchor marker does not exist.
+  # Existence only — it never judges whether the anchored block still says
+  # what the citing comment claims, so ordinary edits cannot turn it red.
+  #
+  # It scans every YAML file in the checkout, but this pipeline only runs on
+  # the paths in `when:` above, so an edit confined to ansible/ or esphome/ is
+  # checked on the next PR that does touch kubernetes/ or .woodpecker/.
+  # Self-test: scripts/ci/test-yaml-citations.sh, in ci-scripts-test.yml.
+  - name: yaml-citations
+    depends_on: [] # see note above `steps:`
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh scripts/ci/yaml-citations.sh
@@ -8,15 +8,45 @@
 # against adversarial PR input. Running it here fails the offending PR's checks
 # before it can merge.
 #
+# scripts/ci/test-promtool-rules.sh is the same idea for the alert-rule gate.
+# What it tests is not whether promtool evaluates PromQL — ci-manifests-test.yml
+# does that with the real binary against the real rules — but whether
+# promtool-rules.sh REFUSES TO REPORT SUCCESS when it has checked nothing: zero
+# manifests discovered, zero suites discovered, a `rule_files` path that does not
+# resolve (promtool warns, loads zero rules, and passes every assertion), a
+# mutation the suites no longer catch. Those are the states a green pipeline
+# cannot tell apart from real coverage, so they need a test that asserts the
+# failure. It stubs promtool, so it needs neither the binary nor the heavy image.
+#
+# Both are unfiltered by path on purpose: this pipeline is two steps on one
+# already-cached image, so running it everywhere is free, and a guard whose test
+# only runs when someone remembers to touch the right directory is not a guard.
+#
 # Only in homelab (scripts/ci/ lives here); the file is not deployed to the
-# mirrored repos, so it does not run on their PRs. The test is offline (no token,
-# no network, no cluster) — the ci-claude image just provides sh + jq, and it is
-# already pulled by pr-review.yml on the same event, so the runner has it cached.
+# mirrored repos, so it does not run on their PRs. All tests here are offline (no
+# token, no network, no cluster) — the ci-claude image just provides sh + jq, and
+# it is already pulled by pr-review.yml on the same event, so the runner has it
+# cached.
 when:
  - event: pull_request

+# `depends_on: []` on every step so they run in parallel and one failure does not hide
+# the other. Same reasoning as the note above `steps:` in ci-manifests-test.yml.
 steps:
  - name: model-selection-tests
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    depends_on: []
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh
+
+  - name: promtool-rules-guard-tests
+    depends_on: []
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
+    commands:
+      - sh scripts/ci/test-promtool-rules.sh
+
+  - name: yaml-citations-guard-tests
+    depends_on: []
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
+    commands:
+      - sh scripts/ci/test-yaml-citations.sh
@@ -13,7 +13,7 @@ when:

 steps:
  - name: validate-secrets
-    image: alpine:3.24.1
+    image: alpine:3.24.2@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
    environment:
      TF_VAR_proxmox_api_url:
        from_secret: proxmox_api_url
@@ -26,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -18,12 +18,12 @@ when:

 steps:
  - name: iac-scan
-    image: aquasec/trivy:latest
+    image: aquasec/trivy:latest@sha256:af6acf9a6b85dfe389a1941505c0ce9efef52a4719635e1a962f022a3d855daa
    commands:
      - trivy fs --format json -o trivy-iac.json --severity CRITICAL,HIGH --ignore-unfixed .

  - name: image-scans
-    image: aquasec/trivy:latest
+    image: aquasec/trivy:latest@sha256:af6acf9a6b85dfe389a1941505c0ce9efef52a4719635e1a962f022a3d855daa
    commands:
      - trivy image --download-db-only
      - |
@@ -70,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -17,7 +17,7 @@ when:

 steps:
  - name: checkout-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -54,7 +54,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: distill
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -89,14 +89,14 @@ steps:
          -p

  - name: audit
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    commands:
      # Findings are report lines in output/, not a pipeline failure.
      - python3 -B notes/bin/wiki_audit.py --vault "$$PWD/notes"

  - name: consolidate
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
@@ -145,7 +145,7 @@ steps:
        exit $$rc

  - name: push-notes
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.289
    pull: true
    environment:
      WIKI_BOT_TOKEN:
@@ -0,0 +1,7 @@
+# Contributing
+
+## Citing other YAML from a comment
+
+- Mark the block you cite with a comment line `# anchor: <name>` (kebab-case, unique in that file). If the block is inside a `|` block scalar, put the marker just above the key, never inside the scalar.
+- Cite it as `path.yaml#<name>`, with the path relative to the citing file's directory or to the repo root. Never cite `path.yaml:<line>`, because line numbers go stale on any edit above them.
+- `scripts/ci/yaml-citations.sh` enforces this in `ci-manifests-test.yml`: it fails on line-number citations and on citations whose file or anchor does not exist.
@@ -0,0 +1,70 @@
+# homelab
+
+GitOps source for the Renner homelab cluster. Flux applies `master` to
+production, so every merge is a production change. Changes go in by pull
+request only; see [CONTRIBUTING.md](CONTRIBUTING.md) for YAML conventions.
+
+## Change rules that protect Home Assistant
+
+Owner decision (REN-240, REN-243). Home Assistant runs the house, so it
+gets two rules on top of normal review.
+
+### 1. Disruptive work runs only 02:00–05:00 Europe/Berlin
+
+Planned work that can restart Home Assistant or disrupt the cluster is
+merged and applied only between **02:00 and 05:00 Europe/Berlin**. That
+covers:
+
+- node drain, node reboot, Talos upgrade
+- CNI (Cilium, Multus) and cluster-wide network policy
+- storage (Rook/Ceph, CSI, snapshot controller, CNPG operator)
+- Flux core (`kubernetes/clusters/*/flux-system/`, Flux controllers)
+- Authentik
+- OpenBao, including secret rotation that restarts consumers
+- anything in `kubernetes/apps/home-assistant/`
+
+A PR in this list states in its description that it waits for the window.
+Flux applies on merge, so "wait for the window" means **merge** in the
+window, not just apply. If unsure whether a change is disruptive, treat it
+as disruptive.
+
+Not covered: Home Assistant image bumps by Flux image automation. They stay
+fully automatic by owner decision. If one breaks Home Assistant, roll it
+back with [RUNBOOK-home-assistant-image-rollback.md](kubernetes/RUNBOOK-home-assistant-image-rollback.md).
+An incident rollback is not planned work; the owner decides its timing.
+
+### 2. HA-impact review before merge
+
+Every PR that touches `kubernetes/apps/home-assistant/`, or makes a
+cluster-wide disruptive change from the list above, needs an **HA-impact
+review** before merge: a named reviewer states in the PR what Home
+Assistant will notice (restart, lost integration, lost history, none) and
+approves.
+
+- Reviewer: Hestia, the Home Assistant maintainer.
+- Until Hestia exists: the owner does the HA-impact review.
+
+This is in addition to the normal review and merge by Argus. Argus does
+not merge such a PR without the HA-impact approval.
+
+## Agent runtime images
+
+Pod images for the Paperclip Kubernetes sandbox (REN-284). Each is the stock
+`ghcr.io/paperclipai/agent-runtime-claude` (pinned by digest) plus a tool
+set. Base user (1000, non-root), entrypoint and cmd are unchanged; the
+images run with a read-only root FS. Source in `ci/images/agent-runtime-*`,
+built by `.woodpecker/build-agent-runtime.yml` (re-run alone: deployment to
+target `build-agent-runtime`), versions bumped by Renovate. Both also
+carry `claude-agent-acp` for the ACP engine, pinned and patched like
+Paperclip ships it, SDK override included
+(`ci/images/agent-runtime-common/`); bump it by hand
+together with Paperclip. All tools are on the default `PATH`.
+
+| Image | `imageRegistry` | Adds |
+|---|---|---|
+| `gitea.renner.uno/konairius/agent-runtime-go/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-go` | Go toolchain, golangci-lint, govulncheck |
+| `gitea.renner.uno/konairius/agent-runtime-ops/agent-runtime-claude:v1` | `gitea.renner.uno/konairius/agent-runtime-ops` | kubectl, flux, kustomize, helm |
+
+The plugin keeps the name `agent-runtime-claude:v1` and swaps only host and
+org, hence the nested path. Every build also pushes a date tag
+(`:YYYY-MM-DD`) as the rollback point. `konairius` is public: no pull secret.
@@ -0,0 +1,66 @@
+diff --git a/dist/acp-agent.js b/dist/acp-agent.js
+index 0214a509eb5e7b777a69e4e34637941c275d27b1..403be02df66e25eee29e7d5bef14627011e02b30 100644
+--- a/dist/acp-agent.js
+++ b/dist/acp-agent.js
+@@ -2922,9 +2922,19 @@ export class ClaudeAcpAgent {
+                                             amount: message.total_cost_usd,
+                                             currency: "USD",
+                                         },
+-                                        ...(message.origin && {
+-                                            _meta: { "_claude/origin": message.origin },
+-                                        }),
+                                        _meta: {
+                                            ...(message.origin && { "_claude/origin": message.origin }),
+                                            // ACP's core Usage shape exposes context occupancy and
+                                            // cost, but not the billable input/output split. Keep the
+                                            // SDK's bounded aggregate in extension metadata so ACPX can
+                                            // normalize it without persisting a raw provider message.
+                                            usage: {
+                                                input_tokens: message.usage.input_tokens,
+                                                output_tokens: message.usage.output_tokens,
+                                                cache_read_input_tokens: message.usage.cache_read_input_tokens,
+                                                cache_creation_input_tokens: message.usage.cache_creation_input_tokens,
+                                            },
+                                        },
+                                     },
+                                 });
+                             }
+@@ -5311,6 +5321,10 @@ export class ClaudeAcpAgent {
+             settingSources: ["user", "project", "local"],
+             ...(thinking !== undefined && { thinking }),
+             ...userProvidedOptions,
+            // Paperclip Runner owns the complete session context. Its isolated
+            // user root contains only assigned skills; project/local settings
+            // would reintroduce host prompts, plugins, skills, and MCP servers.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && { settingSources: ["user"] }),
+             ...(settings && { settings }),
+             env,
+             // Override certain fields that must be controlled by ACP
+@@ -5318,7 +5332,9 @@ export class ClaudeAcpAgent {
+             includePartialMessages: true,
+             forwardSubagentText,
+             mcpServers: {
+-                ...(userProvidedOptions?.mcpServers || {}),
+                ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1"
+                    ? {}
+                    : (userProvidedOptions?.mcpServers || {})),
+                 ...mcpServers,
+                 ...(fileChangeAuditSupport
+                     ? { [FILE_CHANGE_AUDIT_SERVER_NAME]: fileChangeAuditSupport.mcpServer }
+@@ -5329,6 +5345,16 @@ export class ClaudeAcpAgent {
+             allowDangerouslySkipPermissions: ALLOW_BYPASS,
+             permissionMode: initialPermissionMode,
+             canUseTool: this.canUseTool(sessionId),
+            // The runner pins this actual MCP endpoint before provider launch.
+            // Exempt only task delivery/control tools at the SDK dispatch boundary,
+            // not arbitrary provider permission metadata or MCP name prefixes.
+            ...(process.env.PAPERCLIP_ACPX_ISOLATED_CONTEXT === "1" && {
+                allowedTools: mcpServers.paperclip?.type === "http"
+                    && mcpServers.paperclip.url === process.env.PAPERCLIP_ACPX_TASK_TOOL_BRIDGE_URL
+                    ? ["paperclip_finish", "paperclip_block", "read_current_wake_comments", "request_human_input"]
+                        .map((tool) => `mcp__paperclip__${tool}`)
+                    : [],
+            }),
+             // Forward MCP elicitation requests onto ACP elicitation. Only attached
+             // when the client advertised support, so non-supporting clients keep the
+             // SDK's default (auto-decline) behavior. (AskUserQuestion is handled in
@@ -0,0 +1,9 @@
+{
+  "private": true,
+  "dependencies": {
+    "@agentclientprotocol/claude-agent-acp": "0.73.0"
+  },
+  "overrides": {
+    "@anthropic-ai/claude-agent-sdk": "0.3.280"
+  }
+}
@@ -0,0 +1,68 @@
+# Paperclip agent runtime + Go toolchain (REN-284 / REN-286). Runs as the pod
+# image of the `k8s-go` sandbox environment.
+#
+# Every tool comes from its upstream image, pinned by tag AND digest, so the
+# dockerfile manager in Renovate bumps version and digest together. A curl +
+# sha256 pin (as in ci/images/k8s) would break on every Renovate bump.
+#
+# Both binaries are static; they run on the Ubuntu 22.04 base unchanged.
+FROM golang:1.27.1-bookworm@sha256:69a7b9788769bec032d238959b61854e9ae87f57be9029ec04e9885fabf99195 AS go
+
+# govulncheck (REN-488) has no upstream image: build it in the go stage, static.
+# The version pins the module; go verifies it against sum.golang.org. Renovate
+# does not track it (no regex manager here): bump by hand.
+FROM go AS govulncheck
+RUN CGO_ENABLED=0 GOBIN=/out go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
+
+FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS golangci-lint
+
+# upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
+# digest current but cannot order commit tags; a newer base is a manual bump.
+FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518742e33ce2f@sha256:33ad285f9476b8637d6d940c08d577797030125675bdf4f497a8ed1f79261c8b
+
+COPY --from=go /usr/local/go /usr/local/go
+COPY --from=golangci-lint /usr/bin/golangci-lint /usr/local/bin/golangci-lint
+COPY --from=govulncheck /out/govulncheck /usr/local/bin/govulncheck
+# go and gofmt also in /usr/local/bin: a process started with the default PATH
+# (not the image ENV) still finds them.
+USER root
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/go \
+    && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
+USER 1000:1000
+
+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000
+
+# The sandbox pod has a read-only root FS. Writable mounts: $HOME
+# (/home/paperclip, 1Gi), $HOME/.cache (1Gi), /tmp (2Gi), /workspace.
+# - GOPATH (module cache) goes to /tmp, the biggest one.
+# - GOCACHE and the golangci-lint cache keep their default under $HOME/.cache.
+# - GOTOOLCHAIN=local: a go.mod asking for a newer Go fails clearly instead of
+#   downloading a toolchain through the egress allow-list.
+ENV PATH=/usr/local/go/bin:$PATH \
+    GOPATH=/tmp/go \
+    GOTOOLCHAIN=local
+
+# Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
+# The PATH check uses the base image's default PATH, not the ENV above.
+RUN go version && golangci-lint version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in go gofmt golangci-lint govulncheck claude-agent-acp; do command -v "$t"; done'
@@ -0,0 +1,51 @@
+# Paperclip agent runtime + cluster tools (REN-284 / REN-286). Runs as the pod
+# image of the `k8s-ops` sandbox environment.
+#
+# Every tool comes from its upstream image, pinned by tag AND digest, so the
+# dockerfile manager in Renovate bumps version and digest together. A curl +
+# sha256 pin (as in ci/images/k8s) would break on every Renovate bump.
+#
+# kubectl and flux match the ci-k8s image. All four binaries are static; they
+# run on the Ubuntu 22.04 base unchanged.
+FROM registry.k8s.io/kubectl:v1.37.1@sha256:b7cab618e281b1ee7484e7b706a96e2135fbb6e072c2a573a7dab4e87d7f2385 AS kubectl
+FROM ghcr.io/fluxcd/flux-cli:v2.9.6@sha256:b1ac18156f227af9a524b842a96f2c20986c7a779ef721df3ba4e4540f449d76 AS flux
+FROM registry.k8s.io/kustomize/kustomize:v5.8.1@sha256:899fcd3bc898160e62bcaf82932b0cb29ba38d16272353db2e7acbba82129429 AS kustomize
+FROM alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2 AS helm
+
+# upstream publishes no `v1` tag, only git-<sha> tags. Renovate keeps the
+# digest current but cannot order commit tags; a newer base is a manual bump.
+FROM ghcr.io/paperclipai/agent-runtime-claude:git-38d8f371722b315d2fb3bbaa512518742e33ce2f@sha256:33ad285f9476b8637d6d940c08d577797030125675bdf4f497a8ed1f79261c8b
+
+COPY --from=kubectl /bin/kubectl /usr/local/bin/kubectl
+COPY --from=flux /usr/local/bin/flux /usr/local/bin/flux
+COPY --from=kustomize /app/kustomize /usr/local/bin/kustomize
+COPY --from=helm /usr/bin/helm /usr/local/bin/helm
+
+# ACP server the Paperclip ACP engine starts in the pod (REN-302). Version and
+# patch match what Paperclip ships: packages/adapters/claude-local/package.json
+# pins 0.73.0 and applies patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch
+# (isolated context, task-tool allow-list, usage metadata). Not bumped by
+# Renovate on purpose: bump only together with Paperclip, and replace the patch.
+# Paperclip also overrides the SDK (package.json pnpm overrides:
+# claude-agent-sdk 0.3.280); a plain `npm install -g` ignores that and pulls an
+# older SDK. So install from a package.json with the same override into a
+# fixed prefix, and link the binary into /usr/local/bin.
+COPY agent-runtime-common/claude-agent-acp.package.json /opt/claude-agent-acp/package.json
+COPY agent-runtime-common/claude-agent-acp-0.73.0.patch /tmp/claude-agent-acp.patch
+USER root
+RUN cd /opt/claude-agent-acp \
+    && npm install --omit=dev --no-audit --no-fund \
+    && git -C node_modules/@agentclientprotocol/claude-agent-acp apply /tmp/claude-agent-acp.patch \
+    && npm ls @anthropic-ai/claude-agent-sdk | grep -q 'claude-agent-sdk@0.3.280' \
+    && ln -s /opt/claude-agent-acp/node_modules/.bin/claude-agent-acp /usr/local/bin/claude-agent-acp \
+    && rm /tmp/claude-agent-acp.patch \
+    && npm cache clean --force
+USER 1000:1000
+
+# Read-only root FS: kubectl (~/.kube/cache) and helm (~/.cache, ~/.config)
+# write only under $HOME, which the sandbox pod mounts writable. No ENV needed.
+
+# Base user (1000:1000), entrypoint (tini) and cmd (agent shim) are inherited.
+RUN kubectl version --client && flux version --client && kustomize version && helm version \
+    && env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+       sh -ec 'for t in kubectl flux kustomize helm claude-agent-acp; do command -v "$t"; done'
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine plus jq. Nothing application-specific — a general-purpose base for any
 # workload that needs to read JSON from a shell script.
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine plus the openssl CLI. Nothing application-specific — named for what it
 # is, so anything needing `openssl s_client` / `x509` from a shell script can use
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 RUN apk add --no-cache \
      ansible \
@@ -1,4 +1,4 @@
-FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
 # step — the image is the binary plus its runtime.
@@ -1,4 +1,4 @@
-FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2
+FROM node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1

 # Default to the latest Claude Code. Pinning a fixed version is the wrong
 # default here: old releases eventually stop working against the API, so a
@@ -28,10 +28,10 @@
 # amd64 only, matching the ARCH the installer already hardcoded and the cluster's
 # node architecture. Add a second stage if a different arch is ever needed.

-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d AS fetch
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS fetch

-ARG CNI_PLUGINS_VERSION=v1.5.1
-ARG CNI_PLUGINS_SHA256=77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b
+ARG CNI_PLUGINS_VERSION=v1.9.1
+ARG CNI_PLUGINS_SHA256=b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303

 # busybox wget and tar are already in the base; nothing is installed here
 # either. sha256sum -c fails the BUILD if the artifact does not match, which is
@@ -61,7 +61,7 @@ RUN set -eu; \
      [ -x "/plugins/$p" ] || { echo "FATAL: /plugins/$p missing or not executable" >&2; exit 1; }; \
    done

-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Alpine rather than scratch: the installer needs a shell to copy the binaries
 # onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
@@ -23,7 +23,7 @@
 # namespace, so can0 is only reachable with network_mode: host. That is also
 # how the local MQTT broker on 127.0.0.1:1883 is reached. CAN_RAW sockets need
 # no capabilities, so both run unprivileged.
-FROM python:3.13-slim AS build
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151 AS build

 # Pins live here, in the artifact, rather than in Ansible variables.
 #
@@ -56,7 +56,7 @@ RUN git clone --quiet https://github.com/MyHomeMyData/E3onCAN.git /opt/e3oncan \
 && rm -rf /opt/e3oncan/.git


-FROM python:3.13-slim
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151

 ARG OPEN3E_REF=6553c10453a6bae1210908f38a5af2481a46a3e5
 ARG E3ONCAN_REF=461dcbdf97e8d72d2edb9929c63b39382af6c3a1
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 ARG TALOS_VERSION=v1.13.10
 ARG VELERO_VERSION=v1.16.0
@@ -2,7 +2,7 @@
 # domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
 # public label. Standard library only, so the image is the interpreter plus one
 # file; runs unprivileged as its own user.
-FROM python:3.13-slim
+FROM python:3.14-slim@sha256:c3e521df8b2b498a7a682e7e18676771cb80c6b75b8699af886b2d554ce40151

 RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
 COPY controller.py /opt/controller.py
@@ -0,0 +1,87 @@
+# Paperclip server image (REN-401)
+
+Upstream `paperclipai/paperclip` at the commit in `upstream.env`, plus every
+`*.patch` in this directory. Built by `.woodpecker/build-paperclip.yml`,
+pushed as `gitea.renner.uno/konairius/paperclip:build.<unix-ts>`, rolled out
+by Flux image automation (`kubernetes/infrastructure/image-automation/paperclip.yaml`).
+
+## Why
+
+k8s sandbox runs fail: the run path never sets the lease `remoteCwd`, so file
+sync into the pod fails (paperclipai/paperclip#13864). The fix is
+paperclipai/paperclip#13733, open and not merged.
+`pr-13733-lease-remote-cwd.patch` is that PR's diff (head `5f320bf6`).
+
+#13733 covers only the run path. Adapter environment tests (binding an agent
+to a k8s environment) sync managed config before any workspace is realized,
+so the lease still has no `remoteCwd` and the Claude test fails with
+`claude_managed_config_dir_failed` (REN-409). The fix is
+paperclipai/paperclip#14769, also open: the plugin sets `remoteCwd:
+/workspace` when it acquires or resumes a lease.
+`pr-14769-probe-remote-cwd.patch` is that PR's diff (head `10dc7371`).
+
+Claude auto memory is lost after every run (REN-541): it lives under
+`CLAUDE_CONFIG_DIR`, which Paperclip makes fresh per run and deletes.
+`ren-541-claude-auto-memory.patch` makes the ACPX engine write
+`autoMemoryDirectory: <AGENT_HOME>/memory` into the per-run
+`.claude/settings.local.json`. `AGENT_HOME` is restored before and saved after
+every run, local and in k8s pods. The value is not part of the session
+fingerprint, so resume is unchanged. No upstream PR yet; rename the patch to
+`pr-<n>-…` once there is one.
+
+Claude sessions never resume with a managed AI connection (REN-563): the
+server gives every run a fresh temp home (`HOME`, `XDG_*`,
+`CLAUDE_CONFIG_DIR`, …) and deletes it after the run.
+`ren-563-acpx-claude-session-resume.patch` fixes both halves for local runs:
+the ACPX engine hashes a fixed placeholder for those paths in the session
+fingerprint (`adapterEnvHash`), as the server's own fingerprint already does,
+and links `CLAUDE_CONFIG_DIR/projects` (the Claude transcripts) to
+`<stateDir>/claude-projects`, which persists. Both must ship together: a
+stable fingerprint without the transcript makes the resumed turn fail with
+"Resource not found" (homelab#1111, reverted in #1113). Remote runs are
+unchanged. No upstream PR yet.
+
+A cancelled or failed k8s pod run blocks its card for good (REN-841). The
+owner's next comment waits on `remote_cleanup`, because the server continues
+only when every remote lease carries a provider termination receipt
+(`explicit-native-continuation.ts`), and the Kubernetes plugin's
+`onEnvironmentReleaseLease` / `onEnvironmentDestroyLease` returned `void`.
+`ren-841-k8s-termination-receipt.patch` makes both return
+`{ providerLeaseId, state: "destroyed" }`, but only after the Sandbox CR or
+Job (and, on destroy, the pod) reads 404, polling for up to 20 s, below the
+server's 30 s plugin RPC timeout. If that check times out or fails, the hook
+returns no receipt, as before. No upstream PR yet.
+
+## Files
+
+- `upstream.env` — pinned commit and the version string stamped into the
+  image. Bump both together.
+- `prepare.sh` — fetches the commit, applies the patches, writes
+  `Dockerfile.kaniko` (upstream's `COPY --parents` line split into plain
+  `COPY` lines; kaniko does not support `--parents`), and adds the build of
+  the Kubernetes sandbox provider plugin to the `build` stage (REN-405):
+  upstream's `production` target ships only its source, so the plugin fails
+  with "no longer exposes a Paperclip manifest". The build fails if
+  `dist/manifest.js` is missing.
+- `*.patch` — applied in name order with `git apply`.
+
+## Moving to a newer upstream commit
+
+Set `PAPERCLIP_COMMIT` to the new master commit and `PAPERCLIP_VERSION` to
+`git describe --tags --match 'v*' --long <commit>`, then run
+`sh prepare.sh /tmp/src`: it fails if a patch no longer applies.
+
+## Exit: when every patch here is merged upstream
+
+Renovate cannot watch an upstream PR, so this is a manual step. Once
+upstream nightly contains all of them:
+
+1. `kubernetes/infrastructure/image-automation/paperclip.yaml`: back to
+   `ghcr.io/paperclipai/paperclip` with `filterTags.pattern: '^nightly$'`
+   (the version before REN-401 in git history).
+2. `renovate.json`: swap `gitea.renner.uno/konairius/paperclip` back to
+   `ghcr.io/paperclipai/paperclip` in the image-automation exclusion list.
+3. Delete this directory and `.woodpecker/build-paperclip.yml`.
+
+Image automation then rewrites `kubernetes/apps/paperclip/paperclip.yaml`
+to the nightly digest by itself.
@@ -0,0 +1,93 @@
+diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts
+index f3d1fc659..812cae357 100644
+--- a/server/src/__tests__/environment-run-orchestrator.test.ts
+++ b/server/src/__tests__/environment-run-orchestrator.test.ts
+@@ -430,6 +430,51 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
+     expect(result.persistedExecutionWorkspace).toEqual(updatedEw);
+   });
+ 
+  it("persists the realized cwd as remoteCwd when realization returns no workspaceRealization payload", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({ environment: makeEnvironment("sandbox") }),
+    );
+
+    expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(
+      "lease-1",
+      expect.objectContaining({ remoteCwd: "/workspace" }),
+    );
+    // Nothing to record about the realization shape, so the execution workspace
+    // row is left untouched.
+    expect(mockUpdateExecutionWorkspace).not.toHaveBeenCalled();
+  });
+
+  it("does not rewrite lease metadata when the realized cwd already matches the lease", async () => {
+    const runtime = makeMockRuntime({
+      realizeWorkspace: vi.fn().mockResolvedValue({
+        cwd: "/workspace",
+        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
+      }),
+    } as Partial<EnvironmentRuntimeService>);
+    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
+
+    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
+
+    await orchestrator.realizeForRun(
+      makeRealizeInput({
+        environment: makeEnvironment("sandbox"),
+        lease: makeLease({ metadata: { remoteCwd: "/workspace" } }),
+      }),
+    );
+
+    expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled();
+  });
+
+   it("runs a remote provision command after workspace realization when configured", async () => {
+     mockBuildWorkspaceRealizationRequest.mockReturnValue({
+       version: 1,
+diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts
+index 97cca667e..e76f64913 100644
+--- a/server/src/services/environment-run-orchestrator.ts
+++ b/server/src/services/environment-run-orchestrator.ts
+@@ -485,16 +485,29 @@ export function environmentRunOrchestrator(
+     }
+ 
+     // Step 3: Persist realization metadata on lease and execution workspace
+-    if (Object.keys(workspaceRealization).length > 0) {
+    const hasWorkspaceRealization = Object.keys(workspaceRealization).length > 0;
+    // A driver that realizes the workspace reports the directory it prepared
+    // through the realization `cwd`. Persist it on the lease as `remoteCwd`, so
+    // every later step that receives only the lease — native file sync, a
+    // resumed lease, a second run against the same lease — confines itself to
+    // the same root. Without it a driver whose realization returns no
+    // `workspaceRealization` payload leaves the lease without a root at all,
+    // and the sync step fails even though realization succeeded.
+    const currentRemoteCwd =
+      typeof lease.metadata?.remoteCwd === "string" ? lease.metadata.remoteCwd.trim() : "";
+    const realizedRemoteCwd =
+      realizedWorkspaceCwd && realizedWorkspaceCwd !== currentRemoteCwd ? realizedWorkspaceCwd : null;
+    if (hasWorkspaceRealization || realizedRemoteCwd) {
+       const nextLeaseMetadata = {
+         ...(lease.metadata ?? {}),
+-        workspaceRealization,
+        ...(hasWorkspaceRealization ? { workspaceRealization } : {}),
+        ...(realizedRemoteCwd ? { remoteCwd: realizedRemoteCwd } : {}),
+       };
+       const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, nextLeaseMetadata);
+       if (updatedLease) {
+         lease = updatedLease;
+       }
+-      if (persistedExecutionWorkspace) {
+      if (hasWorkspaceRealization && persistedExecutionWorkspace) {
+         const updatedEw = await executionWorkspacesSvc.update(persistedExecutionWorkspace.id, {
+           metadata: {
+             ...(persistedExecutionWorkspace.metadata ?? {}),
@@ -0,0 +1,86 @@
+diff --git a/packages/plugins/sandbox-providers/kubernetes/README.md b/packages/plugins/sandbox-providers/kubernetes/README.md
+index a51c6ecee60e..72618cb11296 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/README.md
+++ b/packages/plugins/sandbox-providers/kubernetes/README.md
+@@ -140,6 +140,8 @@ Every agent pod is:
+ 
+ Plus per-namespace `pod-security.kubernetes.io/enforce: restricted` and a deny-all NetworkPolicy baseline with explicit egress allow-list (DNS, paperclip-server, configured FQDNs/CIDRs).
+ 
+The lease starts with `/workspace` as the native file-sync confinement root. Adapter environment tests can upload managed config before the server realizes a workspace. When resuming a lease, the provider keeps a previously recorded `remoteCwd` so a custom realized workspace is not replaced with the default.
+
+ The per-run Secret carrying the bootstrap token and adapter API keys has `ownerReferences` pointing at the owning Job, so a single `kubectl delete job …` cascades cleanly to the Pod and Secret.
+ 
+ ## Optional Kata-FC microVM isolation
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+index 8ae47de8ecac..98016385da05 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+@@ -454,6 +454,9 @@ const plugin = definePlugin({
+       secretName,
+       phase: "Pending",
+       backend: config.backend,
+      // Adapter environment probes sync managed assets before workspace
+      // realization. Give native sync the pod's mounted confinement root now.
+      remoteCwd: "/workspace",
+       scopedNetworkPolicyName,
+       scopedNetworkEgress,
+       // Native file sync streams over a pod exec; only the sandbox-cr backend
+@@ -529,6 +532,10 @@ const plugin = definePlugin({
+       secretName,
+       phase: check.phase,
+       backend: leaseBackend,
+      remoteCwd:
+        typeof params.leaseMetadata?.remoteCwd === "string" && params.leaseMetadata.remoteCwd.trim()
+          ? params.leaseMetadata.remoteCwd.trim()
+          : "/workspace",
+       scopedNetworkPolicyName:
+         typeof params.leaseMetadata?.scopedNetworkPolicyName === "string"
+           ? params.leaseMetadata.scopedNetworkPolicyName
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/types.ts b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+index 5626daa93846..ef3d0d889a4c 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/types.ts
+@@ -90,6 +90,8 @@ export interface KubernetesLeaseMetadata {
+   phase: "Pending" | "Running" | "Succeeded" | "Failed";
+   /** Which backend provisioned this lease. */
+   backend: "sandbox-cr" | "job";
+  /** Native sync confinement root; present before workspace realization for probes. */
+  remoteCwd: string;
+   scopedNetworkPolicyName: string | null;
+   scopedNetworkEgress: {
+     allowFqdns: string[];
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+index 7ad0ca4807e5..4c23410d4d15 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+@@ -80,6 +80,7 @@ describe("onEnvironmentResumeLease", () => {
+         secretName: "pc-abc-env",
+         phase: "Running",
+         backend: "sandbox-cr",
+        remoteCwd: "/workspace",
+         resumedLease: true,
+         // sandbox-cr has a pod-exec channel, so native file sync stays enabled.
+         nativeFileSyncUnsupported: false,
+@@ -87,6 +88,22 @@ describe("onEnvironmentResumeLease", () => {
+     );
+   });
+ 
+  it("keeps a custom realized cwd when resuming a sandbox lease", async () => {
+    h.clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockResolvedValue(readySandboxCr("pc-abc-pod")) },
+      core: { readNamespacedPod: vi.fn().mockResolvedValue({ status: { phase: "Running" } }) },
+    };
+    const lease = await plugin.definition.onEnvironmentResumeLease!({
+      driverKey: "kubernetes",
+      companyId: "acme",
+      environmentId: "env-1",
+      config: CONFIG,
+      providerLeaseId: "pc-abc",
+      leaseMetadata: leaseMetadata({ remoteCwd: "/workspace/custom" }),
+    });
+    expect(lease.metadata?.remoteCwd).toBe("/workspace/custom");
+  });
+
+   it("flags a resumed job-backend lease as native-sync-unsupported so the server keeps the base64 fallback", async () => {
+     h.clients = {
+       batch: {
@@ -0,0 +1,59 @@
+#!/bin/sh
+# Fetch upstream Paperclip at the pinned commit, apply our patches and make
+# the Dockerfile buildable with kaniko. Usage: prepare.sh <target-dir>
+set -eu
+HERE=$(cd "$(dirname "$0")" && pwd)
+. "$HERE/upstream.env"
+OUT=$1
+
+rm -rf "$OUT"
+git init -q "$OUT"
+cd "$OUT"
+git fetch -q --depth 1 https://github.com/paperclipai/paperclip.git "$PAPERCLIP_COMMIT"
+git -c advice.detachedHead=false checkout -q FETCH_HEAD
+test "$(git rev-parse HEAD)" = "$PAPERCLIP_COMMIT"
+
+for p in "$HERE"/*.patch; do
+  echo "applying $(basename "$p")"
+  git apply "$p"
+done
+
+# kaniko does not know `COPY --parents` (BuildKit only). Upstream uses it for
+# one line: every sandbox provider's package.json. Replace it with one plain
+# COPY per provider directory, which copies the same files to the same place.
+grep -c -- '^COPY --parents packages/plugins/sandbox-providers/\./\*/package.json packages/plugins/sandbox-providers/$' Dockerfile | grep -qx 1
+awk '
+  /^COPY --parents packages\/plugins\/sandbox-providers\// {
+    while (("ls -d packages/plugins/sandbox-providers/*/package.json" | getline f) > 0) {
+      d = f; sub(/package.json$/, "", d)
+      print "COPY " f " " d
+    }
+    next
+  }
+  { print }
+' Dockerfile > Dockerfile.kaniko
+if grep -q -- '--parents' Dockerfile.kaniko; then
+  echo "Dockerfile.kaniko still uses COPY --parents" >&2
+  exit 1
+fi
+
+# Build the Kubernetes sandbox provider into the image (REN-405). Upstream's
+# `production` target ships only its source; the plugin loader needs
+# dist/manifest.js. Same commands as upstream's `cloud-plugins` stage, run in
+# the `build` stage right after the server build, so production's
+# `COPY --from=build /app /app` carries dist/ and node_modules along. No extra
+# stage: kaniko would snapshot the whole build stage again for it.
+export ANCHOR='RUN test -f server/dist/index.js || (echo "ERROR: server build output missing" && exit 1)'
+grep -c -F -x -- "$ANCHOR" Dockerfile.kaniko | grep -qx 1
+K=packages/plugins/sandbox-providers/kubernetes
+awk -v k="$K" '
+  { print }
+  $0 == ENVIRON["ANCHOR"] {
+    print "RUN pnpm -C " k " install --ignore-workspace --no-lockfile \\"
+    print "  && pnpm -C " k " build \\"
+    print "  && test -f " k "/dist/manifest.js"
+  }
+' Dockerfile.kaniko > Dockerfile.kaniko.new
+mv Dockerfile.kaniko.new Dockerfile.kaniko
+grep -q -- "test -f $K/dist/manifest.js" Dockerfile.kaniko
+rm -rf .git
@@ -0,0 +1,118 @@
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+index a26f980..112e2eb 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.test.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+@@ -2544,6 +2544,37 @@ describe("shared ACPX engine runtime behavior", () => {
+     expect(note).toBeTruthy();
+   });
+ 
+  it("points Claude auto memory at the run's AGENT_HOME without changing the session fingerprint", async () => {
+    const root = await makeTempRoot();
+    const stateDir = path.join(root, "state");
+    const cwd = path.join(root, "worktree");
+    await fs.mkdir(cwd, { recursive: true });
+    const stableHome = path.join(root, "agent-home");
+    const runAgentHome = (runId: string) => path.join(root, "file-sync", "runs", runId, "live");
+    const readSettings = async () =>
+      JSON.parse(await fs.readFile(path.join(cwd, ".claude", "settings.local.json"), "utf8")) as {
+        autoMemoryDirectory?: string;
+        permissions?: { additionalDirectories?: string[] };
+      };
+
+    const first = await runExecutor(
+      { agent: "claude", stateDir, cwd },
+      { context: { paperclipWorkspace: { cwd, agentHome: runAgentHome("a"), agentHomeForPermissions: stableHome } } },
+    );
+    expect((await readSettings()).autoMemoryDirectory).toBe(path.join(runAgentHome("a"), "memory"));
+
+    const second = await runExecutor(
+      { agent: "claude", stateDir, cwd },
+      { context: { paperclipWorkspace: { cwd, agentHome: runAgentHome("b"), agentHomeForPermissions: stableHome } } },
+    );
+    const written = await readSettings();
+    expect(written.autoMemoryDirectory).toBe(path.join(runAgentHome("b"), "memory"));
+    expect(written.permissions?.additionalDirectories).toContain(stableHome);
+    expect(written.permissions?.additionalDirectories).not.toContain(runAgentHome("b"));
+    expect(second.result.sessionParams?.configFingerprint).toBeTypeOf("string");
+    expect(second.result.sessionParams?.configFingerprint).toBe(first.result.sessionParams?.configFingerprint);
+  });
+
+   it("merges Paperclip allowlist into an existing .claude/settings.local.json without losing user entries", async () => {
+     const root = await makeTempRoot();
+     const stateDir = path.join(root, "state");
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts
+index b1dc569..e5681b8 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.ts
+@@ -1558,6 +1558,7 @@ interface PaperclipClaudeSettingsResult {
+   additionalDirectories: string[];
+   defaultMode: string;
+   overrodeDontAsk: boolean;
+  autoMemoryDirectory: string | null;
+ }
+ 
+ function uniqueSorted(values: Array<string | null | undefined>): string[] {
+@@ -1571,10 +1572,19 @@ function uniqueSorted(values: Array<string | null | undefined>): string[] {
+ // denies every non-allowlisted tool and never reaches `canUseTool`), and we
+ // widen the SDK's Read sandbox to include the Paperclip state dirs the agent
+ // needs to talk to its own control plane.
+//
+// It also points Claude's auto memory at `<AGENT_HOME>/memory`. The default
+// location lives under CLAUDE_CONFIG_DIR, which is a per-run temp home, so
+// memories were lost after every run. AGENT_HOME is restored before and saved
+// after each run, locally and in remote sandboxes. Claude ignores
+// `autoMemoryDirectory` only in checked-in project settings, not local ones.
+ async function writePaperclipClaudeSettings(input: {
+   cwd: string;
+   stateDir: string;
+   agentHome: string;
+  // The run's actual AGENT_HOME (the in-sandbox path for remote runs). It may
+  // differ from `agentHome`, which stays stable for session identity.
+  runAgentHome: string;
+   companyId: string;
+ }): Promise<PaperclipClaudeSettingsResult> {
+   const filePath = path.join(input.cwd, ".claude", "settings.local.json");
+@@ -1630,7 +1640,14 @@ async function writePaperclipClaudeSettings(input: {
+     additionalDirectories: mergedAdditionalDirectories,
+     defaultMode,
+   };
+-  const next: Record<string, unknown> = { ...existing, permissions: nextPermissions };
+  const autoMemoryDirectory = input.runAgentHome
+    ? path.posix.join(input.runAgentHome, "memory")
+    : null;
+  const next: Record<string, unknown> = {
+    ...existing,
+    permissions: nextPermissions,
+    ...(autoMemoryDirectory ? { autoMemoryDirectory } : {}),
+  };
+   await writeFileAtomically({
+     target: filePath,
+     contents: `${JSON.stringify(next, null, 2)}\n`,
+@@ -1642,6 +1659,7 @@ async function writePaperclipClaudeSettings(input: {
+     additionalDirectories: mergedAdditionalDirectories,
+     defaultMode,
+     overrodeDontAsk,
+    autoMemoryDirectory,
+   };
+ }
+ 
+@@ -2064,12 +2082,18 @@ async function buildRuntime(input: {
+       // by the company root already granted in these settings; the process env
+       // still receives this run's actual AGENT_HOME.
+       agentHome: asString(workspaceContext.agentHomeForPermissions, agentHome),
+      // Auto memory follows the actual AGENT_HOME so it is saved with the run.
+      // It is not part of the session fingerprint: a moved memory directory
+      // must not invalidate a resumable conversation.
+      runAgentHome: agentHome,
+       companyId: agent.companyId,
+     });
+     skillCommandNotes.push(
+       `Wrote Paperclip-managed Claude settings to ${paperclipClaudeSettings.filePath} (defaultMode=${paperclipClaudeSettings.defaultMode}${
+         paperclipClaudeSettings.overrodeDontAsk ? "; overrode user dontAsk" : ""
+-      }, +${paperclipClaudeSettings.additionalDirectories.length} read root(s), +${paperclipClaudeSettings.allow.length} allow rule(s)).`,
+      }, +${paperclipClaudeSettings.additionalDirectories.length} read root(s), +${paperclipClaudeSettings.allow.length} allow rule(s)${
+        paperclipClaudeSettings.autoMemoryDirectory ? `, autoMemoryDirectory=${paperclipClaudeSettings.autoMemoryDirectory}` : ""
+      }).`,
+     );
+   } else if (acpxAgent === "codex") {
+     // Step 2 — codex-home.seed: the codex managed-home + skills preparation.
@@ -0,0 +1,128 @@
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+index 112e2eb..004baf0 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.test.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts
+@@ -1857,6 +1857,45 @@ describe("shared ACPX engine runtime behavior", () => {
+     expect(changed.result.sessionParams?.configFingerprint).not.toBe(first.result.sessionParams?.configFingerprint);
+   });
+ 
+  it("keeps the per-run managed AI home out of session identity and keeps Claude transcripts", async () => {
+    const root = await makeTempRoot();
+    const stateDir = path.join(root, "state");
+    const config = { agentCommand: "node ./fake-acp.js", stateDir };
+    async function withManagedHome(home: string, claudeConfigDir = path.join(home, "provider")) {
+      return runExecutor({
+        ...config,
+        managedAiConnection: { identity: "grant-1:shared:abc" },
+        env: {
+          HOME: home,
+          XDG_CONFIG_HOME: path.join(home, "config"),
+          XDG_DATA_HOME: path.join(home, "data"),
+          CODEX_HOME: path.join(home, "provider"),
+          GROK_HOME: path.join(home, "provider"),
+          CLAUDE_CONFIG_DIR: claudeConfigDir,
+        },
+      }, { context: { taskId: "issue-1" } });
+    }
+    const fp = (run: Awaited<ReturnType<typeof runExecutor>>) => run.result.sessionParams?.configFingerprint;
+    const first = await withManagedHome(path.join(root, "paperclip-ai-a"));
+    const second = await withManagedHome(path.join(root, "paperclip-ai-b"));
+    const overridden = await withManagedHome(path.join(root, "paperclip-ai-c"), path.join(root, "custom-claude"));
+    expect(fp(first)).toBeTypeOf("string");
+    expect(fp(second)).toBe(fp(first));
+    expect(fp(overridden)).not.toBe(fp(first));
+
+    // The transcript dir of each managed home points at one dir in the state
+    // dir, so a resumed session finds its transcript after the home is deleted.
+    const transcripts = path.join(stateDir, "claude-projects");
+    expect(await fs.readlink(path.join(root, "paperclip-ai-a", "provider", "projects"))).toBe(transcripts);
+    expect(await fs.readlink(path.join(root, "paperclip-ai-b", "provider", "projects"))).toBe(transcripts);
+    await expect(fs.lstat(path.join(root, "custom-claude", "projects"))).rejects.toThrow();
+
+    // Without a managed AI connection the same paths are user config and still count.
+    const plainA = await runExecutor({ ...config, env: { HOME: path.join(root, "home-a") } }, { context: { taskId: "issue-1" } });
+    const plainB = await runExecutor({ ...config, env: { HOME: path.join(root, "home-b") } }, { context: { taskId: "issue-1" } });
+    expect(fp(plainB)).not.toBe(fp(plainA));
+  });
+
+   it("busts the session fingerprint when a stable configured PAPERCLIP_* value rotates", async () => {
+     const root = await makeTempRoot();
+     const stateDir = path.join(root, "state");
+diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts
+index e5681b8..0fd486c 100644
+--- a/packages/adapter-utils/src/acpx-engine/execute.ts
+++ b/packages/adapter-utils/src/acpx-engine/execute.ts
+@@ -1663,6 +1663,26 @@ async function writePaperclipClaudeSettings(input: {
+   };
+ }
+ 
+// Claude keeps conversation transcripts in CLAUDE_CONFIG_DIR/projects. A
+// managed AI home is a per-run temp dir that the server deletes after the run,
+// so the next run could not resume ("Resource not found"). Point `projects` at
+// a directory in the agent's state dir, like ~/.claude/projects on a normal
+// install. The server's recursive cleanup removes the link, not its target.
+async function linkManagedClaudeTranscripts(input: {
+  claudeConfigDir: string;
+  stateDir: string;
+}): Promise<string> {
+  const target = path.join(input.stateDir, "claude-projects");
+  await fs.mkdir(input.claudeConfigDir, { recursive: true, mode: 0o700 });
+  await fs.mkdir(target, { recursive: true, mode: 0o700 });
+  const link = path.join(input.claudeConfigDir, "projects");
+  const current = await fs.readlink(link).catch(() => null);
+  if (current === target) return target;
+  await fs.rm(link, { recursive: true, force: true });
+  await fs.symlink(target, link, "dir");
+  return target;
+}
+
+ // Cross the CLI's staging seam for a runner-backed remote sandbox: ship the
+ // workspace (and, in PR 2, the per-adapter managed-home `assets`) into the
+ // sandbox and obtain the in-sandbox `workspaceRemoteDir` plus the non-null
+@@ -2027,6 +2047,30 @@ async function buildRuntime(input: {
+     // are absent from tempKeysApplied and keep their compatibility protection.
+     if (!scratchKeys.has(key) || value !== scratch.dir) resolvedAdapterEnv[key] = value;
+   }
+  // A managed AI connection gets a fresh server-made temp home on every run
+  // (HOME, XDG_*, and the provider home under it). Hash a fixed placeholder for
+  // those paths so the session can resume; the server's own fingerprint does
+  // the same (managedAiSessionFingerprintConfig). Values that differ from the
+  // managed layout are real overrides and stay in the hash. A Claude session
+  // can only resume if its transcript survives the run, so this pairs with
+  // linkManagedClaudeTranscripts below. Remote runs keep their transcript in
+  // the sandbox and are left as they are.
+  const managedHome = config.managedAiConnection && !executionTargetIsRemote ? resolvedAdapterEnv.HOME : undefined;
+  if (managedHome) {
+    const managedLayout: Record<string, string> = {
+      HOME: "",
+      XDG_CONFIG_HOME: "config",
+      XDG_DATA_HOME: "data",
+      CODEX_HOME: "provider",
+      GROK_HOME: "provider",
+      CLAUDE_CONFIG_DIR: "provider",
+    };
+    for (const [key, sub] of Object.entries(managedLayout)) {
+      if (resolvedAdapterEnv[key] === (sub ? path.join(managedHome, sub) : managedHome)) {
+        resolvedAdapterEnv[key] = sub ? `<managed-ai-home>/${sub}` : "<managed-ai-home>";
+      }
+    }
+  }
+   if (authToken) env.PAPERCLIP_API_KEY = authToken;
+   // For the claude agent, set model via ANTHROPIC_MODEL at startup rather than
+   // via session/set_config_option — the ACP server's set_config_option handler
+@@ -2088,6 +2132,15 @@ async function buildRuntime(input: {
+       runAgentHome: agentHome,
+       companyId: agent.companyId,
+     });
+    // Same condition as the fingerprint placeholder above: a resumable session
+    // must find its transcript in the next run's managed home.
+    if (resolvedAdapterEnv.CLAUDE_CONFIG_DIR === "<managed-ai-home>/provider" && env.CLAUDE_CONFIG_DIR) {
+      const transcriptsDir = await linkManagedClaudeTranscripts({
+        claudeConfigDir: env.CLAUDE_CONFIG_DIR,
+        stateDir,
+      });
+      skillCommandNotes.push(`Linked Claude transcripts to ${transcriptsDir}.`);
+    }
+     skillCommandNotes.push(
+       `Wrote Paperclip-managed Claude settings to ${paperclipClaudeSettings.filePath} (defaultMode=${paperclipClaudeSettings.defaultMode}${
+         paperclipClaudeSettings.overrodeDontAsk ? "; overrode user dontAsk" : ""
@@ -0,0 +1,384 @@
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts b/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+index 87e18ae..2224e60 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/lease-lifecycle.ts
+@@ -21,6 +21,7 @@ import { deleteJob, findPodForJob, getJobStatus } from "./job-orchestrator.js";
+ import {
+   deleteSandboxCr,
+   findPodForSandbox,
+  getSandboxCrStatus,
+   waitForSandboxReady,
+ } from "./sandbox-cr-orchestrator.js";
+ 
+@@ -191,3 +192,60 @@ export async function destroyLeaseResources(
+     );
+   }
+ }
+
+export interface WorkloadGoneInput {
+  namespace: string;
+  /** Workload resource name (Sandbox CR name or Job name) == providerLeaseId. */
+  name: string;
+  backend: "sandbox-cr" | "job";
+  podName?: string | null;
+  /** Stays below the server's 30 s default plugin RPC timeout. */
+  timeoutMs?: number;
+  pollMs?: number;
+}
+
+/**
+ * Poll until the workload (and the named pod, if any) returns 404. Deletes use
+ * Foreground propagation, so the workload disappears only after its pod is
+ * gone. True is the evidence behind a `destroyed` termination receipt; false
+ * (still present at the deadline, or a read failed) means the caller must not
+ * claim one. Never throws: the delete itself already succeeded.
+ */
+export async function waitForWorkloadGone(
+  clients: KubeClients,
+  input: WorkloadGoneInput,
+): Promise<boolean> {
+  const deadline = Date.now() + (input.timeoutMs ?? 20_000);
+  const pollMs = input.pollMs ?? 500;
+  const gone = async (read: () => Promise<unknown>): Promise<boolean> => {
+    try {
+      await read();
+      return false;
+    } catch (err) {
+      if (isKubeNotFoundError(err)) return true;
+      throw err;
+    }
+  };
+  try {
+    return await pollUntilGone();
+  } catch {
+    return false;
+  }
+
+  async function pollUntilGone(): Promise<boolean> {
+    for (;;) {
+      const workloadGone = await gone(() =>
+        input.backend === "sandbox-cr"
+          ? getSandboxCrStatus(clients, input.namespace, input.name)
+          : getJobStatus(clients, input.namespace, input.name),
+      );
+      const podName = input.podName;
+      const podGone = !podName || await gone(() =>
+        clients.core.readNamespacedPod({ namespace: input.namespace, name: podName }),
+      );
+      if (workloadGone && podGone) return true;
+      if (Date.now() >= deadline) return false;
+      await new Promise((resolve) => setTimeout(resolve, pollMs));
+    }
+  }
+}
+diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+index 9801638..bf5ce0a 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts
+@@ -15,6 +15,7 @@ import type {
+   PluginEnvironmentSyncInParams,
+   PluginEnvironmentSyncOutParams,
+   PluginEnvironmentSyncResult,
+  PluginEnvironmentTerminationReceipt,
+   PluginEnvironmentValidateConfigParams,
+   PluginEnvironmentValidationResult,
+ } from "@paperclipai/plugin-sdk";
+@@ -38,7 +39,11 @@ import {
+ } from "./sandbox-cr-orchestrator.js";
+ import { execInPod, execInPodStreaming, wrapCommandWithEnv } from "./pod-exec.js";
+ import { performSyncIn, performSyncOut, type PodStreamExec } from "./file-sync.js";
+-import { checkLeaseResumable, destroyLeaseResources } from "./lease-lifecycle.js";
+import {
+  checkLeaseResumable,
+  destroyLeaseResources,
+  waitForWorkloadGone,
+} from "./lease-lifecycle.js";
+ import {
+   appendNetworkEgressDenyHint,
+   createScopedNetworkEgressPolicyOrReleaseWorkload,
+@@ -578,7 +583,7 @@ const plugin = definePlugin({
+ 
+   async onEnvironmentReleaseLease(
+     params: PluginEnvironmentReleaseLeaseParams,
+-  ): Promise<void> {
+  ): Promise<PluginEnvironmentTerminationReceipt | void> {
+     if (!params.providerLeaseId) return;
+     const config = kubernetesProviderConfigSchema.parse(params.config);
+     const namespace =
+@@ -613,11 +618,22 @@ const plugin = definePlugin({
+         ?? (err as { code?: number; statusCode?: number }).statusCode;
+       if (code !== 404) throw err;
+     }
+
+    // The server grants continuation after a stopped run only with a
+    // termination receipt; without one a cancelled run's queued comment
+    // waits on `remote_cleanup` forever (REN-841). Claim `destroyed` only
+    // once the workload is confirmed gone.
+    const gone = await waitForWorkloadGone(clients, {
+      namespace,
+      name: params.providerLeaseId,
+      backend: leaseBackend,
+    });
+    if (gone) return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
+   },
+ 
+   async onEnvironmentDestroyLease(
+     params: PluginEnvironmentDestroyLeaseParams,
+-  ): Promise<void> {
+  ): Promise<PluginEnvironmentTerminationReceipt | void> {
+     if (!params.providerLeaseId) return;
+     const config = kubernetesProviderConfigSchema.parse(params.config);
+     const namespace =
+@@ -658,6 +674,13 @@ const plugin = definePlugin({
+       podName,
+       secretName,
+     });
+    const gone = await waitForWorkloadGone(clients, {
+      namespace,
+      name: params.providerLeaseId,
+      backend: leaseBackend,
+      podName,
+    });
+    if (gone) return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
+   },
+ 
+   async onEnvironmentExecute(
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+index af0875f..d775799 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/lease-lifecycle.test.ts
+@@ -3,6 +3,7 @@ import {
+   checkLeaseResumable,
+   destroyLeaseResources,
+   isKubeNotFoundError,
+  waitForWorkloadGone,
+ } from "../../src/lease-lifecycle.js";
+ 
+ const SANDBOX_GROUP = "agents.x-k8s.io";
+@@ -346,3 +347,64 @@ describe("destroyLeaseResources", () => {
+     ).rejects.toThrow("forbidden");
+   });
+ });
+
+describe("waitForWorkloadGone", () => {
+  const input = { namespace: "ns", name: "pc-abc", backend: "sandbox-cr" as const, pollMs: 1 };
+
+  it("is true once the Sandbox CR and the pod return 404", async () => {
+    const getCr = vi.fn()
+      .mockResolvedValueOnce(readySandboxCr("pc-abc-pod"))
+      .mockRejectedValue(notFound());
+    const readPod = vi.fn().mockRejectedValue(notFound());
+    const clients = { custom: { getNamespacedCustomObject: getCr }, core: { readNamespacedPod: readPod } };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, podName: "pc-abc-pod" }),
+    ).resolves.toBe(true);
+    expect(getCr).toHaveBeenCalledTimes(2);
+    expect(getCr).toHaveBeenCalledWith(expect.objectContaining({
+      group: SANDBOX_GROUP, version: SANDBOX_VERSION, plural: SANDBOX_PLURAL, namespace: "ns", name: "pc-abc",
+    }));
+  });
+
+  it("is false while the workload still exists at the deadline", async () => {
+    const clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockResolvedValue(readySandboxCr()) },
+    };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, timeoutMs: 5 }),
+    ).resolves.toBe(false);
+  });
+
+  it("is false while the pod still exists even if the workload is gone", async () => {
+    const clients = {
+      custom: { getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()) },
+      core: { readNamespacedPod: vi.fn().mockResolvedValue({ status: { phase: "Running" } }) },
+    };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, podName: "pc-abc-pod", timeoutMs: 5 }),
+    ).resolves.toBe(false);
+  });
+
+  it("checks the Job for job-backend leases", async () => {
+    const readJob = vi.fn().mockRejectedValue(notFound());
+    const clients = { batch: { readNamespacedJobStatus: readJob } };
+
+    await expect(
+      waitForWorkloadGone(clients as never, { ...input, backend: "job" }),
+    ).resolves.toBe(true);
+    expect(readJob).toHaveBeenCalledWith({ namespace: "ns", name: "pc-abc" });
+  });
+
+  it("is false (never throws) when a read fails with another error", async () => {
+    const clients = {
+      custom: {
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(Object.assign(new Error("x"), { code: 403 })),
+      },
+    };
+
+    await expect(waitForWorkloadGone(clients as never, input)).resolves.toBe(false);
+  });
+});
+diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+index 4c23410..f4fd073 100644
+--- a/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/plugin-lease-lifecycle.test.ts
+@@ -184,12 +184,19 @@ describe("onEnvironmentDestroyLease", () => {
+     const deletePod = vi.fn().mockResolvedValue({});
+     const deleteSecret = vi.fn().mockResolvedValue({});
+     h.clients = {
+-      custom: { deleteNamespacedCustomObject: deleteCr },
+-      core: { deleteNamespacedPod: deletePod, deleteNamespacedSecret: deleteSecret },
+      custom: {
+        deleteNamespacedCustomObject: deleteCr,
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+      core: {
+        deleteNamespacedPod: deletePod,
+        deleteNamespacedSecret: deleteSecret,
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+       batch: { deleteNamespacedJob: vi.fn() },
+     };
+ 
+-    await plugin.definition.onEnvironmentDestroyLease!({
+    const receipt = await plugin.definition.onEnvironmentDestroyLease!({
+       driverKey: "kubernetes",
+       companyId: "acme",
+       environmentId: "env-1",
+@@ -209,14 +216,47 @@ describe("onEnvironmentDestroyLease", () => {
+       namespace: "paperclip-acme",
+       name: "pc-abc-env",
+     });
+    expect(receipt).toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+   });
+ 
+   it("is idempotent: resolves cleanly when every resource is already gone (404)", async () => {
+     h.clients = {
+-      custom: { deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()) },
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+       core: {
+         deleteNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+         deleteNamespacedSecret: vi.fn().mockRejectedValue(notFound()),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+      batch: { deleteNamespacedJob: vi.fn() },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentDestroyLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns no receipt when the gone check cannot read the cluster", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(
+          Object.assign(new Error("forbidden"), { code: 403 }),
+        ),
+      },
+      core: {
+        deleteNamespacedPod: vi.fn().mockResolvedValue({}),
+        deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+       },
+       batch: { deleteNamespacedJob: vi.fn() },
+     };
+@@ -261,8 +301,12 @@ describe("onEnvironmentDestroyLease", () => {
+       core: {
+         deleteNamespacedPod: vi.fn().mockResolvedValue({}),
+         deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
+        readNamespacedPod: vi.fn().mockRejectedValue(notFound()),
+      },
+      batch: {
+        deleteNamespacedJob: deleteJob,
+        readNamespacedJobStatus: vi.fn().mockRejectedValue(notFound()),
+       },
+-      batch: { deleteNamespacedJob: deleteJob },
+     };
+ 
+     await plugin.definition.onEnvironmentDestroyLease!({
+@@ -280,3 +324,71 @@ describe("onEnvironmentDestroyLease", () => {
+     expect(deleteCr).not.toHaveBeenCalled();
+   });
+ });
+
+describe("onEnvironmentReleaseLease", () => {
+  it("returns a destroyed receipt once the Sandbox CR is gone (REN-841)", async () => {
+    const deleteCr = vi.fn().mockResolvedValue({});
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: deleteCr,
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+    };
+
+    const receipt = await plugin.definition.onEnvironmentReleaseLease!({
+      driverKey: "kubernetes",
+      companyId: "acme",
+      environmentId: "env-1",
+      config: CONFIG,
+      providerLeaseId: "pc-abc",
+      leaseMetadata: leaseMetadata(),
+    });
+
+    expect(deleteCr).toHaveBeenCalledWith(
+      expect.objectContaining({ namespace: "paperclip-acme", name: "pc-abc" }),
+    );
+    expect(receipt).toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns a destroyed receipt when the Sandbox CR was already deleted", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(notFound()),
+      },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toEqual({ providerLeaseId: "pc-abc", state: "destroyed" });
+  });
+
+  it("returns no receipt when the gone check fails", async () => {
+    h.clients = {
+      custom: {
+        deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+        getNamespacedCustomObject: vi.fn().mockRejectedValue(
+          Object.assign(new Error("server error"), { code: 500 }),
+        ),
+      },
+    };
+
+    await expect(
+      plugin.definition.onEnvironmentReleaseLease!({
+        driverKey: "kubernetes",
+        companyId: "acme",
+        environmentId: "env-1",
+        config: CONFIG,
+        providerLeaseId: "pc-abc",
+        leaseMetadata: leaseMetadata(),
+      }),
+    ).resolves.toBeUndefined();
+  });
+});
@@ -0,0 +1,7 @@
+# Upstream paperclipai/paperclip commit this image is built from (REN-401).
+# a027f76 is the master commit the nightly image ran when this was pinned, so
+# the only change against that image is the patch next to this file.
+# PAPERCLIP_VERSION is `git describe --tags --match 'v*' --long` of that
+# commit, the value upstream stamps into its own images. Bump both together.
+PAPERCLIP_COMMIT=a027f76a726e8a556674eded807c2de0e55f5cc0
+PAPERCLIP_VERSION=v2026.916.0-219-ga027f76a7
@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 ARG TOFU_VERSION=1.11.0

@@ -1,4 +1,4 @@
-FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6

 # Everything `bootstrap.sh vmtest` needs, so the rebuild test can run as a Job in
 # the cluster instead of on a workstation. The workstation has 31G and the test
@@ -16,7 +16,7 @@ credentials are out of scope (see "Deferred").
 |---|---|
 | Execution model | **B — server-local runs** (upstream default): agent CLIs are child processes of the server, in the server container. Trial only. |
 | Upgrade path | If Paperclip is kept: **C** — `@paperclipai/plugin-kubernetes` sandbox provider + kubernetes-sigs agent-sandbox controller (`agents.x-k8s.io`), per-run Pods and Secrets. Separate spec. |
-| Claude auth | **One shared Claude subscription AI connection**, created once via Paperclip's local sign-in (`claude auth login` into `ai-local-logins/<id>`); Paperclip stores and refreshes it. Server-env `CLAUDE_CODE_OAUTH_TOKEN` is not used: `stripAiAuthBindings` blanks it for managed runs, and upstream accepts no pasted setup-token (the device-code flow needs a Daytona sandbox). |
+| Claude auth | **One shared Claude subscription AI connection**, created once via Paperclip's local sign-in (`claude auth login` into `ai-local-logins/<id>`), then its value replaced with a long-lived `claude setup-token` token. Paperclip stores the value as company secret "My Claude subscription" and injects it per run as `CLAUDE_CODE_OAUTH_TOKEN`; it never refreshes Anthropic credentials, so the login's ~8h access token expires (runs then fail with "ACP agent reported a terminal access failure", 0 tokens). Renew by rotating that secret with a new setup-token (~yearly). Server-env `CLAUDE_CODE_OAUTH_TOKEN` is not used: `stripAiAuthBindings` blanks it for managed runs, and upstream accepts no pasted setup-token (the device-code flow needs a Daytona sandbox). |
 | Bot credentials | Out of scope. Future "AIR-Manager" Paperclip role handles permission requests. |
 | Exposure | LAN-only, behind an Authentik forward-auth gate, then Paperclip's own login. |

@@ -64,7 +64,32 @@ All modeled on `kubernetes/apps/paperless/` unless stated.
 - Starts as root: upstream `docker-entrypoint.sh` remaps the `node` user, chowns
  `/paperclip`, then drops to uid 1000 via `gosu`. Caps limited to CHOWN,
  DAC_OVERRIDE, FOWNER, SETGID, SETUID; `fsGroup: 1000`.
- `automountServiceAccountToken: false` (all agents share the pod filesystem).
+- `serviceAccountName: paperclip-prometheus-read` with
+  `automountServiceAccountToken: true`. All agents share the pod filesystem, so all
+  of them can read that token — it is mounted anyway because the Role behind it
+  grants exactly `get` on `services/proxy` for
+  `kube-prometheus-stack-prometheus:9090` and
+  `kube-prometheus-stack-alertmanager:9093` in `monitoring`, and nothing else
+  (`kubernetes/apps/paperclip/prometheus-read-rbac.yaml`, REN-36 then REN-95).
+  It was `false` until then.
+  Three things about that grant are worth knowing before you use or debug it:
+  - **It scopes the HTTP method, not the endpoint.** The apiserver maps GET→`get`,
+    POST→`create`, PUT→`update` on a service-proxy request, so `get` alone refuses
+    `POST /-/reload` and `POST /-/quit` in the authorizer. It does *not* narrow
+    which GET endpoints are reachable: the whole read surface of both services is,
+    including `/api/v1/status/config` (scrape credentials there render as
+    `<secret>`) and the web UI.
+  - **Force GET in the caller.** Prometheus accepts both GET and POST on
+    `/api/v1/query`; under this Role only the GET form works, so a client library
+    that switches to POST for long queries will 403 for a reason that has nothing
+    to do with the query.
+  - **If reads start 403ing, check the Service name against `resourceNames`
+    first.** `resourceNames` is matched verbatim, and
+    `kubernetes/apps/monitoring/helmrelease-prometheus.yaml` floats the chart at
+    `>=72.0.0` — a release rename breaks this path silently,
+    with everything still looking correct in git. The port-qualified form is the
+    only one granted: `svc`, `svc:9090` and `http:svc:9090` are three different
+    strings to the authorizer.
 - Port 3100; `Service paperclip` ClusterIP :3100.
 - Probes: startup and readiness on `GET /api/health` with `Host: localhost` (the
  private-hostname guard rejects the pod IP); liveness is a TCP check, since
@@ -164,7 +189,8 @@ available since v0.35.0). First use of digest reflection in this repo.
   **as the node user** (the entrypoint makes exec sessions root, and root-owned 0600
   credentials are unreadable to the server):
   `kubectl exec -it -n paperclip deploy/paperclip -- gosu node sh -c 'export CLAUDE_CONFIG_DIR=<shown path> && mkdir -p "$CLAUDE_CONFIG_DIR" && claude auth login'`
-   Repeat after loss of the PVC or database.
+   Then Settings → Secrets → "My Claude subscription": replace the value with the
+   output of `claude setup-token`. Repeat both after loss of the database.

 ## Verification
 - Flux Kustomization `paperclip` Ready; CNPG `pclp-pg` healthy; first ScheduledBackup
@@ -165,6 +165,21 @@ permanent recovery.

 ---

+## Velero restore-test: 80% OSD capacity gate
+
+The monthly Velero restore-test restores into the production Ceph pools. Before
+each schedule's restore it projects the restore's raw size (DataUpload
+`totalBytes` x the pool's replication/EC factor, spread over all OSDs) onto
+current OSD usage from Prometheus. If the fullest OSD would go above **80%**
+(`OSD_MAX_PCT` in `restore-test-cronjob.yaml`), that schedule is not restored:
+the Job log shows it as **SKIP** with the projected %, and the summary lists it
+under "Skipped (backup NOT restore-tested)". A SKIP does not fail the Job, so
+read the summary each month. If the projection cannot be made (Prometheus
+down, unsized DataUpload), the schedule is not restored and counts as FAIL.
+Background: REN-441, REN-442.
+
+---
+
 ## CNPG restore-test (verify backups are restorable — non-destructive)

 The monthly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
@@ -0,0 +1,71 @@
+# Runbook — roll back a Home Assistant image bump
+
+Use when an automatic Home Assistant update (a
+`chore(image-automation): bump ghcr.io/home-assistant/home-assistant:<new>`
+commit on `master`) breaks Home Assistant. The owner approves the rollback
+and its timing. Atlas writes the PR; Argus merges it.
+
+## Why a plain `git revert` is not enough
+
+Image automation (`kubernetes/infrastructure/image-automation/`) re-scans
+the registry every hour and rewrites the tag every 30 minutes to the
+highest release matching the `home-assistant` ImagePolicy. A plain revert
+of the bump commit would be bumped forward again within about an hour.
+
+So the rollback commit does two things at once: it reverts the tag **and**
+drops the `# {"$imagepolicy": ...}` marker from that image line. Without
+the marker the Setters strategy has nothing to rewrite, so the old tag
+stays pinned. It is still one commit, and reverting that commit restores
+automation.
+
+## Before you roll back
+
+1. Find the bump commit:
+   ```sh
+   git log --oneline -5 origin/master -- kubernetes/apps/home-assistant/deployment.yaml
+   ```
+2. Check the release notes of the broken version for a **database schema**
+   change. Home Assistant migrates the recorder schema (CNPG cluster
+   `ha-pg`) on start and cannot downgrade it. An older version started on a
+   newer schema runs without recorder history until the schema matches
+   again. If the bump crossed a schema change, tell the owner before going
+   on: the choice is roll back without history, or restore the recorder
+   database from backup as a separate step.
+
+## Roll back (one commit, one PR)
+
+```sh
+git worktree add -b atlas/ha-rollback-<old> "$WT" origin/master && cd "$WT"
+git revert --no-commit <bump-sha>
+# In kubernetes/apps/home-assistant/deployment.yaml, delete the trailing
+#   # {"$imagepolicy": "flux-system:home-assistant"}
+# from the home-assistant image line. Leave every other line alone.
+git diff --cached --stat   # expect: 1 file, 1 line changed
+git commit -m "fix(home-assistant): pin <old>, pause image automation (bump to <new> broke HA)"
+git push -u origin HEAD
+```
+
+The PR description states: broken version, pinned version, symptom, schema
+check result, and "revert this commit to resume automatic updates".
+HA-impact review and merge follow [README.md](../README.md#2-ha-impact-review-before-merge).
+
+On merge Flux applies the Deployment (strategy `Recreate`): Home Assistant
+restarts once, about one to two minutes of downtime.
+
+## Verify
+
+```sh
+kubectl -n home-assistant get deploy home-assistant \
+  -o jsonpath='{.spec.template.spec.containers[?(@.name=="home-assistant")].image}'
+kubectl -n home-assistant rollout status deploy/home-assistant
+```
+
+Read-only commands only (see REN-240 protection rule 1). One hour after
+merge, confirm no new `chore(image-automation)` commit touched
+`deployment.yaml`.
+
+## Resume automatic updates
+
+When a fixed Home Assistant release is out, revert the rollback commit. The
+marker comes back, and image automation bumps to the newest release on its
+next pass. That bump is itself a Home Assistant restart.
@@ -48,7 +48,7 @@ spec:
        # cert when doing server-side OIDC token exchange. Bundle both
        # roots into one PEM since NODE_EXTRA_CA_CERTS accepts one file.
        - name: fetch-le-staging-roots
-          image: curlimages/curl:8.10.1
+          image: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777
          command:
            - sh
            - -c
@@ -70,7 +70,7 @@ spec:
              mountPath: /ca
      containers:
        - name: audiobookshelf
-          image: ghcr.io/advplyr/audiobookshelf:2.36.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
+          image: ghcr.io/advplyr/audiobookshelf:2.37.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
          ports:
            - name: http
              containerPort: 80
@@ -2121,6 +2121,74 @@ data:
            - url: https://dash.renner.uno/login/generic_oauth
              matching_mode: strict

+      # Matrix (REN-567): upstream login for MAS (account.matrix.renner.uno).
+      # Git owns this provider, its application and the member list.
+      # Public client + PKCE, so there is no client secret to share.
+      # RS256 with the built-in self-signed keypair: a public client has no
+      # secret for HS256, and MAS verifies the ID token signature.
+      - id: matrix-users-group
+        model: authentik_core.group
+        identifiers:
+          name: Matrix Users
+        state: present
+        attrs:
+          is_superuser: false
+          # Every member can sign in to Matrix and gets an account named
+          # after their Authentik username. Add/remove members HERE.
+          users:
+            - !Find [authentik_core.user, [username, k.renner]]
+
+      - id: matrix-mas-provider
+        model: authentik_providers_oauth2.oauth2provider
+        identifiers:
+          name: Matrix (MAS)
+        state: present
+        attrs:
+          authentication_flow: !KeyOf passkey-passwordless-flow
+          authorization_flow: !KeyOf passkey-stepup-flow
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          client_type: public
+          client_id: matrix-mas
+          # Authentik 2026.5 checks the grant type against this list, and a
+          # provider created by a blueprint starts with it empty, so every
+          # authorize fails with invalid_request ("Invalid grant_type for
+          # provider"). Providers made in the UI get the defaults.
+          grant_types:
+            - authorization_code
+            - refresh_token
+          signing_key: !Find [authentik_crypto.certificatekeypair, [name, authentik Self-signed Certificate]]
+          property_mappings:
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-profile]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]]
+          redirect_uris:
+            # The path ends in the provider ULID from apps/matrix/mas.yaml.
+            - url: https://account.matrix.renner.uno/upstream/callback/01M4480KWZGN9HBNB5SSNGT2M1
+              matching_mode: strict
+
+      - model: authentik_core.application
+        identifiers:
+          slug: matrix
+        state: present
+        attrs:
+          name: Matrix
+          slug: matrix
+          provider: !KeyOf matrix-mas-provider
+          meta_launch_url: https://element.renner.uno
+          meta_description: "Matrix chat (Element)"
+
+      # Only the Matrix Users group passes.
+      - model: authentik_policies.policybinding
+        identifiers:
+          target: !Find [authentik_core.application, [slug, matrix]]
+          group: !KeyOf matrix-users-group
+          order: 0
+        state: present
+        attrs:
+          enabled: true
+          negate: false
+          failure_result: false
+
      # Superseded by the single passkey-webauthn-validation stage above.
      # Their flowstagebindings are FK-CASCADEd away with them; the
      # identification stage's webauthn_stage FK is SET_NULL and is
@@ -5,6 +5,11 @@ metadata:
  namespace: evcc
  labels:
    app.kubernetes.io/name: evcc
+  annotations:
+    # The MQTT client certificate reaches evcc as env (below), which does not
+    # follow a renewal. cert-manager renews it every ~60 days; Reloader then
+    # restarts evcc so it picks up the new PEMs before the old ones expire.
+    secret.reloader.stakater.com/reload: mqtt-evcc-client-tls
 spec:
  replicas: 1
  # RWO PVC (`evcc-data`, ceph-block) can't be mounted twice, so the
@@ -23,13 +28,36 @@ spec:
      enableServiceLinks: false
      containers:
        - name: evcc
-          image: evcc/evcc:0.310.1
+          image: evcc/evcc:0.316.2@sha256:070c8a8912c5107e8424cef60bf8c3d3cac0eb32aa28d1b021d226545f0aac7b
          # evcc auto-discovers /etc/evcc.yaml on its default search path;
          # the file is delivered (with the MQTT password) via the
          # sops-encrypted evcc-config Secret mounted below.
          env:
            - name: TZ
              value: Europe/Berlin
+            # MQTT by client certificate (register account evcc-client,
+            # REN-953) on the mTLS listener. These override the mqtt block of
+            # evcc.yaml (viper AutomaticEnv, checked against 0.316.2); the
+            # file's user/password are ignored there, as the broker takes the
+            # username from the certificate CN. evcc wants the PEM text, not
+            # a path.
+            - name: EVCC_MQTT_BROKER
+              value: tls://mosquitto.home-assistant.svc.cluster.local:8884
+            - name: EVCC_MQTT_CACERT
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-evcc-client-tls
+                  key: ca.crt
+            - name: EVCC_MQTT_CLIENTCERT
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-evcc-client-tls
+                  key: tls.crt
+            - name: EVCC_MQTT_CLIENTKEY
+              valueFrom:
+                secretKeyRef:
+                  name: mqtt-evcc-client-tls
+                  key: tls.key
          ports:
            - name: http
              containerPort: 7070
@@ -30,7 +30,42 @@ metadata:
 #     Web Service 9 = Music Assistant
 #     Web Service 10 = TRMNL
 #     Web Service 11 = Paperless (gate-level only, see its entry)
+#     Web Service 12 = Element Web
+#     Web Service 13 = Matrix (Synapse client API)
+#     Web Service 14 = Matrix Authentication Service
+#
+# CORRECTION (REN-21, 2026-09-25) to the Web Service 11 comment further down,
+# which claims that an in-cluster probe of the paperless ClusterIP "needs a
+# CiliumNetworkPolicy first — the monitoring probes get away without one only
+# because their targets share a namespace with the prober".
+#
+# That is wrong, and it was wrong when written. It is stated here rather than
+# fixed in place because the sentence lives inside the `config.yaml: |` block
+# scalar below: any edit to it changes ConfigMap DATA, and
+# `configmap.reloader.stakater.com/reload: gatus-config`
+# (deployment.yaml#reloader) would then roll the gatus Deployment — a
+# production restart for a comment. This header is outside `data:`, so it is
+# inert; the served config is byte-identical.
+#
+#   * When written, the paperless baseline admitted the whole monitoring
+#     namespace on every port, and monitoring is where blackbox-exporter runs.
+#     That admit is gone (REN-28); the probe is now admitted by the dedicated
+#     ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
+#     scoped to blackbox-exporter on the paperless pod's :8000. Cilium unions
+#     the allow rules of every CNP selecting an endpoint, so nothing else is
+#     required.
+#   * The in-cluster paperless probe therefore exists:
+#     ../monitoring-rules/probes-blackbox.yaml#paperless-app-health, target
+#     http://paperless.paperless.svc.cluster.local:80/accounts/login/
+#   * The claim does NOT generalise to gatus itself. Gatus runs in `apps`,
+#     which is a different source namespace; a gatus-sourced in-cluster check
+#     would need its own allow, exactly as
+#     cnp-cross-namespace-allow-apps-gatus-to-monitored-services describes.
+#
+# The Web Service 11 entry below stays a reachability-only check on purpose.
+# The app-health half of the pair is the blackbox probe named above.
 data:
+  # anchor: gatus-endpoints
  config.yaml: |
    storage:
      type: sqlite
@@ -196,6 +231,46 @@ data:
        conditions:
          - "[STATUS] == any(200, 301, 302)"

+      # Matrix (REN-663): Element Web, the Synapse client API and MAS.
+      - name: Web Service 12
+        group: services
+        url: https://element.renner.uno/config.json
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
+      - name: Web Service 13
+        group: services
+        url: https://matrix.renner.uno/_matrix/client/versions
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
+      - name: Web Service 14
+        group: services
+        url: https://account.matrix.renner.uno/.well-known/openid-configuration
+        interval: 60s
+        ui:
+          hide-hostname: true
+          hide-url: true
+          hide-port: true
+          hide-conditions: true
+          hide-errors: true
+        conditions:
+          - "[STATUS] == 200"
+
      # Removed 2026-05-21: Jellyfin (media.renner.uno), Wiki (wiki.renner.uno),
      # InfluxDB (influx.renner.uno) — hostnames NXDOMAIN, no Ingress; services
      # no longer deployed. Re-add when/if they come back.
@@ -6,6 +6,7 @@ metadata:
  labels:
    app: gatus
  annotations:
+    # anchor: reloader
    # Gatus reads its config once at startup and never re-reads the file, so
    # without this a ConfigMap change reconciles cleanly and then does nothing:
    # git and the running checks drift apart silently, with no failed resource
@@ -6,6 +6,8 @@ kind: Kustomization
 # and verified. See MIGRATION-gitea-woodpecker.md.
 resources:
  - statefulset.yaml
+  # The live size of the data volume; see the file (REN-583).
+  - pvc.yaml
  - service.yaml
  - ingressroute-ssh.yaml
  - ingress.yaml
@@ -0,0 +1,39 @@
+# The live data volume of gitea-0, owned here so its size is set in git
+# (REN-583). The StatefulSet controller created this claim from the
+# volumeClaimTemplates in statefulset.yaml, but that template is immutable on
+# an existing StatefulSet, so changing the size there cannot be applied. A
+# PVC's resources.requests.storage CAN grow in place (ceph-block has
+# allowVolumeExpansion: true; ceph-csi grows the RBD image and the filesystem
+# online, no pod restart). Flux applies this over the existing claim with
+# server-side apply and the resize follows.
+#
+# - Name, namespace, accessModes, storageClassName and volumeMode must match
+#   the live claim exactly: they are immutable, a mismatch fails the whole
+#   gitea Kustomization (no change to the live claim).
+# - volumeName must stay: kustomize-controller has owned spec.volumeName
+#   since the namespace move pre-created this claim (eb23d3ac, #878).
+#   Without it, server-side apply tries to clear the field and the API server
+#   rejects the whole claim as an immutable-spec change.
+# - Grow only. A smaller value is rejected by the API server.
+# - prune: disabled, so removing this file from git never deletes the claim
+#   and its data. To hand ownership back, delete the file; the claim stays.
+#
+# Size history: 20Gi -> 40Gi 2026-07-22 (#336, live resize by hand);
+# 40Gi -> 60Gi 2026-10-03 (REN-583, after the volume filled again on REN-580).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: data-gitea-0
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes: ["ReadWriteOnce"]
+  storageClassName: ceph-block
+  volumeMode: Filesystem
+  volumeName: pvc-e7c56db2-481d-40cb-b94a-b264b7b4b5be
+  resources:
+    requests:
+      storage: 60Gi
@@ -37,7 +37,7 @@ spec:
          # both handled as explicit steps in MIGRATION-gitea-woodpecker.md. Pinned
          # to match the LXC for a clean SQLite lift; no imagepolicy marker (an
          # auto-bump across versions on live SQLite data must never be applied).
-          image: gitea/gitea:1.21.4-rootless
+          image: gitea/gitea:1.21.4-rootless@sha256:600f1f1e1d090c763866db16882b5eac5c6a80b7664ee53bec0806c656c0033f
          env:
            - name: GITEA_WORK_DIR
              value: /var/lib/gitea
@@ -147,14 +147,10 @@ spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: ceph-block
        resources:
-          # Expanded 20Gi -> 40Gi on 2026-07-22 (ceph-block online resize) to
-          # clear a 100%-full outage (#336). The live PVC was resized directly.
-          # NOTE: volumeClaimTemplates is immutable on an existing StatefulSet,
-          # so bumping this value cannot be `kubectl apply`-ed in place — Flux
-          # will reject the whole object. Landing this size change requires a
-          # one-time `kubectl delete statefulset gitea -n gitea --cascade=orphan`
-          # (pod + PVC keep running) so Flux re-creates the STS from this
-          # template and re-adopts them. After the cleanup_packages GC reclaims
-          # the ~10G of orphaned blobs, real usage is ~9G, so 40Gi is headroom.
+          # This template only applies when the claim does not exist yet. The
+          # live size of data-gitea-0 is set in pvc.yaml (REN-583), not here:
+          # volumeClaimTemplates is immutable on an existing StatefulSet, so a
+          # change to this value cannot be applied and Flux would reject the
+          # whole object. Leave it at 40Gi.
          requests:
            storage: 40Gi
@@ -1,15 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: home-assistant-backup-password
-  namespace: home-assistant
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: home-assistant-backup-password # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: apps/home-assistant/backup-password
@@ -60,7 +60,7 @@ spec:
        # repo's configuration.yaml reads this key, so HA does not start
        # without it. Removing this init container takes the recorder down.
        - name: render-db-url
-          image: busybox:1.36
+          image: busybox:1.38@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e
          command:
            - sh
            - -c
@@ -95,7 +95,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.3 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.9.4 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -159,7 +159,7 @@ spec:
        # substitution to POD_IP so nginx-runtime variables like $host
        # / $http_x_forwarded_for in the template stay intact.
        - name: proxy
-          image: nginxinc/nginx-unprivileged:1.27-alpine
+          image: nginxinc/nginx-unprivileged:1.31-alpine@sha256:26b0bf6fbf07297983cb341998d79c831508787de26627dd2a112321b9c3a4af
          env:
            - name: POD_IP
              valueFrom:
@@ -193,7 +193,7 @@ spec:
        # any LAN client can edit, matching the user's trust-the-network
        # model. Runs as root so its writes match HA's root-owned files.
        - name: code-server
-          image: codercom/code-server:4.118.0
+          image: codercom/code-server:4.140.0@sha256:1c9ad972a78450f430155144110da4c02d3627c72a598e9197e2781fb9f92e28
          args:
            - "--auth=none"
            - "--bind-addr=0.0.0.0:8080"
@@ -253,7 +253,7 @@ spec:
        # in the PVC's remote URL; reload auth uses a dedicated long-lived token.
        # Runs as root to match HA's root-owned /config files. See config-sync.sh.
        - name: config-sync
-          image: alpine/git:2.54.0
+          image: alpine/git:2.54.0@sha256:a4bb51f1a3553df194ce679fc1db721d8bfba2046fa1a88fe9d4ac551ffbce25
          command: ["/bin/sh", "/scripts/sync.sh"]
          env:
            - name: SYNC_BRANCH
@@ -36,7 +36,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: esphome
-          image: ghcr.io/esphome/esphome:2026.6.2
+          image: ghcr.io/esphome/esphome:2026.9.1@sha256:776a845d508909bf426d2ee7bdd81796361ebe6a7f091807ce1d6d0b48985413
          imagePullPolicy: IfNotPresent
          # Default entrypoint runs the dashboard (Device Builder) against the
          # mounted /config — no command override needed.
@@ -31,17 +31,70 @@ metadata:
  namespace: home-assistant
 spec:
  instances: 2
+  # switchover, not CNPG's default `restart`: any change that rolls the pods
+  # (walStorage below did) then promotes the already-rolled replica instead of
+  # restarting the primary in place, so the recorder loses seconds of writes
+  # rather than stopDelay plus a pg_wal move plus startup (REN-176).
+  primaryUpdateMethod: switchover
  smartShutdownTimeout: 10
  stopDelay: 60
  imageName: ghcr.io/cloudnative-pg/postgresql:16
  storage:
    storageClass: ceph-block
-    # Sized for the migrated statistics plus 7 days of states at the current
-    # rate: statistics 10.8M rows + statistics_short_term 1.9M (~1.8 GB with
-    # indexes in Postgres) and ~18M states rows for the purge_keep_days:7 window
-    # (~4 GB). 20Gi leaves room for autovacuum bloat and index churn.
-    # ceph-block has allowVolumeExpansion: true, so this can grow in place.
-    size: 20Gi
+    # Sized for WAL headroom during an archiving stall, not for table growth.
+    # The data itself needs far less: migrated statistics 10.8M rows +
+    # statistics_short_term 1.9M (~1.8 GB with indexes in Postgres) and ~18M
+    # states rows for the purge_keep_days:7 window (~4 GB).
+    #
+    # 20Gi left only 6.63 GiB free, and that is the number that matters here.
+    # WAL archiving stalled for 10 minutes and the un-archived backlog reached
+    # 1.84 GiB — 28% of that free space (REN-129).
+    #
+    # That stall was first blamed on Velero repository maintenance. It is not
+    # the cause: REN-136 measured seven nights and the archiver's sustained
+    # drain (~14 segments/min, its serial ceiling) is flat whether Kopia is
+    # running or not, and Kopia's nearest batch drifts across 02:03-02:24Z
+    # while the burst stays pinned to 02:13Z. What is real is a nightly
+    # producer burst — 56-119 segments queued at 02:13-02:15 UTC on all seven
+    # nights, against a steady-state queue of 0. Postgres cannot recycle a WAL
+    # segment until the archiver has shipped it, so a burst that arrives while
+    # archiving is stalled walks the volume toward full, and a full PGDATA
+    # stops the recorder outright.
+    #
+    # 32Gi takes free space to ~18.6 GiB, which puts the same 1.84 GiB backlog
+    # at 10% of headroom rather than 28%. In REN-136's terms: at 6.36 GiB free
+    # one stalled night survives and two fill the volume; ~18.6 GiB carries
+    # about four. Approved by the owner on REN-135.
+    # ceph-block has allowVolumeExpansion: true, so this grows in place.
+    #
+    # Since REN-176 WAL lives on walStorage below, so the stall headroom above
+    # now belongs to that volume. 32Gi stays because a PVC cannot shrink.
+    size: 32Gi
+  # WAL on its own volume (ha-pg-N-wal), approved by the owner on REN-176.
+  #
+  # PERMANENT. The CNPG webhook rejects removing walStorage once set ("walStorage
+  # cannot be disabled once configured"), so reverting this block fails
+  # admission and leaves the home-assistant Kustomization stuck. Undoing it
+  # means a new Cluster recovered from ha-pg-store. Adding it on 1.30.1 needs
+  # no re-clone: the operator creates the PVC, rolls each pod, and the instance
+  # manager moves pg_wal onto it before Postgres starts.
+  #
+  # This does NOT keep the recorder up when WAL fills: a full WAL disk still
+  # stops Postgres. What it does is keep WAL writable if PGDATA fills (a purge
+  # that stops running), and make WAL-volume usage a direct archiver-backlog
+  # signal.
+  #
+  # Sized so stall survival is no worse than the shared 32Gi gives today.
+  # Today: ~18.6 GiB free on PGDATA / 4.66 GiB/day of WAL = ~4.0 days.
+  #   1.86 GiB  normal pg_wal footprint (max_wal_size 2GB)
+  # + 18.64 GiB four days of full archiver outage at 4.66 GiB/day
+  #             (the nightly 56-119 segment burst, REN-136, is inside that rate)
+  # = 20.5 GiB -> 21Gi: (21 - 1.86) / 4.66 = ~4.1 days. 18Gi would give ~3.5,
+  # worse than today. RBD is thin, so the ceiling costs nothing until a stall
+  # fills it. The webhook requires min_wal_size < max_wal_size < this size.
+  walStorage:
+    storageClass: ceph-block
+    size: 21Gi
  plugins:
    - name: barman-cloud.cloudnative-pg.io
      isWALArchiver: true
@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - backup-password-externalsecret.yaml
  - pvcs.yaml
  - deployment.yaml
  - service.yaml
@@ -13,8 +12,9 @@ resources:
  - prometheus-token-externalsecret.yaml
  - config-sync-token-externalsecret.yaml
  - mosquitto-bootstrap.yaml
-  # health broker credential: OpenBao-backed + Class-A rotated (first of the
-  # eight users to migrate off mosquitto-bootstrap.yaml).
+  # Former `health` probe credential. The broker no longer reads it (the
+  # probes use the cert account mosquitto-probe, REN-953); removed with its
+  # rotator in the REN-802 cleanup step.
  - mosquitto-health-credentials-externalsecret.yaml
  # open3e-int broker credential: open3e-esp32 gateway on the Vitocal internal
  # CAN; OpenBao-backed + Class-A rotated.
@@ -69,6 +69,12 @@ resources:
  - ha-pg-objectstore.yaml
  - ha-pg-cluster.yaml
  - ha-pg-scheduled-backup.yaml
+  # MQTT credential management (REN-802): the account controller manages the
+  # production broker's Dynamic Security (REN-900). mosquitto-dynsec-test is
+  # the step-1 test broker, no longer managed; removal follows separately.
+  # Register: mqtt-accounts.yaml (configMapGenerator).
+  - mosquitto-dynsec-test.yaml
+  - mqtt-account-controller.yaml
 configMapGenerator:
  # nginx config for the reverse-proxy sidecar. Hash suffix disabled so
  # the cm name stays stable; editing proxy.conf does not auto-roll the
@@ -104,5 +110,28 @@ configMapGenerator:
    namespace: home-assistant
    files:
      - guard.sh=zigbee2mqtt-discovery-guard.sh
+  # Z2M external converters, copied into /app/data/external_converters by the
+  # seed-data init container on every start. Hash suffix disabled (name stays
+  # stable), so editing a converter does not auto-roll the pod — restart
+  # manually with `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`.
+  - name: zigbee2mqtt-external-converters
+    namespace: home-assistant
+    files:
+      - wetterstation-ws90.js=zigbee2mqtt-ws90-converter.js
+  # MQTT account register, read by mqtt-account-controller on every loop.
+  # Hash suffix disabled (name stays stable); the volume is not a subPath
+  # mount, so an edit reaches the running controller without a restart.
+  - name: mqtt-accounts
+    namespace: home-assistant
+    files:
+      - mqtt-accounts.yaml
+  # shelly-manager api-proxy sidecar config (REN-935). Hash suffix disabled
+  # (name stays stable), so editing shelly-manager-api-proxy.conf does not
+  # auto-roll the pod — restart manually with
+  # `kubectl rollout restart deploy/shelly-manager -n home-assistant`.
+  - name: shelly-manager-api-proxy
+    namespace: home-assistant
+    files:
+      - api.conf.template=shelly-manager-api-proxy.conf
 generatorOptions:
  disableNameSuffixHash: true
@@ -42,6 +42,9 @@ spec:
      labels:
        app.kubernetes.io/name: mdns-relay
      annotations:
+        # Bump to restart the pod and re-attach net1 (DHCP via multus).
+        # 2026-10-06: first attach on containernetworking/plugins v1.9.1.
+        homelab.renner.uno/restarted-for: "2026-10-06 cni-plugins v1.9.1 attach check"
        k8s.v1.cni.cncf.io/networks: mdns-relay-iot
    spec:
      enableServiceLinks: false
@@ -82,7 +85,7 @@ spec:
        # deliver, and the ClusterIP has no backend while HA restarts. The
        # trap forwards SIGTERM so the pod still terminates promptly.
        - name: socat-ipv4
-          image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
+          image: alpine/socat:1.8.1.3@sha256:82ad20f6f6e29b91ff33b6662d24063522b1378f08fd8569cd3ab412cac13f50
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
@@ -124,7 +127,7 @@ spec:
        # depends on the kernel's bindv6only default rather than on anything
        # stated here, so the roles are made explicit instead of inherited.
        - name: socat-ipv6
-          image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
+          image: alpine/socat:1.8.1.3@sha256:82ad20f6f6e29b91ff33b6662d24063522b1378f08fd8569cd3ab412cac13f50
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
@@ -44,6 +44,10 @@ kind: Job
 metadata:
  name: mosquitto-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -55,7 +59,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -112,6 +116,10 @@ kind: Job
 metadata:
  name: mosquitto-admin-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -123,7 +131,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -179,6 +187,10 @@ kind: Job
 metadata:
  name: mosquitto-shellies-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -190,7 +202,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -245,6 +257,10 @@ kind: Job
 metadata:
  name: mosquitto-sml2mqtt-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -256,7 +272,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -313,6 +329,10 @@ kind: Job
 metadata:
  name: mosquitto-open3e-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -324,7 +344,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -383,6 +403,10 @@ kind: Job
 metadata:
  name: mosquitto-zigbee2mqtt-credentials-bootstrap
  namespace: home-assistant
+  annotations:
+    # Job pod templates are immutable; force-recreate so image bumps
+    # apply on Flux re-reconcile. Safe: the Job is idempotent (see above).
+    kustomize.toolkit.fluxcd.io/force: enabled
 spec:
  backoffLimit: 4
  template:
@@ -394,7 +418,7 @@ spec:
      restartPolicy: OnFailure
      containers:
        - name: bootstrap
-          image: alpine:3.21
+          image: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
          command:
            - sh
            - -c
@@ -10,3 +10,38 @@ spec:
    kind: ClusterIssuer
  dnsNames:
    - mqtt.renner.uno
+---
+# Server certificate of the mutual-TLS listener :8884 (REN-802). Issued by
+# mqtt-ca because certificate clients and mqtt-account-controller trust only
+# that CA. The Let's Encrypt certificate above stays on :8883 for password
+# clients. In-cluster names only; an outside cert client (edge-1) is REN-802
+# step 3.
+#
+# Long-lived on purpose: mosquitto reads its certificate only at start, and a
+# broker restart is an outage, so nothing restarts it for a renewal. With 2
+# years and renewal after 1, the broker serves a renewed certificate after the
+# next restart of any kind (image bump, node reboot) within that year; the old
+# one stays valid until then.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mosquitto-mtls
+  namespace: home-assistant
+spec:
+  secretName: mosquitto-mtls
+  duration: 17520h
+  renewBefore: 8760h
+  dnsNames:
+    - mosquitto
+    - mosquitto.home-assistant.svc
+    - mosquitto.home-assistant.svc.cluster.local
+  usages:
+    - digital signature
+    - server auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
@@ -64,6 +64,66 @@ data:
    certfile /mosquitto/tls/tls.crt
    keyfile /mosquitto/tls/tls.key

+    # Mutual TLS for certificate clients (REN-802). Only certificates from
+    # mqtt-ca are accepted, and the certificate's CN is the username. Not on
+    # the LoadBalancer: in-cluster only (Service `mosquitto`).
+    listener 8884
+    max_connections 200
+    cafile /mosquitto/mtls/ca.crt
+    certfile /mosquitto/mtls/tls.crt
+    keyfile /mosquitto/mtls/tls.key
+    require_certificate true
+    use_identity_as_username true
+
    allow_anonymous false
+
+    # Two account sources side by side (REN-802 step 2, tested on 2.0.22 in
+    # REN-900, document `coexistence`):
+    #   Dynamic Security: accounts from the register mqtt-accounts.yaml,
+    #     managed live by mqtt-account-controller over $CONTROL.
+    #   password_file/acl_file: the legacy users, unchanged until each client
+    #     moves to the register (step 3).
+    # Dynamic Security is asked first. For a username it does not know, and
+    # for a request none of its roles match, it defers to the files. A
+    # register entry with the same name as a legacy user therefore replaces
+    # that user's password, while its acl_file block still applies to
+    # whatever the entry's roles do not cover. Delete the legacy user when
+    # its client moves.
+    #
+    # The acl_file check of 2.0 allows every SUBSCRIBE and checks only on
+    # delivery, so the Dynamic Security default must deny publishClientReceive.
+    # Otherwise every Dynamic Security client could read every topic.
+    plugin /usr/lib/mosquitto_dynamic_security.so
+    plugin_opt_config_file /mosquitto/auth/dynamic-security.json
    password_file /mosquitto/auth/passwd
    acl_file /mosquitto/auth/acl
+  # Dynamic Security start file, used until mqtt-account-controller has
+  # written the Secret mqtt-dynsec-seed. No secrets: only the controller's
+  # certificate client and a role that reaches the Dynamic Security API and
+  # nothing else.
+  dynamic-security.json: |
+    {
+      "defaultACLAccess": {
+        "publishClientSend": false,
+        "publishClientReceive": false,
+        "subscribe": false,
+        "unsubscribe": true
+      },
+      "clients": [
+        {
+          "username": "mqtt-account-controller",
+          "roles": [{"rolename": "mqtt-account-controller-admin"}]
+        }
+      ],
+      "groups": [],
+      "roles": [
+        {
+          "rolename": "mqtt-account-controller-admin",
+          "acls": [
+            {"acltype": "publishClientReceive", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "publishClientSend", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "subscribePattern", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true}
+          ]
+        }
+      ]
+    }
@@ -12,7 +12,7 @@ metadata:
    #
    # Listed per-Secret rather than with the blanket auto annotation, so that
    # enrolling another broker user is a visible edit to this line.
-    secret.reloader.stakater.com/reload: mosquitto-health-credentials,mosquitto-open3e-int-credentials
+    secret.reloader.stakater.com/reload: mosquitto-open3e-int-credentials
 spec:
  replicas: 1
  strategy:
@@ -54,7 +54,7 @@ spec:
        fsGroup: 1883
      containers:
        - name: mosquitto
-          # Pinned tag, auto-bumped within the 2.x line by Flux image-automation
+          # Pinned tag, auto-bumped within the 2.0 line by Flux image-automation
          # (infrastructure/image-automation/mosquitto.yaml). The marker tells
          # Flux where to write; patch/minor updates land as git commits and
          # roll the broker. HA re-publishes its own discovery afterwards, but
@@ -80,7 +80,6 @@ spec:
              mosquitto_passwd -b /mosquitto/auth/passwd open3e "$OPEN3E_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd open3e-int "$OPEN3E_INT_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd evcc "$EVCC_PASSWORD"
-              mosquitto_passwd -b /mosquitto/auth/passwd health "$HEALTH_PASSWORD"
              mosquitto_passwd -b /mosquitto/auth/passwd zigbee2mqtt "$ZIGBEE2MQTT_PASSWORD"
              # ACL: HA keeps full access (subscribes to everything); the
              # shellies user is `readwrite shellies/#` (devices publish state
@@ -136,16 +135,6 @@ spec:
              topic readwrite evcc/#
              topic read energy/#

-              # `health` is the broker's own probe credential (see the
-              # liveness/readiness probes below). Scoped to health/# and
-              # nothing else: the probes publish a random value retained and
-              # read it back to prove the broker actually stores+delivers
-              # (not just PUBACKs) — so the user needs read AND write, but
-              # only under health/#. A leak can only touch the throwaway
-              # probe subtree that nothing else subscribes to.
-              user health
-              topic readwrite health/#
-
              # `zigbee2mqtt` is the Zigbee2MQTT bridge (the ZHA replacement).
              # Owns its own `zigbee2mqtt/#` subtree; may WRITE `homeassistant/#`
              # to publish HA MQTT-discovery configs (write-only there, like the
@@ -165,8 +154,21 @@ spec:
              topic write homeassistant/#
              topic read homeassistant/status
              EOF
-              chown 1883:1883 /mosquitto/auth/passwd /mosquitto/auth/acl
-              chmod 0600 /mosquitto/auth/passwd /mosquitto/auth/acl
+              # Dynamic Security start file (REN-802): the controller's seed
+              # if it exists, else the static base file. The seed carries
+              # every register account and its hash, so the broker is in the
+              # desired state before the first client connects. The plugin
+              # rewrites this file on every $CONTROL change, so the broker
+              # user owns it and the directory.
+              if [ -s /mosquitto/seed/dynamic-security.json ]; then
+                cp /mosquitto/seed/dynamic-security.json /mosquitto/auth/dynamic-security.json
+                echo "dynsec: loaded seed from Secret mqtt-dynsec-seed"
+              else
+                cp /mosquitto/config/dynamic-security.json /mosquitto/auth/dynamic-security.json
+                echo "dynsec: no seed yet, loaded base file"
+              fi
+              chown 1883:1883 /mosquitto/auth /mosquitto/auth/passwd /mosquitto/auth/acl /mosquitto/auth/dynamic-security.json
+              chmod 0600 /mosquitto/auth/passwd /mosquitto/auth/acl /mosquitto/auth/dynamic-security.json
              exec mosquitto -c /mosquitto/config/mosquitto.conf
          env:
            - name: HA_PASSWORD
@@ -206,17 +208,6 @@ spec:
                secretKeyRef:
                  name: mosquitto-evcc-credentials
                  key: password
-            # Probe-only credential. Now OpenBao-backed and Class-A rotated (see
-            # mosquitto-health-credentials-externalsecret.yaml); its bootstrap Job
-            # is gone.
-            # Present here for the same reason as the others — the startup
-            # shim hashes it into the passwd file — and reused by the exec
-            # probes below, which run inside this container.
-            - name: HEALTH_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-health-credentials
-                  key: password
            # Zigbee2MQTT bridge credential (mosquitto-zigbee2mqtt-credentials-
            # bootstrap). Hashed into the passwd file here; Z2M reads the same
            # Secret via its init container to authenticate to the broker.
@@ -232,6 +223,9 @@ spec:
            - name: mqtts
              containerPort: 8883
              protocol: TCP
+            - name: mqtts-cert
+              containerPort: 8884
+              protocol: TCP
          volumeMounts:
            - name: config
              mountPath: /mosquitto/config
@@ -241,6 +235,15 @@ spec:
            - name: tls
              mountPath: /mosquitto/tls
              readOnly: true
+            - name: mtls
+              mountPath: /mosquitto/mtls
+              readOnly: true
+            - name: seed
+              mountPath: /mosquitto/seed
+              readOnly: true
+            - name: probe-tls
+              mountPath: /mosquitto/probe-tls
+              readOnly: true
          # Real MQTT round-trip probes, replacing the old tcpSocket checks.
          # A bare TCP accept proved only that a listener existed. Even a
          # QoS-1 PUBACK proves only that the broker acknowledged the publish
@@ -259,14 +262,14 @@ spec:
          # Liveness and readiness use SEPARATE topics (probe/live vs
          # probe/ready) so their rotating retained values never collide.
          #
-          # Uses the dedicated least-privilege `health` user (ACL: readwrite
-          # health/# only — see the passwd/acl build above and the
-          # mosquitto-health-credentials ExternalSecret). Its password is in
-          # this container's env (HEALTH_PASSWORD) because the startup shim
-          # hashes every user into the passwd file; the exec probes run
-          # inside this container and read it directly. A leak of this
-          # credential can only touch the health/# probe subtree, which
-          # nothing else subscribes to.
+          # Logs in as the register account `mosquitto-probe` (mqtt-accounts.yaml,
+          # REN-953): a client certificate from mqtt-ca on the mTLS listener
+          # :8884, ACL readwrite health/# only. No password exists for it.
+          # The Secret mount is not a subPath, so a cert-manager renewal
+          # reaches the next probe without a restart. --insecure skips only
+          # the hostname check, because the server certificate names the
+          # Service and the probe dials 127.0.0.1; the chain is still checked
+          # against mqtt-ca.
          #
          # mosquitto is a single-replica SPOF (HA, Shelly, sml2mqtt, open3e,
          # evcc). Liveness is deliberately slow to fire — failureThreshold 6
@@ -286,8 +289,9 @@ spec:
                - -c
                - |
                  V="probe-$(head -c8 /dev/urandom | od -An -tx1 | tr -d ' ')"
-                  mosquitto_pub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -q 1 -r -t health/probe/ready -m "$V" &&
-                  [ "$(mosquitto_sub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -t health/probe/ready -C 1 -W 5)" = "$V" ]
+                  T="-h 127.0.0.1 -p 8884 --insecure --cafile /mosquitto/probe-tls/ca.crt --cert /mosquitto/probe-tls/tls.crt --key /mosquitto/probe-tls/tls.key"
+                  mosquitto_pub $T -q 1 -r -t health/probe/ready -m "$V" &&
+                  [ "$(mosquitto_sub $T -t health/probe/ready -C 1 -W 5)" = "$V" ]
            periodSeconds: 15
            timeoutSeconds: 10
            failureThreshold: 3
@@ -298,8 +302,9 @@ spec:
                - -c
                - |
                  V="probe-$(head -c8 /dev/urandom | od -An -tx1 | tr -d ' ')"
-                  mosquitto_pub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -q 1 -r -t health/probe/live -m "$V" &&
-                  [ "$(mosquitto_sub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -t health/probe/live -C 1 -W 5)" = "$V" ]
+                  T="-h 127.0.0.1 -p 8884 --insecure --cafile /mosquitto/probe-tls/ca.crt --cert /mosquitto/probe-tls/tls.crt --key /mosquitto/probe-tls/tls.key"
+                  mosquitto_pub $T -q 1 -r -t health/probe/live -m "$V" &&
+                  [ "$(mosquitto_sub $T -t health/probe/live -C 1 -W 5)" = "$V" ]
            periodSeconds: 30
            timeoutSeconds: 10
            failureThreshold: 6
@@ -320,3 +325,23 @@ spec:
        - name: tls
          secret:
            secretName: mosquitto-tls
+        # mqtt-ca server certificate and CA of the :8884 mTLS listener, read
+        # only at start (renewal timing: mosquitto-certificate.yaml).
+        - name: mtls
+          secret:
+            secretName: mosquitto-mtls
+        # Written by mqtt-account-controller. Optional so the broker still
+        # starts with the base file if the Secret is missing. Not in the
+        # reloader annotation: a register change is applied live over
+        # $CONTROL and must never restart the broker.
+        - name: seed
+          secret:
+            secretName: mqtt-dynsec-seed
+            optional: true
+        # Probe certificate of the register account mosquitto-probe, created
+        # by mqtt-account-controller. Not optional: a pod without it fails its
+        # probes, so it waits for the Secret instead.
+        - name: probe-tls
+          secret:
+            secretName: mqtt-mosquitto-probe-tls
+            defaultMode: 0400
@@ -0,0 +1,233 @@
+# Test broker for mqtt-account-controller (REN-802 step 1, REN-888).
+#
+# A second, throwaway mosquitto with the Dynamic Security plugin and a mutual-
+# TLS listener, so the controller can be proven (cert login, reconcile, seed
+# after restart, revocation) without touching the production broker. Nothing
+# in production connects here.
+#
+# Deliberately NOT labelled app.kubernetes.io/name: mosquitto: that label is
+# the selector of the production Services and of mosquitto-world-ingress, so
+# reusing it would put this pod behind mqtt.renner.uno and the IoT VLAN.
+# ClusterIP only; reachable from inside the home-assistant namespace.
+#
+# Removal: delete this file and its kustomization.yaml entry once production
+# runs Dynamic Security (REN-802 step 2).
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mosquitto-dynsec-test-tls
+  namespace: home-assistant
+spec:
+  secretName: mosquitto-dynsec-test-tls
+  dnsNames:
+    - mosquitto-dynsec-test
+    - mosquitto-dynsec-test.home-assistant.svc
+    - mosquitto-dynsec-test.home-assistant.svc.cluster.local
+  usages:
+    - digital signature
+    - server auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mosquitto-dynsec-test-config
+  namespace: home-assistant
+data:
+  mosquitto.conf: |
+    persistence false
+    log_dest stdout
+    log_type error
+    log_type warning
+    log_type notice
+    log_type information
+
+    # Dynamic Security owns authentication and ACLs. The plugin writes its
+    # state back to this file; it is an emptyDir, refilled by the
+    # initContainer on every start.
+    plugin /usr/lib/mosquitto_dynamic_security.so
+    plugin_opt_config_file /mosquitto/dynsec/dynamic-security.json
+
+    # Mutual TLS only. The client certificate's CN is the username; the
+    # Dynamic Security client of that name supplies the roles.
+    listener 8883
+    cafile /mosquitto/tls/ca.crt
+    certfile /mosquitto/tls/tls.crt
+    keyfile /mosquitto/tls/tls.key
+    require_certificate true
+    use_identity_as_username true
+
+    allow_anonymous false
+  # Start file used until the controller has written the Secret
+  # mqtt-dynsec-seed. No secrets: only the controller's certificate client
+  # and a role that reaches the Dynamic Security API and nothing else. Same
+  # shape as the controller's seed renderer (internal/dynsec/config.go).
+  dynamic-security.json: |
+    {
+      "defaultACLAccess": {
+        "publishClientSend": false,
+        "publishClientReceive": true,
+        "subscribe": false,
+        "unsubscribe": true
+      },
+      "clients": [
+        {
+          "username": "mqtt-account-controller",
+          "roles": [{"rolename": "mqtt-account-controller-admin"}]
+        }
+      ],
+      "groups": [],
+      "roles": [
+        {
+          "rolename": "mqtt-account-controller-admin",
+          "acls": [
+            {"acltype": "publishClientReceive", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "publishClientSend", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true},
+            {"acltype": "subscribePattern", "topic": "$CONTROL/dynamic-security/v1/#", "priority": 0, "allow": true}
+          ]
+        }
+      ]
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mosquitto-dynsec-test
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mosquitto-dynsec-test
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mosquitto-dynsec-test
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 10
+      # Runs as the mosquitto UID from the start instead of starting as root
+      # and dropping, so the dynsec file the initContainer writes is the
+      # broker's own.
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1883
+        runAsGroup: 1883
+        fsGroup: 1883
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # Every start: the controller's seed if it exists, else the static
+        # base file. The seed carries every account and password hash, so the
+        # broker is in the desired state before the first client connects.
+        - name: dynsec-seed
+          image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
+          command:
+            - sh
+            - -c
+            - |
+              set -eu
+              if [ -s /seed/dynamic-security.json ]; then
+                cp /seed/dynamic-security.json /mosquitto/dynsec/dynamic-security.json
+                echo "dynsec: loaded seed from Secret mqtt-dynsec-seed"
+              else
+                cp /mosquitto/config/dynamic-security.json /mosquitto/dynsec/dynamic-security.json
+                echo "dynsec: no seed yet, loaded base file"
+              fi
+              chmod 0600 /mosquitto/dynsec/dynamic-security.json
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /mosquitto/config
+              readOnly: true
+            - name: seed
+              mountPath: /seed
+              readOnly: true
+            - name: dynsec
+              mountPath: /mosquitto/dynsec
+      containers:
+        - name: mosquitto
+          # Same tag and image policy as the production broker, so the test
+          # runs the version that production will run.
+          image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
+          command: ["mosquitto", "-c", "/mosquitto/config/mosquitto.conf"]
+          ports:
+            - name: mqtts
+              containerPort: 8883
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /mosquitto/config
+              readOnly: true
+            - name: tls
+              mountPath: /mosquitto/tls
+              readOnly: true
+            - name: dynsec
+              mountPath: /mosquitto/dynsec
+          readinessProbe:
+            tcpSocket:
+              port: mqtts
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: mqtts
+            periodSeconds: 30
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 5m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+      volumes:
+        - name: config
+          configMap:
+            name: mosquitto-dynsec-test-config
+        - name: tls
+          secret:
+            secretName: mosquitto-dynsec-test-tls
+        # Written by mqtt-account-controller. Optional: absent on the very
+        # first start, when the base file is used instead.
+        - name: seed
+          secret:
+            secretName: mqtt-dynsec-seed
+            optional: true
+        - name: dynsec
+          emptyDir:
+            sizeLimit: 4Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mosquitto-dynsec-test
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: mosquitto-dynsec-test
+  ports:
+    - name: mqtts
+      port: 8883
+      targetPort: mqtts
+      protocol: TCP
@@ -12,3 +12,7 @@ spec:
    - name: mqtt
      port: 1883
      targetPort: mqtt
+    # Mutual TLS for certificate clients and mqtt-account-controller (REN-802).
+    - name: mqtts-cert
+      port: 8884
+      targetPort: mqtts-cert
@@ -0,0 +1,317 @@
+# mqtt-account-controller (REN-802, code REN-837, deploy REN-888).
+# Source: konairius/mqtt-account-controller (maintainer Hephaestus). Reconciles
+# the broker's Dynamic Security clients/roles/ACLs, the cert-manager
+# Certificates of `auth: cert` accounts and device passwords to the register
+# mqtt-accounts.yaml, every INTERVAL (2m).
+#
+# Manages the production broker `mosquitto` over its mTLS listener :8884
+# (REN-802 step 2, REN-900). The legacy users in that broker's password_file
+# are not Dynamic Security clients, so the controller never sees or touches
+# them.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+---
+# Find the Certificates it manages (label app.kubernetes.io/managed-by=
+# mqtt-account-controller) in every namespace. List only: no get, watch or
+# write cluster-wide.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: mqtt-account-controller-list-certificates
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: mqtt-account-controller-list-certificates
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: mqtt-account-controller-list-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# Per deliver.namespace: manage the account Certificates there (server-side
+# apply = patch, which creates through `create`). One Role + RoleBinding per
+# namespace that appears as `deliver.namespace` in the register: home-assistant,
+# evcc and paperclip.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "create", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# Secrets in home-assistant, by name only. This namespace holds every broker
+# and HA credential, so an unnamed `delete secrets` grant would let the
+# controller wipe all of them.
+#   mqtt-account-hashes, mqtt-dynsec-seed: its own state (server-side apply
+#     creates them, hence create + patch; resourceNames hold for an apply
+#     that creates, because the name is in the request path).
+#   mqtt-<name>-tls: delete on revocation, one entry per cert account
+#     delivered to this namespace. Add the name with the register entry.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-secrets-home-assistant
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["mqtt-account-hashes", "mqtt-dynsec-seed"]
+    verbs: ["get", "create", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames:
+      - mqtt-test-cert-tls
+      - mqtt-mosquitto-probe-tls
+      - mqtt-zigbee2mqtt-bridge-tls
+      - mqtt-home-assistant-tls
+    verbs: ["delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-secrets-home-assistant
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-secrets-home-assistant
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# deliver.namespace evcc (account evcc-client): its Certificate, and its
+# Secret by name for revocation.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: evcc
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "create", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames:
+      - mqtt-evcc-client-tls
+    verbs: ["delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: evcc
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# deliver.namespace paperclip (account agents-read): its Certificate, and its
+# Secret by name for revocation.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: paperclip
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "create", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames:
+      - mqtt-agents-read-tls
+    verbs: ["delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mqtt-account-controller-certificates
+  namespace: paperclip
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mqtt-account-controller-certificates
+subjects:
+  - kind: ServiceAccount
+    name: mqtt-account-controller
+    namespace: home-assistant
+---
+# The controller's own MQTT identity: CN = Dynamic Security admin client
+# (ADMIN_USER). Not labelled managed-by, so the controller never treats it
+# as an account certificate and never revokes it. Re-read on every TLS
+# handshake, so a renewal needs no restart.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+spec:
+  secretName: mqtt-account-controller-tls
+  commonName: mqtt-account-controller
+  usages:
+    - digital signature
+    - client auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+    rotationPolicy: Always
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: mqtt-ca
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mqtt-account-controller
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: mqtt-account-controller
+  annotations:
+    # The controller reads the shelly-manager API key once at start; the
+    # quarterly rotation of shelly-manager-auth must restart it too.
+    secret.reloader.stakater.com/reload: shelly-manager-auth
+spec:
+  replicas: 1
+  # One writer at a time: two controllers would race on rotations and the
+  # seed Secret.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mqtt-account-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mqtt-account-controller
+    spec:
+      serviceAccountName: mqtt-account-controller
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: controller
+          # Woodpecker builds main-<sha> on every merge to main; no sortable
+          # tag, so bumps are manual (same as ical-proxy).
+          image: gitea.renner.uno/konairius/mqtt-account-controller:main-982231a@sha256:851dd025a8aea352061ac818f9968e1c4a074c8f33af405efde3424382e77ae1
+          env:
+            - name: BROKER_URL
+              value: ssl://mosquitto.home-assistant.svc.cluster.local:8884
+            # Password push to Shellies (accounts with deliver.shelly-manager).
+            # The API-key-only Service (api-proxy sidecar, :8081), like the
+            # agents; same namespace, so the default-deny allows it.
+            - name: SHELLY_MANAGER_URL
+              value: http://shelly-manager-api.home-assistant.svc.cluster.local
+            - name: SHELLY_MANAGER_API_KEY_FILE
+              value: /etc/shelly-manager/api-key
+            # Not :8080: several :8080 ingress grants in the home-assistant
+            # default-deny apply to every pod in the namespace.
+            - name: LISTEN
+              value: ":8081"
+          ports:
+            - name: http
+              containerPort: 8081
+              protocol: TCP
+          # /healthz is 200 while a reconcile succeeded within 3 x INTERVAL.
+          # Readiness only: a restart does not fix an unreachable broker.
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: register
+              mountPath: /etc/mqtt-accounts
+              readOnly: true
+            - name: tls
+              mountPath: /etc/mqtt-tls
+              readOnly: true
+            - name: shelly-manager-key
+              mountPath: /etc/shelly-manager
+              readOnly: true
+          resources:
+            requests:
+              cpu: 5m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+      volumes:
+        # Not a subPath mount, so a register change reaches the running pod
+        # (kubelet sync, ~1 min) and the next loop reads it without a restart.
+        - name: register
+          configMap:
+            name: mqtt-accounts
+        - name: tls
+          secret:
+            secretName: mqtt-account-controller-tls
+        # Only the api-key item, not the shelly-manager admin password.
+        - name: shelly-manager-key
+          secret:
+            secretName: shelly-manager-auth
+            items:
+              - key: api-key
+                path: api-key
@@ -0,0 +1,206 @@
+# MQTT account register (REN-802). The only definition of who may use the
+# broker: mqtt-account-controller reconciles certificates, device passwords
+# and Dynamic Security clients/roles/ACLs to this list. Format: README of
+# konairius/mqtt-account-controller.
+#
+# Delivered to the controller as the ConfigMap `mqtt-accounts` (see the
+# configMapGenerator in kustomization.yaml). An empty list is refused, so it
+# never revokes everyone.
+#
+# The controller manages the production broker `mosquitto` (REN-900). Its
+# legacy users (password_file in mosquitto-deployment.yaml) are not listed
+# here; each moves here with its client (REN-802 step 3). `test-cert` is the
+# certificate test account from REN-888.
+#
+# A cert account needs its Secret `mqtt-<name>-tls` added to the
+# `mqtt-account-controller-secrets-<namespace>` Role in
+# mqtt-account-controller.yaml, or revoking it fails with 403.
+- name: test-cert
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite test/cert/#"]
+# The broker's own liveness/readiness probe (mosquitto-deployment.yaml).
+# Not named `health`: a Dynamic Security client of that name would shadow the
+# legacy password user while the old broker pod still probes with it.
+- name: mosquitto-probe
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite health/#"]
+# Zigbee2MQTT and its two sidecars (discovery-guard, interview-watch), one
+# certificate for the pod. Same ACL as the legacy user `zigbee2mqtt`; not
+# named like it, so this entry cannot shadow the password login of a running
+# Z2M (REN-953).
+- name: zigbee2mqtt-bridge
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite zigbee2mqtt/#", "write homeassistant/#", "read homeassistant/status"]
+# evcc (evcc namespace). Same ACL as the legacy user `evcc`, different name
+# so it cannot shadow the running password login (REN-953). evcc takes the
+# certificate PEMs by env (EVCC_MQTT_*), see apps/evcc/deployment.yaml.
+- name: evcc-client
+  auth: cert
+  deliver: {namespace: evcc}
+  acl: ["read open3e/#", "readwrite evcc/#", "read energy/#"]
+# Home Assistant, via the ghostunnel sidecar `mqtt-tls` in its pod (HA keeps
+# client certificates only as uploaded PEMs in its config entry, so a renewal
+# could not reach it without a reconfigure). Same ACL as the legacy user
+# `homeassistant`; not named like it, so it cannot shadow that login (REN-953).
+- name: home-assistant
+  auth: cert
+  deliver: {namespace: home-assistant}
+  acl: ["readwrite #"]
+# Read-only account for agents (REN-953). One account for all of them: the
+# agents share sandboxes, so per-agent separation would be nominal. The
+# Secret lands in `paperclip`; agents fetch it per run with their kubectl
+# identity (Role mqtt-agents-read in apps/paperclip/agent-kubectl-rbac.yaml),
+# so a renewal needs no restart. Network path: agent sandbox pods ->
+# mosquitto :8884 only.
+- name: agents-read
+  auth: cert
+  deliver: {namespace: paperclip}
+  acl: ["read open3e/#", "read zigbee2mqtt/#", "read homeassistant/#"]
+# Shellies (REN-1017): one password account per device, pushed by the
+# controller through shelly-manager's per-device API. Name shelly-<id> =
+# shelly-manager device id. Same ACL as the legacy user `shellies`, which
+# keeps working until every device has moved. Gen2+ devices reboot once
+# when their password is pushed.
+- name: shelly-5               # Plus i4, the API test device of REN-929
+  auth: password
+  deliver: {shelly-manager: 5}
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+# Only devices that log in as `shellies` today. A device with MQTT off never
+# reports connected, so the controller would push (and reboot it) again
+# every ROTATE_GRACE.
+- name: shelly-6
+  auth: password
+  deliver: {shelly-manager: 6}  # Plus i4 90380c3808b8
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-29
+  auth: password
+  deliver: {shelly-manager: 29}  # Plus i4 b8d61a8c7f24
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-3
+  auth: password
+  deliver: {shelly-manager: 3}  # Pro 3EM ac15186d43c0
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-42
+  auth: password
+  deliver: {shelly-manager: 42}  # 3EM-63 G3 d885ac1c5ab4
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-43
+  auth: password
+  deliver: {shelly-manager: 43}  # 3EM-63 G3 d885ac1ab274
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-10
+  auth: password
+  deliver: {shelly-manager: 10}  # Plug S Gen1 4382e5
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-22
+  auth: password
+  deliver: {shelly-manager: 22}  # Plug S Gen1 87e961
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-25
+  auth: password
+  deliver: {shelly-manager: 25}  # Plug S Gen1 1c1a8f
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-15
+  auth: password
+  deliver: {shelly-manager: 15}  # Plug S G3 8cbfea910578
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-16
+  auth: password
+  deliver: {shelly-manager: 16}  # Plug S G3 b08184a63ef0
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-14
+  auth: password
+  deliver: {shelly-manager: 14}  # Plus 1PM Mini 348518e0be40
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-27
+  auth: password
+  deliver: {shelly-manager: 27}  # Plus 1PM Mini 348518e06c04
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-13
+  auth: password
+  deliver: {shelly-manager: 13}  # Plus 2PM 48551997e624
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-19
+  auth: password
+  deliver: {shelly-manager: 19}  # Plus 2PM 48551999a790
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-21
+  auth: password
+  deliver: {shelly-manager: 21}  # Plus 2PM 0cdc7efa8d00
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-34
+  auth: password
+  deliver: {shelly-manager: 34}  # Plus 2PM 3ce90e2f2d7c
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-2
+  auth: password
+  deliver: {shelly-manager: 2}  # Wall Display 00082266e106
+  # Checked but never rotated on "device not connected" (REN-1066).
+  rotate: false
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-26
+  auth: password
+  deliver: {shelly-manager: 26}  # Power Strip G4 206ef102a6cc
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-1
+  auth: password
+  deliver: {shelly-manager: 1}  # Pro 4PM 9454c5b9c954
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-4
+  auth: password
+  deliver: {shelly-manager: 4}  # Pro 4PM ac15186ca578
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-32
+  auth: password
+  deliver: {shelly-manager: 32}  # Pro 4PM 30c6f7834ec4
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-33
+  auth: password
+  deliver: {shelly-manager: 33}  # Pro 3 fce8c0db5f28
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+# HiluX DS8: HA drives them over MQTT only, and their firmware can hang in
+# the MQTT reconnect path until a power cycle (wiki
+# shelly-hilux-ds8-mqtt-reconnect-hang-bug). One unit at a time; 12 is a
+# bathroom mesh receiver, so its wall switch works without MQTT.
+- name: shelly-12
+  auth: password
+  deliver: {shelly-manager: 12}  # HiluX DS8 34b7da890cfc, Bad Spiegel Rechts
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-9
+  auth: password
+  deliver: {shelly-manager: 9}  # HiluX DS8 34b7da8c03d0, Bad Badewanne
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-31
+  auth: password
+  deliver: {shelly-manager: 31}  # HiluX DS8 34b7da888050, Bad Spiegel Links
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-44
+  auth: password
+  deliver: {shelly-manager: 44}  # HiluX DS8 34b7da894604, Toilette (standalone, no mesh)
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+# Sleeping H&Ts (REN-1107): awake only 8-13 s every 1-2 h, so the
+# controller pushes when the device publishes `online` on its wake topic
+# (shelly-manager topic prefix `shellies/<hostname>`), not in the 2-min loop.
+- name: shelly-35
+  auth: password
+  deliver: {shelly-manager: 35}  # Plus H&T 08b61fcea690
+  wake: shellies/shellyplusht-08b61fcea690/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-36
+  auth: password
+  deliver: {shelly-manager: 36}  # Plus H&T 08b61fce90dc
+  wake: shellies/shellyplusht-08b61fce90dc/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-37
+  auth: password
+  deliver: {shelly-manager: 37}  # H&T G3 84fce63ab4d8
+  wake: shellies/shellyhtg3-84fce63ab4d8/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
+- name: shelly-38
+  auth: password
+  deliver: {shelly-manager: 38}  # H&T G3 ecda3bc0b6a0
+  wake: shellies/shellyhtg3-ecda3bc0b6a0/online
+  acl: ["readwrite shellies/#", "write homeassistant/#"]
@@ -0,0 +1,30 @@
+# API-key-only listener for shelly-manager (REN-931, REN-935). Rendered by the
+# nginx-unprivileged entrypoint (envsubst, filtered to POD_IP) into
+# /etc/nginx/conf.d/ at start.
+#
+# shelly-manager signs in whoever X-authentik-username names when the peer is
+# in Auth__ReverseProxy__TrustedProxies (the pod CIDR). Only the Authentik
+# outpost may reach :8080 directly. Paperclip agent pods reach this :8081
+# instead, which drops the identity header, so every request here falls through
+# to the API key (or the login cookie) and a forged header gets 401.
+#
+# Bound to ${POD_IP}, not 0.0.0.0, so it stays off the net1 macvlan (see the
+# ASPNETCORE_URLS comment in shelly-manager-deployment.yaml).
+server {
+    listen ${POD_IP}:8081;
+    server_tokens off;
+    client_max_body_size 64m;
+
+    location / {
+        proxy_pass http://${POD_IP}:8080;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        # Empty value = nginx does not pass the header at all, whatever the
+        # client's case or how many copies it sent. X-Forwarded-User is the
+        # app's default HeaderName, dropped too in case the setting changes.
+        # Header names with underscores never pass (ignore_invalid_headers).
+        proxy_set_header X-authentik-username "";
+        proxy_set_header X-Forwarded-User "";
+        proxy_read_timeout 300s;
+    }
+}
@@ -61,7 +61,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: shelly-manager
-          image: gitea.renner.uno/k.renner/shelly-manager:build.1784957186 # {"$imagepolicy": "flux-system:shelly-manager"}
+          image: gitea.renner.uno/k.renner/shelly-manager:build.1791385485 # {"$imagepolicy": "flux-system:shelly-manager"}
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
@@ -121,7 +121,9 @@ spec:
            # outpost pod's address is dynamic, so this trusts the cluster pod
            # CIDR and leans on the namespace CNP to keep other pods off the
            # app port. ADMIN_PASSWORD (above) stays as break-glass, e.g. for
-            # port-forward access when authentik is down.
+            # port-forward access when authentik is down. Never open :8080 to
+            # another client: they get the api-proxy sidecar's :8081 (below),
+            # which drops the header (REN-935).
            - name: Auth__ReverseProxy__Enabled
              value: "true"
            - name: Auth__ReverseProxy__HeaderName
@@ -168,7 +170,62 @@ spec:
              memory: 128Mi
            limits:
              memory: 512Mi
+        # API-key-only listener for clients other than the Authentik outpost
+        # (Paperclip agents, REN-931). nginx on ${POD_IP}:8081 proxies to
+        # :8080 and drops X-authentik-username, so the ProxyHeader sign-in
+        # cannot be reached through it. Config and reasoning:
+        # shelly-manager-api-proxy.conf. Service: shelly-manager-api.
+        # No probes: it only serves the agent path, and a probe failure would
+        # also pull the outpost's endpoint out of the shelly-manager Service.
+        - name: api-proxy
+          image: nginxinc/nginx-unprivileged:1.31-alpine@sha256:26b0bf6fbf07297983cb341998d79c831508787de26627dd2a112321b9c3a4af
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NGINX_ENVSUBST_FILTER
+              value: "^POD_IP$"
+          ports:
+            - name: api
+              containerPort: 8081
+              protocol: TCP
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 101
+            runAsGroup: 101
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+          volumeMounts:
+            - name: api-proxy-config
+              mountPath: /etc/nginx/templates
+              readOnly: true
+            # envsubst output of the template above.
+            - name: api-proxy-conf-d
+              mountPath: /etc/nginx/conf.d
+            # pid file and proxy temp paths of the unprivileged image.
+            - name: api-proxy-tmp
+              mountPath: /tmp
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: shelly-manager-data
+        - name: api-proxy-config
+          configMap:
+            name: shelly-manager-api-proxy
+        - name: api-proxy-conf-d
+          emptyDir:
+            medium: Memory
+            sizeLimit: 1Mi
+        - name: api-proxy-tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: 32Mi
@@ -11,3 +11,21 @@ spec:
      port: 80
      targetPort: http
      protocol: TCP
+---
+# API-key-only entry for Paperclip agent pods (REN-931, REN-935): the
+# api-proxy sidecar, which drops the X-authentik-username header. Agents call
+# http://shelly-manager-api.home-assistant.svc/api/v1/... with the API key.
+# The CNPs admit agents to :8081 only; :8080 stays outpost-only.
+apiVersion: v1
+kind: Service
+metadata:
+  name: shelly-manager-api
+  namespace: home-assistant
+spec:
+  selector:
+    app.kubernetes.io/name: shelly-manager
+  ports:
+    - name: http
+      port: 80
+      targetPort: api
+      protocol: TCP
@@ -11,11 +11,11 @@ data:
  # config once the PVC has been seeded — change the on-PVC file (via the Z2M
  # frontend or an edit) instead. Kept here so a fresh PVC comes up configured.
  #
-  # Secrets are NOT in this file: both the MQTT password and the Zigbee
-  # network_key are quoted parseValueRefs (`"!secret <key>"`) resolved from
-  # /app/data/secret.yaml, which the init container renders — the MQTT password
-  # from the broker-credential Secret, and the network_key extracted from the
-  # OpenBao coordinator backup. The non-secret network params (channel/pan_id/
+  # Secrets are NOT in this file: the Zigbee network_key is a quoted
+  # parseValueRef (`"!secret <key>"`) resolved from /app/data/secret.yaml,
+  # which the init container renders from the OpenBao coordinator backup. MQTT
+  # uses the client certificate mounted at /mqtt-tls (REN-953); the Deployment
+  # also sets the mqtt server/ca/cert/key by env, which wins over this file. The non-secret network params (channel/pan_id/
  # ext_pan_id) are in `advanced` below.
  configuration.yaml: |
    # Home Assistant MQTT discovery — Z2M publishes device configs under the
@@ -32,14 +32,12 @@ data:

    mqtt:
      base_topic: zigbee2mqtt
-      server: mqtt://mosquitto:1883
-      user: zigbee2mqtt
-      # MUST be a QUOTED string: Z2M reads secrets via parseValueRef on the
-      # string value (regex `!<file> <key>`), NOT a YAML `!secret` tag. Unquoted
-      # `!secret ...` is parsed as a tag and throws "unknown scalar tag
-      # !<!secret>". The init container renders /app/data/secret.yaml with
-      # `mqtt_password` from the broker-credential Secret.
-      password: "!secret mqtt_password"
+      # Mutual TLS: the broker takes the username from the certificate CN
+      # (zigbee2mqtt-bridge), so no user or password here.
+      server: mqtts://mosquitto:8884
+      ca: /mqtt-tls/ca.crt
+      cert: /mqtt-tls/tls.crt
+      key: /mqtt-tls/tls.key

    # SMLIGHT SLZB-06 (Texas Instruments CC2652, zigpy-znp under ZHA) reached
    # over TCP on the IoT VLAN — the SAME coordinator ZHA uses today
@@ -32,9 +32,11 @@ spec:
        # backup also handles the ZHA<->Z2M ext_pan_id byte-order reversal —
        # which is why we restore a backup rather than hand-set advanced.ext_pan_id).
        # Renders /app/data/secret.yaml (referenced by the config as quoted
-        # "!secret" parseValueRefs) with the MQTT password AND the Zigbee
-        # network_key, the latter extracted from the OpenBao coordinator backup
-        # as a natural-byte-order int array.
+        # "!secret" parseValueRefs) with the Zigbee network_key, extracted from
+        # the OpenBao coordinator backup as a natural-byte-order int array. No
+        # MQTT password since REN-953: `mqtt_password` is absent, so Z2M's
+        # "!secret mqtt_password" resolves to nothing and it logs in by
+        # certificate alone.
        - name: seed-data
          # Our own alpine + jq image, built by build-ci-images.yml. DATED tag,
          # never :latest — rebuilding this base for an unrelated consumer
@@ -114,7 +116,6 @@ spec:
              # hex key "fe0b..2d" -> "0xfe,0x0b,..,0x2d" (natural order)
              NK=$(printf '%s' "$KEY" | sed 's/../0x&,/g; s/,$//')
              cat > /app/data/secret.yaml <<EOF
-              mqtt_password: "${MQTT_PASSWORD}"
              network_key: [${NK}]
              EOF
              if [ ! -f /app/data/configuration.yaml ]; then
@@ -132,12 +133,13 @@ spec:
              else
                echo "existing Z2M state present; NOT touching coordinator_backup.json"
              fi
-          env:
-            - name: MQTT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-zigbee2mqtt-credentials
-                  key: password
+              # External converters: overwritten from git on EVERY start (not
+              # seed-only). Copied rather than mounted because Z2M writes a temp
+              # copy of each converter into this directory while loading it
+              # (dist/extension/externalJS.js), which a read-only mount refuses.
+              mkdir -p /app/data/external_converters
+              cp /external-converters/*.js /app/data/external_converters/
+              echo "copied external converters: $(ls /external-converters)"
          volumeMounts:
            - name: data
              mountPath: /app/data
@@ -147,6 +149,9 @@ spec:
            - name: coordinator-backup
              mountPath: /backup
              readOnly: true
+            - name: external-converters
+              mountPath: /external-converters
+              readOnly: true
          resources:
            requests:
              cpu: 10m
@@ -158,19 +163,36 @@ spec:
          # Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
          # add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
          # follow-up if desired; pinned-and-manual is fine for the initial cutover.
-          image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
+          image: ghcr.io/koenkk/zigbee2mqtt:2.14.2@sha256:addbfb08fcef2e4477de1535636118241e10d13e898fbe4d9faba8a4d7396311
          imagePullPolicy: IfNotPresent
          env:
            - name: TZ
              value: Europe/Berlin
            - name: ZIGBEE2MQTT_DATA
              value: /app/data
+            # MQTT by client certificate (register account zigbee2mqtt-bridge,
+            # REN-953). Env, not the seed template: configuration.yaml on the
+            # PVC is Z2M's own and is never re-seeded. Z2M copies env values
+            # into that file on its next settings write, so a rollback must
+            # set these to the old values, not just drop them. The file keeps
+            # `user: zigbee2mqtt`; the broker ignores it and uses the CN.
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_SERVER
+              value: mqtts://mosquitto:8884
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_CA
+              value: /mqtt-tls/ca.crt
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_CERT
+              value: /mqtt-tls/tls.crt
+            - name: ZIGBEE2MQTT_CONFIG_MQTT_KEY
+              value: /mqtt-tls/tls.key
          ports:
            - name: http
              containerPort: 8080
          volumeMounts:
            - name: data
              mountPath: /app/data
+            - name: mqtt-tls
+              mountPath: /mqtt-tls
+              readOnly: true
          # The frontend only serves once Z2M has started its adapter+MQTT, so
          # an HTTP 200 on / is a decent liveness/readiness signal. Generous
          # startup budget covers coordinator connect + herdsman network adopt.
@@ -194,12 +216,15 @@ spec:
            periodSeconds: 30
            timeoutSeconds: 5
            failureThreshold: 4
+          # Node sizes the V8 heap from this limit: 512Mi gave a 259 MB heap,
+          # and Z2M died on "JavaScript heap out of memory" (exit 134) at
+          # 2026-09-28 17:56Z and 2026-09-29 07:36Z.
          resources:
            requests:
              cpu: 50m
-              memory: 128Mi
+              memory: 256Mi
            limits:
-              memory: 512Mi
+              memory: 1Gi
        # Fires a forced interview whenever a still-un-interviewed device
        # transmits — i.e. at the only moment a sleepy battery device can
        # actually complete one. Z2M itself only retries interviews on a ZDO
@@ -226,19 +251,16 @@ spec:
          imagePullPolicy: IfNotPresent
          command: [sh, /script/watch.sh]
          env:
-            - name: MQTT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-zigbee2mqtt-credentials
-                  key: password
-            # These three must track zigbee2mqtt-config.yaml: MQTT_HOST the
-            # mosquitto Service name, MQTT_USER and BASE_TOPIC the mqtt.user and
-            # mqtt.base_topic it connects with. Set explicitly rather than left
-            # to the script's defaults so the coupling is visible here.
+            # Must track the Z2M container: MQTT_HOST/MQTT_PORT the broker it
+            # connects to, BASE_TOPIC its mqtt.base_topic. Set explicitly
+            # rather than left to the script's defaults so the coupling is
+            # visible here. Logs in with the pod's certificate (/mqtt-tls).
            - name: MQTT_HOST
              value: mosquitto
-            - name: MQTT_USER
-              value: zigbee2mqtt
+            - name: MQTT_PORT
+              value: "8884"
+            - name: MQTT_TLS_DIR
+              value: /mqtt-tls
            - name: BASE_TOPIC
              value: zigbee2mqtt
            - name: STATEDIR
@@ -261,6 +283,9 @@ spec:
            - name: interview-watch-script
              mountPath: /script
              readOnly: true
+            - name: mqtt-tls
+              mountPath: /mqtt-tls
+              readOnly: true
            # Cooldown stamps. An explicit emptyDir rather than the image's
            # /tmp, so it stays writable as non-root without depending on the
            # container filesystem.
@@ -292,19 +317,16 @@ spec:
          imagePullPolicy: IfNotPresent
          command: [sh, /script/guard.sh]
          env:
-            # Reuses the Z2M broker credential: its ACL is `readwrite
-            # zigbee2mqtt/#`, which covers reading bridge/state/bridge/groups and
-            # writing bridge/request/restart. No new broker user needed.
-            - name: MQTT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mosquitto-zigbee2mqtt-credentials
-                  key: password
-            # Must track zigbee2mqtt-config.yaml, same as interview-watch.
+            # Shares the Z2M certificate: its ACL is `readwrite zigbee2mqtt/#`,
+            # which covers reading bridge/state/bridge/groups and writing
+            # bridge/request/restart. Must track the Z2M container, same as
+            # interview-watch.
            - name: MQTT_HOST
              value: mosquitto
-            - name: MQTT_USER
-              value: zigbee2mqtt
+            - name: MQTT_PORT
+              value: "8884"
+            - name: MQTT_TLS_DIR
+              value: /mqtt-tls
            - name: BASE_TOPIC
              value: zigbee2mqtt
            - name: STATEDIR
@@ -320,6 +342,9 @@ spec:
            - name: discovery-guard-script
              mountPath: /script
              readOnly: true
+            - name: mqtt-tls
+              mountPath: /mqtt-tls
+              readOnly: true
            # Restart cooldown stamp. Its own emptyDir rather than the image's
            # /tmp, so it stays writable as non-root.
            - name: discovery-guard-state
@@ -337,12 +362,24 @@ spec:
        - name: config-template
          configMap:
            name: zigbee2mqtt-config
+        - name: external-converters
+          configMap:
+            name: zigbee2mqtt-external-converters
        - name: coordinator-backup
          secret:
            secretName: zigbee2mqtt-coordinator-backup
        - name: interview-watch-script
          configMap:
            name: zigbee2mqtt-interview-watch-script
+        # Client certificate of the register account zigbee2mqtt-bridge,
+        # issued by mqtt-account-controller. Not subPath, so a cert-manager
+        # renewal reaches the files in place; Z2M reads them at connect, the
+        # sidecars on every call. 0444 because the sidecars run as UID 1883
+        # and Z2M as root; only this pod mounts it.
+        - name: mqtt-tls
+          secret:
+            secretName: mqtt-zigbee2mqtt-bridge-tls
+            defaultMode: 0444
        - name: interview-watch-state
          emptyDir:
            medium: Memory
@@ -61,16 +61,19 @@
 # herdsman-only restart. discover() republishes only configs that are missing or
 # whose payload changed, which after a wipe is all of them.
 #
-# Credentials: reuses the existing zigbee2mqtt broker user, whose ACL is
+# Credentials: the pod's client certificate (register account
+# zigbee2mqtt-bridge, REN-953) on the mTLS listener :8884. Its ACL is
 # `readwrite zigbee2mqtt/#` — enough to read bridge/state and bridge/groups and
-# to write bridge/request/restart. No new broker user, and no read grant on
-# homeassistant/# (checking for the discovery configs directly would have needed
-# one, which the ACL denies on purpose).
+# to write bridge/request/restart. No read grant on homeassistant/# (checking
+# for the discovery configs directly would have needed one, which the ACL
+# denies on purpose).
 set -eu

 MQTT_HOST="${MQTT_HOST:-mosquitto}"
-MQTT_PORT="${MQTT_PORT:-1883}"
-MQTT_USER="${MQTT_USER:-zigbee2mqtt}"
+MQTT_PORT="${MQTT_PORT:-8884}"
+# ca.crt, tls.crt, tls.key of the client certificate. The broker takes the
+# username from the certificate CN.
+MQTT_TLS_DIR="${MQTT_TLS_DIR:-/mqtt-tls}"
 BASE_TOPIC="${BASE_TOPIC:-zigbee2mqtt}"
 # Poll cadence. Worst-case time-to-repair is CHECK_INTERVAL + CONFIRM_DELAY.
 CHECK_INTERVAL="${CHECK_INTERVAL:-300}"
@@ -105,7 +108,7 @@ log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] discovery-guard: $*"; }
 # ever paid when the topic is genuinely empty.
 retained() {
  timeout "$((READ_TIMEOUT + 5))" mosquitto_sub -h "$MQTT_HOST" -p "$MQTT_PORT" \
-    -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+    --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" \
    -t "$1" -C 1 -W "$READ_TIMEOUT" 2>/dev/null || true
 }

@@ -152,7 +155,7 @@ while true; do

  log "confirmed — requesting a Zigbee2MQTT restart to re-run discovery"
  if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" \
-       -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+       --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" \
       -t "$BASE_TOPIC/bridge/request/restart" -m '' 2>/dev/null; then
    date -u +%s > "$STAMP"
    log "restart requested"
@@ -53,7 +53,10 @@ set -u
 DATA=/app/data
 DB="$DATA/database.db"
 MQTT_HOST=${MQTT_HOST:-mosquitto}
-MQTT_USER=${MQTT_USER:-zigbee2mqtt}
+MQTT_PORT=${MQTT_PORT:-8884}
+# ca.crt, tls.crt, tls.key of the pod's client certificate (register account
+# zigbee2mqtt-bridge, REN-953). The broker takes the username from its CN.
+MQTT_TLS_DIR=${MQTT_TLS_DIR:-/mqtt-tls}
 # Must track mqtt.base_topic in zigbee2mqtt-config.yaml. Overridable via env so
 # the two can be kept in step from the Deployment rather than by editing this
 # script; the MQTT ACL for this user is scoped to `zigbee2mqtt/#`, so a change
@@ -219,10 +222,6 @@ while :; do
    fi

    log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview"
-    # NOTE: mosquitto_pub can only take a password via argv (-P, or an -L URL);
-    # it has no --pw-file or env option, so this cannot be avoided with this
-    # client. Not a cross-container exposure: the pod does not set
-    # shareProcessNamespace, so /proc is not shared with the Z2M container.
    # Bounded with `timeout`, which also covers a TCP connect that hangs rather
    # than being refused. The read loop is single-threaded, so an unbounded
    # publish would stall it and miss other devices' wake windows.
@@ -230,7 +229,8 @@ while :; do
    # NOT `mosquitto_pub -W`: that flag does not exist for mosquitto_pub (it is
    # a mosquitto_sub option), and passing it makes every publish fail on a
    # usage error — verified against the client in this image.
-    if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+    if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" \
+         --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" \
         -t "$BASE_TOPIC/bridge/request/device/interview" \
         -m "{\"id\":\"$ieee\"}" 2>/dev/null; then
      # Stamp only on a SUCCESSFUL publish. Stamping unconditionally would burn
@@ -0,0 +1,48 @@
+// Wetterstation (WS90) = the built-in zigbee-herdsman-converters definition
+// plus one extra configure step that applies the reporting configuration from
+// zigbee2mqtt-ws90-reporting.md. Z2M runs configure after every interview and
+// on bridge/request/device/configure, so those no longer revert the device to
+// the converter defaults (as happened 2026-09-24..28).
+//
+// The extra step runs after the built-in ones (extend order), so its values
+// win. `version` makes Z2M reconfigure once when this file is first loaded:
+// it re-runs configure when definition.version differs from the device's
+// meta.configured. Bump it whenever REPORTING changes.
+const {definitions} = require("zigbee-herdsman-converters/devices/shelly");
+
+const base = definitions.find((d) => d.model === "WS90");
+if (!base) throw new Error("WS90 definition not found in zigbee-herdsman-converters/devices/shelly");
+
+const SHELLY = 5264;
+
+// [cluster, attribute, reportable_change, manufacturerCode]
+const REPORTING = [
+    ["msTemperatureMeasurement", "measuredValue", 10],
+    ["msRelativeHumidity", "measuredValue", 50],
+    ["msPressureMeasurement", "measuredValue", 1],
+    ["msIlluminanceMeasurement", "measuredValue", 5],
+    ["shellyWS90Wind", "windSpeed", 1, SHELLY],
+    ["shellyWS90Wind", "windDirection", 1, SHELLY],
+    ["shellyWS90Wind", "gustSpeed", 1, SHELLY],
+    ["shellyWS90UV", "uvIndex", 1, SHELLY],
+    ["shellyWS90Rain", "rainStatus", 1, SHELLY],
+    ["shellyWS90Rain", "precipitation", 1, SHELLY],
+];
+
+const reporting = {
+    isModernExtend: true,
+    configure: [
+        async (device) => {
+            const endpoint = device.getEndpoint(1);
+            for (const [cluster, attribute, change, manufacturerCode] of REPORTING) {
+                await endpoint.configureReporting(
+                    cluster,
+                    [{attribute, minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: change}],
+                    manufacturerCode ? {manufacturerCode} : {},
+                );
+            }
+        },
+    ],
+};
+
+module.exports = {...base, version: "ws90-reporting-1", extend: [...base.extend, reporting]};
@@ -2,10 +2,15 @@

 Records the ZCL attribute-reporting configuration applied to the Wetterstation
 (Ecowitt WS90 powered by Shelly, IEEE `0xc02cedfffe958f40`) and how to restore
-it, because **this configuration lives on the device and in Z2M's
-`database.db`, not in git**. Zigbee2MQTT has no settings-schema field for
-reporting configuration, so there is nothing declarative to reconcile — Flux
-cannot restore it and a fresh PVC will not have it.
+it. The configuration itself lives on the device and in Z2M's `database.db`.
+
+**Since 2026-10-01 it is applied from git:** the external converter
+`zigbee2mqtt-ws90-converter.js` extends the built-in WS90 definition with a
+configure step that sets the values below, so every interview and every
+`bridge/request/device/configure` applies them instead of the converter
+defaults. The manual commands under "Re-applying" are only needed if the
+converter is not loaded. Keep the table and `REPORTING` in the converter in
+sync, and bump the converter's `version` when they change.

 ## Applied configuration

@@ -71,14 +76,16 @@ the modernExtend value is the outlier.

 ## When this is lost

-Anything that makes Z2M re-run `configure` on the device overwrites all of it
-with the converter defaults:
+Without the external converter, anything that makes Z2M re-run `configure` on
+the device overwrites all of it with the converter defaults:

- re-pairing or re-joining the device
+- re-pairing, re-joining or re-interviewing the device
 - an explicit `zigbee2mqtt/bridge/request/device/configure`
 - a converter update that changes the definition and triggers reconfiguration

-It also will not exist on a fresh PVC. Re-apply with the commands below.
+With the converter loaded, those same events apply the values above. Check
+that it is loaded: Z2M logs `Successfully configured 'Wetterstation'
+(definition vws90-reporting-1)` after a configure.

 ## Re-applying

@@ -96,25 +103,25 @@ POD=$(kubectl get pod -n home-assistant -l app.kubernetes.io/name=zigbee2mqtt -o
 ```

 ```sh
-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msTemperatureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":10}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msTemperatureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":10}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msRelativeHumidity\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":50}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msRelativeHumidity\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":50}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msPressureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msPressureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msIlluminanceMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":5}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msIlluminanceMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":5}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windDirection\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windDirection\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"gustSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"gustSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90UV\",\"attribute\":\"uvIndex\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90UV\",\"attribute\":\"uvIndex\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"rainStatus\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"rainStatus\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'

-kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"precipitation\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" --cafile "$MQTT_TLS_DIR/ca.crt" --cert "$MQTT_TLS_DIR/tls.crt" --key "$MQTT_TLS_DIR/tls.key" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"precipitation\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
 ```

 These are idempotent — re-sending a request that matches the current
@@ -24,7 +24,7 @@ spec:
          # unchanged, so Kubernetes has no reason to replace anything, and
          # this image is deliberately NOT enrolled in Flux image automation
          # — see kubernetes/infrastructure/image-automation/README.md, which
-          # lists ghcr.io/konairius/ical-proxy as having "only main, latest,
+          # lists gitea.renner.uno/konairius/ical-proxy as having "only main, latest,
          # and main-<sha> commit builds; no release tags", so there is no
          # ImagePolicy watching it. The pods therefore sat on a 21-day-old
          # image while :latest moved underneath them, and the only way to
@@ -35,15 +35,26 @@ spec:
          # what is RUNNING recoverable from this repo rather than "whatever
          # :latest meant the last time a pod happened to restart".
          #
-          # main-99a912b is the merge of konairius/ical-proxy#12, which adds
-          # the summary_match/summary_replace parameters this deployment
-          # needs to strip "| Abfuhrkalender - Landkreis Amberg-Sulzbach"
-          # from every waste collection.
+          # The image comes from the Gitea package registry, built by
+          # Woodpecker (.woodpecker.yml in konairius/ical-proxy) on every
+          # push to main. It is amd64-only and pulls without credentials, so
+          # no imagePullSecret. main-4783abe adds two hardening changes on
+          # top of main-db90a59: an SSRF guard (upstreams and redirect hops
+          # resolving to private/loopback/link-local addresses are refused;
+          # public redirects are still followed, up to 5) and a cap of 20
+          # concurrent /proxy upstream fetches (over the cap: 429 +
+          # Retry-After). main-73b5d97 then raises the server WriteTimeout
+          # from 10s to 40s (the 30s upstream fetch timeout + 10s headroom),
+          # so a slow-but-legitimate upstream is no longer cut off before
+          # /proxy can answer. main-926ead0 caps the upstream response body
+          # at 2 MiB (larger feeds get 413 instead of being buffered whole,
+          # REN-230) and logs upstream fetch failures with the host,
+          # answering 502 instead of 500. The summary_match/summary_replace
+          # parameters this deployment needs are unchanged.
          #
          # Bumping this is a manual, reviewed edit. The tag format is
-          # main-<short-sha> (docker/metadata-action type=sha with the branch
-          # as prefix).
-          image: ghcr.io/konairius/ical-proxy:main-99a912b
+          # main-<short-sha>, the first 7 characters of the main commit.
+          image: gitea.renner.uno/konairius/ical-proxy:main-926ead0
          ports:
            - containerPort: 8080
              name: http
@@ -32,7 +32,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.2.4 # {"$imagepolicy": "flux-system:immich-machine-learning"}
          ports:
            - name: http
              containerPort: 3003
@@ -8,26 +8,11 @@
 # back into `kubernetes/apps/immich/` at retirement is the cleanup.
 #
 # Sizing:
-# - library: 300Gi — LXC 225's /mnt/media/immich is currently ~250 GiB.
-#   StorageClass ceph-filesystem has allowVolumeExpansion=true; grow with
-#   `kubectl patch pvc immich-library -p '{"spec":{"resources":{"requests":{"storage":"<new>"}}}}'`.
+# - library-bulk: 300Gi — the media library is ~250 GiB.
+#   StorageClass ceph-filesystem-bulk has allowVolumeExpansion=true; grow with
+#   `kubectl patch pvc immich-library-bulk -p '{"spec":{"resources":{"requests":{"storage":"<new>"}}}}'`.
 # - ml-cache: 20Gi — face-recognition + CLIP model cache. Survives pod
 #   restarts so ML doesn't re-download on every rollout.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: immich-library
-  namespace: immich
-  annotations:
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 300Gi
---
 # Media root on the EC 2+1 bulk pool; thumbs mount separately below.
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -66,7 +66,7 @@ spec:
        # The library PVC is populated by the seed phase before this
        # Deployment reconciles, so no rsync init is needed here.
        - name: fetch-le-staging-roots
-          image: curlimages/curl:8.10.1
+          image: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777
          command:
            - sh
            - -c
@@ -87,7 +87,7 @@ spec:
              mountPath: /ca
      containers:
        - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.2.4 # {"$imagepolicy": "flux-system:immich-server"}
          ports:
            - name: http
              containerPort: 3001
@@ -57,7 +57,7 @@ spec:
        runAsNonRoot: false
      initContainers:
        - name: wait-for-influxd
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -80,7 +80,7 @@ spec:
              echo "[$(date -u +%H:%M:%S)] $INFLUX_HOST ready and authenticated"
      containers:
        - name: create
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -88,7 +88,7 @@ spec:
        # state). Two-stage so we don't try to create an auth before
        # the chart's `influx setup` has finished.
        - name: wait-for-influxd
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -111,7 +111,7 @@ spec:
              echo "[$(date -u +%H:%M:%S)] $INFLUX_HOST ready and authenticated"
      containers:
        - name: create
-          image: influxdb:2.7.4
+          image: influxdb:2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e
          env:
            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
            - { name: INFLUX_ORG, value: renner }
@@ -133,7 +133,7 @@ spec:
              set -euo pipefail
              log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }

-              # Need jq + kubectl. influxdb:2.7.4 has neither baked in.
+              # Need jq + kubectl. The influxdb image has neither baked in.
              apt-get update -qq
              apt-get install -y --no-install-recommends curl jq ca-certificates >/dev/null
              KV=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
@@ -53,8 +53,9 @@ spec:
  values:
    image:
      repository: influxdb
-      # Match VM 108 for format compatibility with `influx restore`.
-      tag: "2.7.4"
+      # Was pinned to VM 108's 2.7.4 for the one-time `influx restore`; that
+      # migration is done. No downgrade below 2.9 without `influxd downgrade`.
+      tag: "2.9.1@sha256:db0bdab1e5ad5ee899c127b8c13d9c986a3ced78cd9200dd80a1064bf1533b6e"
    # Single replica — InfluxDB OSS doesn't cluster. RWO Ceph-block + Velero
    # backups (pending) is the durability story.
    persistence:
@@ -93,6 +94,12 @@ spec:
    env:
      - name: INFLUXD_STORAGE_WRITE_TIMEOUT
        value: "60s"
+      # 2.9 hashes API tokens on startup by default. Hashed tokens cannot be
+      # read back, so the token Jobs' `influx auth list` reuse would break,
+      # and a downgrade below 2.8 would erase every token (HA, Grafana).
+      # Keep tokens unhashed; enable hashing later as its own change.
+      - name: INFLUXD_USE_HASHED_TOKENS
+        value: "false"
    # The chart defaults to liveness/readiness with initialDelaySeconds=0,
    # so after a crash restart the kubelet kills the pod before shards
    # finish opening. Give it 10 minutes to come up cleanly.
@@ -5,3 +5,4 @@ resources:
  - helmrelease.yaml
  - ha-token-job.yaml
  - grafana-token-job.yaml
+  - volumesnapshot-pre-2.9.yaml
@@ -0,0 +1,18 @@
+---
+# Point-in-time copy of the InfluxDB data PVC, taken before the 2.7.4 -> 2.9.1
+# upgrade (homelab #1130, REN-723). Merge this first and check readyToUse=true
+# before #1130: 2.9 migrates the metadata store, so the way back is
+# `influxd downgrade 2.4` or a restore from this snapshot.
+#
+# Crash-consistent (taken from the live RBD volume), like the velero CSI
+# snapshots. Delete this file once 2.9 has run cleanly; the snapshot goes
+# with it (prune).
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: influxdb-influxdb2-pre-2-9
+  namespace: influxdb
+spec:
+  volumeSnapshotClassName: ceph-block-snapclass
+  source:
+    persistentVolumeClaimName: influxdb-influxdb2
@@ -48,7 +48,7 @@ spec:
            # this sentinel: the entrypoint copies AccountsSettings.json
            # into ephemeral /config-internal rather than symlinking it.
            - name: seed-config
-              image: busybox:1.37.0
+              image: busybox:1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e
              command: ["/bin/sh", "-eu", "-c"]
              args:
                - |
@@ -94,7 +94,7 @@ spec:
            - name: libation
              # Pinned by digest: Spegel caches mutable tag resolutions, so
              # :latest stayed on 13.7.11 despite imagePullPolicy Always.
-              image: docker.io/rmcrackan/libation:14.2.1@sha256:6f9d8ba19dc3803dd6ba295ab5a7abfb6ff3bc06286eebf4ecd8ccbe43245888
+              image: docker.io/rmcrackan/libation:14.2.2@sha256:fe188ac3c4ce63cc305971c76e648748341427e16bcf884011da8a384fd3908c
              # The entrypoint stages config into ephemeral /config-internal
              # by copy (only the DB is symlinked to the PVC), so Audible
              # tokens refreshed during a run are lost at container exit.
@@ -54,7 +54,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: llama-server
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
          args:
            # The one place this model is named; llama-server resolves the repo
            # and downloads it itself, so there is nothing to keep in sync.
@@ -43,7 +43,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: llama-embed
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
          args:
            - --hf-repo
            - gpustack/bge-m3-GGUF:Q8_0
@@ -32,7 +32,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: wiki-embed
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
          args:
            - --hf-repo
            - gpustack/bge-m3-GGUF:Q8_0
@@ -0,0 +1,222 @@
+# matrix-bridge (REN-569, REN-587): Matrix appservice between Element and the
+# Paperclip agents. One room per agent with the owner; owner messages become
+# comments on the agent's chat card, the agent's comments come back to the
+# room. Source: gitea.renner.uno/konairius/matrix-bridge.
+#
+# No volume: the bridge keeps its state in the bot's account data on Synapse.
+# No ingress: only Synapse talks to it (appservice transactions on :8080).
+#
+# Secrets:
+#   matrix-bridge-tokens  as_token (bridge -> Synapse) and hs_token (Synapse
+#                         -> bridge), generated once in-cluster by ESO, plus
+#                         the appservice registration Synapse loads from
+#                         app_service_config_files (synapse.yaml). Both sides
+#                         read the same Secret, so the tokens cannot drift.
+#                         Deleting the Secret regenerates both; Reloader
+#                         restarts Synapse and the bridge.
+#   matrix-bridge         paperclip_api_key from OpenBao
+#                         homelab/apps/matrix-bridge: the Paperclip board key
+#                         "matrix-bridge (board)" (REN-582).
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: matrix-bridge-tokens
+  namespace: matrix
+spec:
+  length: 64
+  digits: 16
+  symbols: 0
+  allowRepeat: true
+  noUpper: false
+  secretKeys:
+    - as_token
+    - hs_token
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: matrix-bridge-tokens
+  namespace: matrix
+spec:
+  # Generate once; Periodic would rotate the tokens every refresh.
+  refreshPolicy: CreatedOnce
+  target:
+    name: matrix-bridge-tokens
+    template:
+      engineVersion: v2
+      data:
+        as_token: "{{ .as_token }}"
+        hs_token: "{{ .hs_token }}"
+        # Agent users are @agent_<urlKey>:matrix.renner.uno; the bot is
+        # @paperclip:matrix.renner.uno.
+        registration.yaml: |
+          id: matrix-bridge
+          url: http://matrix-bridge:8080
+          as_token: "{{ .as_token }}"
+          hs_token: "{{ .hs_token }}"
+          sender_localpart: paperclip
+          rate_limited: false
+          namespaces:
+            users:
+              - exclusive: true
+                regex: "@agent_.*:matrix\\.renner\\.uno"
+            aliases: []
+            rooms: []
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: matrix-bridge-tokens
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: matrix-bridge
+  data:
+    - secretKey: paperclip_api_key
+      remoteRef:
+        key: apps/matrix-bridge
+        property: paperclip_api_key
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+  annotations:
+    secret.reloader.stakater.com/reload: matrix-bridge,matrix-bridge-tokens
+  labels:
+    app.kubernetes.io/name: matrix-bridge
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  # One poller only: two pods would bridge every comment twice.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: matrix-bridge
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: matrix-bridge
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        # `node` in node:24-alpine.
+        runAsUser: 1000
+        runAsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: matrix-bridge
+          image: gitea.renner.uno/konairius/matrix-bridge:build.1791195981
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: MATRIX_HOMESERVER_URL
+              value: http://synapse:8008
+            - name: MATRIX_SERVER_NAME
+              value: matrix.renner.uno
+            - name: MATRIX_OWNER
+              value: "@k.renner:matrix.renner.uno"
+            - name: MATRIX_AS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge-tokens
+                  key: as_token
+            - name: MATRIX_HS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge-tokens
+                  key: hs_token
+            - name: PAPERCLIP_API_URL
+              value: http://paperclip.paperclip.svc.cluster.local:3100
+            - name: PAPERCLIP_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-bridge
+                  key: paperclip_api_key
+            - name: PAPERCLIP_COMPANY_ID
+              value: 72f7b75a-10b0-4084-ad5e-a0cdf31c7caf
+            - name: PAPERCLIP_PROJECT_ID
+              value: 67143c0b-e9a1-44c1-b012-0f8fa25625aa
+            # Card link in the topic of each per-card room (REN-668).
+            - name: PAPERCLIP_PUBLIC_URL
+              value: https://paperclip.renner.uno
+          ports:
+            - name: http
+              containerPort: 8080
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          # /health is 503 when the Paperclip comment poll fails, so it gates
+          # readiness only; a Paperclip outage must not restart the bridge.
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: http
+            periodSeconds: 30
+            failureThreshold: 4
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: matrix-bridge
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: matrix-bridge
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# The bridge calls the Paperclip API in-cluster. Synapse <-> bridge and DNS
+# are covered by matrix-app-egress (part-of: matrix); the Paperclip side of
+# this flow is in apps/paperclip/network-policy.yaml.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: matrix-bridge-paperclip-egress
+  namespace: matrix
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: matrix-bridge
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": paperclip
+            app.kubernetes.io/name: paperclip
+      toPorts:
+        - ports:
+            - port: "3100"
+              protocol: TCP
@@ -0,0 +1,45 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: mx-pg
+  namespace: matrix
+spec:
+  instances: 2
+  smartShutdownTimeout: 10
+  stopDelay: 60
+  imageName: ghcr.io/cloudnative-pg/postgresql:16
+  enableSuperuserAccess: true
+  storage:
+    storageClass: ceph-block
+    size: 5Gi
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: mx-pg-store
+  bootstrap:
+    initdb:
+      database: synapse
+      owner: synapse
+      # Synapse refuses to start on a database whose collation is not C
+      # (its text indexes depend on byte order). Fixed at initdb time; it
+      # cannot be changed on an existing database.
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  postgresql:
+    parameters:
+      shared_buffers: "128MB"
+      max_connections: "50"
+      # Same write-reduction set as the other CNPG clusters; rationale in
+      # apps/trmnl/cluster.yaml.
+      wal_compression: "zstd"
+      archive_timeout: "30min"
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      memory: 512Mi
@@ -0,0 +1,137 @@
+# Element Web at element.renner.uno, locked to matrix.renner.uno: no custom
+# homeserver, no guests, no registration, no integration manager, no
+# analytics (no posthog block), no bug-report endpoint.
+#
+# feature_login_with_qr: "Sign in with QR code" on the welcome page, so a new
+# browser can be linked from a signed-in device. Linking a phone from the
+# desktop (Settings → Sessions → Link new device) needs no flag; it shows
+# when the server offers MSC4108 and the session has cross-signing.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: element-config
+  namespace: matrix
+data:
+  config.json: |
+    {
+      "default_server_config": {
+        "m.homeserver": {
+          "base_url": "https://matrix.renner.uno",
+          "server_name": "matrix.renner.uno"
+        }
+      },
+      "disable_custom_urls": true,
+      "disable_guests": true,
+      "disable_3pid_login": true,
+      "default_country_code": "DE",
+      "room_directory": { "servers": ["matrix.renner.uno"] },
+      "integrations_ui_url": "",
+      "integrations_rest_url": "",
+      "integrations_widgets_urls": [],
+      "bug_report_endpoint_url": "",
+      "show_labs_settings": false,
+      "features": {
+        "feature_login_with_qr": true
+      },
+      "setting_defaults": {
+        "UIFeature.registration": false,
+        "UIFeature.passwordReset": false,
+        "UIFeature.thirdPartyId": false,
+        "UIFeature.identityServer": false,
+        "UIFeature.deactivate": false
+      }
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: element-web
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: element-config
+  labels:
+    app.kubernetes.io/name: element-web
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: element-web
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: element-web
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 101
+        runAsGroup: 101
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: element-web
+          image: ghcr.io/element-hq/element-web:v1.12.30@sha256:3e7dbd4424de9e11c812bc740acf2725b2abe66ea0d0802023b56fd85d3fa09a
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # Unprivileged port: the image runs as nginx, not root.
+            - name: ELEMENT_WEB_PORT
+              value: "8080"
+          ports:
+            - name: http
+              containerPort: 8080
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              memory: 128Mi
+          readinessProbe:
+            httpGet:
+              path: /config.json
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /config.json
+              port: http
+            periodSeconds: 30
+          volumeMounts:
+            # The entrypoint copies /app/config*.json to /tmp and serves it
+            # from there, so the ConfigMap lands at the source path.
+            - name: config
+              mountPath: /app/config.json
+              subPath: config.json
+              readOnly: true
+            # Writable paths for a read-only root: the entrypoint renders the
+            # nginx template into conf.d; pid and temp files go to /tmp.
+            - name: conf-d
+              mountPath: /etc/nginx/conf.d
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: element-config
+        - name: conf-d
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: element-web
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: element-web
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
@@ -0,0 +1,26 @@
+# OpenBao homelab/apps/matrix (seeded by Hermes):
+#   owner_user                 localpart of the owner account (k.renner)
+#   owner_password             the owner's password (MAS password login)
+#   registration_shared_secret random; Synapse's macaroon key fallback
+#   mas_matrix_secret          random; MAS <-> Synapse shared secret
+#   mas_encryption             32 bytes hex; MAS encrypts tokens at rest
+#   mas_rsa_key, mas_ec_key    PEM private keys; MAS signs its tokens
+#
+# The owner reads the login from OpenBao; it never appears in a comment, a
+# manifest or a log. owner_password is the password the account was created
+# with; changing it in OpenBao does not change the account password.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: synapse-secrets
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: synapse-secrets
+  dataFrom:
+    - extract:
+        key: apps/matrix
@@ -0,0 +1,15 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hetzner-s3-creds
+  namespace: matrix
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hetzner-s3-creds
+  dataFrom:
+    - extract:
+        key: infrastructure/hetzner-s3
@@ -0,0 +1,161 @@
+# Public (REN-663): each Ingress carries dns.renner.uno/public, so
+# netbird-service-sync publishes its host through the NetBird proxy
+# (Cloudflare wildcard -> proxy.renner.uno -> TLS passthrough to Traefik) and
+# external-dns-unifi skips it; LAN clients follow the public answer and the UDM
+# hairpins back in. Same path as audiobooks/immich. Federation stays off.
+#
+# Synapse: only the client-facing paths. /_synapse/admin stays off the
+# Ingress (in-cluster only); /_matrix/federation and /_matrix/key are not
+# served by the listener anyway.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: synapse
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - matrix.renner.uno
+      secretName: matrix-tls
+  rules:
+    - host: matrix.renner.uno
+      http:
+        paths:
+          # Legacy login/logout/refresh go to MAS (its compat layer). Longer
+          # paths win in Traefik, so these override /_matrix/client below.
+          - path: /_matrix/client/v3/login
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/v3/logout
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/v3/refresh
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/login
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/logout
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client/r0/refresh
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
+          - path: /_matrix/client
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /_matrix/media
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /_synapse/client
+            pathType: Prefix
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+          - path: /.well-known/matrix/client
+            pathType: Exact
+            backend:
+              service:
+                name: synapse
+                port:
+                  number: 8008
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: element-web
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - element.renner.uno
+      secretName: element-tls
+  rules:
+    - host: element.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: element-web
+                port:
+                  number: 80
+---
+# MAS: login pages, OAuth endpoints and account management.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: mas
+  namespace: matrix
+  labels:
+    # Registered publicly (Cloudflare wildcard -> NetBird proxy); the UDM
+    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
+    dns.renner.uno/public: "true"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-cluster
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - account.matrix.renner.uno
+      secretName: mas-tls
+  rules:
+    - host: account.matrix.renner.uno
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: mas
+                port:
+                  number: 8080
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - externalsecret.yaml
+  - synapse.yaml
+  - mas.yaml
+  - element.yaml
+  - bridge.yaml
+  - ingress.yaml
+  # mx-pg CNPG Postgres + offsite Barman backup. The media store and signing
+  # key are on synapse-data, backed up by Velero (matrix-daily).
+  - cluster.yaml
+  - objectstore.yaml
+  - scheduled-backup.yaml
+  - hetzner-creds-externalsecret.yaml
@@ -0,0 +1,229 @@
+# Matrix Authentication Service (MAS) for matrix.renner.uno (REN-567).
+#
+# Synapse delegates all logins to MAS (matrix_authentication_service in
+# synapse.yaml). Why: Element X only signs in with a password or through
+# MAS. MAS gives it a one-tap passkey login through Authentik
+# (upstream_oauth2 below) and makes QR sign-in possible.
+#
+# Accounts: the owner account was moved from Synapse into MAS by syn2mas
+# (the migrate-to-mas init container in synapse.yaml), with its password
+# hash and sessions. The first Authentik login links to it by localpart
+# (on_conflict: add). The Authentik application admits only the Matrix
+# Users group, so no one else can create an account through it. Password
+# login stays on as a fallback; self-registration and recovery are off.
+#
+# Secrets: OpenBao homelab/apps/matrix (synapse-secrets). The database is
+# `mas` in mx-pg (Barman backup); MAS keeps no other state.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mas-config
+  namespace: matrix
+data:
+  config.yaml: |
+    http:
+      public_base: https://account.matrix.renner.uno/
+      listeners:
+        - name: web
+          resources:
+            - name: discovery
+            - name: human
+            - name: oauth
+            - name: compat
+            - name: graphql
+            - name: assets
+          binds:
+            - address: "[::]:8080"
+        - name: internal
+          resources:
+            - name: health
+          binds:
+            - address: "[::]:8081"
+
+    # Password from the mounted mx-pg-app Secret; nothing in this file.
+    database:
+      host: mx-pg-rw
+      port: 5432
+      username: synapse
+      password_file: /pg/password
+      database: mas
+
+    matrix:
+      kind: synapse
+      homeserver: matrix.renner.uno
+      endpoint: http://synapse:8008/
+      secret_file: /secrets/mas_matrix_secret
+
+    secrets:
+      encryption_file: /secrets/mas_encryption
+      keys:
+        - key_file: /secrets/mas_rsa_key
+        - key_file: /secrets/mas_ec_key
+
+    passwords:
+      enabled: true
+      # Version 1 must be bcrypt with no pepper: syn2mas imports Synapse's
+      # bcrypt hashes under it. New passwords are hashed with argon2id.
+      schemes:
+        - version: 1
+          algorithm: bcrypt
+        - version: 2
+          algorithm: argon2id
+      minimum_complexity: 3
+
+    account:
+      password_registration_enabled: false
+      password_recovery_enabled: false
+      email_change_allowed: false
+      login_with_email_allowed: false
+      account_deactivation_allowed: false
+
+    # No mail server; MAS sends nothing.
+    email:
+      transport: blackhole
+
+    upstream_oauth2:
+      providers:
+        - id: 01M4480KWZGN9HBNB5SSNGT2M1
+          human_name: Authentik (passkey)
+          issuer: https://auth.renner.uno/application/o/matrix/
+          client_id: matrix-mas
+          # Public client + PKCE: no shared client secret to keep in sync.
+          token_endpoint_auth_method: none
+          pkce_method: always
+          scope: "openid profile email"
+          claims_imports:
+            # Authentik username = Matrix localpart. `add` links the first
+            # login to the migrated account of the same name.
+            localpart:
+              action: require
+              template: "{{ user.preferred_username }}"
+              on_conflict: add
+            displayname:
+              action: suggest
+              template: "{{ user.name }}"
+            email:
+              action: ignore
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mas
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: mas-config
+    secret.reloader.stakater.com/reload: synapse-secrets
+  labels:
+    app.kubernetes.io/name: mas
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mas
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mas
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: mas
+          image: ghcr.io/element-hq/matrix-authentication-service:1.26.0@sha256:e089f1048a1d4a9a492ed17b9fe759100f1bd619407b001f5927928d88b780c4
+          # The server applies database migrations and syncs the upstream
+          # providers from the config on start.
+          args: ["server", "--config", "/config/config.yaml"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          ports:
+            - name: http
+              containerPort: 8080
+            - name: internal
+              containerPort: 8081
+          resources:
+            requests:
+              cpu: 20m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: internal
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: internal
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 4
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: pg
+              mountPath: /pg
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: mas-config
+        - name: secrets
+          secret:
+            secretName: synapse-secrets
+            items:
+              - key: mas_matrix_secret
+                path: mas_matrix_secret
+              - key: mas_encryption
+                path: mas_encryption
+              - key: mas_rsa_key
+                path: mas_rsa_key
+              - key: mas_ec_key
+                path: mas_ec_key
+        - name: pg
+          secret:
+            secretName: mx-pg-app
+            items:
+              - key: password
+                path: password
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mas
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: mas
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# `mas` database in mx-pg, owned by the synapse role (one app, one
+# namespace), so MAS reuses mx-pg-app. Barman backs it up with synapse.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: mx-pg-mas
+  namespace: matrix
+spec:
+  cluster:
+    name: mx-pg
+  name: mas
+  owner: synapse
@@ -0,0 +1,30 @@
+---
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: mx-pg-store
+  namespace: matrix
+spec:
+  configuration:
+    # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG
+    # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud
+    # (plugin #646), so at-rest encryption is unavailable; transit is TLS.
+    destinationPath: s3://renner-uno-k8s-cnpg/mx-pg
+    endpointURL: https://fsn1.your-objectstorage.com
+    s3Credentials:
+      accessKeyId:
+        name: hetzner-s3-creds
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: hetzner-s3-creds
+        key: ACCESS_SECRET_KEY
+      # Region is required (REGION=fsn1) — without it boto3's HeadBucket
+      # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s.
+      region:
+        name: hetzner-s3-creds
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  retentionPolicy: "30d"
@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: mx-pg-daily
+  namespace: matrix
+spec:
+  # 03:00 local, same slot as the other CNPG clusters (the Velero PVC
+  # schedule for this namespace runs later, at 09:45).
+  schedule: "0 0 3 * * *"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: mx-pg
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
@@ -0,0 +1,324 @@
+# Synapse homeserver for matrix.renner.uno (REN-567).
+#
+# Private: federation is off at three layers — federation_domain_whitelist
+# is empty, no listener serves the federation/keys resources, and the
+# namespace egress policy lets Synapse reach nothing outside the cluster
+# except the push gateway. Logins and accounts are delegated to MAS
+# (mas.yaml); registration is closed.
+#
+# server_name is permanent: it is part of every user and room ID, and
+# cannot be changed without a new, empty server. matrix.renner.uno needs no
+# .well-known delegation from the apex, which is the WordPress site.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: synapse-config
+  namespace: matrix
+data:
+  homeserver.yaml: |
+    server_name: matrix.renner.uno
+    public_baseurl: https://matrix.renner.uno/
+    # Lets Element X and other clients discover the homeserver from the
+    # server name. No server well-known: there is no federation to point at.
+    serve_client_wellknown: true
+    pid_file: /tmp/homeserver.pid
+    report_stats: false
+
+    listeners:
+      # Client API only (includes media and /.well-known). Leaving out the
+      # `federation` and `keys` resources means this server does not answer
+      # federation at all.
+      - port: 8008
+        type: http
+        tls: false
+        bind_addresses: ["::"]
+        x_forwarded: true
+        resources:
+          - names: [client]
+            compress: false
+
+    # The password comes from PGPASSWORD (libpq reads it), so no credential
+    # is written into this file.
+    database:
+      name: psycopg2
+      args:
+        host: mx-pg-rw
+        port: 5432
+        dbname: synapse
+        user: synapse
+        cp_min: 2
+        cp_max: 10
+
+    # Signing key is generated on first start if missing; it lives on the
+    # data PVC next to the media store, so Velero backs up both.
+    signing_key_path: /data/signing.key
+    media_store_path: /data/media_store
+    max_upload_size: 100M
+    # Also the macaroon key: Synapse falls back to it when
+    # macaroon_secret_key is unset, so rotating it logs every session out.
+    registration_shared_secret_path: /secrets/registration_shared_secret
+
+    # MAS handles login, sessions and accounts. Synapse asks it to check
+    # every access token; password login here must be off.
+    matrix_authentication_service:
+      enabled: true
+      endpoint: http://mas:8080/
+      secret_path: /secrets/mas_matrix_secret
+    password_config:
+      enabled: false
+
+    # Sign in with QR code (Element X): the rendezvous channel between the
+    # new and the signed-in device. Both require MAS. `open` lets the new,
+    # not-yet-signed-in device create the channel; the server is LAN-only.
+    experimental_features:
+      msc4108_enabled: true
+      msc4388_mode: open
+
+    # Federation off.
+    federation_domain_whitelist: []
+    trusted_key_servers: []
+    allow_public_rooms_over_federation: false
+
+    enable_registration: false
+    allow_guest_access: false
+    url_preview_enabled: false
+    user_directory:
+      search_all_users: true
+
+    # Phones get pushes through the Element push gateway on matrix.org.
+    # Only event and room IDs leave the cluster, never message text.
+    push:
+      include_content: false
+
+    # Appservice registrations (bridges), one file each, mounted from a
+    # Secret. matrix-bridge: bridge.yaml (REN-587).
+    app_service_config_files:
+      - /appservices/matrix-bridge/registration.yaml
+---
+# Media store and the server signing key. Not reconstructable: lost media
+# is gone, and a lost signing key invalidates the server's past signatures.
+# Backed up by Velero (matrix-daily); the database by Barman (mx-pg).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: synapse-data
+  namespace: matrix
+  labels:
+    app.kubernetes.io/name: synapse
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ceph-block
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: synapse
+  namespace: matrix
+  annotations:
+    configmap.reloader.stakater.com/reload: synapse-config,mas-config
+    secret.reloader.stakater.com/reload: synapse-secrets,matrix-bridge-tokens
+  labels:
+    app.kubernetes.io/name: synapse
+    app.kubernetes.io/part-of: matrix
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: synapse
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: synapse
+        app.kubernetes.io/part-of: matrix
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 991
+        runAsGroup: 991
+        fsGroup: 991
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # One-time move of the Synapse accounts (owner, password hash,
+        # sessions) into the MAS database. syn2mas needs Synapse offline;
+        # this runs before Synapse starts, and Recreate stops the old pod
+        # first. The marker on the data PVC makes later starts skip it.
+        # Debug image: the plain one has no shell for the marker check.
+        - name: migrate-to-mas
+          image: ghcr.io/element-hq/matrix-authentication-service:1.26.0-debug@sha256:46f7480f5ea4243ccb6fd2c447ea859e0230c8b3c0e676ee285d4ae6ca344ee3
+          command:
+            - /busybox/sh
+            - -ec
+            - |
+              if [ -f /data/.syn2mas-done ]; then
+                echo "syn2mas already done"; exit 0
+              fi
+              mas-cli syn2mas migrate \
+                --config /mas-config/config.yaml \
+                --synapse-config /config/homeserver.yaml \
+                --synapse-database-uri postgresql:
+              touch /data/.syn2mas-done
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # libpq-style variables for the Synapse database; the URI above
+            # is empty on purpose.
+            - name: PGHOST
+              value: mx-pg-rw
+            - name: PGUSER
+              value: synapse
+            - name: PGDATABASE
+              value: synapse
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mx-pg-app
+                  key: password
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: mas-config
+              mountPath: /mas-config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: pg
+              mountPath: /pg
+              readOnly: true
+            - name: data
+              mountPath: /data
+      containers:
+        - name: synapse
+          image: ghcr.io/element-hq/synapse:v1.162.0@sha256:6b84a7bbac36f080b2d2e51e0289cf1b08b349598ea44a558df38d558f2c2311
+          # Bypass /start.py: it expects to run as root and template a config
+          # from env. Default mode is generate-missing-and-run, which creates
+          # the signing key on first start.
+          command:
+            - python
+            - -m
+            - synapse.app.homeserver
+            - --config-path
+            - /config/homeserver.yaml
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mx-pg-app
+                  key: password
+          ports:
+            - name: http
+              containerPort: 8008
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+          # First start runs every schema migration against an empty
+          # database; give it up to 10 min.
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+            failureThreshold: 60
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 4
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: secrets
+              mountPath: /secrets
+              readOnly: true
+            - name: data
+              mountPath: /data
+            - name: tmp
+              mountPath: /tmp
+            - name: appservice-matrix-bridge
+              mountPath: /appservices/matrix-bridge
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: synapse-config
+        - name: secrets
+          secret:
+            secretName: synapse-secrets
+            items:
+              - key: registration_shared_secret
+                path: registration_shared_secret
+              - key: mas_matrix_secret
+                path: mas_matrix_secret
+              - key: mas_encryption
+                path: mas_encryption
+              - key: mas_rsa_key
+                path: mas_rsa_key
+              - key: mas_ec_key
+                path: mas_ec_key
+        - name: mas-config
+          configMap:
+            name: mas-config
+        - name: pg
+          secret:
+            secretName: mx-pg-app
+            items:
+              - key: password
+                path: password
+        - name: data
+          persistentVolumeClaim:
+            claimName: synapse-data
+        - name: tmp
+          emptyDir: {}
+        - name: appservice-matrix-bridge
+          secret:
+            secretName: matrix-bridge-tokens
+            items:
+              - key: registration.yaml
+                path: registration.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: synapse
+  namespace: matrix
+spec:
+  selector:
+    app.kubernetes.io/name: synapse
+  ports:
+    - name: http
+      port: 8008
+      targetPort: http
@@ -1,8 +1,8 @@
+---
 # Cilium's agent ServiceMonitor, declared here rather than by the chart.
 #
-# Reproduces what cilium 1.20.1 renders with prometheus.serviceMonitor.enabled,
-# taken field for field off the live object so scraping is unchanged: same name,
-# same namespace, same selector, same 10s interval and node relabeling.
+# Reproduces what cilium renders with prometheus.serviceMonitor.enabled: same
+# name, same namespace, same 10s interval and node relabeling.
 #
 # It lives here because the chart's template is gated on monitoring.coreos.com/v1
 # existing at install time, and on an empty cluster it does not — those CRDs
@@ -22,13 +22,16 @@ metadata:
    app.kubernetes.io/part-of: cilium
 spec:
  selector:
+    # The headless `cilium-agent` Service that the HelmRelease renders with
+    # prometheus.metricsService (REN-166). It carries k8s-app: cilium, which
+    # targetLabels below copies onto every series; the dashboard the chart
+    # ships queries cilium_bpf_map_pressure{k8s_app="cilium"}.
    matchLabels:
      app.kubernetes.io/name: cilium-agent
+      app.kubernetes.io/part-of: cilium
  namespaceSelector:
    matchNames:
      - kube-system
-  # The chart's Service keeps rendering (prometheus.enabled), so the target and
-  # its `metrics` port are there regardless of who declares the ServiceMonitor.
  targetLabels:
    - k8s-app
  endpoints:
@@ -0,0 +1,82 @@
+# Scrape the external-secrets (ESO) controller's own metrics. Until this
+# existed, nothing in this cluster could see a failed ExternalSecret sync:
+# `externalsecret_status_condition` held 0 series, so no rule could be written
+# about a key that silently stops syncing (REN-100).
+#
+# This is the scrape only. The alert rule is deliberately NOT in this change —
+# its `for:` window has to be tuned against real data, and that data does not
+# exist until this has been live for a while. See REN-103.
+#
+# A PodMonitor, NOT the chart's `serviceMonitor.enabled`, and NOT a hand-written
+# ServiceMonitor. Both alternatives were checked against chart 2.11.0:
+#
+#   * A hand-written ServiceMonitor has nothing to select. `metrics.service.enabled`
+#     defaults to false and infrastructure/external-secrets/helmrelease.yaml
+#     overrides nothing, so templates/service.yaml renders no Service for the
+#     controller. The container port exists (`name: metrics`, containerPort 8080)
+#     but nothing fronts it — the same trap that forced a PodMonitor for the Flux
+#     controllers.
+#   * Setting `serviceMonitor.enabled: true` would work, but it triggers a Helm
+#     UPGRADE of ESO, which re-renders the Deployment and re-applies its CRDs
+#     (`crds: CreateReplace` in the HelmRelease). ESO is what all 54 ExternalSecret
+#     manifests in this repo depend on. A PodMonitor gets the same series while
+#     touching the release not at all, which is the trade this repo already made
+#     for Velero's node-agent (velero-podmonitor.yaml) and for Flux.
+#
+# THIS NEEDS A NETWORK POLICY CHANGE, unlike the Flux PodMonitor: the cluster is
+# default-deny and external-secrets-ns.yaml admitted only intra-namespace traffic,
+# host/remote-node and ICMP — no `monitoring` grant, so this scrape would have
+# been dropped and the metric would have stayed absent. The matching ingress rule
+# is added to infrastructure/network-policies/external-secrets-ns.yaml in the
+# same pull request. Without it this file is decoration.
+#
+# NO TLS AND NO AUTH ON THIS SCRAPE, and that is the chart's posture rather than
+# an omission here. In 2.11.0 `metrics.listen.secure.enabled` and
+# `metrics.listen.auth.enabled` are both false in the chart's values.yaml and
+# infrastructure/external-secrets/helmrelease.yaml overrides neither, so the
+# controller serves /metrics over plain HTTP to anything that can reach :8080.
+# Hence no `scheme`, no `tlsConfig` and no `bearerTokenSecret` below: adding
+# them would break the scrape, not harden it. What limits exposure is the
+# network policy — a single port from a single namespace — so turning on
+# `secure.enabled` here would mean a Helm upgrade of ESO to close a hole the
+# CiliumNetworkPolicy already closes.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: external-secrets
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: external-secrets
+    # Not required for discovery — podMonitorSelector is empty, see
+    # apps/monitoring/helmrelease-prometheus.yaml — but set for consistency with
+    # the other monitors in this directory.
+    release: kube-prometheus-stack
+spec:
+  # Lives in `monitoring` rather than in `external-secrets`, matching
+  # external-dns-servicemonitor.yaml and the Flux PodMonitor. Every object in
+  # the external-secrets namespace is owned by the ESO HelmRelease; keeping this
+  # out of that namespace means this change cannot interact with the release.
+  namespaceSelector:
+    matchNames:
+      - external-secrets
+  selector:
+    matchLabels:
+      # The CONTROLLER only. The webhook and cert-controller carry
+      # `app.kubernetes.io/name: external-secrets-webhook` and
+      # `-cert-controller` respectively (see the chart's
+      # external-secrets-webhook.selectorLabels / -cert-controller.selectorLabels
+      # helpers), so an exact match on the unsuffixed name excludes both. That is
+      # intended: the `externalsecret_*` series are emitted by the controller, and
+      # the other two would only add idle targets.
+      app.kubernetes.io/name: external-secrets
+      app.kubernetes.io/instance: external-secrets
+  podMetricsEndpoints:
+    # Named container port, not a hardcoded 8080, so this follows
+    # `metrics.listen.port` if it is ever overridden in the HelmRelease.
+    - port: metrics
+      # 60s, not the chart's 30s default: `externalsecret_status_condition` is a
+      # slow-moving gauge over ~54 ExternalSecrets, and the alert built on it in
+      # REN-103 will use a multi-minute `for:` window. 30s would double the
+      # samples for no gain in detection time.
+      interval: 60s
+      scrapeTimeout: 10s
@@ -0,0 +1,91 @@
+# Scrape the Flux controllers' own metrics. Until this existed Prometheus held
+# no `gotk_*` series at all, so nothing about Flux's own behaviour was
+# queryable and no rule could be written about it (REN-97).
+#
+# THIS IS THE SCRAPE ONLY, and the readiness alert is not merely deferred —
+# it cannot be written from this endpoint. `gotk_reconcile_condition` and
+# `gotk_suspend_status` are still DECLARED by fluxcd/pkg/runtime (v0.110.3,
+# runtime/metrics/recorder.go), but at the tags pinned in gotk-components.yaml
+# no controller calls the recorder that populates them: grepping the six
+# tagged source trees for RecordReadiness / RecordCondition / RecordSuspend
+# returns zero call sites, and an unpopulated GaugeVec exports no series. So
+# the readiness metrics would stay empty however this file is written. Flux
+# moved per-object readiness to kube-state-metrics custom-resource-state
+# (`gotk_resource_info{ready=...}`); that is REN-102, and it is a prerequisite
+# for the not-Ready alert rather than a follow-up to it.
+#
+# What this scrape does yield, and why it is still worth merging on its own:
+#   * `gotk_reconcile_duration_seconds` per kind/name/namespace — reconcile
+#     latency and, by its absence, an object that has stopped being reconciled
+#     at all. RecordDuration IS called, but by FIVE of the six controllers, not
+#     all six: measured post-merge, 170 series from kustomize (33), source (43),
+#     image-reflector (68), image-automation (1) and helm (25), and ZERO from
+#     notification-controller. That is not a gap waiting to fill in — its three
+#     reconcilers are registered but never walked (controller_runtime_reconcile_
+#     total for controller=alert/provider/receiver is 0 over 56 h of uptime with
+#     2 Alerts and 1 Provider live), so the family is absent from that pod's
+#     /metrics entirely and five is the permanent value. DO NOT write a rule of
+#     the form `count(... gotk_reconcile_duration_seconds ...) < 6`, or an
+#     absent() per pod: it fires forever on notification-controller. Liveness for
+#     all six pods is `up{job="monitoring/flux-controllers"}` (below); that pod's
+#     reconcile health is `controller_runtime_reconcile_errors_total`. REN-127.
+#   * controller-runtime's own series: reconcile totals and errors per
+#     controller, workqueue depth and retry counts, leader election.
+#   * the target itself, so `up{job="monitoring/flux-controllers"}` and the
+#     chart's default TargetDown rule start covering the Flux control plane.
+#   * the scrape REN-102 needs in place anyway.
+#
+# A PodMonitor, NOT a ServiceMonitor, and that is forced rather than preferred:
+# gotk-components.yaml creates Services for only two of the six controllers
+# (source-controller, notification-controller), and both publish port 80 ->
+# targetPort `http`, which is the artifact/webhook server on :9090 — not the
+# metrics port. kustomize-, helm-, image-reflector- and image-automation-
+# controller have no Service at all, so there is nothing for a ServiceMonitor to
+# select. All six pods do carry the same two labels and the same `http-prom`
+# (:8080) container port, so one PodMonitor covers the set.
+#
+# NO NETWORK POLICY CHANGE IS NEEDED, verified rather than assumed, and unlike
+# external-secrets-podmonitor.yaml which did need one: flux-system has no
+# CiliumNetworkPolicy of its own (infrastructure/network-policies/ has no
+# flux-system file), and gotk-components.yaml ships an `allow-scraping`
+# NetworkPolicy admitting every namespace to :8080 (`from: [namespaceSelector:
+# {}]`, port 8080). So `monitoring` is already admitted, as it was for REN-21.
+# Nothing in the Flux install is touched.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: flux-controllers
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: flux
+    # Not required for discovery — podMonitorSelector is empty, see
+    # apps/monitoring/helmrelease-prometheus.yaml — but set for consistency
+    # with the other monitors in this directory.
+    release: kube-prometheus-stack
+spec:
+  # Lives in `monitoring` with a namespaceSelector rather than in flux-system,
+  # matching external-dns-servicemonitor.yaml and external-secrets-podmonitor.yaml.
+  # Flux owns every object in flux-system via gotk-components.yaml; keeping this
+  # out of that namespace means this change cannot interact with the Flux install
+  # itself.
+  namespaceSelector:
+    matchNames:
+      - flux-system
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: flux
+      app.kubernetes.io/instance: flux-system
+  podMetricsEndpoints:
+    # Matches the controllers' `http-prom` port (containerPort 8080). Present on
+    # all six Deployments in gotk-components.yaml; a future controller without
+    # it simply yields no target rather than a broken scrape config.
+    - port: http-prom
+      path: /metrics
+      # 60s, not the 30s used elsewhere here, matching the ESO PodMonitor: the
+      # useful series are slow-moving per-object gauges and histograms over a
+      # few hundred Flux objects, and any rule built on them (REN-102) will use
+      # a multi-minute `for:`. 30s would double the samples for no gain in
+      # detection time — and Prometheus is already near its retention cap
+      # (REN-107).
+      interval: 60s
+      scrapeTimeout: 10s
--- a/Show More
+++ b/Show More