fix(ci): tag images with a dated literal, not the commit SHA

The SHA tag was a workaround for not being able to compute a date in a shell-less build step. That problem is gone — the build timestamp now lives in the image at /etc/image-build-date — so the tag no longer has to carry it. A SHA reads as noise at the point of use and breaks the existing convention every consumer pin follows (ci-k8s:2026-08-10, ci-ansible:2026-04-29, alpine-jq:2026-07-30). Dated literals keep the pins uniform and are bumped in the commit that changes an image, which is the same trigger the `when:` path filters already use.
build(ci): record the build timestamp in ci-claude too
2026-08-23 21:14:51 +02:00 · 2026-08-23 20:59:29 +02:00 · 2026-08-23 20:57:15 +02:00 · 2026-08-23 20:52:06 +02:00 · 2026-08-21 20:28:21 +00:00 · 2026-08-21 22:27:10 +02:00
30 changed files with 795 additions and 158 deletions
@@ -14,6 +14,12 @@
 # Flux-deployed CronJobs, so it is tracked separately in #330 rather than
 # bundled here. Note this pipeline runs no LLM agent, so the exfil path that
 # motivated the other cutovers does not apply to it.
+#
+# The build steps run the plain kaniko executor, which has no shell: they pass
+# flags via `entrypoint` and read credentials from DOCKER_CONFIG. Registry auth
+# is written once by the registry-auth step below. Dated tags are literals: bump
+# them in the commit that changes an image. The build timestamp itself lives in
+# the image at /etc/image-build-date, written by a RUN in the Containerfile.
 when:
  - event: push
    path:
@@ -24,10 +30,22 @@ when:
    evaluate: 'PIPELINE == "build-ci-images"'

 steps:
-  - name: build-ansible
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
-    directory: ci/images/ansible
+  - name: registry-auth
+    image: alpine:3.24.1
    depends_on: []
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p .docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > .docker/config.json
+
+  - name: build-ansible
+    image: gcr.io/kaniko-project/executor:v1.24.0
+    directory: ci/images/ansible
+    depends_on: [registry-auth]
    # Only rebuild when THIS image's source (or the pipeline file) changed, or
    # on a manual run. Without this every step ran on any matching push, so a
    # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
@@ -37,68 +55,47 @@ steps:
      - event: push
        path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-ansible:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/ci-ansible:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/ci-ansible:latest

  - name: build-k8s
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/k8s
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-k8s:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/ci-k8s:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-tofu
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/tofu
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-tofu:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-tofu:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/ci-tofu:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/ci-tofu:latest

  # NOTE: ci-claude is deliberately NOT built here. It tracks upstream Claude
  # Code releases on its own cadence and is built + version-tagged by
@@ -108,28 +105,21 @@ steps:
  # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
  # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
  - name: build-secret-rotator
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: secret-rotator
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/secret-rotator:$TAG \
-          --destination gitea.renner.uno/gitea_admin/secret-rotator:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/secret-rotator:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/secret-rotator:latest

  # Runtime image, not CI tooling: a general-purpose alpine + jq base for
  # workloads that need to read JSON from a shell script. Named for what it IS,
@@ -137,28 +127,21 @@ steps:
  # container, but nothing about it is zigbee-specific. Exists so tools come from
  # the image rather than an `apk add` on every container start.
  - name: build-alpine-jq
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/alpine-jq
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
-          --destination gitea.renner.uno/gitea_admin/alpine-jq:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/alpine-jq:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/alpine-jq:latest

  # Runtime image, not CI tooling: upstream containernetworking/plugins binaries
  # baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
@@ -166,28 +149,21 @@ steps:
  # critical path of every node rebuild. The download now happens here, once,
  # where the sha256 is checked — see the Containerfile for the full rationale.
  - name: build-cni-plugins
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/cni-plugins
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
-          --destination gitea.renner.uno/gitea_admin/cni-plugins:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/cni-plugins:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/cni-plugins:latest

  # Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
  # needing s_client/x509. Named for what it is; first consumer is the paperless
@@ -195,28 +171,21 @@ steps:
  # every paperless pod start. The openssl CLI is genuinely absent from the alpine
  # base, so this one needs an image rather than a busybox substitute.
  - name: build-alpine-openssl
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/alpine-openssl
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
-          --destination gitea.renner.uno/gitea_admin/alpine-openssl:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/alpine-openssl:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/alpine-openssl:latest

  # Heating CAN readers: open3e (Vitocal/VX3) + E3onCAN (E380 meter), one image
  # run as two services on edge-1. Replaces pip-into-venv on the host, where an
@@ -224,25 +193,18 @@ steps:
  # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
  # builds the same way and lands in the same registry so the audit sees it.
  - name: build-heating-can
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/heating-can
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
-          --destination gitea.renner.uno/gitea_admin/heating-can:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/heating-can:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/heating-can:latest
@@ -22,3 +22,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # unanchored extraction therefore yields a device link key, and herdsman then
 # forms a NEW Zigbee network and drops every device.
 RUN apk add --no-cache jq
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -19,3 +19,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # for paperless to trust. busybox has no s_client equivalent, so openssl is not
 # substitutable there.
 RUN apk add --no-cache openssl
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -24,3 +24,8 @@ RUN pip3 install --break-system-packages --no-cache-dir \
 RUN ansible-galaxy collection install \
      community.proxmox \
      community.docker
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -44,3 +44,9 @@ RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \
      /usr \
      /root \
      /tmp
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# version was built, this says when. Deliberately the LAST layer and its own
+# layer, so the byte-stable npm layer above still dedups in the registry — that
+# dedup is what #351 was for, and a few dozen bytes here does not affect it.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -66,3 +66,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # Alpine rather than scratch: the installer needs a shell to copy the binaries
 # onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
 COPY --from=fetch /plugins/ /opt/cni-plugins/
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -120,3 +120,8 @@ USER open3e
 WORKDIR /var/lib/open3e

 ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -42,3 +42,8 @@ RUN curl -sfL -o /tmp/velero.tar.gz \
 && tar -xz -C /tmp -f /tmp/velero.tar.gz \
 && install "/tmp/velero-${VELERO_VERSION}-linux-amd64/velero" /usr/local/bin/velero \
 && rm -rf "/tmp/velero-${VELERO_VERSION}-linux-amd64" /tmp/velero.tar.gz
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -31,3 +31,8 @@ RUN pip3 install --break-system-packages --no-cache-dir \
 RUN ansible-galaxy collection install \
      community.proxmox \
      community.docker
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -43,6 +43,44 @@ spec:
    parameters:
      shared_buffers: "256MB"
      max_connections: "400"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 3.2 GB/day here, most of it padding.
+      #
+      # 15min, NOT the 30min used elsewhere in this rollout. This cluster holds
+      # credentials, sessions and enrolment state, so its RPO is worth more than
+      # the difference: 30min would save a further 0.3 GB/day, which does not
+      # justify doubling the worst-case loss window on this data.
+      #
+      # It does carry a synchronous standby (instances: 2), so the realistic
+      # exposure is losing both instances, not one -- but the conservative value
+      # is cheap here and the aggressive one buys almost nothing.
+      archive_timeout: "15min"
    # Async replication makes CNPG switchover slow because the standby has
    # to chase missing WAL via barman-cloud and replay queued records
    # before it can promote. With one synchronous standby, the standby is
@@ -29,6 +29,52 @@ spec:
    parameters:
      shared_buffers: "128MB"
      max_connections: "50"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # THE dominant write source on this cluster, by three orders of magnitude.
+      #
+      # A WAL segment is a fixed 16 MiB file. archive_timeout forces a segment
+      # switch on a timer so the archive never lags reality by more than that
+      # interval -- and the switched-out segment is written and archived at its
+      # full 16 MiB regardless of how few bytes of real WAL it contains.
+      #
+      # CNPG defaults this to 5min, i.e. up to 288 segments/day = 4.6 GB/day.
+      # Measured here: this database is 21 MB and generates 4.5 MB/day of actual
+      # WAL, while its RBD volume takes 211 MB/hour. Essentially every byte
+      # written is padding -- roughly 1100x amplification.
+      #
+      # 30min cuts that to 48 segments/day. The cost is RPO: on total loss of
+      # the cluster, up to 30min of Grafana state is unrecoverable rather than
+      # 5min. Grafana's state here is dashboards, annotations and preferences,
+      # and the write rate says it changes ~4.5 MB/day -- so that window is
+      # nearly always empty anyway.
+      #
+      # Applied to gf-pg FIRST, alone, as a staged rollout: it is the lowest-risk
+      # cluster and has the cleanest signal, since its real WAL is small enough
+      # that any change in volume is unambiguously this parameter.
+      archive_timeout: "30min"
  resources:
    requests:
      cpu: 100m
@@ -55,7 +55,21 @@ spec:
      owner: homeassistant
  postgresql:
    parameters:
-      shared_buffers: "256MB"
+      # 512MB, not the 256MB this shipped with. Measured after 15h on Postgres:
+      # buffers_backend was 629,704 of 939,025 total page writes — 67% written
+      # by backend processes rather than the checkpointer or bgwriter. That
+      # ratio means the working set does not fit in shared_buffers, so backends
+      # evict dirty pages themselves and the same pages are written out
+      # repeatedly instead of coalescing in memory.
+      #
+      # The recorder touches a wide set of pages continuously (~30 state
+      # changes/sec across states, state_attributes and five indexes), which is
+      # exactly the workload that punishes a small buffer pool.
+      #
+      # Memory headroom checked before raising: the instance was using 383Mi of
+      # a 1Gi limit with 256MB of that being shared_buffers, so the limit goes
+      # to 1.5Gi to keep the same margin.
+      shared_buffers: "512MB"
      max_connections: "50"
      # The rest of this block is tuned to MINIMISE WRITE VOLUME, which is the
      # entire reason for the migration. Defaults would give a correct database
@@ -71,12 +85,70 @@ spec:
      # Compress full-page images in WAL. Recorder writes are many small row
      # inserts scattered across pages, so full-page writes after each
      # checkpoint dominate WAL volume; compression cuts that directly.
-      wal_compression: "on"
+      #
+      # Full-page images really are ~62% of WAL here: 61.9% measured as a
+      # steady-state delta over one checkpoint cycle, against 62.4% lifetime.
+      #
+      # But switching pglz -> zstd changed NOTHING. Measured by
+      # pg_current_wal_lsn advance over 20min windows: 4911 and 4976 MB/day
+      # before, 5002 MB/day after.
+      #
+      # The reason is that `wal_compression: "on"` was never "no compression" --
+      # it is pglz, which was already compressing these page images. zstd only
+      # changes the ratio, and on recorder pages that margin is too small to
+      # measure. The original framing of this change (that the clusters "were
+      # not getting compression") was wrong; they were.
+      #
+      # Note the 62% figure is an upper bound: it is computed as
+      # wal_fpi * 8192 / wal_bytes, which assumes uncompressed 8 KiB pages. With
+      # compression active the true byte share is lower, which is consistent
+      # with there being little headroom left for a better algorithm.
+      #
+      # Kept because zstd is not worse than pglz and costs nothing. It is simply
+      # not a lever. The real one on this cluster was archive_timeout, below.
+      #
+      # zstd, not "on": the boolean spelling resolves to pglz, the original
+      # 2016-era algorithm, which is both slower and weaker than the two
+      # alternatives PostgreSQL 16 accepts here (lz4, zstd). zstd gives the
+      # best ratio on 8 KiB page images, and the CPU cost lands on a cluster
+      # that is IO-bound on QLC NVMe, not CPU-bound.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
      # Fewer checkpoints => fewer full-page writes, since a page is only
-      # fully-logged on its first modification after a checkpoint. 15min with a
-      # 2GB ceiling keeps recovery time sane while cutting repeat full-page
-      # images for the hot rows the recorder rewrites constantly.
-      checkpoint_timeout: "15min"
+      # fully-logged on its first modification after a checkpoint.
+      #
+      # That reasoning did not show up in WAL volume: 15min -> 30min moved the
+      # WAL rate not at all (4911/4976 -> 5002 MB/day). The likely reason is
+      # that this workload is append-mostly -- a page is dirtied while it fills
+      # and then not touched again, so its full-page image is paid once
+      # regardless of how often checkpoints run. Interval only helps pages that
+      # are re-dirtied across many cycles, which is not what a recorder does.
+      #
+      # Kept, but NOT because WAL improved. checkpoint_timeout also governs
+      # data-file writes at checkpoint, and those are unmeasured here: this
+      # volume takes ~11 GB/day against 4.9 GB/day of WAL, so most of its
+      # traffic is outside what the WAL measurement covers. Revisit with a
+      # data-file measurement before treating this value as justified.
+      #
+      # Recovery cost is bounded by how much WAL a single interval produces,
+      # not by the interval itself: at the measured rate 30min is well under
+      # 200MB of replay, i.e. seconds. max_wal_size stays 2GB precisely so the
+      # timeout stays the binding trigger -- if size were the binding one, the
+      # interval change would do nothing.
+      checkpoint_timeout: "30min"
      max_wal_size: "2GB"
      min_wal_size: "512MB"
      # full_page_writes is deliberately LEFT ON (the default). RBD can tear a
@@ -87,4 +159,4 @@ spec:
      cpu: 200m
      memory: 512Mi
    limits:
-      memory: 1Gi
+      memory: 1536Mi
@@ -42,7 +42,43 @@ spec:
    parameters:
      shared_buffers: "512MB"
      max_wal_size: "2GB"
-      wal_compression: "on"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
  resources:
    requests:
      cpu: 500m
@@ -27,7 +27,7 @@ spec:
        # Pinned to 7.4-alpine to match the LXC 225 compose exactly.
        # Any redis major-version bump is a separate migration.
        - name: redis
-          image: redis:7.4.10-alpine # {"$imagepolicy": "flux-system:redis-immich"}
+          image: redis:7.4.11-alpine # {"$imagepolicy": "flux-system:redis-immich"}
          args:
            - redis-server
            - --save
@@ -13,6 +13,10 @@ resources:
  - prometheusrule-netbird-ingress.yaml
  - prometheusrule-velero.yaml
  - prometheusrule-woodpecker-rotation.yaml
+  # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
+  # alerted if that stopped — HetznerBackupEndpointDown only probes the S3
+  # endpoint, which proves reachability, not that anything was written.
+  - prometheusrule-cnpg-backup.yaml
  - alloy-servicemonitor.yaml
  - cnpg-podmonitors.yaml
  - home-assistant-servicemonitor.yaml
@@ -12,18 +12,43 @@ spec:
    - name: ceph.rules
      rules:
        - alert: CephHealthWarn
-          # ceph_health_status: 0 = HEALTH_OK, 1 = HEALTH_WARN, 2 = HEALTH_ERR.
-          # 6h is deliberately long. Ceph warns for self-clearing conditions —
-          # a slow BlueStore op during the Sunday restore sweep, a mon briefly
-          # below its disk threshold — and paging on those trains people to
-          # ignore the alert. What matters is a warning that does not clear.
-          expr: ceph_health_status == 1
+          # Per-check rather than ceph_health_status, so one permanently-active
+          # check cannot mask every other Ceph warning.
+          #
+          # AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a
+          # kernel supporting aes256k over the CSI mount path (#527).
+          #
+          # POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes
+          # `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked
+          # maintenance and not just data risk — do not exclude it for a pool
+          # that is unreplicated deliberately.
+          #
+          # 6h: Ceph warns for self-clearing conditions. Only a warning that
+          # does not clear is worth paging on.
+          expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
          for: 6h
          labels:
            severity: warning
          annotations:
-            summary: "Ceph has been HEALTH_WARN for 6h"
-            description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            summary: "Ceph health check {{ $labels.name }} active for 6h"
+            description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+
+        - alert: CephPoolLostRedundancy
+          # Names the pool, which POOL_NO_REDUNDANCY does not.
+          #
+          # There is no pool-size metric, but bytes_used/stored IS the
+          # replication factor. The >1e6 floor drops pools too small to measure
+          # meaningfully, empty ones included.
+          #
+          # No pool is exempt.
+          expr: |
+            ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Ceph pool {{ $labels.pool_id }} is running unreplicated"
+            description: "Pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning a single OSD loss destroys data. No pool is meant to be unreplicated — an unreplicated pool also blocks ALL OSD maintenance cluster-wide, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."

        - alert: CephHealthError
          # HEALTH_ERR means data availability or redundancy is at risk.
@@ -0,0 +1,78 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cnpg-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: cnpg-backup.rules
+      rules:
+        # WHY THIS FILE EXISTS
+        # Seven Postgres clusters back up to Hetzner and NOTHING alerted if that
+        # stopped. The only related rule was HetznerBackupEndpointDown, which
+        # probes the S3 endpoint — it proves the bucket is reachable, not that
+        # anything was written to it. Same failure shape as the WordPress backup
+        # that failed nightly for three days against a bucket that was declared
+        # in tofu but never applied: the status surface looked fine because
+        # nothing checked the thing that mattered.
+
+        - alert: CNPGWalArchiveBacklog
+          # `ready` counts WAL segments sitting in pg_wal/archive_status waiting
+          # to be archived. Steady state is 0 across every cluster (verified
+          # 2026-08-19). A sustained backlog means the archiver is failing:
+          # point-in-time recovery is silently broken from that moment, and
+          # pg_wal keeps growing until it fills the PVC and takes Postgres down.
+          #
+          # This is the load-bearing alert of the two. It depends only on the
+          # instance-level collector, so unlike the backup-age metrics below it
+          # is unaffected by the missing Backup CRs (#522).
+          #
+          # Threshold: >5 segments sustained for 30m. Brief non-zero values are
+          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
+          # 80 MiB of un-archived WAL, well below anything that threatens the
+          # volume but far above idle noise.
+          expr: cnpg_collector_pg_wal_archive_status{value="ready"} > 5
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "{{ $value }} WAL segments have been waiting to archive for over 30 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."
+
+        - alert: CNPGBackupFailed
+          # cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0
+          # when no failure is recorded. Guard on > 0 so the epoch-zero default
+          # is not read as "failed in 1970".
+          expr: |
+            cnpg_collector_last_failed_backup_timestamp > 0
+            and
+            (time() - cnpg_collector_last_failed_backup_timestamp) < 86400
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CNPG backup failed in the last 24h on {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "A base backup for {{ $labels.namespace }}/{{ $labels.pod }} failed within the last 24 hours. Verify what is actually in the bucket rather than trusting CR status — see the barman-cloud-backup-list recipe in kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md."
+
+        # DELIBERATELY ABSENT: a backup-age alert, which is the one you would
+        # most want here.
+        #
+        # The natural expression is
+        #   (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600
+        # and it CANNOT be used today: that metric reads exactly 0 on every
+        # instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2,
+        # pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any
+        # successful backup anywhere, because the Backup CRs it derives that
+        # from are being deleted within a minute of completing — see #522.
+        #
+        # Backups themselves DO work; that was verified independently by
+        # listing the bucket with barman-cloud-backup-list and restoring a WAL
+        # segment back out of S3. The gap is in the record, not the data.
+        #
+        # Adding the rule now would fire permanently for all seven clusters and
+        # be silenced within a day, which is worse than not having it. Add it as
+        # part of resolving #522, once the metric reflects reality.
@@ -402,6 +402,52 @@ spec:
        limits:
          memory: 256Mi

+    # Drop the apiserver's high-cardinality latency histograms.
+    #
+    # Measured 2026-08-20: Prometheus held 298,236 active series, of which
+    # ~108,000 — over a third — were apiserver/etcd `_bucket` histograms. Each
+    # histogram fans out to one series per bucket x verb x resource x scope, so
+    # a single metric like apiserver_request_duration_seconds_bucket was 23,810
+    # series on its own. For comparison, every Home Assistant metric combined is
+    # ~4,100.
+    #
+    # That cardinality is the dominant driver of Prometheus write volume
+    # (~11 GB/day of TSDB + WAL, doubled again by Ceph replication), and none of
+    # it is observable value here: per-bucket apiserver latency quantiles are for
+    # clusters with SLAs and dozens of nodes, not three.
+    #
+    # ONLY metrics with ZERO rule dependencies are dropped. Verified against the
+    # live rule set before writing this — each metric checked individually for
+    # references in alert and recording rules:
+    #
+    #   apiserver_request_duration_seconds_bucket        rules=0   23,810 series
+    #   apiserver_request_body_size_bytes_bucket         rules=0   23,200
+    #   etcd_request_duration_seconds_bucket             rules=0   15,840
+    #   apiserver_watch_list_duration_seconds_bucket     rules=0   11,538
+    #   apiserver_watch_cache_read_wait_seconds_bucket   rules=0    8,232
+    #   apiserver_response_sizes_bucket                  rules=0    7,592
+    #   apiserver_watch_events_sizes_bucket              rules=0    5,220
+    #
+    # apiserver_request_sli_duration_seconds_bucket is DELIBERATELY KEPT: 23
+    # recording rules depend on it, including the apiserver availability SLO and
+    # the KubeAPIErrorBudgetBurn burn-rate series. Dropping it would silently
+    # break that alerting, which is the failure mode this comment exists to
+    # prevent if someone later "tidies up" this regex.
+    #
+    # Only the `_bucket` series are dropped. The matching `_count` and `_sum`
+    # remain, so request rates and mean latencies still work — what is lost is
+    # per-bucket quantiles.
+    #
+    # etcd_* appears here even though kubeEtcd is disabled below: these are the
+    # apiserver's own etcd-client metrics, exposed on the apiserver endpoint, not
+    # a separate etcd scrape.
+    kubeApiServer:
+      serviceMonitor:
+        metricRelabelings:
+          - action: drop
+            sourceLabels: [__name__]
+            regex: (apiserver_request_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|etcd_request_duration_seconds_bucket|apiserver_watch_list_duration_seconds_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket)
+
    # Disable etcd/scheduler/controller-manager scraping (Talos doesn't expose them by default)
    kubeEtcd:
      enabled: false
@@ -34,6 +34,42 @@ spec:
    parameters:
      shared_buffers: "256MB"
      max_connections: "200"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.1 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if both instances
+      # are lost. The documents themselves live on the media PVC, not in
+      # Postgres -- this database holds their metadata.
+      archive_timeout: "30min"
    # See ak-pg cluster.yaml for the rationale on synchronous replication.
    synchronous:
      method: any
@@ -208,9 +208,25 @@ spec:
            seccompProfile:
              type: RuntimeDefault
      volumes:
+        # paperless-data-block (RWO, ceph-block), NOT paperless-data
+        # (RWX, ceph-filesystem). Only this Deployment ever mounted the volume,
+        # and its contents are single-writer state — the Whoosh index,
+        # classification model, celerybeat schedule, logs. Being on CephFS meant
+        # every access went through the MDS: 22 requests/sec sustained with zero
+        # CI running, ~15 journal events/sec against 2 directory commits per
+        # 180s, i.e. cap/inode churn rather than file writes. That was most of
+        # the constant 7-8 GB/day on cephfs-metadata, and that pool is size 3.
+        #
+        # `consume` and `media` below STAY on CephFS: the ftp-scanner writes to
+        # those from another node, so they genuinely need RWX.
+        #
+        # paperless-data is retained (prune: disabled) as the rollback. Do not
+        # delete it until search has been confirmed working — that is what
+        # proves the Whoosh index survived the copy, not merely that paperless
+        # starts. Migration record: MIGRATION-data-to-block.md.
        - name: data
          persistentVolumeClaim:
-            claimName: paperless-data
+            claimName: paperless-data-block
        - name: media
          persistentVolumeClaim:
            claimName: paperless-media
@@ -25,7 +25,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: redis
-          image: redis:8.10.0-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
+          image: redis:8.10.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
          args:
            - redis-server
            - --save
@@ -32,7 +32,44 @@ spec:
    parameters:
      shared_buffers: "128MB"
      max_connections: "50"
-      wal_compression: "on"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
  resources:
    requests:
      cpu: 100m
@@ -50,12 +50,39 @@ spec:
            # clone + subsequent steps share the workspace PVC and the k8s backend
            # can schedule them on different nodes -> an RWO volume then fails to
            # mount on the second node. CephFS (RWX) lets the workspace follow
-            # the pipeline across nodes. The -ephemeral variant (reclaim Delete,
-            # rook-ceph-cluster/storageclass.yaml) is REQUIRED here: workspaces
-            # are throwaway, and the Retain class leaked one Released PV per
-            # finished pipeline (issue #262 follow-up).
+            # the pipeline across nodes. Reclaim MUST be Delete: workspaces are
+            # throwaway, and a Retain class leaked one Released PV per finished
+            # pipeline (issue #262 follow-up).
+            #
+            # ceph-filesystem-ci, not ceph-filesystem-ephemeral: a dedicated
+            # CephFS whose pools are BOTH size 1 (infrastructure/
+            # rook-ceph-cluster/cephfs-ci.yaml). The shared filesystem
+            # replicates data0 2x and metadata 3x, which is the correct trade
+            # for real data and pure waste for a git clone that is deleted
+            # minutes later.
+            #
+            # Measured 2026-08-21, polling both pools every 15s (a 5-minute
+            # interval misses an entire pipeline -- PVC created, used and
+            # deleted inside ~2 minutes):
+            #   CI active  1.5min: data0 143.2 MB, metadata 17.2 MB
+            #   CI idle   16.0min: data0   0.0 MB, metadata 11.0 MB
+            # data0 write volume is 100% CI. Per pipeline this moves the drive
+            # cost from 71.6x2 + 8.2x3 = ~168 MB to ~80 MB.
+            #
+            # ceph-filesystem-ephemeral is NOT repointed, and the reason is a
+            # technical constraint rather than a safety one. Its other consumer
+            # is velero/restore-test.sh, whose probe PVCs are snapshot CLONES:
+            # `ceph fs subvolume snapshot clone` operates within one filesystem,
+            # so a snapshot taken in `cephfs` cannot be cloned into `cephfs-ci`.
+            #
+            # Those probes are just as throwaway as a CI workspace -- the
+            # authoritative copy is the backup on Hetzner, and a lost probe just
+            # means re-running the test -- so they are equally undeserving of
+            # replication. Landing them unreplicated would mean a second size-1
+            # data pool inside `cephfs`, and restore-test runs WEEKLY against
+            # CI's ~96 pipelines/day, so it is not worth the complexity.
            - name: WOODPECKER_BACKEND_K8S_STORAGE_CLASS
-              value: ceph-filesystem-ephemeral
+              value: ceph-filesystem-ci
            - name: WOODPECKER_BACKEND_K8S_STORAGE_RWX
              value: "true"
            # MUST be a JSON object literal — the backend json.Unmarshal's this
@@ -56,6 +56,44 @@ spec:
    parameters:
      shared_buffers: "128MB"
      max_connections: "50"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
  resources:
    requests:
      cpu: 100m
@@ -917,11 +917,27 @@ subjects:
  name: plugin-barman-cloud
  namespace: database
 ---
+# stringData, NOT base64 `data`. An image reference is not a secret, and
+# encoding it here had two real costs: no reviewer sees a version change in a
+# diff, and no $imagepolicy marker can be attached to a base64 blob. The result
+# was that automation advanced the controller to v0.14.0 while this sidecar sat
+# at v0.12.0 — a two-minor skew in the component that performs every base backup
+# and WAL archive, invisible until someone read the pod spec (#535).
+#
+# The Secret KIND is kept because the controller reads it via secretKeyRef
+# (deployment env SIDECAR_IMAGE); changing it to a ConfigMap would need a matching
+# edit to that reference and diverge further from upstream. stringData is
+# accepted identically by the API server and serialised into `data` on write.
+#
+# The name carries a kustomize hash suffix from upstream's secretGenerator. It is
+# static here, so editing the value does NOT change the name and the deployment's
+# secretKeyRef stays valid.
+#
+# IF YOU RE-IMPORT THE UPSTREAM MANIFEST: upstream ships this as base64 `data`
+# with no marker. Re-apply this block, or the skew silently returns.
 apiVersion: v1
-data:
-  SIDECAR_IMAGE: |
-    Z2hjci5pby9jbG91ZG5hdGl2ZS1wZy9wbHVnaW4tYmFybWFuLWNsb3VkLXNpZGVjYXI6dj
-    AuMTIuMA==
+stringData:
+  SIDECAR_IMAGE: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar:v0.14.0 # {"$imagepolicy": "flux-system:cnpg-barman-plugin-sidecar"}
 kind: Secret
 metadata:
  name: plugin-barman-cloud-kg5c2795f7
@@ -28,3 +28,40 @@ spec:
  policy:
    semver:
      range: "0.x"
+---
+# The SIDECAR image, tracked separately from the controller above.
+#
+# These are two DIFFERENT images that must move together:
+#   plugin-barman-cloud          the controller, in the database namespace
+#   plugin-barman-cloud-sidecar  injected into every Postgres pod
+#
+# Only the controller was tracked, so automation walked it from v0.12.0 to
+# v0.14.0 while the sidecar stayed at v0.12.0 — a two-minor skew in the
+# component that performs every base backup and WAL archive (#535). The skew was
+# invisible because the sidecar tag was base64-encoded inside a Secret, where no
+# $imagepolicy marker can reach it and no reviewer would see it in a diff.
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageRepository
+metadata:
+  name: cnpg-barman-plugin-sidecar
+  namespace: flux-system
+spec:
+  image: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar
+  interval: 1h
+---
+# Same filter and range as the controller policy deliberately: the two images
+# are released together and must not be allowed to drift apart again.
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImagePolicy
+metadata:
+  name: cnpg-barman-plugin-sidecar
+  namespace: flux-system
+spec:
+  imageRepositoryRef:
+    name: cnpg-barman-plugin-sidecar
+  filterTags:
+    pattern: '^v(?P<ver>\d+\.\d+\.\d+)$'
+    extract: '$ver'
+  policy:
+    semver:
+      range: "0.x"
@@ -17,6 +17,17 @@ spec:
    modules:
      - name: pg_autoscaler
        enabled: true
+  # Rotates daemon keys to aes256k for CVE-2025-30156 (#527). Opt-in; disabled
+  # by default. Increment keyGeneration to rotate again.
+  #
+  # Daemon keys only — aes256k over the CSI mount path needs kernel 7.0+ and
+  # these nodes run 6.18, so CSI stays on aes and the client-key checks stay
+  # raised. keyType is unset so Rook picks it.
+  security:
+    cephx:
+      daemon:
+        keyRotationPolicy: KeyGeneration
+        keyGeneration: 2
  # Bump CephFS subvolume cloner pool from 4 → 16. Velero CSI Snapshot
  # Data Movement provisions a backupPVC per source PVC by cloning the
  # VolumeSnapshot, and a 4-thread default stalls migrate-immich's
@@ -0,0 +1,64 @@
+---
+# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces. Each
+# pipeline gets a PVC that is created, filled and deleted within ~2 minutes;
+# keeping that churn off `cephfs` spares its size-3 metadataPool.
+#
+# size 2, not 1: an unreplicated pool makes `ceph osd ok-to-stop` refuse for
+# every OSD, blocking all OSD maintenance cluster-wide. size 1 is viable only
+# once these pools have OSDs of their own.
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystem
+metadata:
+  name: cephfs-ci
+  namespace: rook-ceph
+spec:
+  metadataPool:
+    replicated:
+      size: 2
+  dataPools:
+    - name: data0
+      replicated:
+        size: 2
+  metadataServer:
+    activeCount: 1
+    # No standby-replay, unlike `cephfs`. A standby exists there so a failed
+    # MDS does not hang every CephFS mount cluster-wide; here the only casualty
+    # is a running pipeline, and a second daemon to protect throwaway work is
+    # not worth its memory.
+    activeStandby: false
+    resources:
+      # Deliberately NOT the 4Gi that `cephfs` needs. That figure is driven by
+      # immich's millions of cached inodes; a CI workspace is a few thousand
+      # short-lived files that vanish with the pipeline.
+      requests:
+        cpu: 100m
+        memory: 512Mi
+      limits:
+        memory: 1Gi
+---
+# CI-only class. `ceph-filesystem-ephemeral` is deliberately left pointing at
+# the main filesystem: velero/restore-test.sh uses it to restore REAL data for
+# verification, which must not land on unreplicated pools.
+#
+# A new class rather than repointing the old one — StorageClass.parameters are
+# immutable, so `fsName` cannot be edited in place.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ceph-filesystem-ci
+provisioner: rook-ceph.cephfs.csi.ceph.com
+parameters:
+  clusterID: rook-ceph
+  fsName: cephfs-ci
+  pool: cephfs-ci-data0
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+# Delete, for the same reason ceph-filesystem-ephemeral uses it: with Retain,
+# every finished pipeline leaks a Released PV pinning a subvolume (191 of them
+# by 2026-07-16, issue #262 follow-up).
+reclaimPolicy: Delete
+allowVolumeExpansion: true
@@ -4,5 +4,6 @@ resources:
  - cephcluster.yaml
  - toolbox.yaml
  - storageclass.yaml
+  - cephfs-ci.yaml
  - volumesnapshotclass.yaml
  - objectstore.yaml
@@ -21,8 +21,14 @@ COPY src ./src
 RUN cargo build --release --target x86_64-unknown-linux-musl \
 && strip target/x86_64-unknown-linux-musl/release/secret-rotator

+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Written here because the final
+# image is scratch and has no shell to run `date` in.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /build/image-build-date
+
 FROM scratch
 COPY --from=build /build/target/x86_64-unknown-linux-musl/release/secret-rotator /secret-rotator
+COPY --from=build /build/image-build-date /etc/image-build-date

 # Matches runAsUser on the rotation Jobs. There is no /etc/passwd in a scratch
 # image, so this is a bare uid by necessity — which is also all Kubernetes needs.