fix(ci): tag images with a dated literal, not the commit SHA

The SHA tag was a workaround for not being able to compute a date in a shell-less build step. That problem is gone — the build timestamp now lives in the image at /etc/image-build-date — so the tag no longer has to carry it. A SHA reads as noise at the point of use and breaks the existing convention every consumer pin follows (ci-k8s:2026-08-10, ci-ansible:2026-04-29, alpine-jq:2026-07-30). Dated literals keep the pins uniform and are bumped in the commit that changes an image, which is the same trigger the `when:` path filters already use.
build(ci): record the build timestamp in ci-claude too
2026-08-23 21:14:51 +02:00 · 2026-08-23 20:59:29 +02:00 · 2026-08-23 20:57:15 +02:00 · 2026-08-23 20:52:06 +02:00 · 2026-08-21 20:28:21 +00:00 · 2026-08-21 22:27:10 +02:00
24 changed files with 419 additions and 148 deletions
@@ -14,6 +14,12 @@
 # Flux-deployed CronJobs, so it is tracked separately in #330 rather than
 # bundled here. Note this pipeline runs no LLM agent, so the exfil path that
 # motivated the other cutovers does not apply to it.
+#
+# The build steps run the plain kaniko executor, which has no shell: they pass
+# flags via `entrypoint` and read credentials from DOCKER_CONFIG. Registry auth
+# is written once by the registry-auth step below. Dated tags are literals: bump
+# them in the commit that changes an image. The build timestamp itself lives in
+# the image at /etc/image-build-date, written by a RUN in the Containerfile.
 when:
  - event: push
    path:
@@ -24,10 +30,22 @@ when:
    evaluate: 'PIPELINE == "build-ci-images"'

 steps:
-  - name: build-ansible
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
-    directory: ci/images/ansible
+  - name: registry-auth
+    image: alpine:3.24.1
    depends_on: []
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p .docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > .docker/config.json
+
+  - name: build-ansible
+    image: gcr.io/kaniko-project/executor:v1.24.0
+    directory: ci/images/ansible
+    depends_on: [registry-auth]
    # Only rebuild when THIS image's source (or the pipeline file) changed, or
    # on a manual run. Without this every step ran on any matching push, so a
    # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
@@ -37,68 +55,47 @@ steps:
      - event: push
        path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-ansible:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/ci-ansible:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/ci-ansible:latest

  - name: build-k8s
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/k8s
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-k8s:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/ci-k8s:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-tofu
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/tofu
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/ci-tofu:$TAG \
-          --destination gitea.renner.uno/gitea_admin/ci-tofu:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/ci-tofu:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/ci-tofu:latest

  # NOTE: ci-claude is deliberately NOT built here. It tracks upstream Claude
  # Code releases on its own cadence and is built + version-tagged by
@@ -108,28 +105,21 @@ steps:
  # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
  # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
  - name: build-secret-rotator
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: secret-rotator
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/secret-rotator:$TAG \
-          --destination gitea.renner.uno/gitea_admin/secret-rotator:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/secret-rotator:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/secret-rotator:latest

  # Runtime image, not CI tooling: a general-purpose alpine + jq base for
  # workloads that need to read JSON from a shell script. Named for what it IS,
@@ -137,28 +127,21 @@ steps:
  # container, but nothing about it is zigbee-specific. Exists so tools come from
  # the image rather than an `apk add` on every container start.
  - name: build-alpine-jq
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/alpine-jq
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
-          --destination gitea.renner.uno/gitea_admin/alpine-jq:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/alpine-jq:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/alpine-jq:latest

  # Runtime image, not CI tooling: upstream containernetworking/plugins binaries
  # baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
@@ -166,28 +149,21 @@ steps:
  # critical path of every node rebuild. The download now happens here, once,
  # where the sha256 is checked — see the Containerfile for the full rationale.
  - name: build-cni-plugins
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/cni-plugins
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
-          --destination gitea.renner.uno/gitea_admin/cni-plugins:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/cni-plugins:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/cni-plugins:latest

  # Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
  # needing s_client/x509. Named for what it is; first consumer is the paperless
@@ -195,28 +171,21 @@ steps:
  # every paperless pod start. The openssl CLI is genuinely absent from the alpine
  # base, so this one needs an image rather than a busybox substitute.
  - name: build-alpine-openssl
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/alpine-openssl
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
-          --destination gitea.renner.uno/gitea_admin/alpine-openssl:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/alpine-openssl:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/alpine-openssl:latest

  # Heating CAN readers: open3e (Vitocal/VX3) + E3onCAN (E380 meter), one image
  # run as two services on edge-1. Replaces pip-into-venv on the host, where an
@@ -224,25 +193,18 @@ steps:
  # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
  # builds the same way and lands in the same registry so the audit sees it.
  - name: build-heating-can
-    image: gcr.io/kaniko-project/executor:v1.23.2-debug
+    image: gcr.io/kaniko-project/executor:v1.24.0
    directory: ci/images/heating-can
-    depends_on: []
+    depends_on: [registry-auth]
    when:
      - event: manual
      - event: push
        path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
-          --destination gitea.renner.uno/gitea_admin/heating-can:latest
+      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
+    entrypoint:
+      - /kaniko/executor
+      - --context=.
+      - --dockerfile=Containerfile
+      - --destination=gitea.renner.uno/gitea_admin/heating-can:2026-08-23
+      - --destination=gitea.renner.uno/gitea_admin/heating-can:latest
@@ -22,3 +22,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # unanchored extraction therefore yields a device link key, and herdsman then
 # forms a NEW Zigbee network and drops every device.
 RUN apk add --no-cache jq
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -19,3 +19,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # for paperless to trust. busybox has no s_client equivalent, so openssl is not
 # substitutable there.
 RUN apk add --no-cache openssl
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -24,3 +24,8 @@ RUN pip3 install --break-system-packages --no-cache-dir \
 RUN ansible-galaxy collection install \
      community.proxmox \
      community.docker
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -44,3 +44,9 @@ RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \
      /usr \
      /root \
      /tmp
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# version was built, this says when. Deliberately the LAST layer and its own
+# layer, so the byte-stable npm layer above still dedups in the registry — that
+# dedup is what #351 was for, and a few dozen bytes here does not affect it.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -66,3 +66,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # Alpine rather than scratch: the installer needs a shell to copy the binaries
 # onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
 COPY --from=fetch /plugins/ /opt/cni-plugins/
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -120,3 +120,8 @@ USER open3e
 WORKDIR /var/lib/open3e

 ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -42,3 +42,8 @@ RUN curl -sfL -o /tmp/velero.tar.gz \
 && tar -xz -C /tmp -f /tmp/velero.tar.gz \
 && install "/tmp/velero-${VELERO_VERSION}-linux-amd64/velero" /usr/local/bin/velero \
 && rm -rf "/tmp/velero-${VELERO_VERSION}-linux-amd64" /tmp/velero.tar.gz
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -31,3 +31,8 @@ RUN pip3 install --break-system-packages --no-cache-dir \
 RUN ansible-galaxy collection install \
      community.proxmox \
      community.docker
+
+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Last layer, so it only invalidates
+# itself.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
@@ -67,6 +67,20 @@ spec:
      # and slower than the lz4/zstd this build also accepts). Compresses the
      # full-page images that dominate WAL volume after each checkpoint.
      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 3.2 GB/day here, most of it padding.
+      #
+      # 15min, NOT the 30min used elsewhere in this rollout. This cluster holds
+      # credentials, sessions and enrolment state, so its RPO is worth more than
+      # the difference: 30min would save a further 0.3 GB/day, which does not
+      # justify doubling the worst-case loss window on this data.
+      #
+      # It does carry a synchronous standby (instances: 2), so the realistic
+      # exposure is losing both instances, not one -- but the conservative value
+      # is cheap here and the aggressive one buys almost nothing.
+      archive_timeout: "15min"
    # Async replication makes CNPG switchover slow because the standby has
    # to chase missing WAL via barman-cloud and replay queued records
    # before it can promote. With one synchronous standby, the standby is
@@ -53,6 +53,28 @@ spec:
      # and slower than the lz4/zstd this build also accepts). Compresses the
      # full-page images that dominate WAL volume after each checkpoint.
      wal_compression: "zstd"
+      # THE dominant write source on this cluster, by three orders of magnitude.
+      #
+      # A WAL segment is a fixed 16 MiB file. archive_timeout forces a segment
+      # switch on a timer so the archive never lags reality by more than that
+      # interval -- and the switched-out segment is written and archived at its
+      # full 16 MiB regardless of how few bytes of real WAL it contains.
+      #
+      # CNPG defaults this to 5min, i.e. up to 288 segments/day = 4.6 GB/day.
+      # Measured here: this database is 21 MB and generates 4.5 MB/day of actual
+      # WAL, while its RBD volume takes 211 MB/hour. Essentially every byte
+      # written is padding -- roughly 1100x amplification.
+      #
+      # 30min cuts that to 48 segments/day. The cost is RPO: on total loss of
+      # the cluster, up to 30min of Grafana state is unrecoverable rather than
+      # 5min. Grafana's state here is dashboards, annotations and preferences,
+      # and the write rate says it changes ~4.5 MB/day -- so that window is
+      # nearly always empty anyway.
+      #
+      # Applied to gf-pg FIRST, alone, as a staged rollout: it is the lowest-risk
+      # cluster and has the cleanest signal, since its real WAL is small enough
+      # that any change in volume is unambiguously this parameter.
+      archive_timeout: "30min"
  resources:
    requests:
      cpu: 100m
@@ -86,8 +86,26 @@ spec:
      # inserts scattered across pages, so full-page writes after each
      # checkpoint dominate WAL volume; compression cuts that directly.
      #
-      # Measured on this cluster: full-page images are 62.4% of WAL bytes, so
-      # this is the single largest lever on recorder write volume.
+      # Full-page images really are ~62% of WAL here: 61.9% measured as a
+      # steady-state delta over one checkpoint cycle, against 62.4% lifetime.
+      #
+      # But switching pglz -> zstd changed NOTHING. Measured by
+      # pg_current_wal_lsn advance over 20min windows: 4911 and 4976 MB/day
+      # before, 5002 MB/day after.
+      #
+      # The reason is that `wal_compression: "on"` was never "no compression" --
+      # it is pglz, which was already compressing these page images. zstd only
+      # changes the ratio, and on recorder pages that margin is too small to
+      # measure. The original framing of this change (that the clusters "were
+      # not getting compression") was wrong; they were.
+      #
+      # Note the 62% figure is an upper bound: it is computed as
+      # wal_fpi * 8192 / wal_bytes, which assumes uncompressed 8 KiB pages. With
+      # compression active the true byte share is lower, which is consistent
+      # with there being little headroom left for a better algorithm.
+      #
+      # Kept because zstd is not worse than pglz and costs nothing. It is simply
+      # not a lever. The real one on this cluster was archive_timeout, below.
      #
      # zstd, not "on": the boolean spelling resolves to pglz, the original
      # 2016-era algorithm, which is both slower and weaker than the two
@@ -95,9 +113,35 @@ spec:
      # best ratio on 8 KiB page images, and the CPU cost lands on a cluster
      # that is IO-bound on QLC NVMe, not CPU-bound.
      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
      # Fewer checkpoints => fewer full-page writes, since a page is only
-      # fully-logged on its first modification after a checkpoint. Doubling the
-      # interval halves how often each hot page pays that 8 KiB toll.
+      # fully-logged on its first modification after a checkpoint.
+      #
+      # That reasoning did not show up in WAL volume: 15min -> 30min moved the
+      # WAL rate not at all (4911/4976 -> 5002 MB/day). The likely reason is
+      # that this workload is append-mostly -- a page is dirtied while it fills
+      # and then not touched again, so its full-page image is paid once
+      # regardless of how often checkpoints run. Interval only helps pages that
+      # are re-dirtied across many cycles, which is not what a recorder does.
+      #
+      # Kept, but NOT because WAL improved. checkpoint_timeout also governs
+      # data-file writes at checkpoint, and those are unmeasured here: this
+      # volume takes ~11 GB/day against 4.9 GB/day of WAL, so most of its
+      # traffic is outside what the WAL measurement covers. Revisit with a
+      # data-file measurement before treating this value as justified.
      #
      # Recovery cost is bounded by how much WAL a single interval produces,
      # not by the interval itself: at the measured rate 30min is well under
@@ -46,6 +46,20 @@ spec:
      # and slower than the lz4/zstd this build also accepts). Compresses the
      # full-page images that dominate WAL volume after each checkpoint.
      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
      # Write-reduction tuning. These databases are largely idle, and the
      # Postgres defaults favour throughput over storage that wears out.
      #
@@ -27,7 +27,7 @@ spec:
        # Pinned to 7.4-alpine to match the LXC 225 compose exactly.
        # Any redis major-version bump is a separate migration.
        - name: redis
-          image: redis:7.4.10-alpine # {"$imagepolicy": "flux-system:redis-immich"}
+          image: redis:7.4.11-alpine # {"$imagepolicy": "flux-system:redis-immich"}
          args:
            - redis-server
            - --save
@@ -12,18 +12,43 @@ spec:
    - name: ceph.rules
      rules:
        - alert: CephHealthWarn
-          # ceph_health_status: 0 = HEALTH_OK, 1 = HEALTH_WARN, 2 = HEALTH_ERR.
-          # 6h is deliberately long. Ceph warns for self-clearing conditions —
-          # a slow BlueStore op during the Sunday restore sweep, a mon briefly
-          # below its disk threshold — and paging on those trains people to
-          # ignore the alert. What matters is a warning that does not clear.
-          expr: ceph_health_status == 1
+          # Per-check rather than ceph_health_status, so one permanently-active
+          # check cannot mask every other Ceph warning.
+          #
+          # AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a
+          # kernel supporting aes256k over the CSI mount path (#527).
+          #
+          # POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes
+          # `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked
+          # maintenance and not just data risk — do not exclude it for a pool
+          # that is unreplicated deliberately.
+          #
+          # 6h: Ceph warns for self-clearing conditions. Only a warning that
+          # does not clear is worth paging on.
+          expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
          for: 6h
          labels:
            severity: warning
          annotations:
-            summary: "Ceph has been HEALTH_WARN for 6h"
-            description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            summary: "Ceph health check {{ $labels.name }} active for 6h"
+            description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+
+        - alert: CephPoolLostRedundancy
+          # Names the pool, which POOL_NO_REDUNDANCY does not.
+          #
+          # There is no pool-size metric, but bytes_used/stored IS the
+          # replication factor. The >1e6 floor drops pools too small to measure
+          # meaningfully, empty ones included.
+          #
+          # No pool is exempt.
+          expr: |
+            ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Ceph pool {{ $labels.pool_id }} is running unreplicated"
+            description: "Pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning a single OSD loss destroys data. No pool is meant to be unreplicated — an unreplicated pool also blocks ALL OSD maintenance cluster-wide, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."

        - alert: CephHealthError
          # HEALTH_ERR means data availability or redundancy is at risk.
@@ -58,6 +58,18 @@ spec:
      # and slower than the lz4/zstd this build also accepts). Compresses the
      # full-page images that dominate WAL volume after each checkpoint.
      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.1 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if both instances
+      # are lost. The documents themselves live on the media PVC, not in
+      # Postgres -- this database holds their metadata.
+      archive_timeout: "30min"
    # See ak-pg cluster.yaml for the rationale on synchronous replication.
    synchronous:
      method: any
@@ -25,7 +25,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: redis
-          image: redis:8.10.0-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
+          image: redis:8.10.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
          args:
            - redis-server
            - --save
@@ -36,6 +36,20 @@ spec:
      # and slower than the lz4/zstd this build also accepts). Compresses the
      # full-page images that dominate WAL volume after each checkpoint.
      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
      # Write-reduction tuning. These databases are largely idle, and the
      # Postgres defaults favour throughput over storage that wears out.
      #
@@ -50,12 +50,39 @@ spec:
            # clone + subsequent steps share the workspace PVC and the k8s backend
            # can schedule them on different nodes -> an RWO volume then fails to
            # mount on the second node. CephFS (RWX) lets the workspace follow
-            # the pipeline across nodes. The -ephemeral variant (reclaim Delete,
-            # rook-ceph-cluster/storageclass.yaml) is REQUIRED here: workspaces
-            # are throwaway, and the Retain class leaked one Released PV per
-            # finished pipeline (issue #262 follow-up).
+            # the pipeline across nodes. Reclaim MUST be Delete: workspaces are
+            # throwaway, and a Retain class leaked one Released PV per finished
+            # pipeline (issue #262 follow-up).
+            #
+            # ceph-filesystem-ci, not ceph-filesystem-ephemeral: a dedicated
+            # CephFS whose pools are BOTH size 1 (infrastructure/
+            # rook-ceph-cluster/cephfs-ci.yaml). The shared filesystem
+            # replicates data0 2x and metadata 3x, which is the correct trade
+            # for real data and pure waste for a git clone that is deleted
+            # minutes later.
+            #
+            # Measured 2026-08-21, polling both pools every 15s (a 5-minute
+            # interval misses an entire pipeline -- PVC created, used and
+            # deleted inside ~2 minutes):
+            #   CI active  1.5min: data0 143.2 MB, metadata 17.2 MB
+            #   CI idle   16.0min: data0   0.0 MB, metadata 11.0 MB
+            # data0 write volume is 100% CI. Per pipeline this moves the drive
+            # cost from 71.6x2 + 8.2x3 = ~168 MB to ~80 MB.
+            #
+            # ceph-filesystem-ephemeral is NOT repointed, and the reason is a
+            # technical constraint rather than a safety one. Its other consumer
+            # is velero/restore-test.sh, whose probe PVCs are snapshot CLONES:
+            # `ceph fs subvolume snapshot clone` operates within one filesystem,
+            # so a snapshot taken in `cephfs` cannot be cloned into `cephfs-ci`.
+            #
+            # Those probes are just as throwaway as a CI workspace -- the
+            # authoritative copy is the backup on Hetzner, and a lost probe just
+            # means re-running the test -- so they are equally undeserving of
+            # replication. Landing them unreplicated would mean a second size-1
+            # data pool inside `cephfs`, and restore-test runs WEEKLY against
+            # CI's ~96 pipelines/day, so it is not worth the complexity.
            - name: WOODPECKER_BACKEND_K8S_STORAGE_CLASS
-              value: ceph-filesystem-ephemeral
+              value: ceph-filesystem-ci
            - name: WOODPECKER_BACKEND_K8S_STORAGE_RWX
              value: "true"
            # MUST be a JSON object literal — the backend json.Unmarshal's this
@@ -80,6 +80,20 @@ spec:
      # and slower than the lz4/zstd this build also accepts). Compresses the
      # full-page images that dominate WAL volume after each checkpoint.
      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
  resources:
    requests:
      cpu: 100m
@@ -17,6 +17,17 @@ spec:
    modules:
      - name: pg_autoscaler
        enabled: true
+  # Rotates daemon keys to aes256k for CVE-2025-30156 (#527). Opt-in; disabled
+  # by default. Increment keyGeneration to rotate again.
+  #
+  # Daemon keys only — aes256k over the CSI mount path needs kernel 7.0+ and
+  # these nodes run 6.18, so CSI stays on aes and the client-key checks stay
+  # raised. keyType is unset so Rook picks it.
+  security:
+    cephx:
+      daemon:
+        keyRotationPolicy: KeyGeneration
+        keyGeneration: 2
  # Bump CephFS subvolume cloner pool from 4 → 16. Velero CSI Snapshot
  # Data Movement provisions a backupPVC per source PVC by cloning the
  # VolumeSnapshot, and a 4-thread default stalls migrate-immich's
@@ -0,0 +1,64 @@
+---
+# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces. Each
+# pipeline gets a PVC that is created, filled and deleted within ~2 minutes;
+# keeping that churn off `cephfs` spares its size-3 metadataPool.
+#
+# size 2, not 1: an unreplicated pool makes `ceph osd ok-to-stop` refuse for
+# every OSD, blocking all OSD maintenance cluster-wide. size 1 is viable only
+# once these pools have OSDs of their own.
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystem
+metadata:
+  name: cephfs-ci
+  namespace: rook-ceph
+spec:
+  metadataPool:
+    replicated:
+      size: 2
+  dataPools:
+    - name: data0
+      replicated:
+        size: 2
+  metadataServer:
+    activeCount: 1
+    # No standby-replay, unlike `cephfs`. A standby exists there so a failed
+    # MDS does not hang every CephFS mount cluster-wide; here the only casualty
+    # is a running pipeline, and a second daemon to protect throwaway work is
+    # not worth its memory.
+    activeStandby: false
+    resources:
+      # Deliberately NOT the 4Gi that `cephfs` needs. That figure is driven by
+      # immich's millions of cached inodes; a CI workspace is a few thousand
+      # short-lived files that vanish with the pipeline.
+      requests:
+        cpu: 100m
+        memory: 512Mi
+      limits:
+        memory: 1Gi
+---
+# CI-only class. `ceph-filesystem-ephemeral` is deliberately left pointing at
+# the main filesystem: velero/restore-test.sh uses it to restore REAL data for
+# verification, which must not land on unreplicated pools.
+#
+# A new class rather than repointing the old one — StorageClass.parameters are
+# immutable, so `fsName` cannot be edited in place.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ceph-filesystem-ci
+provisioner: rook-ceph.cephfs.csi.ceph.com
+parameters:
+  clusterID: rook-ceph
+  fsName: cephfs-ci
+  pool: cephfs-ci-data0
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+# Delete, for the same reason ceph-filesystem-ephemeral uses it: with Retain,
+# every finished pipeline leaks a Released PV pinning a subvolume (191 of them
+# by 2026-07-16, issue #262 follow-up).
+reclaimPolicy: Delete
+allowVolumeExpansion: true
@@ -4,5 +4,6 @@ resources:
  - cephcluster.yaml
  - toolbox.yaml
  - storageclass.yaml
+  - cephfs-ci.yaml
  - volumesnapshotclass.yaml
  - objectstore.yaml
@@ -21,8 +21,14 @@ COPY src ./src
 RUN cargo build --release --target x86_64-unknown-linux-musl \
 && strip target/x86_64-unknown-linux-musl/release/secret-rotator

+# Build timestamp, recorded in the image rather than the tag: the tag says which
+# source revision was built, this says when. Written here because the final
+# image is scratch and has no shell to run `date` in.
+RUN date -u +%Y-%m-%dT%H:%M:%SZ > /build/image-build-date
+
 FROM scratch
 COPY --from=build /build/target/x86_64-unknown-linux-musl/release/secret-rotator /secret-rotator
+COPY --from=build /build/image-build-date /etc/image-build-date

 # Matches runAsUser on the rotation Jobs. There is no /etc/passwd in a scratch
 # image, so this is a bare uid by necessity — which is also all Kubernetes needs.