fix: [alert] KubeJobFailed (wordpress) (#550)

.woodpecker/build-ci-images.yml

@@ -14,12 +14,6 @@
 # Flux-deployed CronJobs, so it is tracked separately in #330 rather than
 # bundled here. Note this pipeline runs no LLM agent, so the exfil path that
 # motivated the other cutovers does not apply to it.
-#
-# The build steps run the plain kaniko executor, which has no shell: they pass
-# flags via `entrypoint` and read credentials from DOCKER_CONFIG. Registry auth
-# is written once by the registry-auth step below. Dated tags are literals: bump
-# them in the commit that changes an image. The build timestamp itself lives in
-# the image at /etc/image-build-date, written by a RUN in the Containerfile.
 when:
   - event: push
     path:
@@ -30,22 +24,10 @@ when:
     evaluate: 'PIPELINE == "build-ci-images"'
 
 steps:
-  - name: registry-auth
-    image: alpine:3.24.1
-    depends_on: []
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p .docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > .docker/config.json
-
   - name: build-ansible
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: ci/images/ansible
-    depends_on: [registry-auth]
+    depends_on: []
     # Only rebuild when THIS image's source (or the pipeline file) changed, or
     # on a manual run. Without this every step ran on any matching push, so a
     # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
@@ -55,47 +37,68 @@ steps:
       - event: push
         path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/ci-ansible:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/ci-ansible:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/ci-ansible:$TAG \
+          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest
 
   - name: build-k8s
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: ci/images/k8s
-    depends_on: [registry-auth]
+    depends_on: []
     when:
       - event: manual
       - event: push
         path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/ci-k8s:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/ci-k8s:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/ci-k8s:$TAG \
+          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest
 
   - name: build-tofu
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: ci/images/tofu
-    depends_on: [registry-auth]
+    depends_on: []
     when:
       - event: manual
       - event: push
         path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/ci-tofu:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/ci-tofu:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/ci-tofu:$TAG \
+          --destination gitea.renner.uno/gitea_admin/ci-tofu:latest
 
   # NOTE: ci-claude is deliberately NOT built here. It tracks upstream Claude
   # Code releases on its own cadence and is built + version-tagged by
@@ -105,21 +108,28 @@ steps:
   # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
   # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
   - name: build-secret-rotator
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: secret-rotator
-    depends_on: [registry-auth]
+    depends_on: []
     when:
       - event: manual
       - event: push
         path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/secret-rotator:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/secret-rotator:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/secret-rotator:$TAG \
+          --destination gitea.renner.uno/gitea_admin/secret-rotator:latest
 
   # Runtime image, not CI tooling: a general-purpose alpine + jq base for
   # workloads that need to read JSON from a shell script. Named for what it IS,
@@ -127,21 +137,28 @@ steps:
   # container, but nothing about it is zigbee-specific. Exists so tools come from
   # the image rather than an `apk add` on every container start.
   - name: build-alpine-jq
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: ci/images/alpine-jq
-    depends_on: [registry-auth]
+    depends_on: []
     when:
       - event: manual
       - event: push
         path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/alpine-jq:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/alpine-jq:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
+          --destination gitea.renner.uno/gitea_admin/alpine-jq:latest
 
   # Runtime image, not CI tooling: upstream containernetworking/plugins binaries
   # baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
@@ -149,21 +166,28 @@ steps:
   # critical path of every node rebuild. The download now happens here, once,
   # where the sha256 is checked — see the Containerfile for the full rationale.
   - name: build-cni-plugins
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: ci/images/cni-plugins
-    depends_on: [registry-auth]
+    depends_on: []
     when:
       - event: manual
       - event: push
         path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/cni-plugins:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/cni-plugins:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
+          --destination gitea.renner.uno/gitea_admin/cni-plugins:latest
 
   # Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
   # needing s_client/x509. Named for what it is; first consumer is the paperless
@@ -171,21 +195,28 @@ steps:
   # every paperless pod start. The openssl CLI is genuinely absent from the alpine
   # base, so this one needs an image rather than a busybox substitute.
   - name: build-alpine-openssl
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: ci/images/alpine-openssl
-    depends_on: [registry-auth]
+    depends_on: []
     when:
       - event: manual
       - event: push
         path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/alpine-openssl:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/alpine-openssl:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
+          --destination gitea.renner.uno/gitea_admin/alpine-openssl:latest
 
   # Heating CAN readers: open3e (Vitocal/VX3) + E3onCAN (E380 meter), one image
   # run as two services on edge-1. Replaces pip-into-venv on the host, where an
@@ -193,18 +224,25 @@ steps:
   # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
   # builds the same way and lands in the same registry so the audit sees it.
   - name: build-heating-can
-    image: gcr.io/kaniko-project/executor:v1.24.0
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
     directory: ci/images/heating-can
-    depends_on: [registry-auth]
+    depends_on: []
     when:
       - event: manual
       - event: push
         path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
     environment:
-      DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
-    entrypoint:
-      - /kaniko/executor
-      - --context=.
-      - --dockerfile=Containerfile
-      - --destination=gitea.renner.uno/gitea_admin/heating-can:2026-08-23
-      - --destination=gitea.renner.uno/gitea_admin/heating-can:latest
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
+          --destination gitea.renner.uno/gitea_admin/heating-can:latest

ci/images/alpine-jq/Containerfile

@@ -22,8 +22,3 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # unanchored extraction therefore yields a device link key, and herdsman then
 # forms a NEW Zigbee network and drops every device.
 RUN apk add --no-cache jq
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Last layer, so it only invalidates
-# itself.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

ci/images/alpine-openssl/Containerfile

@@ -19,8 +19,3 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # for paperless to trust. busybox has no s_client equivalent, so openssl is not
 # substitutable there.
 RUN apk add --no-cache openssl
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Last layer, so it only invalidates
-# itself.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

ci/images/ansible/Containerfile

@@ -24,8 +24,3 @@ RUN pip3 install --break-system-packages --no-cache-dir \
 RUN ansible-galaxy collection install \
       community.proxmox \
       community.docker
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Last layer, so it only invalidates
-# itself.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

ci/images/claude/Containerfile

@@ -44,9 +44,3 @@ RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \
       /usr \
       /root \
       /tmp
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# version was built, this says when. Deliberately the LAST layer and its own
-# layer, so the byte-stable npm layer above still dedups in the registry — that
-# dedup is what #351 was for, and a few dozen bytes here does not affect it.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

ci/images/cni-plugins/Containerfile

@@ -66,8 +66,3 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 # Alpine rather than scratch: the installer needs a shell to copy the binaries
 # onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
 COPY --from=fetch /plugins/ /opt/cni-plugins/
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Last layer, so it only invalidates
-# itself.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

ci/images/heating-can/Containerfile

@@ -120,8 +120,3 @@ USER open3e
 WORKDIR /var/lib/open3e
 
 ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Last layer, so it only invalidates
-# itself.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

ci/images/k8s/Containerfile

@@ -42,8 +42,3 @@ RUN curl -sfL -o /tmp/velero.tar.gz \
  && tar -xz -C /tmp -f /tmp/velero.tar.gz \
  && install "/tmp/velero-${VELERO_VERSION}-linux-amd64/velero" /usr/local/bin/velero \
  && rm -rf "/tmp/velero-${VELERO_VERSION}-linux-amd64" /tmp/velero.tar.gz
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Last layer, so it only invalidates
-# itself.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

ci/images/tofu/Containerfile

@@ -31,8 +31,3 @@ RUN pip3 install --break-system-packages --no-cache-dir \
 RUN ansible-galaxy collection install \
       community.proxmox \
       community.docker
-
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Last layer, so it only invalidates
-# itself.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - prometheusrule-cilium.yaml
   - prometheusrule-hetzner-backup.yaml
   - prometheusrule-loki.yaml
+  - prometheusrule-mariadb-backup.yaml
   - prometheusrule-netbird-availability.yaml
   - prometheusrule-netbird-backup.yaml
   - prometheusrule-netbird-ingress.yaml

kubernetes/apps/monitoring-rules/prometheusrule-mariadb-backup.yaml

@@ -0,0 +1,51 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: mariadb-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: mariadb-backup.rules
+      rules:
+        - alert: WordpressMariadbBackupStale
+          # The nightly 02:45 logical dump (kubernetes/apps/wordpress/mariadb.yaml)
+          # is the only transaction-consistent copy of www.renner.uno's database
+          # — Velero snapshots the PVC, which for a running DB is not a restore
+          # point you would trust.
+          #
+          # KubeJobFailed already covers a failed Job, but only while the Job
+          # object survives. On #550 it cleared when the failed Job was
+          # garbage-collected, not when a backup succeeded — so a failure that
+          # is GC'd before anyone looks leaves no signal at all.
+          # lastSuccessfulTime lives on the CronJob and outlives Job GC.
+          #
+          # 26h = one missed cycle plus 2h slack, matching NetbirdVpsBackupStale.
+          # It counts any Job owned by the CronJob, so a manual
+          # `kubectl create job --from=cronjob/...` re-run clears it too.
+          expr: |
+            time() - kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"} > 26 * 3600
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "WordPress MariaDB backup has not succeeded in 26h+"
+            description: "The wordpress-mariadb-backup CronJob's last successful run is over 26 hours old. Check the newest Job's pod logs in the wordpress namespace: `kubectl -n wordpress get jobs`. The 2026-08-19 failure was `The specified bucket does not exist.` against renner-uno-k8s-mariadb while the endpoint itself probed healthy, so a green HetznerBackupEndpointDown does not clear this."
+
+        - alert: WordpressMariadbBackupNeverRan
+          # The staleness expression above goes blind when the metric is
+          # absent, which is what happens once the operator recreates the
+          # CronJob. Without this guard "never backed up" is quieter than
+          # "backed up late".
+          expr: |
+            kube_cronjob_created{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
+            unless
+            kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            summary: "WordPress MariaDB backup has no recorded successful run"
+            description: "The wordpress-mariadb-backup CronJob exists but has no lastSuccessfulTime: either it has not completed once since the Backup CR was recreated, or every run since has failed. Check `kubectl -n wordpress get jobs` and that the hetzner-s3-creds ExternalSecret is synced."

secret-rotator/Containerfile

@@ -21,14 +21,8 @@ COPY src ./src
 RUN cargo build --release --target x86_64-unknown-linux-musl \
  && strip target/x86_64-unknown-linux-musl/release/secret-rotator
 
-# Build timestamp, recorded in the image rather than the tag: the tag says which
-# source revision was built, this says when. Written here because the final
-# image is scratch and has no shell to run `date` in.
-RUN date -u +%Y-%m-%dT%H:%M:%SZ > /build/image-build-date
-
 FROM scratch
 COPY --from=build /build/target/x86_64-unknown-linux-musl/release/secret-rotator /secret-rotator
-COPY --from=build /build/image-build-date /etc/image-build-date
 
 # Matches runAsUser on the rotation Jobs. There is no /etc/passwd in a scratch
 # image, so this is a bare uid by necessity — which is also all Kubernetes needs.