fix: [alert] CephHealthWarn (rook-ceph) (#562)

Updates managed by image-automation-controller.

.woodpecker/deploy-infrastructure.yml

@@ -13,7 +13,7 @@ when:
 
 steps:
   - name: validate-secrets
-    image: alpine:3.21
+    image: alpine:3.24.1
     environment:
       TF_VAR_proxmox_api_url:
         from_secret: proxmox_api_url

kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml

@@ -176,3 +176,22 @@ data:
         friendly_name: Treppenhaus Haustür Schalter 1
       '0x001788010b00d011':
         friendly_name: Treppenhaus Haustür Schalter 2
+      # Ecowitt WS90 weather station. Joined after the ZHA-mirrored block above
+      # was written, hence listed apart from it.
+      #
+      # The station transmits every 8.8s, as a burst of seven per-cluster ZCL
+      # frames; Z2M publishes its full cumulative state once per frame. debounce
+      # merges a burst into one publish carrying the last value per attribute,
+      # and must stay below the transmit interval — a window that never sees
+      # silence never flushes.
+      #
+      # The firmware drops all readings to zero for a few seconds every few
+      # days. Merged, that surfaces as a single all-zero frame; unmerged, the
+      # zeros spread across the seven frames and none of them is individually
+      # distinguishable from a valid reading.
+      #
+      # Z2M reads this only at startup (requiresRestart), and this file is
+      # SEED-ONLY per the header — the running instance takes it from its PVC.
+      '0xc02cedfffe958f40':
+        friendly_name: Wetterstation
+        debounce: 2

kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml

@@ -158,7 +158,7 @@ spec:
           # Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
           # add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
           # follow-up if desired; pinned-and-manual is fine for the initial cutover.
-          image: ghcr.io/koenkk/zigbee2mqtt:2.12.1
+          image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
           imagePullPolicy: IfNotPresent
           env:
             - name: TZ

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - prometheusrule-cilium.yaml
   - prometheusrule-hetzner-backup.yaml
   - prometheusrule-loki.yaml
+  - prometheusrule-mariadb-backup.yaml
   - prometheusrule-netbird-availability.yaml
   - prometheusrule-netbird-backup.yaml
   - prometheusrule-netbird-ingress.yaml

kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -25,13 +25,36 @@ spec:
           #
           # 6h: Ceph warns for self-clearing conditions. Only a warning that
           # does not clear is worth paging on.
+          #
+          # BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT
+          # excluded (#557). Ceph holds the check for
+          # bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so
+          # one multi-second stall outlives the 6h `for` and always pages,
+          # with latency back to normal long before anyone looks. Measured:
+          # three episodes in the 30d to 2026-08-24 (Sun Jul 26, Aug 2 and
+          # Aug 23), each ran ~24-26h and cleared untouched.
+          #
+          # All began inside the weekly Velero restore-test window (Sun 08:00Z
+          # plus up to 6h, see velero/restore-test-cronjob.yaml) — a sustained
+          # restore that outruns the SLC cache on these QLC NVMe. Expected for
+          # the hardware, but not excludable: this is also the earliest signal
+          # of a genuinely failing drive, and all three are at 80-87%
+          # endurance used (see cephcluster.yaml).
+          #
+          # No metric names the stalling OSD: ceph_health_detail carries no osd
+          # label, and ceph_daemon_health_metrics{type="SLOW_OPS"} reads 0 at
+          # every scrape (a stall is shorter than the scrape interval). Only
+          # the OSD's own log has it, hence the Loki query in the description.
+          # Jul 26 and Aug 2 hit osd.0+1+2 together, which is load-driven;
+          # Aug 23 hit osd.1 alone in one burst. If osd.1 alone repeats,
+          # suspect that drive rather than the restore load.
           expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
           for: 6h
           labels:
             severity: warning
           annotations:
             summary: "Ceph health check {{ $labels.name }} active for 6h"
-            description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the Sunday 08:00-14:00Z Velero restore-test window (08:00 start, 6h activeDeadlineSeconds cap), which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). No metric names the OSD — get it from Loki, which also works after the check clears: `{namespace=\"rook-ceph\",pod=~\"rook-ceph-osd-.*\"} |= \"slow operation observed\"`. While still active, `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` adds the op detail."
 
         - alert: CephPoolLostRedundancy
           # Names the pool, which POOL_NO_REDUNDANCY does not.

kubernetes/apps/monitoring-rules/prometheusrule-mariadb-backup.yaml

@@ -0,0 +1,51 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: mariadb-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: mariadb-backup.rules
+      rules:
+        - alert: WordpressMariadbBackupStale
+          # The nightly 02:45 logical dump (kubernetes/apps/wordpress/mariadb.yaml)
+          # is the only transaction-consistent copy of www.renner.uno's database
+          # — Velero snapshots the PVC, which for a running DB is not a restore
+          # point you would trust.
+          #
+          # KubeJobFailed already covers a failed Job, but only while the Job
+          # object survives. On #550 it cleared when the failed Job was
+          # garbage-collected, not when a backup succeeded — so a failure that
+          # is GC'd before anyone looks leaves no signal at all.
+          # lastSuccessfulTime lives on the CronJob and outlives Job GC.
+          #
+          # 26h = one missed cycle plus 2h slack, matching NetbirdVpsBackupStale.
+          # It counts any Job owned by the CronJob, so a manual
+          # `kubectl create job --from=cronjob/...` re-run clears it too.
+          expr: |
+            time() - kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"} > 26 * 3600
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "WordPress MariaDB backup has not succeeded in 26h+"
+            description: "The wordpress-mariadb-backup CronJob's last successful run is over 26 hours old. Check the newest Job's pod logs in the wordpress namespace: `kubectl -n wordpress get jobs`. The 2026-08-19 failure was `The specified bucket does not exist.` against renner-uno-k8s-mariadb while the endpoint itself probed healthy, so a green HetznerBackupEndpointDown does not clear this."
+
+        - alert: WordpressMariadbBackupNeverRan
+          # The staleness expression above goes blind when the metric is
+          # absent, which is what happens once the operator recreates the
+          # CronJob. Without this guard "never backed up" is quieter than
+          # "backed up late".
+          expr: |
+            kube_cronjob_created{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
+            unless
+            kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            summary: "WordPress MariaDB backup has no recorded successful run"
+            description: "The wordpress-mariadb-backup CronJob exists but has no lastSuccessfulTime: either it has not completed once since the Backup CR was recreated, or every run since has failed. Check `kubectl -n wordpress get jobs` and that the hetzner-s3-creds ExternalSecret is synced."

kubernetes/apps/woodpecker/agent.yaml

@@ -34,7 +34,7 @@ spec:
           # Image automation, range 3.x (see image-automation/woodpecker.yaml).
           # MUST stay version-matched with woodpecker-server (gRPC parity) — both
           # track the same range/filter so they bump together on each release.
-          image: woodpeckerci/woodpecker-agent:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
+          image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
           env:
             - name: WOODPECKER_SERVER
               value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000

kubernetes/apps/woodpecker/server.yaml

@@ -40,7 +40,7 @@ spec:
           # minor + patch auto-update within 3.x; only the 4.0 major is held for
           # a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
           # -bump hazard is gone). Stays in lockstep with the agent.
-          image: woodpeckerci/woodpecker-server:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
+          image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
           env:
             - name: WOODPECKER_HOST
               value: https://ci.renner.uno