fix: [alert] CephHealthWarn (rook-ceph) (#557)

Updates managed by image-automation-controller.

kubernetes/apps/home-assistant/deployment.yaml

@@ -84,7 +84,7 @@ spec:
           env:
             - name: DB_URI
               valueFrom:
-                secretKeyRef: { name: ha-pg-app, key: fqdn-uri }
+                secretKeyRef: {name: ha-pg-app, key: fqdn-uri}
           volumeMounts:
             - name: config
               mountPath: /config
@@ -96,7 +96,7 @@ spec:
               memory: 32Mi
       containers:
         - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.8.2 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
           env:
             - name: TZ
               value: Europe/Berlin

kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -12,18 +12,57 @@ spec:
     - name: ceph.rules
       rules:
         - alert: CephHealthWarn
-          # ceph_health_status: 0 = HEALTH_OK, 1 = HEALTH_WARN, 2 = HEALTH_ERR.
-          # 6h is deliberately long. Ceph warns for self-clearing conditions —
-          # a slow BlueStore op during the Sunday restore sweep, a mon briefly
-          # below its disk threshold — and paging on those trains people to
-          # ignore the alert. What matters is a warning that does not clear.
-          expr: ceph_health_status == 1
+          # Per-check rather than ceph_health_status, so one permanently-active
+          # check cannot mask every other Ceph warning.
+          #
+          # AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a
+          # kernel supporting aes256k over the CSI mount path (#527).
+          #
+          # POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes
+          # `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked
+          # maintenance and not just data risk — do not exclude it for a pool
+          # that is unreplicated deliberately.
+          #
+          # 6h: Ceph warns for self-clearing conditions. Only a warning that
+          # does not clear is worth paging on.
+          #
+          # BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT
+          # excluded (#557). Ceph holds the check for
+          # bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so
+          # one multi-second stall outlives the 6h `for` and always pages,
+          # with latency back to normal long before anyone looks. Measured:
+          # both episodes in the 30d to 2026-08-23 ran ~26h and cleared
+          # untouched, the first ~24h after the last stall.
+          #
+          # Both began inside the weekly Velero restore-test window (Sun
+          # 08:00-13:00Z) — a 5h sustained restore that outruns the SLC cache
+          # on these QLC NVMe. Expected for the hardware, but not excludable:
+          # this is also the earliest signal of a genuinely failing drive, and
+          # all three are at 80-87% endurance used (see cephcluster.yaml).
+          expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
           for: 6h
           labels:
             severity: warning
           annotations:
-            summary: "Ceph has been HEALTH_WARN for 6h"
-            description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            summary: "Ceph health check {{ $labels.name }} active for 6h"
+            description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the Sunday 08:00-13:00Z Velero restore-test window, which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+
+        - alert: CephPoolLostRedundancy
+          # Names the pool, which POOL_NO_REDUNDANCY does not.
+          #
+          # There is no pool-size metric, but bytes_used/stored IS the
+          # replication factor. The >1e6 floor drops pools too small to measure
+          # meaningfully, empty ones included.
+          #
+          # No pool is exempt.
+          expr: |
+            ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Ceph pool {{ $labels.pool_id }} is running unreplicated"
+            description: "Pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning a single OSD loss destroys data. No pool is meant to be unreplicated — an unreplicated pool also blocks ALL OSD maintenance cluster-wide, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."
 
         - alert: CephHealthError
           # HEALTH_ERR means data availability or redundancy is at risk.

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -17,6 +17,17 @@ spec:
     modules:
       - name: pg_autoscaler
         enabled: true
+  # Rotates daemon keys to aes256k for CVE-2025-30156 (#527). Opt-in; disabled
+  # by default. Increment keyGeneration to rotate again.
+  #
+  # Daemon keys only — aes256k over the CSI mount path needs kernel 7.0+ and
+  # these nodes run 6.18, so CSI stays on aes and the client-key checks stay
+  # raised. keyType is unset so Rook picks it.
+  security:
+    cephx:
+      daemon:
+        keyRotationPolicy: KeyGeneration
+        keyGeneration: 2
   # Bump CephFS subvolume cloner pool from 4 → 16. Velero CSI Snapshot
   # Data Movement provisions a backupPVC per source PVC by cloning the
   # VolumeSnapshot, and a 4-thread default stalls migrate-immich's

kubernetes/infrastructure/rook-ceph-cluster/cephfs-ci.yaml

@@ -1,29 +1,11 @@
 ---
-# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces, with both
-# pools at size 1.
+# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces. Each
+# pipeline gets a PVC that is created, filled and deleted within ~2 minutes;
+# keeping that churn off `cephfs` spares its size-3 metadataPool.
 #
-# WHY THIS EXISTS
-# Woodpecker's Kubernetes backend creates a PVC per pipeline, fills it with a
-# checkout plus build output, and deletes it when the pipeline ends. Measured
-# 2026-08-21 by polling both pools at 15s (a 5-minute interval misses a whole
-# pipeline — its PVC is created, used and deleted inside ~2 minutes):
-#
-#   one pipeline, a single 15s interval:  data0 +71.6 MB   metadata +8.2 MB
-#   every interval without a CI pod:      data0   ~0        metadata   ~0
-#
-# CI drove essentially all of it. On the shared filesystem that costs
-# 71.6x2 + 8.2x3 = ~168 MB of drive writes per pipeline. Here it costs ~80 MB.
-#
-# WHY size 1 IS CORRECT HERE, HAVING BEEN REJECTED FOR THE MAIN FILESYSTEM
-# `cephfs` keeps metadataPool at size 3 because losing it destroys the whole
-# filesystem, not the objects on one OSD, and 550 GiB in cephfs-data0 depends
-# on it. None of that applies to this filesystem. Every byte in it is a fresh
-# git clone; the worst case of total loss is that in-flight pipelines fail and
-# get re-run. Replicating throwaway data is the thing being paid for, so it is
-# the thing removed.
-#
-# Losing an OSD takes these PGs offline and kills running pipelines. That is
-# not a regression: CI already breaks during Ceph maintenance.
+# size 2, not 1: an unreplicated pool makes `ceph osd ok-to-stop` refuse for
+# every OSD, blocking all OSD maintenance cluster-wide. size 1 is viable only
+# once these pools have OSDs of their own.
 apiVersion: ceph.rook.io/v1
 kind: CephFilesystem
 metadata:
@@ -32,15 +14,11 @@ metadata:
 spec:
   metadataPool:
     replicated:
-      size: 1
-      # Rook refuses size 1 without this — it is asserting that an unreplicated
-      # pool is deliberate rather than a typo. It is.
-      requireSafeReplicaSize: false
+      size: 2
   dataPools:
     - name: data0
       replicated:
-        size: 1
-        requireSafeReplicaSize: false
+        size: 2
   metadataServer:
     activeCount: 1
     # No standby-replay, unlike `cephfs`. A standby exists there so a failed