diff --git a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml index 795daaa1..693d160d 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml @@ -24,6 +24,27 @@ spec: # gives headroom for parallel restores during cluster rebuild too # (paperless + audiobookshelf + immich + ... all clone at once). cephConfig: + mon: + # Ceph's 30% default assumes the mon has a filesystem to itself, where + # 30% free is a meaningful reserve. Here the mons live on + # dataDirHostPath /var/lib/rook — the node's /var, shared with ~29 GB + # of containerd image and snapshot storage — and the mon DB itself is + # ~106 MB. So the default measures container churn, not Ceph: k8s-2 + # trips MON_DISK_LOW whenever the image cache fills, and image GC does + # not clear it (most of /var/lib/containerd is overlayfs snapshots held + # by running containers, which GC cannot reclaim — see #434). + # 20% of a 60 GB /var still leaves ~12 GB against a 106 MB database. + # mon_data_avail_crit stays at the 5% default, and CephMonDiskLow in + # monitoring-rules watches /var directly for the real risk, which is the + # node filling rather than the mon growing. + # + # BARE-METAL MIGRATION: revisit this. Give the mons their own volume and + # the default 30% becomes meaningful again, at which point this override + # should be dropped rather than carried over. It cannot be done on the + # current cluster — every StorageClass is Ceph-backed, so a mon PVC + # would be circular, and Rook only applies mon.volumeClaimTemplate to + # newly created mons, never migrating existing ones. + mon_data_avail_warn: "20" mgr: mgr/volumes/max_concurrent_clones: "16" # QLC tune for the Crucial P2 NVMe per host. Baseline 2026-05-05: