diff --git a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml index c7356f65..11a9ba3d 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml @@ -47,26 +47,27 @@ spec: # (paperless + audiobookshelf + immich + ... all clone at once). cephConfig: mon: - # Ceph's 30% default assumes the mon has a filesystem to itself, where - # 30% free is a meaningful reserve. Here the mons live on - # dataDirHostPath /var/lib/rook — the node's /var, shared with ~29 GB - # of containerd image and snapshot storage — and the mon DB itself is - # ~106 MB. So the default measures container churn, not Ceph: k8s-2 - # trips MON_DISK_LOW whenever the image cache fills, and image GC does - # not clear it (most of /var/lib/containerd is overlayfs snapshots held - # by running containers, which GC cannot reclaim — see #434). - # 20% of a 60 GB /var still leaves ~12 GB against a 106 MB database. - # mon_data_avail_crit stays at the 5% default, and CephMonDiskLow in - # monitoring-rules watches /var directly for the real risk, which is the - # node filling rather than the mon growing. + # The Ceph default, stated rather than left out. Removing the line would + # not restore it: `ceph config` keeps a value that only disappears from + # this file, so the old 20 would sit in the mon config database + # indefinitely — the same trap the bluestore settings below describe, + # and the one that crash-looped an OSD during the network provider + # change when public_network survived being deleted here. # - # BARE-METAL MIGRATION: revisit this. Give the mons their own volume and - # the default 30% becomes meaningful again, at which point this override - # should be dropped rather than carried over. It cannot be done on the - # current cluster — every StorageClass is Ceph-backed, so a mon PVC - # would be circular, and Rook only applies mon.volumeClaimTemplate to - # newly created mons, never migrating existing ones. - mon_data_avail_warn: "20" + # It was 20 because the mons share the node's /var with containerd, and + # on the VMs that was ~60 GB: 30% free measured image and snapshot churn + # rather than anything about Ceph, so k8s-2 tripped MON_DISK_LOW whenever + # the image cache filled and GC could not reclaim it (#434). + # + # Bare metal removed the premise. /var is now ~931 GB per node at 7-8% + # used, so 30% free is a real reserve again and the ~106 MB mon database + # is irrelevant against it. Reaching 70% would be a genuine signal about + # the node rather than noise about the image cache. + # + # mon_data_avail_crit stays at the 5% default, and CephMonDiskLow in + # monitoring-rules still watches /var directly for the real risk, which + # is the node filling rather than the mon growing. + mon_data_avail_warn: "30" # Stop compacting the whole RocksDB prefix on every paxos trim. # # Upstream default is true — "Compact a certain prefix (including paxos)