fix: [alert] CephMonDiskLow (monitoring) (#569)

Reviewed-on: k.renner/homelab#570

kubernetes/apps/immich/machine-learning.yaml

@@ -6,7 +6,17 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-machine-learning
 spec:
-  replicas: 1
+  # TEMPORARY — RESTORE TO 1 ONCE k8s-1 HAS REJOINED.
+  #
+  # Down for the jupiter bare-metal conversion. With one of three nodes out,
+  # the remaining pair have no room: cluster CPU requests exceed two nodes'
+  # capacity, and k8s-3 runs ~5.8 GiB above the kubelet's eviction threshold,
+  # so every pod that does not need to be running is one less thing competing
+  # for both. This one is the best candidate: 500m reserved against a measured
+  # 2m idle draw, and inference bursts hard enough that lowering the request
+  # instead would be the wrong trade. Photo processing pauses; nothing else
+  # depends on it.
+  replicas: 0
   strategy:
     type: RollingUpdate
   selector:

kubernetes/apps/immich/server.yaml

@@ -36,7 +36,11 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-server
 spec:
-  replicas: 1
+  # TEMPORARY — RESTORE TO 1 ONCE k8s-1 HAS REJOINED. See the note on
+  # immich-machine-learning; immich is offline for the jupiter conversion.
+  # Its Postgres cluster is deliberately left running, so nothing needs
+  # restoring beyond these two replica counts.
+  replicas: 0
   strategy:
     type: Recreate
   selector:

kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -93,5 +93,5 @@ spec:
           labels:
             severity: warning
           annotations:
-            summary: "{{ $labels.instance }} /var below 30% free — Ceph mon will warn"
-            description: "/var on {{ $labels.instance }} is under 30% free, which is Ceph's mon_data_avail_warn threshold, so the mon on this node will report MON_DISK_LOW. Find the actual consumer with `talosctl -n <node> usage /var/lib --depth 1` and then `/var/lib/containerd --depth 1` — note that overlayfs snapshots belong to running containers and are not reclaimable by image GC."
+            summary: "{{ $labels.instance }} /var below 30% free — early warning above Ceph's mon threshold"
+            description: "/var on {{ $labels.instance }} is under 30% free. This is an early-warning margin, not Ceph's threshold: rook-ceph-cluster/cephcluster.yaml overrides mon_data_avail_warn to 20, so the mon on this node reports MON_DISK_LOW only below 20% free. Check whether it actually has, with `ceph_health_detail{name=\"MON_DISK_LOW\"} == 1` in Prometheus, before treating this as a Ceph problem. Find the actual consumer with `talosctl -n <node> usage /var/lib --depth 1` and then `/var/lib/containerd --depth 1` — note that overlayfs snapshots belong to running containers and are not reclaimable by image GC."

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -160,19 +160,34 @@ spec:
         - "10.2.40.0/24"
       cluster:
         - "10.99.0.0/29"
+  # Nodes are listed explicitly rather than useAllNodes, so that a node can be
+  # taken out of the storage fleet by removing it here. Under useAllNodes the
+  # operator re-provisions an OSD on any node whose device still carries a
+  # BlueStore signature, which makes purging an OSD ahead of pulling its drive
+  # impossible: the OSD is recreated within a reconcile and starts backfilling
+  # onto the drive being replaced.
+  #
+  # k8s-1 is absent while jupiter's OSD drive is swapped and the node is
+  # converted to bare metal. RE-ADD IT once the PM983 is in and the node has
+  # rejoined; the cluster runs on two OSDs until then.
+  #
+  # nvme0n1 on k8s-2/k8s-3 is the NVMe drive PCI-passed-through from saturn and
+  # neptune, and is the only block device those guests can see. Bare-metal k8s-1
+  # will see two — the OSD drive in MS-01 slot 1 (PCI 01:00.0) and the Talos
+  # system disk in slot 2 (58:00.0) — so when it is re-added, name its device
+  # deliberately rather than assuming slot 1 keeps enumerating first. The system
+  # disk is separately protected: talos/patches/k8s-1.yaml pins the installer to
+  # it by serial, and Rook skips a disk carrying partitions.
   storage:
-    useAllNodes: true
+    useAllNodes: false
     useAllDevices: false
-    devices:
-      # On k8s-2/k8s-3 this is the NVMe drive PCI-passed-through from saturn
-      # and neptune, and it is the only block device the guest can see. k8s-1
-      # is bare metal on jupiter and sees two: the OSD drive in MS-01 slot 1
-      # (PCI 01:00.0) and the Talos system disk in slot 2 (58:00.0). Slot 1
-      # enumerates first, so nvme0n1 is still the OSD drive there — but that
-      # is enumeration order, not a guarantee. The system disk is protected
-      # from the other direction instead: talos/patches/k8s-1.yaml pins the
-      # installer to it by serial, and Rook skips a disk carrying partitions.
-      - name: nvme0n1
+    nodes:
+      - name: k8s-2
+        devices:
+          - name: nvme0n1
+      - name: k8s-3
+        devices:
+          - name: nvme0n1
     config:
       osdsPerDevice: "1"
   # Storage layer must never be evicted by kubelet under node pressure