kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 # come from the kube-prometheus-stack chart. Flux applies this Kustomization
 # only after `monitoring` is Ready (chart installed → CRDs present).
 resources:
+  - prometheusrule-ceph.yaml
   - prometheusrule-cilium.yaml
   - prometheusrule-hetzner-backup.yaml
   - prometheusrule-loki.yaml

kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -0,0 +1,53 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ceph
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: ceph.rules
+      rules:
+        - alert: CephHealthWarn
+          # ceph_health_status: 0 = HEALTH_OK, 1 = HEALTH_WARN, 2 = HEALTH_ERR.
+          # 6h is deliberately long. Ceph warns for self-clearing conditions —
+          # a slow BlueStore op during the Sunday restore sweep, a mon briefly
+          # below its disk threshold — and paging on those trains people to
+          # ignore the alert. What matters is a warning that does not clear.
+          expr: ceph_health_status == 1
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ceph has been HEALTH_WARN for 6h"
+            description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+
+        - alert: CephHealthError
+          # HEALTH_ERR means data availability or redundancy is at risk.
+          expr: ceph_health_status == 2
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Ceph is HEALTH_ERR"
+            description: "The Ceph cluster is in HEALTH_ERR — availability or redundancy is at risk. Inspect immediately with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` and `ceph osd df tree`."
+
+        - alert: CephMonDiskLow
+          # Named explicitly rather than left to CephHealthWarn, because the
+          # mons sit on the node's /var alongside containerd's image cache:
+          # this fires for a full image cache, not for Ceph growth. Cleared by
+          # kubelet image GC (imageGCHighThresholdPercent in
+          # kubernetes/talos/patches/cluster.yaml), which must stay below
+          # Ceph's mon_data_avail_warn of 30% free.
+          expr: |
+            100 * node_filesystem_avail_bytes{mountpoint="/var"}
+              / node_filesystem_size_bytes{mountpoint="/var"} < 30
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.instance }} /var below 30% free — Ceph mon will warn"
+            description: "/var on {{ $labels.instance }} is under 30% free, which is Ceph's mon_data_avail_warn threshold, and the mon on this node reports MON_DISK_LOW below it. Usually containerd's image cache; check whether kubelet image GC is reclaiming with `talosctl -n <node> usage /var/lib --depth 1`."

kubernetes/infrastructure/rook-ceph-cluster/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cephcluster.yaml
+  - toolbox.yaml
   - storageclass.yaml
   - volumesnapshotclass.yaml
   - objectstore.yaml

kubernetes/infrastructure/rook-ceph-cluster/toolbox.yaml

@@ -0,0 +1,172 @@
+# Upstream Rook toolbox, taken verbatim from
+# https://raw.githubusercontent.com/rook/rook/v1.20.4/deploy/examples/toolbox.yaml
+# (matching the operator image docker.io/rook/ceph:v1.20.4), with three
+# deliberate changes:
+#   - the image is pinned to the cluster's own Ceph version and carries the
+#     same imagepolicy marker as cephcluster.yaml, so the client never drifts
+#     ahead of the mons it talks to
+#   - resource requests/limits, since this is a long-lived idle pod
+#   - no other divergence; keep it that way so upstream diffs stay readable
+#
+# This is the only admin path to Ceph internals. Without it `ceph status`,
+# `ceph health detail`, `ceph osd df tree`, `ceph osd perf` and `ceph device
+# ls` are unreachable during an incident: the mons, OSDs and mgr mount no
+# admin keyring, and the operator's /etc/ceph is empty.
+#
+# It mounts the client.admin keyring from the rook-ceph-mon secret, so it is
+# a full-privilege credential holder, not a read-only one — Rook offers no
+# lesser-privileged toolbox. Ingress to it is gated by the namespace
+# default-deny-ingress CNP (network-policies/rook-ceph-ns.yaml).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rook-ceph-tools
+  namespace: rook-ceph
+  labels:
+    app: rook-ceph-tools
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rook-ceph-tools
+  template:
+    metadata:
+      labels:
+        app: rook-ceph-tools
+    spec:
+      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccountName: rook-ceph-default
+      containers:
+        - name: rook-ceph-tools
+          image: quay.io/ceph/ceph:v19.2.5 # {"$imagepolicy": "flux-system:ceph"}
+          command:
+            - /bin/bash
+            - -c
+            - |
+              # Replicate the script from toolbox.sh inline so the ceph image
+              # can be run directly, instead of requiring the rook toolbox
+              CEPH_CONFIG="/etc/ceph/ceph.conf"
+              MON_CONFIG="/etc/rook/mon-endpoints"
+              KEYRING_FILE="/etc/ceph/keyring"
+              CONFIG_OVERRIDE="/etc/rook-config-override/config"
+
+              # create a ceph config file in its default location so ceph/rados tools can be used
+              # without specifying any arguments
+              write_endpoints() {
+                endpoints=$(cat ${MON_CONFIG})
+
+                # filter out the mon names
+                # external cluster can have numbers or hyphens in mon names, handling them in regex
+                # shellcheck disable=SC2001
+                mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
+
+                DATE=$(date)
+                echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
+                  cat <<EOF > ${CEPH_CONFIG}
+              [global]
+              mon_host = ${mon_endpoints}
+
+              [client.admin]
+              keyring = ${KEYRING_FILE}
+              EOF
+
+                # Merge the config override if it exists and is not empty
+                if [ -f "${CONFIG_OVERRIDE}" ] && [ -s "${CONFIG_OVERRIDE}" ]; then
+                  echo "$DATE merging config override from ${CONFIG_OVERRIDE}"
+                  echo "" >> ${CEPH_CONFIG}
+                  cat ${CONFIG_OVERRIDE} >> ${CEPH_CONFIG}
+                  echo "" >> ${CEPH_CONFIG}
+                fi
+              }
+
+              # watch the endpoints config file and update if the mon endpoints ever change
+              watch_endpoints() {
+                # get the timestamp for the target of the soft link
+                real_path=$(realpath ${MON_CONFIG})
+                initial_time=$(stat -c %Z "${real_path}")
+                while true; do
+                  real_path=$(realpath ${MON_CONFIG})
+                  latest_time=$(stat -c %Z "${real_path}")
+
+                  if [[ "${latest_time}" != "${initial_time}" ]]; then
+                    write_endpoints
+                    initial_time=${latest_time}
+                  fi
+
+                  sleep 10
+                done
+              }
+
+              # read the secret from an env var (for backward compatibility), or from the secret file
+              ceph_secret=${ROOK_CEPH_SECRET}
+              if [[ "$ceph_secret" == "" ]]; then
+                ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring)
+              fi
+
+              # create the keyring file
+              cat <<EOF > ${KEYRING_FILE}
+              [${ROOK_CEPH_USERNAME}]
+              key = ${ceph_secret}
+              EOF
+
+              # write the initial config file
+              write_endpoints
+
+              # continuously update the mon endpoints if they fail over
+              watch_endpoints
+          imagePullPolicy: IfNotPresent
+          tty: true
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2016
+            runAsGroup: 2016
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: ROOK_CEPH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: rook-ceph-mon
+                  key: ceph-username
+          resources:
+            requests:
+              cpu: 10m
+              memory: 128Mi
+            limits:
+              memory: 512Mi
+          volumeMounts:
+            - mountPath: /etc/ceph
+              name: ceph-config
+            - name: mon-endpoint-volume
+              mountPath: /etc/rook
+            - name: ceph-admin-secret
+              mountPath: /var/lib/rook-ceph-mon
+              readOnly: true
+            - name: rook-config-override
+              mountPath: /etc/rook-config-override
+              readOnly: true
+      volumes:
+        - name: ceph-admin-secret
+          secret:
+            secretName: rook-ceph-mon
+            optional: false
+            items:
+              - key: ceph-secret
+                path: secret.keyring
+        - name: mon-endpoint-volume
+          configMap:
+            name: rook-ceph-mon-endpoints
+            items:
+              - key: data
+                path: mon-endpoints
+        - name: rook-config-override
+          configMap:
+            name: rook-config-override
+            optional: true
+        - name: ceph-config
+          emptyDir: {}
+      tolerations:
+        - key: "node.kubernetes.io/unreachable"
+          operator: "Exists"
+          effect: "NoExecute"
+          tolerationSeconds: 5

kubernetes/talos/patches/cluster.yaml

@@ -26,19 +26,23 @@ machine:
     nodeIP:
       validSubnets:
         - 10.2.40.0/24
-    # Lower the image GC thresholds from the kubelet defaults (85/80)
-    # so containerd's image cache is pruned before /var crosses Ceph's
-    # mon_data_avail_warn line (mon DB warns when its hostpath has
-    # <30% free). k8s-2 and k8s-3 sit at 77-79% used because of pod
-    # density (75 pods on k8s-2 vs 42 on k8s-1) — unused image layers
-    # accumulate untouched until the 85% high watermark, which the
-    # node never hits, so GC never fires. 70/60 keeps /var safely
-    # under 30%-free without raising the VM disk again. Spegel P2P
-    # pull-through softens the re-pull cost when GC'd images are
-    # needed back.
+    # Lower the image GC thresholds from the kubelet defaults (85/80) so
+    # containerd's image cache is pruned before /var crosses Ceph's
+    # mon_data_avail_warn line — the mon DB warns when its hostpath has
+    # <=30% free, and mons live on /var/lib/rook (hostPath, no PVC).
+    # High must stay strictly below that line, not on it: a high of 70
+    # means GC first fires at exactly 30% free, which is the warn point
+    # itself, so every cache fill-up trips MON_DISK_LOW before kubelet
+    # reclaims anything. 60/50 fires at 40% free and reclaims to 50%,
+    # keeping 10 points of margin.
+    # containerd is the dominant local consumer of /var (~29 GB of 60 on
+    # k8s-2, which carries 75 pods against 42 on k8s-1) and most of it is
+    # reclaimable — 179 images cached against 60 in use. Spegel P2P
+    # pull-through softens the re-pull cost when GC'd images are needed
+    # back.
     extraConfig:
-      imageGCHighThresholdPercent: 70
-      imageGCLowThresholdPercent: 60
+      imageGCHighThresholdPercent: 60
+      imageGCLowThresholdPercent: 50
 cluster:
   controlPlane:
     endpoint: https://k8s.renner.uno:6443