diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index 77c38b74..d0f39b1e 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization # come from the kube-prometheus-stack chart. Flux applies this Kustomization # only after `monitoring` is Ready (chart installed → CRDs present). resources: + - prometheusrule-ceph.yaml - prometheusrule-cilium.yaml - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml new file mode 100644 index 00000000..0f3296a4 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml @@ -0,0 +1,53 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: ceph + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus resource selects on this label; + # without it the rule is created but never loaded into Prometheus. + release: kube-prometheus-stack +spec: + groups: + - name: ceph.rules + rules: + - alert: CephHealthWarn + # ceph_health_status: 0 = HEALTH_OK, 1 = HEALTH_WARN, 2 = HEALTH_ERR. + # 6h is deliberately long. Ceph warns for self-clearing conditions — + # a slow BlueStore op during the Sunday restore sweep, a mon briefly + # below its disk threshold — and paging on those trains people to + # ignore the alert. What matters is a warning that does not clear. + expr: ceph_health_status == 1 + for: 6h + labels: + severity: warning + annotations: + summary: "Ceph has been HEALTH_WARN for 6h" + description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`." + + - alert: CephHealthError + # HEALTH_ERR means data availability or redundancy is at risk. + expr: ceph_health_status == 2 + for: 5m + labels: + severity: critical + annotations: + summary: "Ceph is HEALTH_ERR" + description: "The Ceph cluster is in HEALTH_ERR — availability or redundancy is at risk. Inspect immediately with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` and `ceph osd df tree`." + + - alert: CephMonDiskLow + # Named explicitly rather than left to CephHealthWarn, because the + # mons sit on the node's /var alongside containerd's image cache: + # this fires for a full image cache, not for Ceph growth. Cleared by + # kubelet image GC (imageGCHighThresholdPercent in + # kubernetes/talos/patches/cluster.yaml), which must stay below + # Ceph's mon_data_avail_warn of 30% free. + expr: | + 100 * node_filesystem_avail_bytes{mountpoint="/var"} + / node_filesystem_size_bytes{mountpoint="/var"} < 30 + for: 30m + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} /var below 30% free — Ceph mon will warn" + description: "/var on {{ $labels.instance }} is under 30% free, which is Ceph's mon_data_avail_warn threshold, and the mon on this node reports MON_DISK_LOW below it. Usually containerd's image cache; check whether kubelet image GC is reclaiming with `talosctl -n usage /var/lib --depth 1`." diff --git a/kubernetes/infrastructure/rook-ceph-cluster/kustomization.yaml b/kubernetes/infrastructure/rook-ceph-cluster/kustomization.yaml index 64fbc38a..9b797189 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/kustomization.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cephcluster.yaml + - toolbox.yaml - storageclass.yaml - volumesnapshotclass.yaml - objectstore.yaml diff --git a/kubernetes/infrastructure/rook-ceph-cluster/toolbox.yaml b/kubernetes/infrastructure/rook-ceph-cluster/toolbox.yaml new file mode 100644 index 00000000..60a1d5a3 --- /dev/null +++ b/kubernetes/infrastructure/rook-ceph-cluster/toolbox.yaml @@ -0,0 +1,172 @@ +# Upstream Rook toolbox, taken verbatim from +# https://raw.githubusercontent.com/rook/rook/v1.20.4/deploy/examples/toolbox.yaml +# (matching the operator image docker.io/rook/ceph:v1.20.4), with three +# deliberate changes: +# - the image is pinned to the cluster's own Ceph version and carries the +# same imagepolicy marker as cephcluster.yaml, so the client never drifts +# ahead of the mons it talks to +# - resource requests/limits, since this is a long-lived idle pod +# - no other divergence; keep it that way so upstream diffs stay readable +# +# This is the only admin path to Ceph internals. Without it `ceph status`, +# `ceph health detail`, `ceph osd df tree`, `ceph osd perf` and `ceph device +# ls` are unreachable during an incident: the mons, OSDs and mgr mount no +# admin keyring, and the operator's /etc/ceph is empty. +# +# It mounts the client.admin keyring from the rook-ceph-mon secret, so it is +# a full-privilege credential holder, not a read-only one — Rook offers no +# lesser-privileged toolbox. Ingress to it is gated by the namespace +# default-deny-ingress CNP (network-policies/rook-ceph-ns.yaml). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rook-ceph-tools + namespace: rook-ceph + labels: + app: rook-ceph-tools +spec: + replicas: 1 + selector: + matchLabels: + app: rook-ceph-tools + template: + metadata: + labels: + app: rook-ceph-tools + spec: + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: rook-ceph-default + containers: + - name: rook-ceph-tools + image: quay.io/ceph/ceph:v19.2.5 # {"$imagepolicy": "flux-system:ceph"} + command: + - /bin/bash + - -c + - | + # Replicate the script from toolbox.sh inline so the ceph image + # can be run directly, instead of requiring the rook toolbox + CEPH_CONFIG="/etc/ceph/ceph.conf" + MON_CONFIG="/etc/rook/mon-endpoints" + KEYRING_FILE="/etc/ceph/keyring" + CONFIG_OVERRIDE="/etc/rook-config-override/config" + + # create a ceph config file in its default location so ceph/rados tools can be used + # without specifying any arguments + write_endpoints() { + endpoints=$(cat ${MON_CONFIG}) + + # filter out the mon names + # external cluster can have numbers or hyphens in mon names, handling them in regex + # shellcheck disable=SC2001 + mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g') + + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" + cat < ${CEPH_CONFIG} + [global] + mon_host = ${mon_endpoints} + + [client.admin] + keyring = ${KEYRING_FILE} + EOF + + # Merge the config override if it exists and is not empty + if [ -f "${CONFIG_OVERRIDE}" ] && [ -s "${CONFIG_OVERRIDE}" ]; then + echo "$DATE merging config override from ${CONFIG_OVERRIDE}" + echo "" >> ${CEPH_CONFIG} + cat ${CONFIG_OVERRIDE} >> ${CEPH_CONFIG} + echo "" >> ${CEPH_CONFIG} + fi + } + + # watch the endpoints config file and update if the mon endpoints ever change + watch_endpoints() { + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z "${real_path}") + while true; do + real_path=$(realpath ${MON_CONFIG}) + latest_time=$(stat -c %Z "${real_path}") + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + + sleep 10 + done + } + + # read the secret from an env var (for backward compatibility), or from the secret file + ceph_secret=${ROOK_CEPH_SECRET} + if [[ "$ceph_secret" == "" ]]; then + ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring) + fi + + # create the keyring file + cat < ${KEYRING_FILE} + [${ROOK_CEPH_USERNAME}] + key = ${ceph_secret} + EOF + + # write the initial config file + write_endpoints + + # continuously update the mon endpoints if they fail over + watch_endpoints + imagePullPolicy: IfNotPresent + tty: true + securityContext: + runAsNonRoot: true + runAsUser: 2016 + runAsGroup: 2016 + capabilities: + drop: ["ALL"] + env: + - name: ROOK_CEPH_USERNAME + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-username + resources: + requests: + cpu: 10m + memory: 128Mi + limits: + memory: 512Mi + volumeMounts: + - mountPath: /etc/ceph + name: ceph-config + - name: mon-endpoint-volume + mountPath: /etc/rook + - name: ceph-admin-secret + mountPath: /var/lib/rook-ceph-mon + readOnly: true + - name: rook-config-override + mountPath: /etc/rook-config-override + readOnly: true + volumes: + - name: ceph-admin-secret + secret: + secretName: rook-ceph-mon + optional: false + items: + - key: ceph-secret + path: secret.keyring + - name: mon-endpoint-volume + configMap: + name: rook-ceph-mon-endpoints + items: + - key: data + path: mon-endpoints + - name: rook-config-override + configMap: + name: rook-config-override + optional: true + - name: ceph-config + emptyDir: {} + tolerations: + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 diff --git a/kubernetes/talos/patches/cluster.yaml b/kubernetes/talos/patches/cluster.yaml index 0acd9b79..f606e0f7 100644 --- a/kubernetes/talos/patches/cluster.yaml +++ b/kubernetes/talos/patches/cluster.yaml @@ -26,19 +26,23 @@ machine: nodeIP: validSubnets: - 10.2.40.0/24 - # Lower the image GC thresholds from the kubelet defaults (85/80) - # so containerd's image cache is pruned before /var crosses Ceph's - # mon_data_avail_warn line (mon DB warns when its hostpath has - # <30% free). k8s-2 and k8s-3 sit at 77-79% used because of pod - # density (75 pods on k8s-2 vs 42 on k8s-1) — unused image layers - # accumulate untouched until the 85% high watermark, which the - # node never hits, so GC never fires. 70/60 keeps /var safely - # under 30%-free without raising the VM disk again. Spegel P2P - # pull-through softens the re-pull cost when GC'd images are - # needed back. + # Lower the image GC thresholds from the kubelet defaults (85/80) so + # containerd's image cache is pruned before /var crosses Ceph's + # mon_data_avail_warn line — the mon DB warns when its hostpath has + # <=30% free, and mons live on /var/lib/rook (hostPath, no PVC). + # High must stay strictly below that line, not on it: a high of 70 + # means GC first fires at exactly 30% free, which is the warn point + # itself, so every cache fill-up trips MON_DISK_LOW before kubelet + # reclaims anything. 60/50 fires at 40% free and reclaims to 50%, + # keeping 10 points of margin. + # containerd is the dominant local consumer of /var (~29 GB of 60 on + # k8s-2, which carries 75 pods against 42 on k8s-1) and most of it is + # reclaimable — 179 images cached against 60 in use. Spegel P2P + # pull-through softens the re-pull cost when GC'd images are needed + # back. extraConfig: - imageGCHighThresholdPercent: 70 - imageGCLowThresholdPercent: 60 + imageGCHighThresholdPercent: 60 + imageGCLowThresholdPercent: 50 cluster: controlPlane: endpoint: https://k8s.renner.uno:6443