From 1fcda644c064fb63c654afdea45fb11ba3b89eb4 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Fri, 12 Jun 2026 23:22:20 +0200 Subject: [PATCH] talos: lower kubelet image GC thresholds to 70/60 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CP VMs are seeing Ceph `MON_DISK_LOW` on mons a,b because their shared /var (xfs on /dev/vda5) sits at 77-79% used — past Ceph's default `mon_data_avail_warn` line of <30% free. Mon DB itself is 118 MiB; the disk pressure is pod density (k8s-2 hosts 75 pods vs k8s-1's 42), driving containerd image cache and /var/log/pods. Kubelet's default image GC thresholds (85% high / 80% low) sit above where we land, so GC never fires and unused image layers accumulate indefinitely. Lowering to 70/60 forces eviction before /var crosses the mon warn line, with Spegel's P2P pull-through cache softening the re-pull cost on the rare case a GC'd image is needed back. Requires a node reboot to take effect (Talos `extraConfig` change). Roll one CP at a time with `kubectl drain` + Ceph `HEALTH_OK` wait between nodes, per the standing drain/Ceph-wait discipline. Co-Authored-By: Claude Opus 4.7 (1M context) --- kubernetes/talos/patches/cluster.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kubernetes/talos/patches/cluster.yaml b/kubernetes/talos/patches/cluster.yaml index 95fe88c0..70843132 100644 --- a/kubernetes/talos/patches/cluster.yaml +++ b/kubernetes/talos/patches/cluster.yaml @@ -26,6 +26,19 @@ machine: nodeIP: validSubnets: - 10.2.40.0/24 + # Lower the image GC thresholds from the kubelet defaults (85/80) + # so containerd's image cache is pruned before /var crosses Ceph's + # mon_data_avail_warn line (mon DB warns when its hostpath has + # <30% free). k8s-2 and k8s-3 sit at 77-79% used because of pod + # density (75 pods on k8s-2 vs 42 on k8s-1) — unused image layers + # accumulate untouched until the 85% high watermark, which the + # node never hits, so GC never fires. 70/60 keeps /var safely + # under 30%-free without raising the VM disk again. Spegel P2P + # pull-through softens the re-pull cost when GC'd images are + # needed back. + extraConfig: + imageGCHighThresholdPercent: 70 + imageGCLowThresholdPercent: 60 cluster: controlPlane: endpoint: https://k8s.renner.uno:6443 -- 2.54.0