fix: [alert] KubeContainerWaiting (home-assistant) (#592)

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -77,6 +77,21 @@ spec:
       # redirects the socket directly to the backend pod instead of sending
       # traffic out to the VIP and back (which fails with L2 announcements).
       lbExternalClusterIP: true
+      # VLAN ids whose tagged frames are passed to the stack instead of being
+      # dropped by Cilium's VLAN filter. Unset, the filter drops every tagged
+      # frame the datapath sees — which is what strands the v30iot macvlan
+      # pods on a converted node: there `ens19` (VLAN 30) and `mgmt0`
+      # (VLAN 40) are VLAN sub-interfaces of the enp2s0f0np0 trunk, so a
+      # macvlan child's DHCP DISCOVER leaves tagged and dies as
+      # DROP_VLAN_FILTER. The CNI DHCP daemon reports that as "no buffer
+      # space available" (a tc drop surfaces to an AF_PACKET sender as
+      # ENOBUFS, not as a timeout) and the pod sits in ContainerCreating
+      # forever — issue #592, predicted in RUNBOOK-baremetal-conversion.md.
+      # Lists only the two VLANs a node terminates; [0] would bypass the
+      # filter for every id. Takes effect on agent restart.
+      # Keep in lockstep with vlan-bpf-bypass in
+      # kubernetes/talos/patches/cilium-cni.yaml and kubernetes/talos/cilium.yaml.
+      vlanBypass: [30, 40]
     cni:
       # Multus (kubernetes/infrastructure/multus) runs alongside Cilium and
       # writes 00-multus.conf to /etc/cni/net.d so kubelet picks multus

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -167,44 +167,37 @@ spec:
   # impossible: the OSD is recreated within a reconcile and starts backfilling
   # onto the drive being replaced.
   #
-  # All three nodes are bare metal and see three NVMe drives each, so they name
-  # their OSDs by /dev/disk/by-id path. A bare name would be actively wrong:
-  # Talos enumerates these machines' drives in a different order than Proxmox
-  # did, and the kernel name lands on the Kingston SYSTEM disk or on EPHEMERAL
-  # depending on the boot. Rook would probably decline the system disk for
-  # carrying partitions, but that is luck rather than intent. The by-id path is
-  # derived from model and serial and does not depend on enumeration at all.
-  #
-  # k8s-3 demonstrated why: it enumerated the Kingston as nvme0n1 on one boot
-  # and the Crucial as nvme0n1 on the very next one.
+  # k8s-1 and k8s-2 are bare metal and see three NVMe drives each, so they name
+  # their OSDs by /dev/disk/by-id path instead. A bare name would be actively
+  # wrong: Talos enumerates these machines' drives in a different order than
+  # Proxmox did, and nvme0n1 there is the Kingston SYSTEM disk. Rook would
+  # probably decline it for carrying partitions, but that is luck rather than
+  # intent. The by-id path is derived from model and serial and does not depend
+  # on enumeration at all.
   #
   #   k8s-1 (jupiter)
-  #   50026B7382B48B86  Kingston  — Talos system disk
-  #   2332E86B749A      Crucial   — Talos EPHEMERAL (/var)
-  #   S437NX0M400426    PM983     — this OSD
+  #   nvme0n1  50026B7382B48B86  Kingston  — Talos system disk
+  #   nvme1n1  2332E86B749A      Crucial   — Talos EPHEMERAL (/var)
+  #   nvme2n1  S437NX0M400426    PM983     — this OSD
   #
   #   k8s-2 (saturn)
-  #   50026B7382DA5F6E  Kingston  — Talos system disk
-  #   2332E86B74BC      Crucial   — Talos EPHEMERAL (/var)
-  #   S437NX0M403161    PM983     — this OSD
-  #
-  #   k8s-3 (neptune)
-  #   50026B7382B48B90  Kingston  — Talos system disk
-  #   2326E6E8E67D      Crucial   — Talos EPHEMERAL (/var)
-  #   S437NF0M501176    PM983     — this OSD
+  #   nvme0n1  50026B7382DA5F6E  Kingston  — Talos system disk
+  #   nvme1n1  2332E86B74BC      Crucial   — Talos EPHEMERAL (/var)
+  #   nvme2n1  S437NX0M403161    PM983     — this OSD
   storage:
     useAllNodes: false
     useAllDevices: false
     nodes:
+      # k8s-3 is absent while neptune's OSD drive is swapped and the node is
+      # converted to bare metal. RE-ADD IT once the PM983 is in and the node has
+      # rejoined, naming the device by /dev/disk/by-id as the other two do —
+      # bare metal sees three NVMe drives and the bare name is not the OSD.
       - name: k8s-1
         devices:
           - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M400426
       - name: k8s-2
         devices:
           - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M403161
-      - name: k8s-3
-        devices:
-          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NF0M501176
     config:
       osdsPerDevice: "1"
   # Storage layer must never be evicted by kubelet under node pressure

kubernetes/infrastructure/rook-ceph-cluster/storageclass.yaml

@@ -5,24 +5,31 @@ metadata:
   namespace: rook-ceph
 spec:
   failureDomain: host
-  # size 3. This pool sat at 2 while the OSDs were Crucial P3 QLC drives: with
-  # 3 OSDs on 3 hosts, size=3 put every write on every drive, so per-drive write
-  # volume equalled total cluster write volume and all three wore at an
-  # identical rate (measured 2026-08-19: 80/82/87% endurance used, ~0.13
-  # points/day, all three projected to exhaust between Nov 2026 and Jan 2027 —
-  # see #434). Replication does not protect against that, because the wear was
-  # correlated by construction rather than independent.
+  # size 2, NOT 3. With 3 OSDs on 3 hosts, size=3 meant every write landed on
+  # every drive in the cluster: per-drive write volume equalled total cluster
+  # write volume, and all three drives wore at an identical rate and would have
+  # reached end-of-rated-life together (measured 2026-08-19: 80/82/87% endurance
+  # used, ~0.13 points/day, all three exhausted between Nov 2026 and Jan 2027 —
+  # see #434). Replication does not protect against that failure mode, because
+  # wear-out here is correlated by construction rather than independent.
   #
-  # The OSDs are Samsung PM983 enterprise drives now, at 2-3% of rated
-  # endurance with a far higher write budget, so wear-out is no longer the
-  # binding constraint and the correlation argument no longer applies.
+  # size 2 cuts per-drive write volume by a third and desynchronises the wear.
   #
-  # min_size is left at the size-3 default of 2. The explicit min_size 1
-  # override existed only because Ceph does not lower min_size when size is
-  # reduced, which would have left the pool 2/2 — where any single OSD down
+  # min_size 1 is REQUIRED and deliberate. Ceph does not lower min_size when
+  # size is reduced — it was 2 here, and leaving it would make the pool 2/2,
+  # where any single OSD down (every rolling node reboot, every OSD restart)
   # blocks IO for its PGs and hangs every RBD volume in the cluster.
+  #
+  # The trade this accepts: while the pool is degraded (one OSD down), a second
+  # drive failing loses data, and with all three drives at 80%+ endurance that
+  # window is not hypothetical. Mitigations that already exist: CNPG barman and
+  # Velero both ship offsite to Hetzner (opentofu/object_storage.tf), so the
+  # exposure is the RPO of those backups, not total loss. Keep node reboots
+  # serialised and wait for HEALTH_OK between them.
   replicated:
-    size: 3
+    size: 2
+  parameters:
+    min_size: "1"
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
@@ -80,26 +87,39 @@ metadata:
   name: cephfs
   namespace: rook-ceph
 spec:
-  # size 3 with no min_size override. This pool holds ~253 MiB against the data
-  # pool's 280 GiB, and losing it would destroy the whole filesystem rather
-  # than just the objects on the lost OSD, so it is the last pool that should
-  # ever run thin.
+  # TEMPORARY — RESTORE TO size 3 / no min_size OVERRIDE ONCE SATURN AND
+  # NEPTUNE HAVE BOTH BEEN CONVERTED TO BARE METAL. It stays at 2/1 across all
+  # three conversions rather than being toggled per node: each change
+  # re-replicates the pool, and the window reopens with the next node anyway.
   #
-  # It drops to 2/1 whenever one of the three hosts is out for longer than a
-  # reboot. size 3 on two remaining hosts leaves every PG undersized at two
-  # copies with min_size 2, so a single further OSD loss falls below min_size
-  # and blocks IO on every CephFS mount cluster-wide. At 2/1 the same two
-  # copies are active+clean instead, and a further loss degrades rather than
-  # blocks. See RUNBOOK-baremetal-conversion.md.
+  # The standing trade for this pool is size 3: it holds ~253 MiB against the
+  # data pool's 280 GiB, so it contributes nothing measurable to drive wear —
+  # the entire reason data0 dropped to 2 — while losing it would destroy the
+  # whole filesystem, not just the objects on the lost OSD. Paying 3x on a
+  # quarter of a gigabyte to keep that intact is the right trade in steady
+  # state.
+  #
+  # It does not hold while one of three hosts is out. size 3 on two remaining
+  # hosts leaves every PG undersized at two copies with min_size 2, so a single
+  # further OSD loss drops the pool below min_size and blocks IO on every
+  # CephFS mount cluster-wide. At 2/1 the same two copies are active+clean
+  # instead, and a further loss degrades rather than blocks — availability
+  # bought at the cost of accepting writes onto a single copy while degraded.
+  #
+  # See RUNBOOK-baremetal-conversion.md.
   metadataPool:
     replicated:
-      size: 3
+      size: 2
+    parameters:
+      min_size: "1"
   dataPools:
     - name: data0
-      # size 3 for the same reasons as replicapool above — see the comment
-      # there for why this pool sat at 2 while the Crucials backed the OSDs.
+      # size 2 / min_size 1 for the same reasons as replicapool above — see
+      # the comment there for the full rationale and the accepted trade.
       replicated:
-        size: 3
+        size: 2
+      parameters:
+        min_size: "1"
   metadataServer:
     activeCount: 1
     activeStandby: true

kubernetes/talos/cilium.yaml

@@ -306,6 +306,12 @@ data:
   # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic
   # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
   bpf-map-dynamic-size-ratio: "0.0025"
+  # VLAN ids whose tagged frames bypass the eBPF datapath rather than being
+  # dropped by the VLAN filter. 30 (v30iot, the macvlan parent) and 40 (mgmt)
+  # are VLAN sub-interfaces of the trunk on a bare-metal node. Keep in lockstep
+  # with patches/cilium-cni.yaml and values.bpf.vlanBypass in
+  # kubernetes/infrastructure/cilium/helmrelease.yaml.
+  vlan-bpf-bypass: "30,40"
   # bpf-policy-map-max specifies the maximum number of entries in endpoint
   # policy map (per endpoint). Keep in lockstep with patches/cilium-cni.yaml and
   # values.bpf.policyMapMax in kubernetes/infrastructure/cilium/helmrelease.yaml.

kubernetes/talos/patches/cilium-cni.yaml

@@ -324,6 +324,13 @@ cluster:
           # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic
           # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
           bpf-map-dynamic-size-ratio: "0.0025"
+          # VLAN ids whose tagged frames bypass the eBPF datapath rather than being
+          # dropped by the VLAN filter. 30 (v30iot, the macvlan parent) and 40 (mgmt)
+          # are VLAN sub-interfaces of the trunk on a bare-metal node, so without this
+          # a cold rebuild comes up with the v30iot macvlan pods unable to DHCP. Keep
+          # in lockstep with kubernetes/talos/cilium.yaml and values.bpf.vlanBypass in
+          # kubernetes/infrastructure/cilium/helmrelease.yaml.
+          vlan-bpf-bypass: "30,40"
           # bpf-policy-map-max specifies the maximum number of entries in endpoint
           # policy map (per endpoint). Keep in lockstep with values.bpf.policyMapMax
           # in kubernetes/infrastructure/cilium/helmrelease.yaml, and with the third

kubernetes/talos/patches/k8s-2.yaml

@@ -1,6 +1,7 @@
 # k8s-2 runs Talos directly on saturn's MS-01 hardware. Structurally identical
 # to k8s-1.yaml — see that file for the reasoning behind each choice; only the
-# serials, MACs and addresses differ.
+# serials, MACs and addresses differ. k8s-3 is still a Proxmox VM and keeps the
+# VM-era layout until neptune is converted.
 machine:
   network:
     hostname: k8s-2

kubernetes/talos/patches/k8s-3.yaml

@@ -1,82 +1,31 @@
-# k8s-3 runs Talos directly on neptune's MS-01 hardware. Structurally identical
-# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
-# serials and addresses differ. neptune was the last node converted, so all
-# three control-plane nodes now share this layout and no Proxmox remains.
 machine:
   network:
     hostname: k8s-3
+    # Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
+    interfaces:
+      - deviceSelector:
+          busPath: "0000:06:12.0"
+        dhcp: true
+        vip:
+          ip: 10.2.40.60
+      # v30iot: link only, no host addressing — see k8s-1.yaml.
+      - deviceSelector:
+          busPath: "0000:06:13.0"
+        dhcp: false
+      # See k8s-1.yaml for context on the dedicated Ceph cluster network.
+      - deviceSelector:
+          driver: i40e
+        addresses:
+          - 10.99.0.3/29
+        mtu: 9000
   sysctls:
     # v30iot NIC carries no host addressing — see k8s-1.yaml.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    # Three NVMe drives are visible on bare metal. Select the installer target
-    # by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
-    # order the kernel enumerates them in — Talos and Proxmox do not agree on
-    # that order.
-    diskSelector:
-      serial: 50026B7382B48B90
+    disk: /dev/vda
   certSANs:
     - k8s-3.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: neptune
 cluster:
   allowSchedulingOnControlPlanes: true
----
-# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
-# drive is at 83% of its rated endurance and is finished as an OSD, but
-# everything EPHEMERAL holds is reconstructible, so it is consumable in this
-# role. Available spare is still 100% and it reports no media errors.
-#
-# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
-# solely when the volume does not yet exist; applied later it is silently
-# ignored and relocating needs a disk wipe and a boot from install media.
-#
-# Matched by serial deliberately: a broad selector such as
-# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
-apiVersion: v1alpha1
-kind: VolumeConfig
-name: EPHEMERAL
-provisioning:
-  diskSelector:
-    match: disk.serial == '2326E6E8E67D'
-  minSize: 50GiB
-  grow: true
----
-apiVersion: v1alpha1
-kind: LinkConfig
-name: enp2s0f0np0
-up: true
----
-apiVersion: v1alpha1
-kind: VLANConfig
-name: mgmt0
-vlanID: 40
-parent: enp2s0f0np0
-up: true
----
-apiVersion: v1alpha1
-kind: DHCPv4Config
-name: mgmt0
----
-# Shared control-plane VIP, held by whichever node wins the etcd campaign.
-apiVersion: v1alpha1
-kind: Layer2VIPConfig
-name: 10.2.40.60
-link: mgmt0
----
-# v30iot: link only, no host addressing — see k8s-1.yaml.
-apiVersion: v1alpha1
-kind: VLANConfig
-name: ens19
-vlanID: 30
-parent: enp2s0f0np0
-up: true
----
-# Dedicated Ceph cluster network — see k8s-1.yaml.
-apiVersion: v1alpha1
-kind: LinkConfig
-name: enp2s0f1np1
-up: true
-mtu: 9000
-addresses:
-  - address: 10.99.0.3/29