diff --git a/kubernetes/apps/home-assistant/bgp-service.yaml b/kubernetes/apps/home-assistant/bgp-service.yaml index 54bc86ed..8a847f38 100644 --- a/kubernetes/apps/home-assistant/bgp-service.yaml +++ b/kubernetes/apps/home-assistant/bgp-service.yaml @@ -5,8 +5,9 @@ # directly on that node — no L2 lease holder, no inter-VLAN ARP race. # # Use this URL for any LAN client that can resolve DNS, including -# ESPHome devices fetching TTS audio. The legacy `home-assistant-iot` -# Service at 10.2.30.80 stays for devices with hard-coded IPs. +# ESPHome devices fetching TTS audio. (The legacy `home-assistant-iot` +# Service at 10.2.30.80 was retired on 2026-05-20; no LoadBalancer +# Service holds a 10.2.30.x address any more. Verified 2026-07-09.) apiVersion: v1 kind: Service metadata: diff --git a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml b/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml index 7ce8e9ee..5bc1d9e8 100644 --- a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml +++ b/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml @@ -7,8 +7,19 @@ # resolver browse `_shelly._tcp.local.` / `_http._tcp.local.` and # receive responses. # -# Static IP on v30iot alongside matter-server (.226) and otbr (.227). -# `.228` was free per the 2026-05-13 ping sweep. +# The address is leased from the UniFi DHCP server on v30iot. +# +# This attachment previously pinned 10.2.30.228 via static IPAM, on the +# rationale that "`.228` was free per the 2026-05-13 ping sweep". A ping +# sweep shows which hosts answer at that instant; it says nothing about +# what the DHCP server will lease later, and .228 is inside the pool +# (10.2.30.100-.254). It conflicted with *itself*: Multus randomizes the +# macvlan MAC per sandbox, so on every rolling update the terminating and +# starting pods held .228 under different MACs. Under DHCP each sandbox +# leases its own address and the overlap is harmless. The app binds +# 0.0.0.0 and resolves Shellies by mDNS hostname, so the address is never +# referenced. Requires the cni-dhcp-daemon DaemonSet in +# infrastructure/multus. # # Multus's `master` is an interface name. On every cluster node v30iot # lands on `eth1`: k8s-edge has only two NICs (mgmt=eth0, v30iot=eth1) @@ -36,11 +47,6 @@ spec: "master": "eth1", "mode": "bridge", "ipam": { - "type": "static", - "addresses": [ - { - "address": "10.2.30.228/24" - } - ] + "type": "dhcp" } } diff --git a/kubernetes/apps/matter-server/deployment.yaml b/kubernetes/apps/matter-server/deployment.yaml index 31c03da1..3b49b408 100644 --- a/kubernetes/apps/matter-server/deployment.yaml +++ b/kubernetes/apps/matter-server/deployment.yaml @@ -20,8 +20,8 @@ spec: # macvlan (see network-attachment.yaml). Matter device # commissioning and discovery need real L2 presence on the IoT # subnet — multicast send/receive — which a Cilium-managed pod - # IP or a LoadBalancer Service IP cannot provide. The NAD pins - # 10.2.30.226 statically and uses master: eth1, which is the + # IP or a LoadBalancer Service IP cannot provide. The NAD leases + # the address by DHCP and uses master: eth1, which is the # v30iot interface on every node (k8s-1/2/3 and k8s-edge alike), # so this pod is no longer tied to k8s-edge. k8s.v1.cni.cncf.io/networks: matter-server-iot diff --git a/kubernetes/apps/matter-server/network-attachment.yaml b/kubernetes/apps/matter-server/network-attachment.yaml index 9f8cb0a3..9d60a5d6 100644 --- a/kubernetes/apps/matter-server/network-attachment.yaml +++ b/kubernetes/apps/matter-server/network-attachment.yaml @@ -7,10 +7,20 @@ # send/receive included — which Matter device commissioning and # discovery need on top of the Cilium-managed pod network. # -# 10.2.30.226 is statically allocated on v30iot alongside otbr (.227) -# and shelly-manager (.228). This pod doesn't go through Cilium IPAM — +# The address is leased from the UniFi DHCP server on v30iot, like every +# other device on that VLAN. This pod doesn't go through Cilium IPAM — # the address lives on the IoT VLAN, not the pod CIDR. # +# This attachment previously pinned 10.2.30.226 via static IPAM, an +# address picked because a ping sweep found it idle. It sits inside the +# DHCP pool (10.2.30.100-.254), so the UDM eventually leased it to a +# Miele oven; the pod lost the ARP race and Matter's IPv4 unicast went to +# the appliance. Silently — mDNS is multicast and kept working. Nothing +# references this address (matter-server binds by interface name via +# --primary-interface=net1, and HA reaches it through the ClusterIP +# Service), so there is nothing to pin. Requires the cni-dhcp-daemon +# DaemonSet in infrastructure/multus. +# # No gateway is set on this attachment: default route stays on the # pod's primary (Cilium-managed) interface, so non-IoT egress doesn't # leak through eth1. The macvlan only needs the connected /24 to talk @@ -39,12 +49,7 @@ spec: "master": "eth1", "mode": "bridge", "ipam": { - "type": "static", - "addresses": [ - { - "address": "10.2.30.226/24" - } - ] + "type": "dhcp" } }, { diff --git a/kubernetes/apps/otbr/network-attachment.yaml b/kubernetes/apps/otbr/network-attachment.yaml index f62616fa..a1839383 100644 --- a/kubernetes/apps/otbr/network-attachment.yaml +++ b/kubernetes/apps/otbr/network-attachment.yaml @@ -10,9 +10,17 @@ # multicast on the IoT VLAN, so without this attachment OTBR's BR side # is invisible. # -# 10.2.30.227 is statically allocated on v30iot alongside matter-server -# (.226) and shelly-manager (.228). This pod doesn't go through Cilium -# IPAM — the address lives on the IoT VLAN, not the pod CIDR. +# The address is leased from the UniFi DHCP server on v30iot. This pod +# doesn't go through Cilium IPAM — the address lives on the IoT VLAN, not +# the pod CIDR. +# +# This attachment previously pinned 10.2.30.227 via static IPAM. That +# address is inside the DHCP pool (10.2.30.100-.254) and the UDM leased +# it to a Miele fridge; the two fought over it from 2026-05-13 onward. +# OTBR binds by interface (BACKBONE_INTERFACE=net1, REST_HOST=0.0.0.0) +# and HA reaches it at otbr.home-assistant.svc.cluster.local:8081, so no +# consumer depends on the address. Requires the cni-dhcp-daemon DaemonSet +# in infrastructure/multus. # # No gateway is set: default route stays on the pod's primary # (Cilium-managed) interface so non-IoT egress doesn't leak through @@ -30,11 +38,6 @@ spec: "master": "eth1", "mode": "bridge", "ipam": { - "type": "static", - "addresses": [ - { - "address": "10.2.30.227/24" - } - ] + "type": "dhcp" } } diff --git a/kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml b/kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml new file mode 100644 index 00000000..b89a28e8 --- /dev/null +++ b/kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml @@ -0,0 +1,126 @@ +--- +# Runs the containernetworking `dhcp` IPAM plugin in daemon mode on every +# node. The plugin binary itself is a thin client: when a macvlan NAD sets +# `"ipam": {"type": "dhcp"}`, the CNI invocation connects to the daemon's +# unix socket at /run/cni/dhcp.sock, and the daemon does the actual DHCP +# exchange from inside the container's netns and renews the lease for the +# lifetime of the pod. Without this DaemonSet the plugin fails with +# "failed to connect to dhcp daemon" and the attachment never comes up. +# +# Why DHCP IPAM at all — the 2026-07-09 IP conflict: +# The three v30iot macvlan NADs (matter-server, otbr, shelly-manager) +# used `"ipam": {"type": "static"}` with hand-picked addresses .226/.227/ +# .228, chosen because a ping sweep on 2026-05-13 found them idle. They +# sit inside the UniFi DHCP pool (10.2.30.100-.254), so the UDM leased +# them out from under the pods: +# .226 vs Miele oven (00:1d:63:5a:95:4e) 16 conflict alerts +# .227 vs Miele fridge (00:1d:63:87:9d:b1) 8 conflict alerts +# .228 vs *itself*, rolling-update overlap 4 conflict alerts +# Conflicts fired continuously from 2026-05-13 to 2026-07-09 and were +# never noticed: the attachments are mDNS/multicast-only, and multicast +# reception survives losing the ARP race, so nothing logged an error. +# +# Letting the UDM allocate is the fix. An address is then owned by an +# allocator, as every other address on this network is. Nothing dials +# these pods by IP (verified 2026-07-09 across the repo, all live cluster +# objects incl. decoded Secrets, and HA's stored config entries) — they +# are reached via their ClusterIP Services, and the macvlan exists purely +# for L2 presence. matter-server binds by interface name +# (--primary-interface=net1) and otbr by BACKBONE_INTERFACE=net1, so +# neither cares what the address is. +# +# Note the .228 case: Multus randomizes the macvlan MAC per pod sandbox, +# so during a rolling update the old and new pod held the same *static* +# address with different MACs and conflicted with each other. Under DHCP +# each sandbox gets its own lease, which makes the random MAC a feature — +# do NOT pin a stable MAC here (e.g. via the tuning plugin's `mac` field) +# or rolling updates will collide again on a single reservation. +# +# The dhcp binary ships from the sibling cni-plugins-installer DaemonSet +# (containernetworking/plugins v1.5.1, whole tarball extracted). Keep the +# version pins in step. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cni-dhcp-daemon + namespace: kube-system + labels: + app: cni-dhcp-daemon +spec: + selector: + matchLabels: + app: cni-dhcp-daemon + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: cni-dhcp-daemon + spec: + # Must share the host netns so the daemon can setns() into each + # container netns handed to it over the socket. + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + initContainers: + # The daemon binds /run/cni/dhcp.sock but will not create a stale + # socket's parent, and refuses to start if the socket file is left + # over from an unclean shutdown (bind: address already in use). + - name: clean-socket + image: alpine:3.21 + command: ["sh", "-c", "rm -f /run/cni/dhcp.sock"] + volumeMounts: + - name: run-cni + mountPath: /run/cni + securityContext: + privileged: true + containers: + - name: dhcp + # Any image with a shell works: the binary executed here is the + # host's, bind-mounted from /opt/cni/bin. Pinned to the same + # alpine tag the sibling installer uses. + image: alpine:3.21 + command: + - /opt/cni/bin/dhcp + - daemon + # No -hostprefix: /run/netns is mounted at its host path, and + # alpine symlinks /var/run -> /run, so the /var/run/netns/cni-* + # paths containerd passes as CNI_NETNS resolve unchanged. + securityContext: + privileged: true + volumeMounts: + - name: cnibin + mountPath: /opt/cni/bin + readOnly: true + - name: run-cni + mountPath: /run/cni + - name: run-netns + mountPath: /run/netns + # New netns bind-mounts appear after this container starts; + # without propagation the daemon cannot see them and every + # subsequent pod's attachment times out. + mountPropagation: HostToContainer + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin + type: Directory + - name: run-cni + hostPath: + path: /run/cni + # Talos does not pre-create /run/cni. + type: DirectoryOrCreate + - name: run-netns + hostPath: + path: /run/netns + type: Directory diff --git a/kubernetes/infrastructure/multus/kustomization.yaml b/kubernetes/infrastructure/multus/kustomization.yaml index 197b62a2..a8d1e713 100644 --- a/kubernetes/infrastructure/multus/kustomization.yaml +++ b/kubernetes/infrastructure/multus/kustomization.yaml @@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cni-plugins-installer.yaml + - cni-dhcp-daemon.yaml - multus-daemonset.yaml