From 19a44821a68d10c70017964e7b63871eb05e613d Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 9 Jul 2026 12:02:53 +0200 Subject: [PATCH] Lease v30iot macvlan addresses by DHCP instead of pinning them The three macvlan NADs on the IoT VLAN (matter-server, otbr, shelly-manager) used static IPAM with hand-picked addresses .226/.227/ .228, chosen because a ping sweep on 2026-05-13 found them idle. All three sit inside the UniFi DHCP pool (10.2.30.100-.254), so the UDM leased them out from under the pods. The controller has been logging CLIENT_IP_CONFLICT continuously since 2026-05-13: .226 vs Miele oven (00:1d:63:5a:95:4e) 16 alerts .227 vs Miele fridge (00:1d:63:87:9d:b1) 8 alerts .228 vs itself 4 alerts Nobody noticed because these attachments exist only for mDNS/multicast L2 presence, and multicast reception survives losing the ARP race, so no pod ever logged an error. The .228 case involves no real device at all: Multus randomizes the macvlan MAC per sandbox, so a rolling update had the terminating and starting pods claiming the same static address under different MACs. A ping sweep shows which hosts answer at that instant. It says nothing about what DHCP will lease later. Relocating the addresses to a quieter part of the subnet would preserve the actual defect -- an address that no allocator owns. So remove the addresses instead: let the UDM allocate, as it does for every other device on this VLAN. Nothing depends on the values. Verified across the repo, every live cluster object including decoded Secrets, and Home Assistant's stored config entries: no consumer references .226/.227/.228, and no UniFi firewall rule, port-forward, or firewall group mentions them. All three pods are reached via their ClusterIP Services; matter-server binds by interface name (--primary-interface=net1) and otbr by BACKBONE_INTERFACE=net1. Adds cni-dhcp-daemon to infrastructure/multus, which the dhcp IPAM plugin requires -- the plugin is only a client of a per-node daemon on /run/cni/dhcp.sock. The dhcp binary is already on the nodes; the sibling cni-plugins-installer extracts the whole v1.5.1 tarball. Flux's apps Kustomization already dependsOn infrastructure, so the daemon lands before the NADs reconcile, and running pods keep their addresses until restarted. No stable MAC is pinned: under DHCP the random per-sandbox MAC is what lets a rolling update take two independent leases rather than collide on one reservation. Also corrects two stale comments: matter-server/deployment.yaml still described the NAD as pinning .226, and bgp-service.yaml claimed the home-assistant-iot Service at 10.2.30.80 was retained for hard-coded clients -- it was retired 2026-05-20 and no LoadBalancer Service holds a 10.2.30.x address any more. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../apps/home-assistant/bgp-service.yaml | 5 +- .../shelly-manager-network-attachment.yaml | 22 +-- kubernetes/apps/matter-server/deployment.yaml | 4 +- .../matter-server/network-attachment.yaml | 21 +-- kubernetes/apps/otbr/network-attachment.yaml | 21 +-- .../multus/cni-dhcp-daemon.yaml | 126 ++++++++++++++++++ .../infrastructure/multus/kustomization.yaml | 1 + 7 files changed, 171 insertions(+), 29 deletions(-) create mode 100644 kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml diff --git a/kubernetes/apps/home-assistant/bgp-service.yaml b/kubernetes/apps/home-assistant/bgp-service.yaml index 54bc86ed..8a847f38 100644 --- a/kubernetes/apps/home-assistant/bgp-service.yaml +++ b/kubernetes/apps/home-assistant/bgp-service.yaml @@ -5,8 +5,9 @@ # directly on that node — no L2 lease holder, no inter-VLAN ARP race. # # Use this URL for any LAN client that can resolve DNS, including -# ESPHome devices fetching TTS audio. The legacy `home-assistant-iot` -# Service at 10.2.30.80 stays for devices with hard-coded IPs. +# ESPHome devices fetching TTS audio. (The legacy `home-assistant-iot` +# Service at 10.2.30.80 was retired on 2026-05-20; no LoadBalancer +# Service holds a 10.2.30.x address any more. Verified 2026-07-09.) apiVersion: v1 kind: Service metadata: diff --git a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml b/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml index 7ce8e9ee..5bc1d9e8 100644 --- a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml +++ b/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml @@ -7,8 +7,19 @@ # resolver browse `_shelly._tcp.local.` / `_http._tcp.local.` and # receive responses. # -# Static IP on v30iot alongside matter-server (.226) and otbr (.227). -# `.228` was free per the 2026-05-13 ping sweep. +# The address is leased from the UniFi DHCP server on v30iot. +# +# This attachment previously pinned 10.2.30.228 via static IPAM, on the +# rationale that "`.228` was free per the 2026-05-13 ping sweep". A ping +# sweep shows which hosts answer at that instant; it says nothing about +# what the DHCP server will lease later, and .228 is inside the pool +# (10.2.30.100-.254). It conflicted with *itself*: Multus randomizes the +# macvlan MAC per sandbox, so on every rolling update the terminating and +# starting pods held .228 under different MACs. Under DHCP each sandbox +# leases its own address and the overlap is harmless. The app binds +# 0.0.0.0 and resolves Shellies by mDNS hostname, so the address is never +# referenced. Requires the cni-dhcp-daemon DaemonSet in +# infrastructure/multus. # # Multus's `master` is an interface name. On every cluster node v30iot # lands on `eth1`: k8s-edge has only two NICs (mgmt=eth0, v30iot=eth1) @@ -36,11 +47,6 @@ spec: "master": "eth1", "mode": "bridge", "ipam": { - "type": "static", - "addresses": [ - { - "address": "10.2.30.228/24" - } - ] + "type": "dhcp" } } diff --git a/kubernetes/apps/matter-server/deployment.yaml b/kubernetes/apps/matter-server/deployment.yaml index 31c03da1..3b49b408 100644 --- a/kubernetes/apps/matter-server/deployment.yaml +++ b/kubernetes/apps/matter-server/deployment.yaml @@ -20,8 +20,8 @@ spec: # macvlan (see network-attachment.yaml). Matter device # commissioning and discovery need real L2 presence on the IoT # subnet — multicast send/receive — which a Cilium-managed pod - # IP or a LoadBalancer Service IP cannot provide. The NAD pins - # 10.2.30.226 statically and uses master: eth1, which is the + # IP or a LoadBalancer Service IP cannot provide. The NAD leases + # the address by DHCP and uses master: eth1, which is the # v30iot interface on every node (k8s-1/2/3 and k8s-edge alike), # so this pod is no longer tied to k8s-edge. k8s.v1.cni.cncf.io/networks: matter-server-iot diff --git a/kubernetes/apps/matter-server/network-attachment.yaml b/kubernetes/apps/matter-server/network-attachment.yaml index 9f8cb0a3..9d60a5d6 100644 --- a/kubernetes/apps/matter-server/network-attachment.yaml +++ b/kubernetes/apps/matter-server/network-attachment.yaml @@ -7,10 +7,20 @@ # send/receive included — which Matter device commissioning and # discovery need on top of the Cilium-managed pod network. # -# 10.2.30.226 is statically allocated on v30iot alongside otbr (.227) -# and shelly-manager (.228). This pod doesn't go through Cilium IPAM — +# The address is leased from the UniFi DHCP server on v30iot, like every +# other device on that VLAN. This pod doesn't go through Cilium IPAM — # the address lives on the IoT VLAN, not the pod CIDR. # +# This attachment previously pinned 10.2.30.226 via static IPAM, an +# address picked because a ping sweep found it idle. It sits inside the +# DHCP pool (10.2.30.100-.254), so the UDM eventually leased it to a +# Miele oven; the pod lost the ARP race and Matter's IPv4 unicast went to +# the appliance. Silently — mDNS is multicast and kept working. Nothing +# references this address (matter-server binds by interface name via +# --primary-interface=net1, and HA reaches it through the ClusterIP +# Service), so there is nothing to pin. Requires the cni-dhcp-daemon +# DaemonSet in infrastructure/multus. +# # No gateway is set on this attachment: default route stays on the # pod's primary (Cilium-managed) interface, so non-IoT egress doesn't # leak through eth1. The macvlan only needs the connected /24 to talk @@ -39,12 +49,7 @@ spec: "master": "eth1", "mode": "bridge", "ipam": { - "type": "static", - "addresses": [ - { - "address": "10.2.30.226/24" - } - ] + "type": "dhcp" } }, { diff --git a/kubernetes/apps/otbr/network-attachment.yaml b/kubernetes/apps/otbr/network-attachment.yaml index f62616fa..a1839383 100644 --- a/kubernetes/apps/otbr/network-attachment.yaml +++ b/kubernetes/apps/otbr/network-attachment.yaml @@ -10,9 +10,17 @@ # multicast on the IoT VLAN, so without this attachment OTBR's BR side # is invisible. # -# 10.2.30.227 is statically allocated on v30iot alongside matter-server -# (.226) and shelly-manager (.228). This pod doesn't go through Cilium -# IPAM — the address lives on the IoT VLAN, not the pod CIDR. +# The address is leased from the UniFi DHCP server on v30iot. This pod +# doesn't go through Cilium IPAM — the address lives on the IoT VLAN, not +# the pod CIDR. +# +# This attachment previously pinned 10.2.30.227 via static IPAM. That +# address is inside the DHCP pool (10.2.30.100-.254) and the UDM leased +# it to a Miele fridge; the two fought over it from 2026-05-13 onward. +# OTBR binds by interface (BACKBONE_INTERFACE=net1, REST_HOST=0.0.0.0) +# and HA reaches it at otbr.home-assistant.svc.cluster.local:8081, so no +# consumer depends on the address. Requires the cni-dhcp-daemon DaemonSet +# in infrastructure/multus. # # No gateway is set: default route stays on the pod's primary # (Cilium-managed) interface so non-IoT egress doesn't leak through @@ -30,11 +38,6 @@ spec: "master": "eth1", "mode": "bridge", "ipam": { - "type": "static", - "addresses": [ - { - "address": "10.2.30.227/24" - } - ] + "type": "dhcp" } } diff --git a/kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml b/kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml new file mode 100644 index 00000000..b89a28e8 --- /dev/null +++ b/kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml @@ -0,0 +1,126 @@ +--- +# Runs the containernetworking `dhcp` IPAM plugin in daemon mode on every +# node. The plugin binary itself is a thin client: when a macvlan NAD sets +# `"ipam": {"type": "dhcp"}`, the CNI invocation connects to the daemon's +# unix socket at /run/cni/dhcp.sock, and the daemon does the actual DHCP +# exchange from inside the container's netns and renews the lease for the +# lifetime of the pod. Without this DaemonSet the plugin fails with +# "failed to connect to dhcp daemon" and the attachment never comes up. +# +# Why DHCP IPAM at all — the 2026-07-09 IP conflict: +# The three v30iot macvlan NADs (matter-server, otbr, shelly-manager) +# used `"ipam": {"type": "static"}` with hand-picked addresses .226/.227/ +# .228, chosen because a ping sweep on 2026-05-13 found them idle. They +# sit inside the UniFi DHCP pool (10.2.30.100-.254), so the UDM leased +# them out from under the pods: +# .226 vs Miele oven (00:1d:63:5a:95:4e) 16 conflict alerts +# .227 vs Miele fridge (00:1d:63:87:9d:b1) 8 conflict alerts +# .228 vs *itself*, rolling-update overlap 4 conflict alerts +# Conflicts fired continuously from 2026-05-13 to 2026-07-09 and were +# never noticed: the attachments are mDNS/multicast-only, and multicast +# reception survives losing the ARP race, so nothing logged an error. +# +# Letting the UDM allocate is the fix. An address is then owned by an +# allocator, as every other address on this network is. Nothing dials +# these pods by IP (verified 2026-07-09 across the repo, all live cluster +# objects incl. decoded Secrets, and HA's stored config entries) — they +# are reached via their ClusterIP Services, and the macvlan exists purely +# for L2 presence. matter-server binds by interface name +# (--primary-interface=net1) and otbr by BACKBONE_INTERFACE=net1, so +# neither cares what the address is. +# +# Note the .228 case: Multus randomizes the macvlan MAC per pod sandbox, +# so during a rolling update the old and new pod held the same *static* +# address with different MACs and conflicted with each other. Under DHCP +# each sandbox gets its own lease, which makes the random MAC a feature — +# do NOT pin a stable MAC here (e.g. via the tuning plugin's `mac` field) +# or rolling updates will collide again on a single reservation. +# +# The dhcp binary ships from the sibling cni-plugins-installer DaemonSet +# (containernetworking/plugins v1.5.1, whole tarball extracted). Keep the +# version pins in step. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cni-dhcp-daemon + namespace: kube-system + labels: + app: cni-dhcp-daemon +spec: + selector: + matchLabels: + app: cni-dhcp-daemon + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: cni-dhcp-daemon + spec: + # Must share the host netns so the daemon can setns() into each + # container netns handed to it over the socket. + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + initContainers: + # The daemon binds /run/cni/dhcp.sock but will not create a stale + # socket's parent, and refuses to start if the socket file is left + # over from an unclean shutdown (bind: address already in use). + - name: clean-socket + image: alpine:3.21 + command: ["sh", "-c", "rm -f /run/cni/dhcp.sock"] + volumeMounts: + - name: run-cni + mountPath: /run/cni + securityContext: + privileged: true + containers: + - name: dhcp + # Any image with a shell works: the binary executed here is the + # host's, bind-mounted from /opt/cni/bin. Pinned to the same + # alpine tag the sibling installer uses. + image: alpine:3.21 + command: + - /opt/cni/bin/dhcp + - daemon + # No -hostprefix: /run/netns is mounted at its host path, and + # alpine symlinks /var/run -> /run, so the /var/run/netns/cni-* + # paths containerd passes as CNI_NETNS resolve unchanged. + securityContext: + privileged: true + volumeMounts: + - name: cnibin + mountPath: /opt/cni/bin + readOnly: true + - name: run-cni + mountPath: /run/cni + - name: run-netns + mountPath: /run/netns + # New netns bind-mounts appear after this container starts; + # without propagation the daemon cannot see them and every + # subsequent pod's attachment times out. + mountPropagation: HostToContainer + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin + type: Directory + - name: run-cni + hostPath: + path: /run/cni + # Talos does not pre-create /run/cni. + type: DirectoryOrCreate + - name: run-netns + hostPath: + path: /run/netns + type: Directory diff --git a/kubernetes/infrastructure/multus/kustomization.yaml b/kubernetes/infrastructure/multus/kustomization.yaml index 197b62a2..a8d1e713 100644 --- a/kubernetes/infrastructure/multus/kustomization.yaml +++ b/kubernetes/infrastructure/multus/kustomization.yaml @@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cni-plugins-installer.yaml + - cni-dhcp-daemon.yaml - multus-daemonset.yaml -- 2.54.0