From df61407f84f253246fa8e2100bee1916ab710365 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 9 Jul 2026 12:43:16 +0200 Subject: [PATCH] Confine the DHCP-leased default route with sbr on the v30iot macvlans PR #135 moved the three v30iot macvlan attachments from static IPAM to DHCP. That was right about ownership and wrong about routing: a leased address arrives with a default gateway (DHCP option 3, 10.2.30.1), and the dhcp IPAM plugin installs it as a *second* default route on net1 at the same metric as eth0's Cilium-managed one. Cluster-internal replies then egress the IoT VLAN and are dropped. The kubelet readiness probe comes from the node address and keeps passing for a few seconds, which makes the pod look healthy; anything from the pod CIDR -- Traefik -- times out. shelly-manager went NotReady and its ingress returned 504. The old static IPAM sidestepped this by simply never setting a gateway, as its comment said: "No gateway is set on this attachment: default route stays on the pod's primary (Cilium-managed) interface, so non-IoT egress doesn't leak through eth1." A leased address comes with a gateway, so it has to be confined rather than omitted. Chain the `sbr` plugin (source-based routing, containernetworking/plugins v1.5.1, already on every node from cni-plugins-installer) after macvlan. It moves net1's routes into a routing table selected by net1's source address, leaving eth0's default alone in the main table. matter-server's chain becomes macvlan -> tuning -> sbr; otbr and shelly-manager convert from a single-plugin config to a plugins array. Only shelly-manager currently runs a post-#135 sandbox, so it is the only pod actually broken right now. matter-server and otbr still hold their pre-#135 static addresses and single eth0 default route -- but their NADs already say dhcp, so they would break the same way on any restart. This change is a strict improvement for them either way. UNVERIFIED, called out in the matter-server NAD: sbr's effect on the IPv6 OMR route. Matter CASE sessions to Thread devices need the fd5f::/64 RIO that OTBR publishes on v30iot. Source-based routing should keep it reachable for traffic sourced from net1's address, but this has not been demonstrated. Check `ip -6 route` inside matter-server after its first restart under this config. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../shelly-manager-network-attachment.yaml | 27 ++++++++++++++----- .../matter-server/network-attachment.yaml | 21 ++++++++++++--- kubernetes/apps/otbr/network-attachment.yaml | 26 +++++++++++++----- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml b/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml index 5bc1d9e8..2afd2b15 100644 --- a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml +++ b/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml @@ -39,14 +39,29 @@ metadata: name: shelly-manager-iot namespace: home-assistant spec: + # Chained CNI: macvlan leases the v30iot address by DHCP, then `sbr` + # moves net1's routes into a source-based routing table. Without sbr the + # DHCP-supplied default gateway (10.2.30.1, option 3) installs a *second* + # default route on net1 at the same metric as eth0's, and cluster-internal + # replies (Traefik on the pod CIDR) egress the IoT VLAN and are dropped — + # the pod goes NotReady and its ingress returns 504. Static IPAM used to + # dodge this by simply never setting a gateway; leased addresses come with + # one, so it has to be confined instead of omitted. config: | { "cniVersion": "0.3.1", "name": "shelly-manager-iot", - "type": "macvlan", - "master": "eth1", - "mode": "bridge", - "ipam": { - "type": "dhcp" - } + "plugins": [ + { + "type": "macvlan", + "master": "eth1", + "mode": "bridge", + "ipam": { + "type": "dhcp" + } + }, + { + "type": "sbr" + } + ] } diff --git a/kubernetes/apps/matter-server/network-attachment.yaml b/kubernetes/apps/matter-server/network-attachment.yaml index 9d60a5d6..64e5fb41 100644 --- a/kubernetes/apps/matter-server/network-attachment.yaml +++ b/kubernetes/apps/matter-server/network-attachment.yaml @@ -31,9 +31,21 @@ kind: NetworkAttachmentDefinition metadata: name: matter-server-iot spec: - # Chained CNI: macvlan creates net1 with the static v30iot address, - # then the `tuning` plugin sets sysctls on the freshly-created - # interface. The kernel default `accept_ra_rt_info_max_plen=0` drops + # Chained CNI: macvlan creates net1 with a DHCP-leased v30iot address, + # the `tuning` plugin sets sysctls on the freshly-created interface, and + # `sbr` confines net1's routes to a source-based routing table. + # + # sbr is not optional here. A leased address arrives with a default + # gateway (DHCP option 3, 10.2.30.1), and the dhcp IPAM plugin installs + # it as a second default route at the same metric as eth0's. Replies to + # cluster-internal peers then egress the IoT VLAN and vanish; the pod + # goes NotReady. Static IPAM avoided this by never setting a gateway. + # + # UNVERIFIED: sbr's effect on the IPv6 OMR route below. Matter CASE + # sessions to Thread devices need the fd5f::/64 RIO that OTBR publishes + # on v30iot. Source-based routing should keep it reachable for traffic + # sourced from net1's address, but that has not been demonstrated — + # confirm `ip -6 route` inside the pod after the first restart. The kernel default `accept_ra_rt_info_max_plen=0` drops # every RA route-info entry, including the Thread OMR prefix # (fd5f:…/64) OTBR publishes to v30iot — Matter CASE sessions to # Thread devices then time out because the pod has no route. Bumping @@ -57,6 +69,9 @@ spec: "sysctl": { "net.ipv6.conf.IFNAME.accept_ra_rt_info_max_plen": "64" } + }, + { + "type": "sbr" } ] } diff --git a/kubernetes/apps/otbr/network-attachment.yaml b/kubernetes/apps/otbr/network-attachment.yaml index a1839383..da114c03 100644 --- a/kubernetes/apps/otbr/network-attachment.yaml +++ b/kubernetes/apps/otbr/network-attachment.yaml @@ -31,13 +31,27 @@ kind: NetworkAttachmentDefinition metadata: name: otbr-iot spec: + # Chained CNI: macvlan leases the v30iot address by DHCP, then `sbr` + # confines net1's routes to a source-based routing table. Without it the + # DHCP-supplied default gateway (10.2.30.1) installs a second default + # route on net1 alongside eth0's, and cluster-internal replies egress the + # IoT VLAN and are dropped. Static IPAM used to sidestep this by never + # setting a gateway; a leased address comes with one. config: | { "cniVersion": "0.3.1", - "type": "macvlan", - "master": "eth1", - "mode": "bridge", - "ipam": { - "type": "dhcp" - } + "name": "otbr-iot", + "plugins": [ + { + "type": "macvlan", + "master": "eth1", + "mode": "bridge", + "ipam": { + "type": "dhcp" + } + }, + { + "type": "sbr" + } + ] } -- 2.54.0