From b077de8556ee39200b100b65a17bd110256c3428 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 23 Jul 2026 13:03:14 +0200 Subject: [PATCH 1/3] fix(talos): remove node host addressing on the v30iot NIC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security + routing. The k8s nodes held a DHCP-leased v4 address and an RA-derived v6 address on the IoT VLAN — a network that must be treated as compromised. That gave every IoT device an L3-reachable path to the node (kubelet, SSH-equivalent apid, any host service) and, as a side effect, installed a connected 10.2.30.0/24 route so node replies to IoT-subnet clients bypassed the UDM entirely — breaking every UDM-DNATed flow (the hairpin NAT: un-translated SYN-ACK -> client kernel RST, verified live 2026-07-23). The NIC (busPath 0000:06:13.0) stays up as the macvlan PARENT for the v30iot pod attachments (matter-server, otbr, shelly-manager), whose children do their own DHCP in their own netns — host addressing was never needed for that. dhcp: false drops the v4 address+route; sysctl net.ipv6.conf.eth1.disable_ipv6=1 drops v6 including link-local, so the host has no addressable presence on the L2 at all. The old comment's claimed purpose (Cilium L2 announce for an IoT LB pool) does not exist — no L2AnnouncementPolicy, BGP-only pool. RESIDUAL (not addressed here): the NIC remains a macvlan parent, so the node kernel still processes frames from the IoT L2. Full node isolation from IoT would require relocating how those three pods reach IoT devices — a separate, larger change. Same suppression pattern the repo already uses (edge-host 10-vlan40, HA macvlan /32) rather than policy routing (Talos 1.12 has no RoutingRuleConfig; that is 1.13+). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014S5CLfizEtzHS51kxJFQgc --- kubernetes/talos/patches/k8s-1.yaml | 23 ++++++++++++++++------- kubernetes/talos/patches/k8s-2.yaml | 8 +++++--- kubernetes/talos/patches/k8s-3.yaml | 8 +++++--- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index c369d7cf..97e1a999 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -17,15 +17,20 @@ machine: dhcp: true vip: ip: 10.2.40.60 - # v30iot: deprioritise its default route so eth0's default - # (metric 1024) wins normal egress and v30iot's only feeds the - # connected /24 used by Cilium L2 announce for the IoT-pool LBs. - # Same pattern as k8s-edge — see patches/k8s-edge.yaml. + # v30iot: LINK ONLY — no host addressing. The NIC is the macvlan + # parent for the v30iot pod attachments (matter/otbr/shelly-manager), + # whose children do their own DHCP on the L2; host addresses were + # never needed for that. Host DHCP (formerly dhcp + routeMetric 4096) + # installed a connected 10.2.30.0/24 route, so node replies to + # IoT-subnet clients bypassed the UDM and broke every UDM-DNATed + # flow (hairpin NAT: un-translated SYN-ACK -> client kernel RST, + # verified live 2026-07-23). The purpose the old comment claimed — + # Cilium L2 announce for an IoT LB pool — does not exist (no + # L2AnnouncementPolicy, no such pool). v6 RA addressing on this NIC + # is disabled via machine.sysctls for the same reason. - deviceSelector: busPath: "0000:06:13.0" - dhcp: true - dhcpOptions: - routeMetric: 4096 + dhcp: false # Dedicated Ceph cluster network on the X710 SFP+ port (PCI passthrough, # mapping x710-port1 in opentofu/vm_150_k8s_1.tf). Separate /29 on VLAN 49, # MTU 9000, no default route — used by Rook for OSD↔OSD replication via @@ -35,6 +40,10 @@ machine: addresses: - 10.99.0.1/29 mtu: 9000 + sysctls: + # v30iot NIC carries no host addressing (see interfaces above); this + # disables v6 entirely (incl. link-local) so the host is unreachable on that L2. + net.ipv6.conf.eth1.disable_ipv6: "1" install: disk: /dev/vda certSANs: diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index eac62727..1d290bb3 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -8,17 +8,19 @@ machine: dhcp: true vip: ip: 10.2.40.60 + # v30iot: link only, no host addressing — see k8s-1.yaml. - deviceSelector: busPath: "0000:06:13.0" - dhcp: true - dhcpOptions: - routeMetric: 4096 + dhcp: false # See k8s-1.yaml for context on the dedicated Ceph cluster network. - deviceSelector: driver: i40e addresses: - 10.99.0.2/29 mtu: 9000 + sysctls: + # v30iot NIC carries no host addressing — see k8s-1.yaml. + net.ipv6.conf.eth1.disable_ipv6: "1" install: disk: /dev/vda certSANs: diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index 08ee28b0..a9dbe1de 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -8,17 +8,19 @@ machine: dhcp: true vip: ip: 10.2.40.60 + # v30iot: link only, no host addressing — see k8s-1.yaml. - deviceSelector: busPath: "0000:06:13.0" - dhcp: true - dhcpOptions: - routeMetric: 4096 + dhcp: false # See k8s-1.yaml for context on the dedicated Ceph cluster network. - deviceSelector: driver: i40e addresses: - 10.99.0.3/29 mtu: 9000 + sysctls: + # v30iot NIC carries no host addressing — see k8s-1.yaml. + net.ipv6.conf.eth1.disable_ipv6: "1" install: disk: /dev/vda certSANs: -- 2.54.0 From 02c6128dc6e751f06e6184ecf089f96ba295ef66 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 23 Jul 2026 13:08:40 +0200 Subject: [PATCH 2/3] docs(talos,shelly): flag the eth1 name dependency; fix stale eth2 comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups (non-blocking): - The disable_ipv6 sysctl keys on the literal 'eth1' while the interface is selected by busPath; a rename would no-op the sysctl SILENTLY and leave the host v6-addressed on IoT. Document the dependency and that the rollout must verify eth1 is addressless per node. - shelly-manager-deployment comment said 'master: eth2' — the NAD uses eth1. Last lingering eth2-means-v30iot reference, corrected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014S5CLfizEtzHS51kxJFQgc --- .../apps/home-assistant/shelly-manager-deployment.yaml | 2 +- kubernetes/talos/patches/k8s-1.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml b/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml index 0221dc2e..8fda5993 100644 --- a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml +++ b/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml @@ -50,7 +50,7 @@ spec: # above is additive. No explicit edge exclusion is needed: the # `workload-class=pinned:NoSchedule` taint on k8s-edge already # keeps this pod off there (we never tolerate it), and the NAD's - # `master: eth2` lines up with v30iot on every k8s-{1,2,3} worker. + # `master: eth1` lines up with v30iot on every k8s-{1,2,3} worker. enableServiceLinks: false # Grants shelly-manager the in-cluster identity used by the firmware # pipeline to create esphome flash Jobs and stream their logs. RBAC is diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index 97e1a999..052679e6 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -43,6 +43,14 @@ machine: sysctls: # v30iot NIC carries no host addressing (see interfaces above); this # disables v6 entirely (incl. link-local) so the host is unreachable on that L2. + # + # NAME DEPENDENCY: the interface above is selected by busPath, but a + # sysctl can only key on the kernel name. `0000:06:13.0` -> `eth1` is + # verified on k8s-{1,2,3} (same fact the three IoT NADs rely on). If PCI + # probe order ever renames it, this sysctl no-ops SILENTLY and the host + # keeps a v6 address on IoT — so the rollout MUST confirm `eth1` is + # addressless per node before trusting it (a wrong name here fails quiet, + # unlike the NADs which fail loud with a NotReady pod). net.ipv6.conf.eth1.disable_ipv6: "1" install: disk: /dev/vda -- 2.54.0 From 5a9693bdd8371f520c841f7ca106713c777cf885 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 23 Jul 2026 13:31:50 +0200 Subject: [PATCH 3/3] fix(music-assistant): macvlan child on v30iot instead of hostNetwork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unblocks the node-unaddressing change: MA ran hostNetwork and its reconcile initContainer fails closed if no 10.2.30.0/24 address exists on a host interface — removing node host addressing would crash-loop it on every node it can schedule to (no edge toleration). MA now attaches net1 via macvlan on v30iot (DHCP-leased), exactly like matter-server/otbr/shelly-manager: its own IoT address, direct L2 adjacency for Snapcast/SlimProto/Sendspin audio sync (macvlan bridge mode puts frames straight on the wire — same property hostNetwork gave, without the node holding an IoT address). The reconcile script is unchanged in logic: it scans the pod's interfaces for the 10.2.30.0/24 address and now finds net1 (Multus attaches it with its lease before any container runs) instead of the host eth1. bind_ip stays 0.0.0.0 -> binds eth0 + net1; publish_ip = net1's leased IP. NAD omits matter's Thread accept_ra tuning (MA has no Thread); keeps sbr (leased default gateway would otherwise blackhole cluster-internal replies). Folded into this PR so the change is atomic: on merge Flux migrates MA to its own IoT lease before the manual node-unaddressing rollout removes the host addresses. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014S5CLfizEtzHS51kxJFQgc --- .../configmap-net-reconcile.yaml | 8 ++- .../apps/music-assistant/deployment.yaml | 33 +++++------ .../apps/music-assistant/kustomization.yaml | 1 + .../music-assistant/network-attachment.yaml | 56 +++++++++++++++++++ 4 files changed, 79 insertions(+), 19 deletions(-) create mode 100644 kubernetes/apps/music-assistant/network-attachment.yaml diff --git a/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml b/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml index 92f3a163..ec587717 100644 --- a/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml +++ b/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml @@ -28,9 +28,11 @@ data: The v30iot IP is DHCP and node-dependent (MA is not node-pinned), so a stored IP is a stale assumption the moment MA reschedules or the lease changes - exactly the failure this replaces. We therefore recompute it from the live - interface on every start. hostNetwork means this initContainer sees eth1 - directly. Fail closed if no v30iot address is present, so MA can never come - up advertising a wrong IP. + interface on every start. The macvlan child net1 (see network-attachment.yaml) + is attached by Multus with its DHCP lease before any container runs, so this + initContainer sees the pod's own v30iot address by scanning its interfaces. + Fail closed if no v30iot address is present, so MA can never come up + advertising a wrong IP. """ import fcntl import ipaddress diff --git a/kubernetes/apps/music-assistant/deployment.yaml b/kubernetes/apps/music-assistant/deployment.yaml index 38dc4397..191dca07 100644 --- a/kubernetes/apps/music-assistant/deployment.yaml +++ b/kubernetes/apps/music-assistant/deployment.yaml @@ -16,19 +16,19 @@ spec: metadata: labels: app.kubernetes.io/name: music-assistant + annotations: + # Multus attaches net1 (macvlan on v30iot, DHCP-leased) for direct + # L2 adjacency to the audio clients (Snapcast / SlimProto / + # Sendspin) — those protocols sync audio across group members and + # require a shared L2 broadcast domain; routing the audio path + # through the CNI overlay breaks the sync timing budget. This + # replaces the former hostNetwork attachment (see + # network-attachment.yaml) so the NODE no longer holds an address + # on the compromised IoT VLAN; MA gets its own leased IoT IP on + # net1 instead. master: eth1 is v30iot on every k8s-{1,2,3}/edge + # worker. + k8s.v1.cni.cncf.io/networks: music-assistant-iot spec: - # Host networking attaches MA to the node's L2 VLANs directly. - # Every node's second NIC (eth1) is on v30iot, so the pod sits - # on the IoT VLAN with the audio clients (Snapcast / SlimProto / - # Chromecast). Those protocols sync audio across group members - # and require all members to share an L2 broadcast domain; - # routing the audio path through the CNI overlay breaks the - # sync timing budget. The visibility / discovery side-effects - # (mDNS, uPnP, DLNA browse) are byproducts of being on the - # VLAN, not the reason for hostNetwork — the reason is L2 - # adjacency for audio sync. - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet enableServiceLinks: false # No nodeSelector/toleration: MA only needs a node on v30iot (all # of them) for audio-sync L2, not the edge USB devices. It's the @@ -47,10 +47,11 @@ spec: # stale on the next reschedule or lease change (the Sendspin server # then fails to bind and dies, with no fallback). # So we recompute the v30iot IP from the live interface every boot and - # write it into /data/settings.json before MA reads it. Runs in the - # pod's hostNetwork namespace, so it sees eth1 directly. Reuses the MA - # image (python + stdlib only; Flux keeps the tag in lockstep via the - # shared image policy) — no extra dependency. + # write it into /data/settings.json before MA reads it. Multus attaches + # net1 (with its DHCP lease) before any container runs, so this sees the + # pod's own v30iot interface directly. Reuses the MA image (python + + # stdlib only; Flux keeps the tag in lockstep via the shared image + # policy) — no extra dependency. - name: reconcile-publish-ip image: ghcr.io/music-assistant/server:2.9.9 # {"$imagepolicy": "flux-system:music-assistant"} command: ["/app/venv/bin/python", "/scripts/reconcile.py"] diff --git a/kubernetes/apps/music-assistant/kustomization.yaml b/kubernetes/apps/music-assistant/kustomization.yaml index 5c801071..d6041780 100644 --- a/kubernetes/apps/music-assistant/kustomization.yaml +++ b/kubernetes/apps/music-assistant/kustomization.yaml @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - network-attachment.yaml - configmap-net-reconcile.yaml - deployment.yaml - ingress.yaml diff --git a/kubernetes/apps/music-assistant/network-attachment.yaml b/kubernetes/apps/music-assistant/network-attachment.yaml new file mode 100644 index 00000000..9b4ca34c --- /dev/null +++ b/kubernetes/apps/music-assistant/network-attachment.yaml @@ -0,0 +1,56 @@ +--- +# music-assistant's second NIC: macvlan attached to the node's v30iot +# interface (eth1 — the connected 10.2.30.0/24 on every k8s-{1,2,3}/edge +# worker). Gives the pod a real leased IP on the IoT VLAN with full L2 +# presence, which the audio-sync protocols (Snapcast / SlimProto / +# Sendspin) require: group members must share an L2 broadcast domain or +# the sync timing budget breaks. macvlan bridge mode puts net1 frames +# straight onto the v30iot wire — the same direct-L2 property MA +# previously got from hostNetwork, without the node itself holding an +# address on the (compromised) IoT VLAN. +# +# This replaces the former hostNetwork attachment. The reconcile +# initContainer (configmap-net-reconcile.yaml) discovers net1's leased +# address the same way it used to discover the host's eth1 address — +# scanning the pod's own interfaces for a 10.2.30.0/24 address — and +# writes it as MA's publish_ip. bind_ip stays 0.0.0.0, which now binds +# eth0 (pod) + net1; players reach the stream/slimproto/sendspin servers +# on net1. Requires the cni-dhcp-daemon DaemonSet in infrastructure/multus. +# +# Mirrors matter-server-iot; the only difference is MA does not speak +# Thread, so the matter NAD's accept_ra_rt_info_max_plen tuning (for the +# OTBR OMR RIO) is omitted here. +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: music-assistant-iot + namespace: music-assistant +spec: + # Chained CNI: macvlan creates net1 with a DHCP-leased v30iot address, + # then `sbr` confines net1's routes to a source-based routing table. + # + # sbr is not optional: a leased address arrives with a default gateway + # (DHCP option 3, 10.2.30.1), and the dhcp IPAM plugin installs it as a + # second default route at eth0's metric. Replies to cluster-internal + # peers (the ClusterIP Service, the HTTP ingress path) would then egress + # the IoT VLAN and vanish, and the pod would go NotReady. sbr keeps IoT + # traffic sourced from net1 on net1 while the pod default route stays on + # eth0. Same rationale as matter-server-iot. + config: | + { + "cniVersion": "0.3.1", + "name": "music-assistant-iot", + "plugins": [ + { + "type": "macvlan", + "master": "eth1", + "mode": "bridge", + "ipam": { + "type": "dhcp" + } + }, + { + "type": "sbr" + } + ] + } -- 2.54.0