diff --git a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml b/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml index 0221dc2e..8fda5993 100644 --- a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml +++ b/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml @@ -50,7 +50,7 @@ spec: # above is additive. No explicit edge exclusion is needed: the # `workload-class=pinned:NoSchedule` taint on k8s-edge already # keeps this pod off there (we never tolerate it), and the NAD's - # `master: eth2` lines up with v30iot on every k8s-{1,2,3} worker. + # `master: eth1` lines up with v30iot on every k8s-{1,2,3} worker. enableServiceLinks: false # Grants shelly-manager the in-cluster identity used by the firmware # pipeline to create esphome flash Jobs and stream their logs. RBAC is diff --git a/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml b/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml index 92f3a163..ec587717 100644 --- a/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml +++ b/kubernetes/apps/music-assistant/configmap-net-reconcile.yaml @@ -28,9 +28,11 @@ data: The v30iot IP is DHCP and node-dependent (MA is not node-pinned), so a stored IP is a stale assumption the moment MA reschedules or the lease changes - exactly the failure this replaces. We therefore recompute it from the live - interface on every start. hostNetwork means this initContainer sees eth1 - directly. Fail closed if no v30iot address is present, so MA can never come - up advertising a wrong IP. + interface on every start. The macvlan child net1 (see network-attachment.yaml) + is attached by Multus with its DHCP lease before any container runs, so this + initContainer sees the pod's own v30iot address by scanning its interfaces. + Fail closed if no v30iot address is present, so MA can never come up + advertising a wrong IP. """ import fcntl import ipaddress diff --git a/kubernetes/apps/music-assistant/deployment.yaml b/kubernetes/apps/music-assistant/deployment.yaml index 38dc4397..191dca07 100644 --- a/kubernetes/apps/music-assistant/deployment.yaml +++ b/kubernetes/apps/music-assistant/deployment.yaml @@ -16,19 +16,19 @@ spec: metadata: labels: app.kubernetes.io/name: music-assistant + annotations: + # Multus attaches net1 (macvlan on v30iot, DHCP-leased) for direct + # L2 adjacency to the audio clients (Snapcast / SlimProto / + # Sendspin) — those protocols sync audio across group members and + # require a shared L2 broadcast domain; routing the audio path + # through the CNI overlay breaks the sync timing budget. This + # replaces the former hostNetwork attachment (see + # network-attachment.yaml) so the NODE no longer holds an address + # on the compromised IoT VLAN; MA gets its own leased IoT IP on + # net1 instead. master: eth1 is v30iot on every k8s-{1,2,3}/edge + # worker. + k8s.v1.cni.cncf.io/networks: music-assistant-iot spec: - # Host networking attaches MA to the node's L2 VLANs directly. - # Every node's second NIC (eth1) is on v30iot, so the pod sits - # on the IoT VLAN with the audio clients (Snapcast / SlimProto / - # Chromecast). Those protocols sync audio across group members - # and require all members to share an L2 broadcast domain; - # routing the audio path through the CNI overlay breaks the - # sync timing budget. The visibility / discovery side-effects - # (mDNS, uPnP, DLNA browse) are byproducts of being on the - # VLAN, not the reason for hostNetwork — the reason is L2 - # adjacency for audio sync. - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet enableServiceLinks: false # No nodeSelector/toleration: MA only needs a node on v30iot (all # of them) for audio-sync L2, not the edge USB devices. It's the @@ -47,10 +47,11 @@ spec: # stale on the next reschedule or lease change (the Sendspin server # then fails to bind and dies, with no fallback). # So we recompute the v30iot IP from the live interface every boot and - # write it into /data/settings.json before MA reads it. Runs in the - # pod's hostNetwork namespace, so it sees eth1 directly. Reuses the MA - # image (python + stdlib only; Flux keeps the tag in lockstep via the - # shared image policy) — no extra dependency. + # write it into /data/settings.json before MA reads it. Multus attaches + # net1 (with its DHCP lease) before any container runs, so this sees the + # pod's own v30iot interface directly. Reuses the MA image (python + + # stdlib only; Flux keeps the tag in lockstep via the shared image + # policy) — no extra dependency. - name: reconcile-publish-ip image: ghcr.io/music-assistant/server:2.9.9 # {"$imagepolicy": "flux-system:music-assistant"} command: ["/app/venv/bin/python", "/scripts/reconcile.py"] diff --git a/kubernetes/apps/music-assistant/kustomization.yaml b/kubernetes/apps/music-assistant/kustomization.yaml index 5c801071..d6041780 100644 --- a/kubernetes/apps/music-assistant/kustomization.yaml +++ b/kubernetes/apps/music-assistant/kustomization.yaml @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - network-attachment.yaml - configmap-net-reconcile.yaml - deployment.yaml - ingress.yaml diff --git a/kubernetes/apps/music-assistant/network-attachment.yaml b/kubernetes/apps/music-assistant/network-attachment.yaml new file mode 100644 index 00000000..9b4ca34c --- /dev/null +++ b/kubernetes/apps/music-assistant/network-attachment.yaml @@ -0,0 +1,56 @@ +--- +# music-assistant's second NIC: macvlan attached to the node's v30iot +# interface (eth1 — the connected 10.2.30.0/24 on every k8s-{1,2,3}/edge +# worker). Gives the pod a real leased IP on the IoT VLAN with full L2 +# presence, which the audio-sync protocols (Snapcast / SlimProto / +# Sendspin) require: group members must share an L2 broadcast domain or +# the sync timing budget breaks. macvlan bridge mode puts net1 frames +# straight onto the v30iot wire — the same direct-L2 property MA +# previously got from hostNetwork, without the node itself holding an +# address on the (compromised) IoT VLAN. +# +# This replaces the former hostNetwork attachment. The reconcile +# initContainer (configmap-net-reconcile.yaml) discovers net1's leased +# address the same way it used to discover the host's eth1 address — +# scanning the pod's own interfaces for a 10.2.30.0/24 address — and +# writes it as MA's publish_ip. bind_ip stays 0.0.0.0, which now binds +# eth0 (pod) + net1; players reach the stream/slimproto/sendspin servers +# on net1. Requires the cni-dhcp-daemon DaemonSet in infrastructure/multus. +# +# Mirrors matter-server-iot; the only difference is MA does not speak +# Thread, so the matter NAD's accept_ra_rt_info_max_plen tuning (for the +# OTBR OMR RIO) is omitted here. +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: music-assistant-iot + namespace: music-assistant +spec: + # Chained CNI: macvlan creates net1 with a DHCP-leased v30iot address, + # then `sbr` confines net1's routes to a source-based routing table. + # + # sbr is not optional: a leased address arrives with a default gateway + # (DHCP option 3, 10.2.30.1), and the dhcp IPAM plugin installs it as a + # second default route at eth0's metric. Replies to cluster-internal + # peers (the ClusterIP Service, the HTTP ingress path) would then egress + # the IoT VLAN and vanish, and the pod would go NotReady. sbr keeps IoT + # traffic sourced from net1 on net1 while the pod default route stays on + # eth0. Same rationale as matter-server-iot. + config: | + { + "cniVersion": "0.3.1", + "name": "music-assistant-iot", + "plugins": [ + { + "type": "macvlan", + "master": "eth1", + "mode": "bridge", + "ipam": { + "type": "dhcp" + } + }, + { + "type": "sbr" + } + ] + } diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index c369d7cf..052679e6 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -17,15 +17,20 @@ machine: dhcp: true vip: ip: 10.2.40.60 - # v30iot: deprioritise its default route so eth0's default - # (metric 1024) wins normal egress and v30iot's only feeds the - # connected /24 used by Cilium L2 announce for the IoT-pool LBs. - # Same pattern as k8s-edge — see patches/k8s-edge.yaml. + # v30iot: LINK ONLY — no host addressing. The NIC is the macvlan + # parent for the v30iot pod attachments (matter/otbr/shelly-manager), + # whose children do their own DHCP on the L2; host addresses were + # never needed for that. Host DHCP (formerly dhcp + routeMetric 4096) + # installed a connected 10.2.30.0/24 route, so node replies to + # IoT-subnet clients bypassed the UDM and broke every UDM-DNATed + # flow (hairpin NAT: un-translated SYN-ACK -> client kernel RST, + # verified live 2026-07-23). The purpose the old comment claimed — + # Cilium L2 announce for an IoT LB pool — does not exist (no + # L2AnnouncementPolicy, no such pool). v6 RA addressing on this NIC + # is disabled via machine.sysctls for the same reason. - deviceSelector: busPath: "0000:06:13.0" - dhcp: true - dhcpOptions: - routeMetric: 4096 + dhcp: false # Dedicated Ceph cluster network on the X710 SFP+ port (PCI passthrough, # mapping x710-port1 in opentofu/vm_150_k8s_1.tf). Separate /29 on VLAN 49, # MTU 9000, no default route — used by Rook for OSD↔OSD replication via @@ -35,6 +40,18 @@ machine: addresses: - 10.99.0.1/29 mtu: 9000 + sysctls: + # v30iot NIC carries no host addressing (see interfaces above); this + # disables v6 entirely (incl. link-local) so the host is unreachable on that L2. + # + # NAME DEPENDENCY: the interface above is selected by busPath, but a + # sysctl can only key on the kernel name. `0000:06:13.0` -> `eth1` is + # verified on k8s-{1,2,3} (same fact the three IoT NADs rely on). If PCI + # probe order ever renames it, this sysctl no-ops SILENTLY and the host + # keeps a v6 address on IoT — so the rollout MUST confirm `eth1` is + # addressless per node before trusting it (a wrong name here fails quiet, + # unlike the NADs which fail loud with a NotReady pod). + net.ipv6.conf.eth1.disable_ipv6: "1" install: disk: /dev/vda certSANs: diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index eac62727..1d290bb3 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -8,17 +8,19 @@ machine: dhcp: true vip: ip: 10.2.40.60 + # v30iot: link only, no host addressing — see k8s-1.yaml. - deviceSelector: busPath: "0000:06:13.0" - dhcp: true - dhcpOptions: - routeMetric: 4096 + dhcp: false # See k8s-1.yaml for context on the dedicated Ceph cluster network. - deviceSelector: driver: i40e addresses: - 10.99.0.2/29 mtu: 9000 + sysctls: + # v30iot NIC carries no host addressing — see k8s-1.yaml. + net.ipv6.conf.eth1.disable_ipv6: "1" install: disk: /dev/vda certSANs: diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index 08ee28b0..a9dbe1de 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -8,17 +8,19 @@ machine: dhcp: true vip: ip: 10.2.40.60 + # v30iot: link only, no host addressing — see k8s-1.yaml. - deviceSelector: busPath: "0000:06:13.0" - dhcp: true - dhcpOptions: - routeMetric: 4096 + dhcp: false # See k8s-1.yaml for context on the dedicated Ceph cluster network. - deviceSelector: driver: i40e addresses: - 10.99.0.3/29 mtu: 9000 + sysctls: + # v30iot NIC carries no host addressing — see k8s-1.yaml. + net.ipv6.conf.eth1.disable_ipv6: "1" install: disk: /dev/vda certSANs: