Stable predictable interface names (net.ifnames=1) — fixes the v30iot NIC shuffle #375
@@ -131,9 +131,9 @@ steps:
|
||||
- |
|
||||
# Ensure Talos image exists on all Proxmox nodes via API
|
||||
TALOS_VERSION="v1.12.6"
|
||||
TALOS_SCHEMATIC="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
|
||||
TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
|
||||
TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
|
||||
TALOS_FILENAME="talos-$TALOS_VERSION-nocloud-amd64.raw"
|
||||
TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
|
||||
API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
|
||||
for node in jupiter saturn neptune; do
|
||||
EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
|
||||
|
||||
@@ -50,7 +50,7 @@ spec:
|
||||
# above is additive. No explicit edge exclusion is needed: the
|
||||
# `workload-class=pinned:NoSchedule` taint on k8s-edge already
|
||||
# keeps this pod off there (we never tolerate it), and the NAD's
|
||||
# `master: eth1` lines up with v30iot on every k8s-{1,2,3} worker.
|
||||
# `master: ens19` (stable predictable name for the v30iot NIC) is present on every k8s-{1,2,3} worker.
|
||||
enableServiceLinks: false
|
||||
# Grants shelly-manager the in-cluster identity used by the firmware
|
||||
# pipeline to create esphome flash Jobs and stream their logs. RBAC is
|
||||
|
||||
@@ -21,18 +21,15 @@
|
||||
# referenced. Requires the cni-dhcp-daemon DaemonSet in
|
||||
# infrastructure/multus.
|
||||
#
|
||||
# Multus's `master` is an interface name. On every cluster node v30iot
|
||||
# lands on `eth1`: k8s-edge has only two NICs (mgmt=eth0, v30iot=eth1)
|
||||
# and on k8s-{1,2,3} the PCI probe order is mgmt virtio (eth0) → v30iot
|
||||
# virtio (eth1) → X710 i40e (eth2). The earlier comment that "eth1 is
|
||||
# the X710" inverted this; empirically verified via `talosctl get links`
|
||||
# (eth1 carries the bus_path=0000:06:13.0 virtio across k8s-{1,2,3}).
|
||||
#
|
||||
# Caveat: this is PCI-probe-order-dependent. Talos v1.12's LinkAliasConfig
|
||||
# doesn't propagate to the kernel `IFLA_IFNAME`, only to Talos-internal
|
||||
# selectors, so an upstream macvlan CNI's `master: <alias>` cannot
|
||||
# resolve it. Until a CNI build that supports altnames or a MAC-based
|
||||
# selector lands, `master: eth1` is the only working option here.
|
||||
# Multus's `master` is an interface name. On the k8s-{1,2,3} workers the
|
||||
# v30iot NIC (busPath 0000:06:13.0) is `ens19` — a STABLE, busPath-derived
|
||||
# predictable name (net.ifnames=1, via the Image Factory schematic). It is
|
||||
# identical across the three identical VMs and does NOT depend on driver
|
||||
# probe order, so it cannot shuffle across reboots. (Before the schematic
|
||||
# change the cluster ran net.ifnames=0, giving driver-probe-order eth0/1/2
|
||||
# that renamed on reboot and broke this master reference — the bug this
|
||||
# fixed. LinkAliasConfig cannot help: it only sets a cosmetic IFLA_IFALIAS
|
||||
# that CNI master can't resolve.)
|
||||
apiVersion: k8s.cni.cncf.io/v1
|
||||
kind: NetworkAttachmentDefinition
|
||||
metadata:
|
||||
@@ -54,7 +51,7 @@ spec:
|
||||
"plugins": [
|
||||
{
|
||||
"type": "macvlan",
|
||||
"master": "eth1",
|
||||
"master": "ens19",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "dhcp"
|
||||
|
||||
@@ -21,9 +21,10 @@ spec:
|
||||
# commissioning and discovery need real L2 presence on the IoT
|
||||
# subnet — multicast send/receive — which a Cilium-managed pod
|
||||
# IP or a LoadBalancer Service IP cannot provide. The NAD leases
|
||||
# the address by DHCP and uses master: eth1, which is the
|
||||
# v30iot interface on every node (k8s-1/2/3 and k8s-edge alike),
|
||||
# so this pod is no longer tied to k8s-edge.
|
||||
# the address by DHCP and uses master: ens19, the stable predictable
|
||||
# v30iot interface name on the k8s-{1,2,3} workers (which run
|
||||
# net.ifnames=1). This pod is pinned off k8s-edge by the
|
||||
# workload-class taint (no toleration), so edge's naming is moot.
|
||||
k8s.v1.cni.cncf.io/networks: matter-server-iot
|
||||
spec:
|
||||
# Cluster pod network is the primary attachment (eth0, used by HA
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
---
|
||||
# matter-server's second NIC: macvlan attached to k8s-edge's v30iot
|
||||
# interface (eth1 — verified via `talosctl -n k8s-edge get routes`,
|
||||
# 10.2.30.0/24 is connected on eth1; the second NIC defined in
|
||||
# vm_153_k8s_edge.tf via `extra_network_bridges = ["v30iot"]`). Gives
|
||||
# the pod a real IP on 10.2.30.0/24 with full L2 presence — multicast
|
||||
# send/receive included — which Matter device commissioning and
|
||||
# discovery need on top of the Cilium-managed pod network.
|
||||
# matter-server's second NIC: macvlan attached to the v30iot interface
|
||||
# (`ens19` — the stable busPath-derived name for 0000:06:13.0 on the
|
||||
# k8s-{1,2,3} workers, net.ifnames=1). Gives the pod a real IP on
|
||||
# 10.2.30.0/24 with full L2 presence — multicast send/receive included —
|
||||
# which Matter device commissioning and discovery need on top of the
|
||||
# Cilium-managed pod network. (This pod is pinned off k8s-edge by the
|
||||
# workload-class taint, so only the worker naming matters.)
|
||||
#
|
||||
# The address is leased from the UniFi DHCP server on v30iot, like every
|
||||
# other device on that VLAN. This pod doesn't go through Cilium IPAM —
|
||||
@@ -23,7 +23,7 @@
|
||||
#
|
||||
# No gateway is set on this attachment: default route stays on the
|
||||
# pod's primary (Cilium-managed) interface, so non-IoT egress doesn't
|
||||
# leak through eth1. The macvlan only needs the connected /24 to talk
|
||||
# leak through ens19. The macvlan only needs the connected /24 to talk
|
||||
# to devices on the IoT subnet; that's exactly what static IPAM with
|
||||
# just the address gives it.
|
||||
apiVersion: k8s.cni.cncf.io/v1
|
||||
@@ -58,7 +58,7 @@ spec:
|
||||
"plugins": [
|
||||
{
|
||||
"type": "macvlan",
|
||||
"master": "eth1",
|
||||
"master": "ens19",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "dhcp"
|
||||
|
||||
@@ -17,9 +17,10 @@ data:
|
||||
|
||||
- publish_ip is the *literal* IP advertised to players (Sendspin / Snapcast /
|
||||
SlimProto). It must be a v30iot address the players can reach. MA's
|
||||
auto-detect always picks the mgmt default-route IP instead (eth1/v30iot
|
||||
carries routeMetric 4096, so it is never the default route), which players
|
||||
on the IoT VLAN cannot route to.
|
||||
auto-detect always picks the pod eth0 (cluster CIDR) address instead —
|
||||
the pod's default route — which players on the IoT VLAN cannot route to.
|
||||
The v30iot address lives on the net1 macvlan child (see
|
||||
network-attachment.yaml).
|
||||
- bind_ip must be 0.0.0.0. The Sendspin server (aiosendspin, port 8927) has
|
||||
NO all-interfaces fallback: if bind_ip is a specific IP that is not local
|
||||
it fails to bind and the server never starts, and its proxy then targets
|
||||
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
# replaces the former hostNetwork attachment (see
|
||||
# network-attachment.yaml) so the NODE no longer holds an address
|
||||
# on the compromised IoT VLAN; MA gets its own leased IoT IP on
|
||||
# net1 instead. master: eth1 is v30iot on every k8s-{1,2,3}/edge
|
||||
# net1 instead. master: ens19 is the v30iot NIC on every k8s-{1,2,3}/edge
|
||||
# worker.
|
||||
k8s.v1.cni.cncf.io/networks: music-assistant-iot
|
||||
spec:
|
||||
@@ -40,10 +40,9 @@ spec:
|
||||
# MA advertises a stored publish_ip to players (Sendspin / Snapcast /
|
||||
# SlimProto) and binds its stream + Sendspin servers to a stored
|
||||
# bind_ip. Both must track this node's *current* v30iot address, but:
|
||||
# - MA's auto-detect picks the mgmt default-route IP (v30iot/eth1 has
|
||||
# routeMetric 4096, never the default route), which IoT players
|
||||
# can't reach; and
|
||||
# - eth1 is DHCP and MA is not node-pinned, so any hard-coded IP goes
|
||||
# - MA's auto-detect picks the pod eth0 (cluster CIDR) address,
|
||||
# which IoT players can't reach; and
|
||||
# - net1 is DHCP and MA is not node-pinned, so any hard-coded IP goes
|
||||
# stale on the next reschedule or lease change (the Sendspin server
|
||||
# then fails to bind and dies, with no fallback).
|
||||
# So we recompute the v30iot IP from the live interface every boot and
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
# music-assistant's second NIC: macvlan attached to the node's v30iot
|
||||
# interface (eth1 — the connected 10.2.30.0/24 on every k8s-{1,2,3}/edge
|
||||
# worker). Gives the pod a real leased IP on the IoT VLAN with full L2
|
||||
# interface (ens19 — the connected 10.2.30.0/24 on the k8s-{1,2,3}
|
||||
# workers). Gives the pod a real leased IP on the IoT VLAN with full L2
|
||||
# presence, which the audio-sync protocols (Snapcast / SlimProto /
|
||||
# Sendspin) require: group members must share an L2 broadcast domain or
|
||||
# the sync timing budget breaks. macvlan bridge mode puts net1 frames
|
||||
@@ -11,7 +11,7 @@
|
||||
#
|
||||
# This replaces the former hostNetwork attachment. The reconcile
|
||||
# initContainer (configmap-net-reconcile.yaml) discovers net1's leased
|
||||
# address the same way it used to discover the host's eth1 address —
|
||||
# address the same way it used to discover the host's ens19 address —
|
||||
# scanning the pod's own interfaces for a 10.2.30.0/24 address — and
|
||||
# writes it as MA's publish_ip. bind_ip stays 0.0.0.0, which now binds
|
||||
# eth0 (pod) + net1; players reach the stream/slimproto/sendspin servers
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
"plugins": [
|
||||
{
|
||||
"type": "macvlan",
|
||||
"master": "eth1",
|
||||
"master": "ens19",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "dhcp"
|
||||
|
||||
@@ -41,7 +41,7 @@ spec:
|
||||
k8s.v1.cni.cncf.io/networks: otbr-iot
|
||||
spec:
|
||||
# No nodeSelector/toleration: OTBR only needs v30iot L2 presence
|
||||
# (macvlan master: eth1, present on every node), not the edge USB
|
||||
# (macvlan master: ens19, present on every node), not the edge USB
|
||||
# devices, and its Thread RCP is reached over the network
|
||||
# (slzb-thread.renner.uno). Dropping the workload-class=pinned
|
||||
# toleration lets the edge taint keep it off the tiny edge box.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
# OTBR's second NIC: macvlan attached to k8s-edge's v30iot interface
|
||||
# (eth1; same parent matter-server uses). Mirrors the matter-server NAD
|
||||
# OTBR's second NIC: macvlan attached to the v30iot interface
|
||||
# (`ens19`; same parent matter-server uses). Mirrors the matter-server NAD
|
||||
# pattern. OTBR needs real L2 presence on the IoT subnet to:
|
||||
# - advertise the Thread Border Router service via mDNS/SRP so Matter
|
||||
# commissioners and Thread devices on the LAN can find it
|
||||
@@ -24,7 +24,7 @@
|
||||
#
|
||||
# No gateway is set: default route stays on the pod's primary
|
||||
# (Cilium-managed) interface so non-IoT egress doesn't leak through
|
||||
# eth1. The macvlan only needs the connected /24 to talk to devices on
|
||||
# ens19. The macvlan only needs the connected /24 to talk to devices on
|
||||
# the IoT subnet.
|
||||
apiVersion: k8s.cni.cncf.io/v1
|
||||
kind: NetworkAttachmentDefinition
|
||||
@@ -44,7 +44,7 @@ spec:
|
||||
"plugins": [
|
||||
{
|
||||
"type": "macvlan",
|
||||
"master": "eth1",
|
||||
"master": "ens19",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "dhcp"
|
||||
|
||||
@@ -20,9 +20,9 @@ machine:
|
||||
node.kubernetes.io/exclude-from-external-load-balancers: ""
|
||||
kubelet:
|
||||
# Constrain kubelet's auto-picked node IP to the mgmt LAN. Without
|
||||
# this, kubelet picks the first non-loopback IP and depending on
|
||||
# interface ordering can land on eth1/v30iot (10.2.30.0/24).
|
||||
# Control-plane traffic + hostNetwork pod IPs must stay on mgmt LAN.
|
||||
# this, kubelet picks the first non-loopback IP, which could land on
|
||||
# the v30iot NIC (ens19, 10.2.30.0/24). Control-plane traffic +
|
||||
# hostNetwork pod IPs must stay on mgmt LAN.
|
||||
nodeIP:
|
||||
validSubnets:
|
||||
- 10.2.40.0/24
|
||||
|
||||
@@ -44,14 +44,14 @@ machine:
|
||||
# v30iot NIC carries no host addressing (see interfaces above); this
|
||||
# disables v6 entirely (incl. link-local) so the host is unreachable on that L2.
|
||||
#
|
||||
# NAME DEPENDENCY: the interface above is selected by busPath, but a
|
||||
# sysctl can only key on the kernel name. `0000:06:13.0` -> `eth1` is
|
||||
# verified on k8s-{1,2,3} (same fact the three IoT NADs rely on). If PCI
|
||||
# probe order ever renames it, this sysctl no-ops SILENTLY and the host
|
||||
# keeps a v6 address on IoT — so the rollout MUST confirm `eth1` is
|
||||
# addressless per node before trusting it (a wrong name here fails quiet,
|
||||
# unlike the NADs which fail loud with a NotReady pod).
|
||||
net.ipv6.conf.eth1.disable_ipv6: "1"
|
||||
# Keyed on the STABLE predictable name `ens19` (busPath 0000:06:13.0 =
|
||||
# v30iot). Predictable naming is re-enabled via the Image Factory
|
||||
# schematic (net.ifnames=1; see .woodpecker/deploy-infrastructure.yml),
|
||||
# so `ens19` is topology-derived and does NOT shuffle across reboots —
|
||||
# unlike the former `eth0/1/2` (driver-probe order) which renamed on a
|
||||
# k8s-3 reboot and silently mis-targeted this sysctl onto mgmt. The four
|
||||
# v30iot macvlan NADs key on the same `ens19`.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
install:
|
||||
disk: /dev/vda
|
||||
certSANs:
|
||||
|
||||
@@ -20,7 +20,7 @@ machine:
|
||||
mtu: 9000
|
||||
sysctls:
|
||||
# v30iot NIC carries no host addressing — see k8s-1.yaml.
|
||||
net.ipv6.conf.eth1.disable_ipv6: "1"
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
install:
|
||||
disk: /dev/vda
|
||||
certSANs:
|
||||
|
||||
@@ -20,7 +20,7 @@ machine:
|
||||
mtu: 9000
|
||||
sysctls:
|
||||
# v30iot NIC carries no host addressing — see k8s-1.yaml.
|
||||
net.ipv6.conf.eth1.disable_ipv6: "1"
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
install:
|
||||
disk: /dev/vda
|
||||
certSANs:
|
||||
|
||||
+11
-3
@@ -1,5 +1,13 @@
|
||||
locals {
|
||||
talos_version = "v1.12.6"
|
||||
talos_filename = "talos-${local.talos_version}-nocloud-amd64.raw"
|
||||
talos_image = "local:import/${local.talos_filename}"
|
||||
talos_version = "v1.12.6"
|
||||
# Image Factory schematic (predictable naming: net.ifnames=1). Keep in
|
||||
# sync with TALOS_SCHEMATIC in .woodpecker/deploy-infrastructure.yml —
|
||||
# that pipeline downloads the image, this local names it for VM import,
|
||||
# and both MUST agree on the filename. The schematic hash is embedded in
|
||||
# the filename so a schematic bump changes the name, forcing a fresh
|
||||
# download instead of silently reusing a stale cached image (which would
|
||||
# reintroduce net.ifnames=0 on the next VM (re)provision).
|
||||
talos_schematic = "88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
|
||||
talos_filename = "talos-${local.talos_version}-${substr(local.talos_schematic, 0, 12)}-nocloud-amd64.raw"
|
||||
talos_image = "local:import/${local.talos_filename}"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user