feat(talos): node network on VLAN 46, etcd pinned to VLAN 40 (#576 step 1)

- node0: VLANConfig on the uplink trunk, VLAN 46, static fd02:46::1x/64
  and 10.2.46.1x/24, accept_ra 0. Closed like pod0. Unused until the node
  IP moves there (step 3).
- cluster.etcd.advertisedSubnets pinned to 10.2.40.0/24 so a reboot
  cannot pick a node0 address: Talos never updates an existing member's
  peer URLs.

Applied live with talosctl patch --mode=no-reboot, k8s-3 first. Node IPs
unchanged, etcd 3/3 healthy, node0 reachable node to node over v6 and v4.

Refs #576

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -118,14 +118,21 @@ spec:
       # traffic out to the VIP and back (which fails with L2 announcements).
       lbExternalClusterIP: true
     cni:
-      # Multus (kubernetes/infrastructure/multus) runs alongside Cilium and
-      # writes 00-multus.conf to /etc/cni/net.d so kubelet picks multus
-      # first; multus then delegates to Cilium's 05-cilium.conflist via
-      # clusterNetwork. The default `exclusive: true` makes cilium-cni's
-      # install step rename anything alphabetically before its own config
-      # to *.cilium_bak on every reconcile, which kept nuking multus.
-      # Setting it false lets multus's config coexist.
+      # Multus is the only writer of /etc/cni/net.d (#832): Cilium's conflist
+      # goes to /etc/cni/net.d/multus/, which the container runtime does not
+      # read, and Multus delegates to it via clusterNetwork. Without this a
+      # pod created after a reboot, before Multus rewrote 00-multus.conf, came
+      # up with Cilium alone. See kubernetes/infrastructure/multus.
+      confPath: /etc/cni/net.d/multus
+      # The default `exclusive: true` renames other configs in the conf dir to
+      # *.cilium_bak on every reconcile, which kept nuking multus.
       exclusive: false
+    # One agent at a time: the chart default of 2 restarts two of three
+    # nodes' datapath at once.
+    updateStrategy:
+      type: RollingUpdate
+      rollingUpdate:
+        maxUnavailable: 1
     # Talos-specific settings
     securityContext:
       capabilities:

kubernetes/infrastructure/multus/multus-daemonset.yaml

@@ -111,10 +111,18 @@ metadata:
   name: multus
   namespace: kube-system
 ---
+# Multus is the only writer of /etc/cni/net.d (#832). Cilium writes its conflist
+# to /etc/cni/net.d/multus/, a subdirectory the container runtime does not read
+# (libcni skips directories); Multus picks it up there via multusAutoconfigDir
+# and writes 00-multus.conf. Until that file exists the runtime has no network
+# config at all, so no pod can be created with Cilium alone after a reboot.
+#
+# The "-subdir" names date from the node-by-node rollout; renaming would roll
+# Multus again for nothing.
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: multus-daemon-config
+  name: multus-daemon-config-subdir
   namespace: kube-system
   labels:
     tier: node
@@ -127,7 +135,7 @@ data:
         "logLevel": "verbose",
         "logToStderr": true,
         "cniConfigDir": "/host/etc/cni/net.d",
-        "multusAutoconfigDir": "/host/etc/cni/net.d",
+        "multusAutoconfigDir": "/host/etc/cni/net.d/multus",
         "multusConfigFile": "auto",
         "socketDir": "/host/run/multus/"
     }
@@ -135,16 +143,16 @@ data:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kube-multus-ds
+  name: kube-multus-ds-subdir
   namespace: kube-system
   labels:
     tier: node
     app: multus
-    name: multus
+    name: multus-subdir
 spec:
   selector:
     matchLabels:
-      name: multus
+      name: multus-subdir
   updateStrategy:
     type: RollingUpdate
   template:
@@ -152,7 +160,7 @@ spec:
       labels:
         tier: node
         app: multus
-        name: multus
+        name: multus-subdir
     spec:
       # Upstream ships this DaemonSet with no priority class, so it runs at 0
       # and can preempt nothing. Any event that makes the pod need a fresh
@@ -224,6 +232,10 @@ spec:
           volumeMounts:
             - name: cni
               mountPath: /host/etc/cni/net.d
+            # Exists before Multus starts, so it waits for Cilium's conflist
+            # instead of exiting on a missing directory.
+            - name: cni-cilium-conf
+              mountPath: /host/etc/cni/net.d/multus
             # multus-daemon expects that cnibin path must be identical between pod and container host.
             # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
             # not to any other directory, like '/opt/bin' or '/usr/bin'.
@@ -279,6 +291,10 @@ spec:
         - name: cni
           hostPath:
             path: /etc/cni/net.d
+        - name: cni-cilium-conf
+          hostPath:
+            path: /etc/cni/net.d/multus
+            type: DirectoryOrCreate
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
@@ -287,7 +303,7 @@ spec:
             path: /
         - name: multus-daemon-config
           configMap:
-            name: multus-daemon-config
+            name: multus-daemon-config-subdir
             items:
               - key: daemon-config.json
                 path: daemon-config.json

kubernetes/infrastructure/multus/whereabouts-reconciler.yaml

@@ -39,9 +39,11 @@ spec:
         name: whereabouts-reconciler
     spec:
       serviceAccountName: whereabouts
-      # Every node here is a control plane node.
+      # Every node here is a control plane node. Keyed: a keyless Exists would
+      # also tolerate node.kubernetes.io/unschedulable and ignore cordons.
       tolerations:
-        - operator: Exists
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
           effect: NoSchedule
       containers:
         - name: whereabouts-reconciler

kubernetes/talos/patches/cluster.yaml

@@ -58,6 +58,12 @@ machine:
       imageGCHighThresholdPercent: 70
       imageGCLowThresholdPercent: 60
 cluster:
+  # etcd advertises on the management VLAN until the node↔node move (#576
+  # step 3). Pinned so the node0 addresses cannot be picked on a reboot: Talos
+  # never updates an existing member's peer URLs.
+  etcd:
+    advertisedSubnets:
+      - 10.2.40.0/24
   # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its
   # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443
   # there. Setting it again in this patch restated the same value in a second

kubernetes/talos/patches/k8s-1.yaml

@@ -17,6 +17,8 @@ machine:
     # pod0 is a closed L2 fabric between the three nodes: statically
     # addressed, no gateway, nothing to learn from an RA.
     net.ipv6.conf.pod0.accept_ra: "0"
+    # node0 is closed the same way.
+    net.ipv6.conf.node0.accept_ra: "0"
     # ingress0 keeps RAs, at the default accept_ra of 2: an IPv6 default
     # router is a link-local address learned only from the RA, and with
     # net.ipv6.conf.all.forwarding=1 — which Cilium sets — Linux ignores RAs
@@ -177,6 +179,19 @@ addresses:
   - address: 10.2.48.11/24
   - address: fd02:48::11/64
 ---
+# Node network, VLAN 46: traffic among the three nodes themselves (#576) — the
+# node IP, etcd, apiserver↔kubelet. Closed like pod0: statically addressed, no
+# gateway, never routed via the UDM. Unused until the node IP moves here.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::11/64
+  - address: 10.2.46.11/24
+---
 # Ingress network, VLAN 47. World-to-pod: the BGP session, the next hop for
 # every advertised LoadBalancer VIP, and the masquerade source for pod egress.
 # Publicly routed, which is what separates it from pod0.

kubernetes/talos/patches/k8s-2.yaml

@@ -16,6 +16,7 @@ machine:
     # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
     # and only the default route is held back — see k8s-1.yaml.
     net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.node0.accept_ra: "0"
     net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
@@ -122,6 +123,17 @@ addresses:
   - address: 10.2.48.12/24
   - address: fd02:48::12/64
 ---
+# Node network, VLAN 46 — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::12/64
+  - address: 10.2.46.12/24
+---
 # Ingress network, VLAN 47. World-to-pod, publicly routed, statically
 # addressed — see k8s-1.yaml for why it is not leased.
 apiVersion: v1alpha1

kubernetes/talos/patches/k8s-3.yaml

@@ -12,6 +12,7 @@ machine:
     # ingress0 keeps RAs: it learns its IPv6 router and its address from them,
     # and only the default route is held back — see k8s-1.yaml.
     net.ipv6.conf.pod0.accept_ra: "0"
+    net.ipv6.conf.node0.accept_ra: "0"
     net.ipv6.conf.ingress0.accept_ra_defrtr: "0"
     # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
     net.ipv4.conf.all.arp_ignore: "1"
@@ -109,6 +110,17 @@ addresses:
   - address: 10.2.48.13/24
   - address: fd02:48::13/64
 ---
+# Node network, VLAN 46 — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: node0
+vlanID: 46
+parent: uplink
+up: true
+addresses:
+  - address: fd02:46::13/64
+  - address: 10.2.46.13/24
+---
 # Ingress network, VLAN 47. World-to-pod, publicly routed, statically
 # addressed — see k8s-1.yaml for why it is not leased.
 apiVersion: v1alpha1