kubernetes/bootstrap.sh

@@ -39,6 +39,7 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 # needs is read out of this file at run time so there is no second copy of a
 # version or a value to keep in step. See PROPOSAL-cilium-single-owner.md.
 CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml"
+K8SGW_HR="$REPO_ROOT/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml"
 ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging"
 NODES=(k8s-1 k8s-2 k8s-3)
 # No dedicated workers since the k8s-edge VM was retired (2026-07-02) —
@@ -1320,16 +1321,93 @@ EOF
     yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES"
     grep -q 'policyMapMax' "$CILIUM_VALUES" \
         || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR"
+    # The site's part, from the cilium-site ConfigMap in the site patch: the
+    # HelmRelease reads it through valuesFrom, under its own values.
+    CILIUM_SITE_VALUES=$(mktemp)
+    yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "cilium-site") | .contents' \
+        "$TALOS_DIR/patches/site-${SITE}.yaml" \
+        | yq -r '.data["values.yaml"]' > "$CILIUM_SITE_VALUES"
+    grep -q 'routingMode' "$CILIUM_SITE_VALUES" \
+        || fail "no cilium-site values (routingMode) in talos/patches/site-${SITE}.yaml"
+
+    # Cilium's cni.confPath is a subdirectory of /etc/cni/net.d that Multus
+    # creates (#832). /etc/cni is read-only to the kubelet on Talos, so the
+    # agent's DirectoryOrCreate mount cannot make it, and on an empty cluster
+    # Multus is not there yet. A hostNetwork pod per node creates it; there is
+    # no pod network before Cilium.
+    cni_conf_dir=$(yq -r '.cni.confPath' "$CILIUM_VALUES")
+    if [[ "$cni_conf_dir" == /etc/cni/net.d/* ]]; then
+        step 0 "Creating $cni_conf_dir on every node..."
+        # kube-controller-manager creates kube-system/default shortly after the API answers.
+        for i in $(seq 1 30); do
+            kubectl -n kube-system get serviceaccount default >/dev/null && break
+            sleep 5
+            [[ $i -eq 30 ]] && fail "kube-system/default ServiceAccount never appeared"
+        done
+        # A pod bound to a node that has not registered yet is garbage collected.
+        for node in "${ALL_NODES[@]}"; do
+            for i in $(seq 1 60); do
+                kubectl get node "$node" >/dev/null && break
+                sleep 5
+                [[ $i -eq 60 ]] && fail "$node never registered with the API server"
+            done
+        done
+        for node in "${ALL_NODES[@]}"; do
+            kubectl apply -f - >/dev/null <<EOF || fail "could not create the cni-confdir pod on $node"
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cni-confdir-$node
+  namespace: kube-system
+spec:
+  nodeName: $node
+  hostNetwork: true
+  restartPolicy: Never
+  tolerations:
+    - operator: Exists
+  containers:
+    - name: mkdir
+      image: docker.io/library/busybox:1.37
+      command: ["mkdir", "-p", "/host/${cni_conf_dir#/etc/cni/net.d/}"]
+      volumeMounts:
+        - name: netd
+          mountPath: /host
+  volumes:
+    - name: netd
+      hostPath:
+        path: /etc/cni/net.d
+        type: Directory
+EOF
+        done
+        for node in "${ALL_NODES[@]}"; do
+            kubectl -n kube-system wait --for=jsonpath='{.status.phase}'=Succeeded \
+                "pod/cni-confdir-$node" --timeout=5m >/dev/null \
+                || fail "creating $cni_conf_dir on $node did not succeed (kubectl -n kube-system describe pod cni-confdir-$node)"
+            kubectl -n kube-system delete pod "cni-confdir-$node" >/dev/null
+        done
+        ok
+    fi
 
     helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null
     helm repo update cilium >/dev/null
     helm upgrade --install cilium cilium/cilium \
         --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \
         --namespace kube-system \
+        -f "$CILIUM_SITE_VALUES" \
         -f "$CILIUM_VALUES" \
         --set prometheus.serviceMonitor.enabled=false \
         --wait --timeout 10m
-    rm -f "$CILIUM_VALUES"
+    rm -f "$CILIUM_VALUES" "$CILIUM_SITE_VALUES"
+    ok
+
+    # Multus is the only writer of /etc/cni/net.d (#832); without it the kubelet
+    # reports "cni plugin not initialized" and nothing schedules, Flux included.
+    # Applied from the manifests Flux owns, which adopts them.
+    step 0 "Installing Multus (the CNI entry point)..."
+    kubectl apply -k "$REPO_ROOT/kubernetes/infrastructure/multus" >/dev/null \
+        || fail "could not apply kubernetes/infrastructure/multus"
+    kubectl -n kube-system rollout status daemonset/kube-multus-ds-subdir --timeout=10m >/dev/null \
+        || fail "Multus never became ready (kubectl -n kube-system get pods -l app=multus)"
     ok
 
     step 0 "Waiting for K8s nodes..."
@@ -1368,6 +1446,32 @@ EOF
         fi
     done
 
+    # The nodes' resolver asks k8s-gateway first (ResolverConfig in the site
+    # patch), and CoreDNS forwards external names to the node. Until k8s-gateway
+    # answers, every lookup waits out the node's 2 s read timeout and CoreDNS
+    # gives up first: pods resolve nothing, Flux included. So it is installed
+    # here, from the values Flux will reconcile (the HelmRelease, under the
+    # site's k8s-gateway-site values, as valuesFrom merges them), and Flux
+    # adopts the release.
+    step 0 "Installing k8s-gateway (the nodes' first resolver)..."
+    k8sgw_values=$(mktemp)
+    k8sgw_site_values=$(mktemp)
+    yq -y 'select(.kind == "HelmRelease") | .spec.values' "$K8SGW_HR" > "$k8sgw_values"
+    yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "k8s-gateway-site") | .contents' \
+        "$TALOS_DIR/patches/site-${SITE}.yaml" \
+        | yq -r '.data["values.yaml"]' > "$k8sgw_site_values"
+    grep -q 'dnsPolicy' "$k8sgw_site_values" \
+        || fail "no k8s-gateway-site values (dnsPolicy) in talos/patches/site-${SITE}.yaml"
+    helm upgrade --install k8s-gateway \
+        "$(yq -r 'select(.kind == "OCIRepository") | .spec.url' "$K8SGW_HR")" \
+        --namespace kube-system \
+        -f "$k8sgw_site_values" \
+        -f "$k8sgw_values" \
+        --wait --timeout 5m >/dev/null \
+        || fail "k8s-gateway did not come up (kubectl -n kube-system get pods -l app.kubernetes.io/name=k8s-gateway)"
+    rm -f "$k8sgw_values" "$k8sgw_site_values"
+    ok
+
     # Stash configs for subsequent phases / export-access step. Skipped under
     # vmtest: those are a throwaway cluster's credentials, and writing them here
     # would leave the flux phase pointed at VMs that no longer exist.

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -184,26 +184,13 @@ spec:
     # Cilium's, the sessions re-establish.
     bgpControlPlane:
       enabled: true
-    # Masquerading exception, not a routing statement: pod traffic to these
-    # ranges leaves unmasqueraded, so LAN services see real pod IPs. That only
-    # works because BGP advertises the pod CIDRs, which is what gives the UDM a
-    # route back. The two settings are one decision and have to move together.
-    #
-    # Pod CIDRs are 10.244.0.0/16 and fd10:244::/48 (talos/patches/cluster.yaml);
-    # the supernets here cover the LAN as well, which is the point.
-    ipv4NativeRoutingCIDR: 10.0.0.0/8
-    ipv6NativeRoutingCIDR: fd00::/8
-    # Pod east-west leaves the node on pod0 (VLAN 48) rather than the
-    # management VLAN. Native routing hands forwarding to the kernel;
-    # autoDirectNodeRoutes installs the remote PodCIDR routes, which is
-    # allowed because all three nodes share pod0's L2 segment.
-    #
-    # directRoutingDevice is explicit on purpose: with several addressed
-    # devices Cilium falls back to the one carrying the Kubernetes InternalIP,
-    # which is mgmt0.
-    #
-    # Encapsulation is gone, so the datapath MTU goes 1450 -> 1500.
-    routingMode: native
-    autoDirectNodeRoutes: true
-    nodePort:
-      directRoutingDevice: pod0
+  # What the site's network can carry decides how pod traffic crosses between
+  # nodes and which destinations see real pod IPs: routingMode,
+  # autoDirectNodeRoutes, the direct-routing device and the native-routing
+  # (masquerade-exempt) CIDRs come from the cilium-site ConfigMap that each
+  # Talos site patch carries (talos/patches/site-*.yaml). They must not be set
+  # in values above, which would override it.
+  valuesFrom:
+    - kind: ConfigMap
+      name: cilium-site
+      valuesKey: values.yaml

kubernetes/infrastructure/k8s-gateway/helmrelease.yaml

@@ -19,13 +19,10 @@ spec:
 # `auth.renner.uno`, `audiobooks.renner.uno`, etc. stay inside the
 # cluster instead of hairpinning through the legacy Caddy VIP.
 #
-# Names not backed by an in-cluster Ingress fall through to the UDM
-# (10.2.40.1 / fd00::1), which serves the split-horizon renner.uno zone —
-# preserving resolution of LXC/VM-hosted services (gitea, immich, …) and
-# physical devices (udm.renner.uno, …) without manual enumeration.
-# NOTE: the old Proxmox DNS HA VIP (10.2.40.53) is DECOMMISSIONED; pointing
-# here dead-ended non-Ingress resolution and broke ~1374 HA entities on a
-# pod restart (2026-07-14). Do not reintroduce it.
+# Names not backed by an in-cluster Ingress fall through to the site's
+# resolver: the pod's own resolv.conf, which the site patch sets through the
+# k8s-gateway-site ConfigMap (deployment.dnsPolicy/dnsConfig). Never the node's
+# resolver: the nodes ask k8s-gateway first, which would loop.
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -95,7 +92,7 @@ spec:
           - name: loop
           - name: reload
           - name: forward
-            parameters: ". 10.2.40.1 fd00::1"
+            parameters: ". /etc/resolv.conf"
           - name: prometheus
             parameters: "0.0.0.0:9153"
     serviceType: ClusterIP
@@ -106,3 +103,10 @@ spec:
       # populated end of the service CIDR (kube-dns is 10.96.0.10).
       clusterIP: 10.96.0.11
       ipFamilyPolicy: PreferDualStack
+  # The site's upstream resolvers (deployment.dnsPolicy/dnsConfig), from the
+  # k8s-gateway-site ConfigMap each Talos site patch carries. Not set in values
+  # above, which would override it.
+  valuesFrom:
+    - kind: ConfigMap
+      name: k8s-gateway-site
+      valuesKey: values.yaml

kubernetes/talos/patches/cluster.yaml

@@ -58,12 +58,7 @@ machine:
       imageGCHighThresholdPercent: 70
       imageGCLowThresholdPercent: 60
 cluster:
-  # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an
-  # existing member's peer URLs, so changing this subnet means leave + rejoin
-  # per member (talosctl etcd leave, reboot).
-  etcd:
-    advertisedSubnets:
-      - 10.2.46.0/24
+  # etcd.advertisedSubnets is in the site patch: it names the site's node segment.
   # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its
   # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443
   # there. Setting it again in this patch restated the same value in a second

kubernetes/talos/patches/k8s-1.yaml

@@ -54,6 +54,12 @@ machine:
     # this label instead of listing hostnames, so which boxes carry storage is
     # stated once, here, by the machine that knows.
     storage.renner.uno/ceph-osd: "true"
+    # This machine has the Intel iGPU. The GPU plugin and everything that needs
+    # gpu.intel.com run only on nodes with this label.
+    hardware.renner.uno/intel-gpu: "true"
+    # This machine is on the IoT VLAN: ens19, declared below in its VLANConfig.
+    # The macvlan workloads on v30iot run only on nodes with this label.
+    net.renner.uno/iot-vlan: "true"
   # Publishes this node's pod0 addresses, declared below in its VLANConfig, so
   # the bird DaemonSet reads its peers from the node list rather than from a
   # copy. Annotations, not labels: a label value cannot contain a colon.

kubernetes/talos/patches/k8s-2.yaml

@@ -34,6 +34,10 @@ machine:
     topology.kubernetes.io/zone: saturn
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
+    # Has the Intel iGPU — see k8s-1.yaml.
+    hardware.renner.uno/intel-gpu: "true"
+    # See k8s-1.yaml.
+    net.renner.uno/iot-vlan: "true"
   # pod0 addresses for the bird peer list — see k8s-1.yaml.
   nodeAnnotations:
     net.renner.uno/pod0-v6: fd02:48::12

kubernetes/talos/patches/k8s-3.yaml

@@ -30,6 +30,10 @@ machine:
     topology.kubernetes.io/zone: neptune
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
+    # Has the Intel iGPU — see k8s-1.yaml.
+    hardware.renner.uno/intel-gpu: "true"
+    # See k8s-1.yaml.
+    net.renner.uno/iot-vlan: "true"
   # pod0 addresses for the bird peer list — see k8s-1.yaml.
   nodeAnnotations:
     net.renner.uno/pod0-v6: fd02:48::13

kubernetes/talos/patches/site-homelab.yaml

@@ -10,18 +10,77 @@ machine:
     # The node IP lives on node0, VLAN 46: the closed node↔node network (#576).
     # Both families are listed so the Node object carries an IPv6 InternalIP at
     # all; without one Cilium detects the node's IPv6 itself and picks a SLAAC
-    # address from the dynamic provider prefix (#789). Talos picks one address
-    # per family, IPv4 first, whatever the order here.
+    # address from the dynamic provider prefix (#789).
+    #
+    # Order is load-bearing: Talos keeps this order, the first entry is the
+    # primary node IP, and kube-apiserver requires its family to match the
+    # first service subnet (IPv4, cluster.yaml).
     nodeIP:
       validSubnets:
-        - fd02:46::/64
         - 10.2.46.0/24
+        - fd02:46::/64
   # The UDM on the ingress segment. Published here rather than as a route
   # because bird owns table 47 and originates the default from it; this is the
   # one place the address is stated.
   nodeAnnotations:
     net.renner.uno/egress-gw-v6: fd02:47::1
     net.renner.uno/egress-gw-v4: 10.2.47.1
+  # Pod traffic is routed into table 47 here (RoutingRuleConfig below); bird
+  # fills that table and runs only on nodes with this label.
+  nodeLabels:
+    net.renner.uno/pod-table: "47"
+cluster:
+  # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an
+  # existing member's peer URLs, so changing this subnet means leave + rejoin
+  # per member (talosctl etcd leave, reboot).
+  etcd:
+    advertisedSubnets:
+      - 10.2.46.0/24
+  # The site's part of Cilium's values, read by the cilium HelmRelease
+  # (valuesFrom) and by bootstrap.sh. Pod east-west leaves on pod0 (VLAN 48):
+  # native routing, and autoDirectNodeRoutes installs the remote PodCIDR routes
+  # because all nodes share pod0's L2 segment. directRoutingDevice is explicit:
+  # otherwise Cilium takes the InternalIP device, mgmt0. The native-routing
+  # CIDRs are a masquerading exception: pod traffic to the LAN keeps its pod
+  # IP, which works because BGP advertises the pod CIDRs to the UDM — the two
+  # settings move together. Talos only creates this ConfigMap; a changed one
+  # is synced with talosctl upgrade-k8s.
+  inlineManifests:
+    - name: cilium-site
+      contents: |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: cilium-site
+          namespace: kube-system
+          labels:
+            reconcile.fluxcd.io/watch: Enabled
+        data:
+          values.yaml: |
+            routingMode: native
+            autoDirectNodeRoutes: true
+            nodePort:
+              directRoutingDevice: pod0
+            ipv4NativeRoutingCIDR: 10.0.0.0/8
+            ipv6NativeRoutingCIDR: fd00::/8
+    # The site's upstream for k8s-gateway (its pod resolv.conf): the UDM.
+    - name: k8s-gateway-site
+      contents: |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: k8s-gateway-site
+          namespace: kube-system
+          labels:
+            reconcile.fluxcd.io/watch: Enabled
+        data:
+          values.yaml: |
+            deployment:
+              dnsPolicy: None
+              dnsConfig:
+                nameservers:
+                  - 10.2.40.1
+                  - fd00::1
 ---
 # All pod traffic goes to table 47, and the table decides — longest prefix
 # match, no exceptions to carve out. bird fills it with the node's attached
@@ -53,9 +112,9 @@ src: 10.244.0.0/16
 table: "47"
 ---
 # Host DNS asks k8s-gateway first, so containerd resolves in-cluster names
-# (gitea.renner.uno) in the cluster. The UDM answers while k8s-gateway is not
-# reachable yet; Talos moves on after a 2 s read timeout. This list replaces
-# the nameservers from DHCP.
+# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right
+# after the CNI, before anything else needs DNS. The UDM is the site's
+# resolver after it. This list replaces the nameservers from DHCP.
 apiVersion: v1alpha1
 kind: ResolverConfig
 nameservers:

kubernetes/talos/patches/site-qemu.yaml

@@ -13,3 +13,37 @@ machine:
     nodeIP:
       validSubnets:
         - 10.5.0.0/24
+cluster:
+  # The site's part of Cilium's values (see site-homelab.yaml). The nodes share
+  # the provisioner's bridge, one L2 segment; Cilium picks its only NIC.
+  inlineManifests:
+    - name: cilium-site
+      contents: |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: cilium-site
+          namespace: kube-system
+          labels:
+            reconcile.fluxcd.io/watch: Enabled
+        data:
+          values.yaml: |
+            routingMode: native
+            autoDirectNodeRoutes: true
+            ipv4NativeRoutingCIDR: 10.0.0.0/8
+            ipv6NativeRoutingCIDR: fd00::/8
+    # The nodes keep the provisioner's resolver (no ResolverConfig here), so
+    # k8s-gateway can use the node's.
+    - name: k8s-gateway-site
+      contents: |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: k8s-gateway-site
+          namespace: kube-system
+          labels:
+            reconcile.fluxcd.io/watch: Enabled
+        data:
+          values.yaml: |
+            deployment:
+              dnsPolicy: Default