diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index f2872850..3970cba5 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -39,6 +39,7 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" # needs is read out of this file at run time so there is no second copy of a # version or a value to keep in step. See PROPOSAL-cilium-single-owner.md. CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml" +K8SGW_HR="$REPO_ROOT/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml" ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging" NODES=(k8s-1 k8s-2 k8s-3) # No dedicated workers since the k8s-edge VM was retired (2026-07-02) — @@ -1320,16 +1321,93 @@ EOF yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES" grep -q 'policyMapMax' "$CILIUM_VALUES" \ || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR" + # The site's part, from the cilium-site ConfigMap in the site patch: the + # HelmRelease reads it through valuesFrom, under its own values. + CILIUM_SITE_VALUES=$(mktemp) + yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "cilium-site") | .contents' \ + "$TALOS_DIR/patches/site-${SITE}.yaml" \ + | yq -r '.data["values.yaml"]' > "$CILIUM_SITE_VALUES" + grep -q 'routingMode' "$CILIUM_SITE_VALUES" \ + || fail "no cilium-site values (routingMode) in talos/patches/site-${SITE}.yaml" + + # Cilium's cni.confPath is a subdirectory of /etc/cni/net.d that Multus + # creates (#832). /etc/cni is read-only to the kubelet on Talos, so the + # agent's DirectoryOrCreate mount cannot make it, and on an empty cluster + # Multus is not there yet. A hostNetwork pod per node creates it; there is + # no pod network before Cilium. + cni_conf_dir=$(yq -r '.cni.confPath' "$CILIUM_VALUES") + if [[ "$cni_conf_dir" == /etc/cni/net.d/* ]]; then + step 0 "Creating $cni_conf_dir on every node..." + # kube-controller-manager creates kube-system/default shortly after the API answers. + for i in $(seq 1 30); do + kubectl -n kube-system get serviceaccount default >/dev/null && break + sleep 5 + [[ $i -eq 30 ]] && fail "kube-system/default ServiceAccount never appeared" + done + # A pod bound to a node that has not registered yet is garbage collected. + for node in "${ALL_NODES[@]}"; do + for i in $(seq 1 60); do + kubectl get node "$node" >/dev/null && break + sleep 5 + [[ $i -eq 60 ]] && fail "$node never registered with the API server" + done + done + for node in "${ALL_NODES[@]}"; do + kubectl apply -f - >/dev/null </dev/null \ + || fail "creating $cni_conf_dir on $node did not succeed (kubectl -n kube-system describe pod cni-confdir-$node)" + kubectl -n kube-system delete pod "cni-confdir-$node" >/dev/null + done + ok + fi helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null helm repo update cilium >/dev/null helm upgrade --install cilium cilium/cilium \ --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \ --namespace kube-system \ + -f "$CILIUM_SITE_VALUES" \ -f "$CILIUM_VALUES" \ --set prometheus.serviceMonitor.enabled=false \ --wait --timeout 10m - rm -f "$CILIUM_VALUES" + rm -f "$CILIUM_VALUES" "$CILIUM_SITE_VALUES" + ok + + # Multus is the only writer of /etc/cni/net.d (#832); without it the kubelet + # reports "cni plugin not initialized" and nothing schedules, Flux included. + # Applied from the manifests Flux owns, which adopts them. + step 0 "Installing Multus (the CNI entry point)..." + kubectl apply -k "$REPO_ROOT/kubernetes/infrastructure/multus" >/dev/null \ + || fail "could not apply kubernetes/infrastructure/multus" + kubectl -n kube-system rollout status daemonset/kube-multus-ds-subdir --timeout=10m >/dev/null \ + || fail "Multus never became ready (kubectl -n kube-system get pods -l app=multus)" ok step 0 "Waiting for K8s nodes..." @@ -1368,6 +1446,32 @@ EOF fi done + # The nodes' resolver asks k8s-gateway first (ResolverConfig in the site + # patch), and CoreDNS forwards external names to the node. Until k8s-gateway + # answers, every lookup waits out the node's 2 s read timeout and CoreDNS + # gives up first: pods resolve nothing, Flux included. So it is installed + # here, from the values Flux will reconcile (the HelmRelease, under the + # site's k8s-gateway-site values, as valuesFrom merges them), and Flux + # adopts the release. + step 0 "Installing k8s-gateway (the nodes' first resolver)..." + k8sgw_values=$(mktemp) + k8sgw_site_values=$(mktemp) + yq -y 'select(.kind == "HelmRelease") | .spec.values' "$K8SGW_HR" > "$k8sgw_values" + yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "k8s-gateway-site") | .contents' \ + "$TALOS_DIR/patches/site-${SITE}.yaml" \ + | yq -r '.data["values.yaml"]' > "$k8sgw_site_values" + grep -q 'dnsPolicy' "$k8sgw_site_values" \ + || fail "no k8s-gateway-site values (dnsPolicy) in talos/patches/site-${SITE}.yaml" + helm upgrade --install k8s-gateway \ + "$(yq -r 'select(.kind == "OCIRepository") | .spec.url' "$K8SGW_HR")" \ + --namespace kube-system \ + -f "$k8sgw_site_values" \ + -f "$k8sgw_values" \ + --wait --timeout 5m >/dev/null \ + || fail "k8s-gateway did not come up (kubectl -n kube-system get pods -l app.kubernetes.io/name=k8s-gateway)" + rm -f "$k8sgw_values" "$k8sgw_site_values" + ok + # Stash configs for subsequent phases / export-access step. Skipped under # vmtest: those are a throwaway cluster's credentials, and writing them here # would leave the flux phase pointed at VMs that no longer exist. diff --git a/kubernetes/infrastructure/cilium/helmrelease.yaml b/kubernetes/infrastructure/cilium/helmrelease.yaml index cae0738c..ec50484e 100644 --- a/kubernetes/infrastructure/cilium/helmrelease.yaml +++ b/kubernetes/infrastructure/cilium/helmrelease.yaml @@ -184,26 +184,13 @@ spec: # Cilium's, the sessions re-establish. bgpControlPlane: enabled: true - # Masquerading exception, not a routing statement: pod traffic to these - # ranges leaves unmasqueraded, so LAN services see real pod IPs. That only - # works because BGP advertises the pod CIDRs, which is what gives the UDM a - # route back. The two settings are one decision and have to move together. - # - # Pod CIDRs are 10.244.0.0/16 and fd10:244::/48 (talos/patches/cluster.yaml); - # the supernets here cover the LAN as well, which is the point. - ipv4NativeRoutingCIDR: 10.0.0.0/8 - ipv6NativeRoutingCIDR: fd00::/8 - # Pod east-west leaves the node on pod0 (VLAN 48) rather than the - # management VLAN. Native routing hands forwarding to the kernel; - # autoDirectNodeRoutes installs the remote PodCIDR routes, which is - # allowed because all three nodes share pod0's L2 segment. - # - # directRoutingDevice is explicit on purpose: with several addressed - # devices Cilium falls back to the one carrying the Kubernetes InternalIP, - # which is mgmt0. - # - # Encapsulation is gone, so the datapath MTU goes 1450 -> 1500. - routingMode: native - autoDirectNodeRoutes: true - nodePort: - directRoutingDevice: pod0 + # What the site's network can carry decides how pod traffic crosses between + # nodes and which destinations see real pod IPs: routingMode, + # autoDirectNodeRoutes, the direct-routing device and the native-routing + # (masquerade-exempt) CIDRs come from the cilium-site ConfigMap that each + # Talos site patch carries (talos/patches/site-*.yaml). They must not be set + # in values above, which would override it. + valuesFrom: + - kind: ConfigMap + name: cilium-site + valuesKey: values.yaml diff --git a/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml b/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml index 20c99f49..903853ee 100644 --- a/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml +++ b/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml @@ -19,13 +19,10 @@ spec: # `auth.renner.uno`, `audiobooks.renner.uno`, etc. stay inside the # cluster instead of hairpinning through the legacy Caddy VIP. # -# Names not backed by an in-cluster Ingress fall through to the UDM -# (10.2.40.1 / fd00::1), which serves the split-horizon renner.uno zone — -# preserving resolution of LXC/VM-hosted services (gitea, immich, …) and -# physical devices (udm.renner.uno, …) without manual enumeration. -# NOTE: the old Proxmox DNS HA VIP (10.2.40.53) is DECOMMISSIONED; pointing -# here dead-ended non-Ingress resolution and broke ~1374 HA entities on a -# pod restart (2026-07-14). Do not reintroduce it. +# Names not backed by an in-cluster Ingress fall through to the site's +# resolver: the pod's own resolv.conf, which the site patch sets through the +# k8s-gateway-site ConfigMap (deployment.dnsPolicy/dnsConfig). Never the node's +# resolver: the nodes ask k8s-gateway first, which would loop. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -95,7 +92,7 @@ spec: - name: loop - name: reload - name: forward - parameters: ". 10.2.40.1 fd00::1" + parameters: ". /etc/resolv.conf" - name: prometheus parameters: "0.0.0.0:9153" serviceType: ClusterIP @@ -106,3 +103,10 @@ spec: # populated end of the service CIDR (kube-dns is 10.96.0.10). clusterIP: 10.96.0.11 ipFamilyPolicy: PreferDualStack + # The site's upstream resolvers (deployment.dnsPolicy/dnsConfig), from the + # k8s-gateway-site ConfigMap each Talos site patch carries. Not set in values + # above, which would override it. + valuesFrom: + - kind: ConfigMap + name: k8s-gateway-site + valuesKey: values.yaml diff --git a/kubernetes/talos/patches/cluster.yaml b/kubernetes/talos/patches/cluster.yaml index 6541a69b..e0e5d41e 100644 --- a/kubernetes/talos/patches/cluster.yaml +++ b/kubernetes/talos/patches/cluster.yaml @@ -58,12 +58,7 @@ machine: imageGCHighThresholdPercent: 70 imageGCLowThresholdPercent: 60 cluster: - # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an - # existing member's peer URLs, so changing this subnet means leave + rejoin - # per member (talosctl etcd leave, reboot). - etcd: - advertisedSubnets: - - 10.2.46.0/24 + # etcd.advertisedSubnets is in the site patch: it names the site's node segment. # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443 # there. Setting it again in this patch restated the same value in a second diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index 1c3fead3..3f7aa0de 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -54,6 +54,12 @@ machine: # this label instead of listing hostnames, so which boxes carry storage is # stated once, here, by the machine that knows. storage.renner.uno/ceph-osd: "true" + # This machine has the Intel iGPU. The GPU plugin and everything that needs + # gpu.intel.com run only on nodes with this label. + hardware.renner.uno/intel-gpu: "true" + # This machine is on the IoT VLAN: ens19, declared below in its VLANConfig. + # The macvlan workloads on v30iot run only on nodes with this label. + net.renner.uno/iot-vlan: "true" # Publishes this node's pod0 addresses, declared below in its VLANConfig, so # the bird DaemonSet reads its peers from the node list rather than from a # copy. Annotations, not labels: a label value cannot contain a colon. diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index a5ceabf1..32b16c79 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -34,6 +34,10 @@ machine: topology.kubernetes.io/zone: saturn # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" + # See k8s-1.yaml. + net.renner.uno/iot-vlan: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::12 diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index a26b57e5..20012676 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -30,6 +30,10 @@ machine: topology.kubernetes.io/zone: neptune # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" + # See k8s-1.yaml. + net.renner.uno/iot-vlan: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::13 diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index 4f790a2d..32df7d7d 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -10,18 +10,77 @@ machine: # The node IP lives on node0, VLAN 46: the closed node↔node network (#576). # Both families are listed so the Node object carries an IPv6 InternalIP at # all; without one Cilium detects the node's IPv6 itself and picks a SLAAC - # address from the dynamic provider prefix (#789). Talos picks one address - # per family, IPv4 first, whatever the order here. + # address from the dynamic provider prefix (#789). + # + # Order is load-bearing: Talos keeps this order, the first entry is the + # primary node IP, and kube-apiserver requires its family to match the + # first service subnet (IPv4, cluster.yaml). nodeIP: validSubnets: - - fd02:46::/64 - 10.2.46.0/24 + - fd02:46::/64 # The UDM on the ingress segment. Published here rather than as a route # because bird owns table 47 and originates the default from it; this is the # one place the address is stated. nodeAnnotations: net.renner.uno/egress-gw-v6: fd02:47::1 net.renner.uno/egress-gw-v4: 10.2.47.1 + # Pod traffic is routed into table 47 here (RoutingRuleConfig below); bird + # fills that table and runs only on nodes with this label. + nodeLabels: + net.renner.uno/pod-table: "47" +cluster: + # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an + # existing member's peer URLs, so changing this subnet means leave + rejoin + # per member (talosctl etcd leave, reboot). + etcd: + advertisedSubnets: + - 10.2.46.0/24 + # The site's part of Cilium's values, read by the cilium HelmRelease + # (valuesFrom) and by bootstrap.sh. Pod east-west leaves on pod0 (VLAN 48): + # native routing, and autoDirectNodeRoutes installs the remote PodCIDR routes + # because all nodes share pod0's L2 segment. directRoutingDevice is explicit: + # otherwise Cilium takes the InternalIP device, mgmt0. The native-routing + # CIDRs are a masquerading exception: pod traffic to the LAN keeps its pod + # IP, which works because BGP advertises the pod CIDRs to the UDM — the two + # settings move together. Talos only creates this ConfigMap; a changed one + # is synced with talosctl upgrade-k8s. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: native + autoDirectNodeRoutes: true + nodePort: + directRoutingDevice: pod0 + ipv4NativeRoutingCIDR: 10.0.0.0/8 + ipv6NativeRoutingCIDR: fd00::/8 + # The site's upstream for k8s-gateway (its pod resolv.conf): the UDM. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: None + dnsConfig: + nameservers: + - 10.2.40.1 + - fd00::1 --- # All pod traffic goes to table 47, and the table decides — longest prefix # match, no exceptions to carve out. bird fills it with the node's attached @@ -53,9 +112,9 @@ src: 10.244.0.0/16 table: "47" --- # Host DNS asks k8s-gateway first, so containerd resolves in-cluster names -# (gitea.renner.uno) in the cluster. The UDM answers while k8s-gateway is not -# reachable yet; Talos moves on after a 2 s read timeout. This list replaces -# the nameservers from DHCP. +# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right +# after the CNI, before anything else needs DNS. The UDM is the site's +# resolver after it. This list replaces the nameservers from DHCP. apiVersion: v1alpha1 kind: ResolverConfig nameservers: diff --git a/kubernetes/talos/patches/site-qemu.yaml b/kubernetes/talos/patches/site-qemu.yaml index fd22df2f..077d9954 100644 --- a/kubernetes/talos/patches/site-qemu.yaml +++ b/kubernetes/talos/patches/site-qemu.yaml @@ -13,3 +13,37 @@ machine: nodeIP: validSubnets: - 10.5.0.0/24 +cluster: + # The site's part of Cilium's values (see site-homelab.yaml). The nodes share + # the provisioner's bridge, one L2 segment; Cilium picks its only NIC. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: native + autoDirectNodeRoutes: true + ipv4NativeRoutingCIDR: 10.0.0.0/8 + ipv6NativeRoutingCIDR: fd00::/8 + # The nodes keep the provisioner's resolver (no ResolverConfig here), so + # k8s-gateway can use the node's. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: Default