From 35015868416a7fbb023f27cc28945bdf8152a5ce Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 2 Nov 2026 08:42:49 +0100 Subject: [PATCH] fix(talos): route LoadBalancer replies through table 47 A reply from a Cilium LoadBalancer VIP reaches the routing decision with the VIP as its source, so the pod-source rules 1000/1001 missed it and main sent it out mgmt0 while the request came in over ingress0, the BGP next hop (k8s-3, 2026-11-02: replies from 10.2.99.90 to an IoT device seen only on mgmt0). Rules 1002/1003 send the pool to table 47. The pool becomes a site fact: each site patch carries lb-pool-site (flux-system), and bgp-pool takes its blocks from it by Flux substitution in infrastructure-crds. The CIDR blocks cover the same addresses as the start/stop ranges did (allowFirstLastIPs defaults to Yes). The qemu and hcloud sites carry the production pool, since Services pin their addresses from it. Refs #1547 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0185Qb5YG28FfnVi1ujMEV5C --- kubernetes/NETWORK-traffic-classes.md | 7 +++++ .../clusters/homelab/infrastructure.yaml | 3 +++ .../infrastructure/post-crds/cilium-bgp.yaml | 10 ++++--- kubernetes/talos/patches/site-hcloud.yaml | 12 +++++++++ kubernetes/talos/patches/site-homelab.yaml | 27 +++++++++++++++++++ kubernetes/talos/patches/site-qemu.yaml | 12 +++++++++ 6 files changed, 67 insertions(+), 4 deletions(-) diff --git a/kubernetes/NETWORK-traffic-classes.md b/kubernetes/NETWORK-traffic-classes.md index 6d99f223..632c5c58 100644 --- a/kubernetes/NETWORK-traffic-classes.md +++ b/kubernetes/NETWORK-traffic-classes.md @@ -22,6 +22,13 @@ Two rules, one per family, send every packet with a pod source to table 47 (`talos/patches/site-homelab.yaml`, priorities 1000/1001, below main). The table decides by longest prefix; no rule carves out exceptions. +A reply from a LoadBalancer VIP is pod traffic as well, but it reaches the +routing decision with the VIP as its source. Two more rules (1002/1003) send +the LoadBalancer pool to table 47, so the reply leaves over `ingress0`, where +the request arrived as the BGP next hop. Without them main sent it out +`mgmt0`. The pool is a site fact: the site patch's `lb-pool-site` ConfigMap, +which `bgp-pool` takes by Flux substitution, and the rules beside it. + bird (`kubernetes/infrastructure/bird/`) owns table 47 and fills it with: - the node's attached networks — `mgmt0`, `ingress0`, `pod0`, `node0` — diff --git a/kubernetes/clusters/homelab/infrastructure.yaml b/kubernetes/clusters/homelab/infrastructure.yaml index 1892cf91..abf68ffe 100644 --- a/kubernetes/clusters/homelab/infrastructure.yaml +++ b/kubernetes/clusters/homelab/infrastructure.yaml @@ -89,6 +89,9 @@ spec: substituteFrom: - kind: ConfigMap name: cluster-settings + # The LoadBalancer pool, from the site's Talos patch (site-*.yaml). + - kind: ConfigMap + name: lb-pool-site dependsOn: - name: infrastructure --- diff --git a/kubernetes/infrastructure/post-crds/cilium-bgp.yaml b/kubernetes/infrastructure/post-crds/cilium-bgp.yaml index 7ed8d510..44900d60 100644 --- a/kubernetes/infrastructure/post-crds/cilium-bgp.yaml +++ b/kubernetes/infrastructure/post-crds/cilium-bgp.yaml @@ -8,6 +8,10 @@ # Selector is opt-in via `lb-pool: bgp`. There is no default pool — # Services without this label do not get a LoadBalancer IP. # +# The ranges are a site fact: lb-pool-site from the site's Talos patch +# (talos/patches/site-*.yaml), by Flux substitution. A CIDR block uses all its +# addresses (allowFirstLastIPs defaults to Yes). +# # IPv4 + IPv6 ranges keep matching last bytes (10.2.99.x ↔ fd99::x) # so each LB has the same "tail" in both families. apiVersion: cilium.io/v2alpha1 @@ -19,10 +23,8 @@ spec: matchLabels: lb-pool: bgp blocks: - - start: "10.2.99.0" - stop: "10.2.99.255" - - start: "fd99::" - stop: "fd99::ff" + - cidr: "${LB_POOL_V4}" + - cidr: "${LB_POOL_V6}" --- # Cilium BGPv2 control-plane config peering each node with the UDM over the # ingress VLAN 47 — never over v30iot, which is the untrusted IoT segment. diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index 1aa2c78b..b162755d 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -67,6 +67,18 @@ cluster: nameservers: - 185.12.64.1 - 185.12.64.2 + # The LoadBalancer pool (see site-homelab.yaml). Services pin their + # addresses from the production pool, so a test run carries the same one. + - name: lb-pool-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: lb-pool-site + namespace: flux-system + data: + LB_POOL_V4: 10.2.99.0/24 + LB_POOL_V6: fd99::/120 --- # Host DNS asks k8s-gateway first, so containerd resolves in-cluster names # (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index f347f3ce..d59c0850 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -85,6 +85,18 @@ cluster: nameservers: - fd02:47::1 - 10.2.47.1 + # The LoadBalancer pool (bgp-pool, infrastructure/post-crds/cilium-bgp.yaml, + # by Flux substitution). The routing rules 1002/1003 below name it too. + - name: lb-pool-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: lb-pool-site + namespace: flux-system + data: + LB_POOL_V4: 10.2.99.0/24 + LB_POOL_V6: fd99::/120 --- # All pod traffic goes to table 47, and the table decides — longest prefix # match, no exceptions to carve out. bird fills it with the node's attached @@ -115,6 +127,21 @@ name: "1001" src: 10.244.0.0/16 table: "47" --- +# A LoadBalancer reply is pod traffic too, but it reaches the routing decision +# with the VIP as its source, so the rules above miss it and main sends it out +# mgmt0 while the request came in over ingress0. The pool is lb-pool-site above. +apiVersion: v1alpha1 +kind: RoutingRuleConfig +name: "1002" +src: fd99::/120 +table: "47" +--- +apiVersion: v1alpha1 +kind: RoutingRuleConfig +name: "1003" +src: 10.2.99.0/24 +table: "47" +--- # Host DNS asks k8s-gateway first, so containerd resolves in-cluster names # (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right # after the CNI, before anything else needs DNS. The UDM is the site's diff --git a/kubernetes/talos/patches/site-qemu.yaml b/kubernetes/talos/patches/site-qemu.yaml index 077d9954..79df6183 100644 --- a/kubernetes/talos/patches/site-qemu.yaml +++ b/kubernetes/talos/patches/site-qemu.yaml @@ -47,3 +47,15 @@ cluster: values.yaml: | deployment: dnsPolicy: Default + # The LoadBalancer pool (see site-homelab.yaml). Services pin their + # addresses from the production pool, so a test run carries the same one. + - name: lb-pool-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: lb-pool-site + namespace: flux-system + data: + LB_POOL_V4: 10.2.99.0/24 + LB_POOL_V6: fd99::/120 -- 2.54.0