From d0da46b0542dd0e7f1c705defdf397c325cf51ee Mon Sep 17 00:00:00 2001 From: hermes-bot Date: Mon, 2 Nov 2026 06:36:08 +0000 Subject: [PATCH] fix(bird): route LoadBalancer VIP replies into table 47 (#1547) A reply from a LoadBalancer VIP reaches the kernel's routing decision with the VIP as its source: Cilium reverses the service NAT in from-container before the packet enters the stack. The pod-source rules 1000/1001 do not match it, so it took main's default over mgmt0 while the request had arrived on ingress0 as the BGP next hop. bird's DaemonSet gets a vip-rules container that reads every enabled CiliumLoadBalancerIPPool and keeps one rule per block, priority 1002, `from lookup 47`, reconciled every minute. The pool stays the only place the prefixes are written. Like the pod rules, a rule into an empty table 47 (bird gone) falls through to main. bgp-pool's blocks move from start/stop to the equivalent CIDRs so each block maps to one rule. Cilium v1.20.4 LB-IPAM compares a modified pool's blocks by first and last address (handlePoolModified, LBRange.EqualCIDR), and allowFirstLastIPs unset keeps .0/.255 and fd99::/fd99::ff allocatable, so the ranges are kept and no service is reallocated. Closes #1547 --- kubernetes/NETWORK-traffic-classes.md | 11 +++- .../bird/daemonset.yaml | 22 +++++++ .../bird/kustomization.yaml | 2 + .../infrastructure-inhouse/bird/vip-rules.sh | 66 +++++++++++++++++++ .../bird/vip-rules.tmpl | 5 ++ .../infrastructure/post-crds/cilium-bgp.yaml | 9 +-- kubernetes/talos/patches/site-homelab.yaml | 3 +- 7 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 kubernetes/infrastructure-inhouse/bird/vip-rules.sh create mode 100644 kubernetes/infrastructure-inhouse/bird/vip-rules.tmpl diff --git a/kubernetes/NETWORK-traffic-classes.md b/kubernetes/NETWORK-traffic-classes.md index 6d99f223..5f3656bf 100644 --- a/kubernetes/NETWORK-traffic-classes.md +++ b/kubernetes/NETWORK-traffic-classes.md @@ -8,7 +8,7 @@ Issue #576. Five classes, each on its own VLAN. |---|---|---| | Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) | | pod↔pod | `pod0`, VLAN 48 | pod-sourced traffic is routed from table 47; bird puts the peers' pod CIDRs there via `pod0` | -| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress takes table 47's default | +| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress and VIP replies take table 47's default | | node↔node | `node0`, VLAN 46 | the node InternalIP and etcd live there; so do apiserver↔kubelet, cilium-health and host→remote-pod traffic | | world↔node | `mgmt0`, VLAN 40 | API VIP for kubectl, talosctl from outside, node egress (DNS, NTP, image pulls) | @@ -41,6 +41,15 @@ address, or a hostNetwork pod — would otherwise take the default to the UDM. The routing decision runs before Cilium's masquerade hook, so the rule sees the pod address and the masquerade then rewrites it to the egress link's. +A reply from a LoadBalancer VIP is the exception: Cilium reverses the service +NAT before the packet reaches the routing decision, so its source is the VIP +and the pod rules miss it. bird's `vip-rules` container adds one rule per +block of every `CiliumLoadBalancerIPPool` (priority 1002, `from +lookup 47`) and keeps them in sync with the pools every minute. The blocks +must be CIDRs; a start/stop block stops the container. Without these rules +the reply takes main's default over `mgmt0` while the request arrived on +`ingress0` (#1547). + Node-originated traffic does not match the rules and uses main, where Cilium routes each peer's pod CIDR via the peer's InternalIP — on `node0`. diff --git a/kubernetes/infrastructure-inhouse/bird/daemonset.yaml b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml index e7d91d64..86f3ef35 100644 --- a/kubernetes/infrastructure-inhouse/bird/daemonset.yaml +++ b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml @@ -14,6 +14,10 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list"] + # vip-rules reads the LoadBalancer pool blocks it routes into table 47. + - apiGroups: ["cilium.io"] + resources: ["ciliumloadbalancerippools"] + verbs: ["get", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -111,6 +115,24 @@ spec: volumeMounts: - name: etc-bird mountPath: /etc/bird + # Routes replies from LoadBalancer VIPs into table 47 (vip-rules.sh). + - name: vip-rules + image: gitea.renner.uno/gitea_admin/bird@sha256:65fc93efead1e9b065d2c78e748f6c3c1ad7216e28b757d9fa4e204fa2ffe9a9 + command: ["/bin/sh", "/config/vip-rules.sh"] + resources: + requests: + cpu: "5m" + memory: "16Mi" + limits: + cpu: "100m" + memory: "64Mi" + securityContext: + capabilities: + drop: ["ALL"] + add: ["NET_ADMIN"] + volumeMounts: + - name: config + mountPath: /config volumes: - name: config configMap: diff --git a/kubernetes/infrastructure-inhouse/bird/kustomization.yaml b/kubernetes/infrastructure-inhouse/bird/kustomization.yaml index 447fbce2..7c310b3e 100644 --- a/kubernetes/infrastructure-inhouse/bird/kustomization.yaml +++ b/kubernetes/infrastructure-inhouse/bird/kustomization.yaml @@ -10,3 +10,5 @@ configMapGenerator: - bird.conf - defines.tmpl - peers.tmpl + - vip-rules.sh + - vip-rules.tmpl diff --git a/kubernetes/infrastructure-inhouse/bird/vip-rules.sh b/kubernetes/infrastructure-inhouse/bird/vip-rules.sh new file mode 100644 index 00000000..a576c322 --- /dev/null +++ b/kubernetes/infrastructure-inhouse/bird/vip-rules.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# Keeps one routing rule per LoadBalancer pool block: from lookup 47. +# +# A reply from a VIP reaches the routing decision with the VIP as its source: +# Cilium reverses the service NAT before the packet enters the stack. The pod +# rules (1000/1001, talos/patches/site-homelab.yaml) do not match it, so +# without these rules it takes main's default over mgmt0 while its request +# arrived on ingress0. The blocks are read from the pools, so the prefix is +# written only there. +# +# Rules are left in place on exit: if bird is gone, table 47 is empty and the +# lookup falls through to main, as it does for the pod rules. +set -eu + +PRIO=1002 +TABLE=47 + +family_of() { + case $1 in *:*) echo 6 ;; *) echo 4 ;; esac +} + +reconcile() { + # An API error skips this round; the rules in place stay. + if ! blocks=$(kubectl get ciliumloadbalancerippools.cilium.io \ + -o go-template-file=/config/vip-rules.tmpl); then + echo "reading the pools failed; retrying" >&2 + return 0 + fi + for b in $blocks; do + case $b in + range:*) + echo "pool block ${b#range:} is a range; only cidr blocks map to a rule" >&2 + return 1 + ;; + esac + done + + for fam in 4 6; do + want=$(for b in $blocks; do + if [ "$(family_of "$b")" = "$fam" ]; then echo "$b"; fi + done | sort -u) + have=$(ip -"$fam" rule show | + awk -v p="$PRIO:" -v t="$TABLE" '$1 == p && $2 == "from" && $4 == "lookup" && $5 == t { print $3 }' | + sort -u) + + for b in $have; do + if ! echo "$want" | grep -qxF "$b"; then + ip -"$fam" rule del priority "$PRIO" from "$b" table "$TABLE" + echo "removed: priority $PRIO from $b table $TABLE" + fi + done + for b in $want; do + if ! echo "$have" | grep -qxF "$b"; then + ip -"$fam" rule add priority "$PRIO" from "$b" table "$TABLE" + echo "added: priority $PRIO from $b table $TABLE" + fi + done + done +} + +trap 'exit 0' TERM INT +while :; do + reconcile + sleep 60 & + wait $! +done diff --git a/kubernetes/infrastructure-inhouse/bird/vip-rules.tmpl b/kubernetes/infrastructure-inhouse/bird/vip-rules.tmpl new file mode 100644 index 00000000..818956b3 --- /dev/null +++ b/kubernetes/infrastructure-inhouse/bird/vip-rules.tmpl @@ -0,0 +1,5 @@ +{{- /* One line per block of every enabled CiliumLoadBalancerIPPool. A start/stop + block renders as range:-, which vip-rules.sh refuses. */ -}} +{{- range .items }}{{ if not .spec.disabled }}{{ range .spec.blocks }} +{{- if .cidr }}{{ .cidr }}{{ else }}range:{{ .start }}-{{ .stop }}{{ end }} +{{ end }}{{ end }}{{ end -}} diff --git a/kubernetes/infrastructure/post-crds/cilium-bgp.yaml b/kubernetes/infrastructure/post-crds/cilium-bgp.yaml index 7ed8d510..e323a924 100644 --- a/kubernetes/infrastructure/post-crds/cilium-bgp.yaml +++ b/kubernetes/infrastructure/post-crds/cilium-bgp.yaml @@ -10,6 +10,9 @@ # # IPv4 + IPv6 ranges keep matching last bytes (10.2.99.x ↔ fd99::x) # so each LB has the same "tail" in both families. +# +# Blocks are CIDRs: bird's vip-rules turns each into a routing rule into +# table 47, so VIP replies leave on ingress0 (infrastructure-inhouse/bird). apiVersion: cilium.io/v2alpha1 kind: CiliumLoadBalancerIPPool metadata: @@ -19,10 +22,8 @@ spec: matchLabels: lb-pool: bgp blocks: - - start: "10.2.99.0" - stop: "10.2.99.255" - - start: "fd99::" - stop: "fd99::ff" + - cidr: "10.2.99.0/24" + - cidr: "fd99::/120" --- # Cilium BGPv2 control-plane config peering each node with the UDM over the # ingress VLAN 47 — never over v30iot, which is the untrusted IoT segment. diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index f347f3ce..0b7a07a5 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -102,7 +102,8 @@ cluster: # # Priorities are below main (32766) and clear of the rule Cilium installs at 9. # The two families are separate rule tables, so the priorities need not match; -# the document name is the priority and must be unique. +# the document name is the priority and must be unique. 1002 is taken by bird's +# vip-rules: replies from LoadBalancer VIPs, whose source is the VIP. apiVersion: v1alpha1 kind: RoutingRuleConfig name: "1000" -- 2.54.0