7 changed files with 112 additions and 6 deletions
@@ -8,7 +8,7 @@ Issue #576. Five classes, each on its own VLAN.
 |---|---|---|
 | Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) |
 | pod↔pod | `pod0`, VLAN 48 | pod-sourced traffic is routed from table 47; bird puts the peers' pod CIDRs there via `pod0` |
-| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress takes table 47's default |
+| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress and VIP replies take table 47's default |
 | node↔node | `node0`, VLAN 46 | the node InternalIP and etcd live there; so do apiserver↔kubelet, cilium-health and host→remote-pod traffic |
 | world↔node | `mgmt0`, VLAN 40 | API VIP for kubectl, talosctl from outside, node egress (DNS, NTP, image pulls) |

@@ -41,6 +41,15 @@ address, or a hostNetwork pod — would otherwise take the default to the UDM.
 The routing decision runs before Cilium's masquerade hook, so the rule sees the
 pod address and the masquerade then rewrites it to the egress link's.

+A reply from a LoadBalancer VIP is the exception: Cilium reverses the service
+NAT before the packet reaches the routing decision, so its source is the VIP
+and the pod rules miss it. bird's `vip-rules` container adds one rule per
+block of every `CiliumLoadBalancerIPPool` (priority 1002, `from <block>
+lookup 47`) and keeps them in sync with the pools every minute. The blocks
+must be CIDRs; a start/stop block stops the container. Without these rules
+the reply takes main's default over `mgmt0` while the request arrived on
+`ingress0` (#1547).
+
 Node-originated traffic does not match the rules and uses main, where Cilium
 routes each peer's pod CIDR via the peer's InternalIP — on `node0`.

@@ -14,6 +14,10 @@ rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list"]
+  # vip-rules reads the LoadBalancer pool blocks it routes into table 47.
+  - apiGroups: ["cilium.io"]
+    resources: ["ciliumloadbalancerippools"]
+    verbs: ["get", "list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -111,6 +115,24 @@ spec:
          volumeMounts:
            - name: etc-bird
              mountPath: /etc/bird
+        # Routes replies from LoadBalancer VIPs into table 47 (vip-rules.sh).
+        - name: vip-rules
+          image: gitea.renner.uno/gitea_admin/bird@sha256:65fc93efead1e9b065d2c78e748f6c3c1ad7216e28b757d9fa4e204fa2ffe9a9
+          command: ["/bin/sh", "/config/vip-rules.sh"]
+          resources:
+            requests:
+              cpu: "5m"
+              memory: "16Mi"
+            limits:
+              cpu: "100m"
+              memory: "64Mi"
+          securityContext:
+            capabilities:
+              drop: ["ALL"]
+              add: ["NET_ADMIN"]
+          volumeMounts:
+            - name: config
+              mountPath: /config
      volumes:
        - name: config
          configMap:
@@ -10,3 +10,5 @@ configMapGenerator:
      - bird.conf
      - defines.tmpl
      - peers.tmpl
+      - vip-rules.sh
+      - vip-rules.tmpl
@@ -0,0 +1,66 @@
+#!/bin/sh
+# Keeps one routing rule per LoadBalancer pool block: from <block> lookup 47.
+#
+# A reply from a VIP reaches the routing decision with the VIP as its source:
+# Cilium reverses the service NAT before the packet enters the stack. The pod
+# rules (1000/1001, talos/patches/site-homelab.yaml) do not match it, so
+# without these rules it takes main's default over mgmt0 while its request
+# arrived on ingress0. The blocks are read from the pools, so the prefix is
+# written only there.
+#
+# Rules are left in place on exit: if bird is gone, table 47 is empty and the
+# lookup falls through to main, as it does for the pod rules.
+set -eu
+
+PRIO=1002
+TABLE=47
+
+family_of() {
+	case $1 in *:*) echo 6 ;; *) echo 4 ;; esac
+}
+
+reconcile() {
+	# An API error skips this round; the rules in place stay.
+	if ! blocks=$(kubectl get ciliumloadbalancerippools.cilium.io \
+		-o go-template-file=/config/vip-rules.tmpl); then
+		echo "reading the pools failed; retrying" >&2
+		return 0
+	fi
+	for b in $blocks; do
+		case $b in
+		range:*)
+			echo "pool block ${b#range:} is a range; only cidr blocks map to a rule" >&2
+			return 1
+			;;
+		esac
+	done
+
+	for fam in 4 6; do
+		want=$(for b in $blocks; do
+			if [ "$(family_of "$b")" = "$fam" ]; then echo "$b"; fi
+		done | sort -u)
+		have=$(ip -"$fam" rule show |
+			awk -v p="$PRIO:" -v t="$TABLE" '$1 == p && $2 == "from" && $4 == "lookup" && $5 == t { print $3 }' |
+			sort -u)
+
+		for b in $have; do
+			if ! echo "$want" | grep -qxF "$b"; then
+				ip -"$fam" rule del priority "$PRIO" from "$b" table "$TABLE"
+				echo "removed: priority $PRIO from $b table $TABLE"
+			fi
+		done
+		for b in $want; do
+			if ! echo "$have" | grep -qxF "$b"; then
+				ip -"$fam" rule add priority "$PRIO" from "$b" table "$TABLE"
+				echo "added: priority $PRIO from $b table $TABLE"
+			fi
+		done
+	done
+}
+
+trap 'exit 0' TERM INT
+while :; do
+	reconcile
+	sleep 60 &
+	wait $!
+done
@@ -0,0 +1,5 @@
+{{- /* One line per block of every enabled CiliumLoadBalancerIPPool. A start/stop
+     block renders as range:<start>-<stop>, which vip-rules.sh refuses. */ -}}
+{{- range .items }}{{ if not .spec.disabled }}{{ range .spec.blocks }}
+{{- if .cidr }}{{ .cidr }}{{ else }}range:{{ .start }}-{{ .stop }}{{ end }}
+{{ end }}{{ end }}{{ end -}}
@@ -10,6 +10,9 @@
 #
 # IPv4 + IPv6 ranges keep matching last bytes (10.2.99.x ↔ fd99::x)
 # so each LB has the same "tail" in both families.
+#
+# Blocks are CIDRs: bird's vip-rules turns each into a routing rule into
+# table 47, so VIP replies leave on ingress0 (infrastructure-inhouse/bird).
 apiVersion: cilium.io/v2alpha1
 kind: CiliumLoadBalancerIPPool
 metadata:
@@ -19,10 +22,8 @@ spec:
    matchLabels:
      lb-pool: bgp
  blocks:
-    - start: "10.2.99.0"
-      stop: "10.2.99.255"
-    - start: "fd99::"
-      stop: "fd99::ff"
+    - cidr: "10.2.99.0/24"
+    - cidr: "fd99::/120"
 ---
 # Cilium BGPv2 control-plane config peering each node with the UDM over the
 # ingress VLAN 47 — never over v30iot, which is the untrusted IoT segment.
@@ -102,7 +102,8 @@ cluster:
 #
 # Priorities are below main (32766) and clear of the rule Cilium installs at 9.
 # The two families are separate rule tables, so the priorities need not match;
-# the document name is the priority and must be unique.
+# the document name is the priority and must be unique. 1002 is taken by bird's
+# vip-rules: replies from LoadBalancer VIPs, whose source is the VIP.
 apiVersion: v1alpha1
 kind: RoutingRuleConfig
 name: "1000"