diff --git a/kubernetes/NETWORK-traffic-classes.md b/kubernetes/NETWORK-traffic-classes.md index 6d99f223..5f3656bf 100644 --- a/kubernetes/NETWORK-traffic-classes.md +++ b/kubernetes/NETWORK-traffic-classes.md @@ -8,7 +8,7 @@ Issue #576. Five classes, each on its own VLAN. |---|---|---| | Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) | | pod↔pod | `pod0`, VLAN 48 | pod-sourced traffic is routed from table 47; bird puts the peers' pod CIDRs there via `pod0` | -| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress takes table 47's default | +| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress and VIP replies take table 47's default | | node↔node | `node0`, VLAN 46 | the node InternalIP and etcd live there; so do apiserver↔kubelet, cilium-health and host→remote-pod traffic | | world↔node | `mgmt0`, VLAN 40 | API VIP for kubectl, talosctl from outside, node egress (DNS, NTP, image pulls) | @@ -41,6 +41,15 @@ address, or a hostNetwork pod — would otherwise take the default to the UDM. The routing decision runs before Cilium's masquerade hook, so the rule sees the pod address and the masquerade then rewrites it to the egress link's. +A reply from a LoadBalancer VIP is the exception: Cilium reverses the service +NAT before the packet reaches the routing decision, so its source is the VIP +and the pod rules miss it. bird's `vip-rules` container adds one rule per +block of every `CiliumLoadBalancerIPPool` (priority 1002, `from +lookup 47`) and keeps them in sync with the pools every minute. The blocks +must be CIDRs; a start/stop block stops the container. Without these rules +the reply takes main's default over `mgmt0` while the request arrived on +`ingress0` (#1547). + Node-originated traffic does not match the rules and uses main, where Cilium routes each peer's pod CIDR via the peer's InternalIP — on `node0`. diff --git a/kubernetes/infrastructure-inhouse/bird/daemonset.yaml b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml index e7d91d64..86f3ef35 100644 --- a/kubernetes/infrastructure-inhouse/bird/daemonset.yaml +++ b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml @@ -14,6 +14,10 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list"] + # vip-rules reads the LoadBalancer pool blocks it routes into table 47. + - apiGroups: ["cilium.io"] + resources: ["ciliumloadbalancerippools"] + verbs: ["get", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -111,6 +115,24 @@ spec: volumeMounts: - name: etc-bird mountPath: /etc/bird + # Routes replies from LoadBalancer VIPs into table 47 (vip-rules.sh). + - name: vip-rules + image: gitea.renner.uno/gitea_admin/bird@sha256:65fc93efead1e9b065d2c78e748f6c3c1ad7216e28b757d9fa4e204fa2ffe9a9 + command: ["/bin/sh", "/config/vip-rules.sh"] + resources: + requests: + cpu: "5m" + memory: "16Mi" + limits: + cpu: "100m" + memory: "64Mi" + securityContext: + capabilities: + drop: ["ALL"] + add: ["NET_ADMIN"] + volumeMounts: + - name: config + mountPath: /config volumes: - name: config configMap: diff --git a/kubernetes/infrastructure-inhouse/bird/kustomization.yaml b/kubernetes/infrastructure-inhouse/bird/kustomization.yaml index 447fbce2..7c310b3e 100644 --- a/kubernetes/infrastructure-inhouse/bird/kustomization.yaml +++ b/kubernetes/infrastructure-inhouse/bird/kustomization.yaml @@ -10,3 +10,5 @@ configMapGenerator: - bird.conf - defines.tmpl - peers.tmpl + - vip-rules.sh + - vip-rules.tmpl diff --git a/kubernetes/infrastructure-inhouse/bird/vip-rules.sh b/kubernetes/infrastructure-inhouse/bird/vip-rules.sh new file mode 100644 index 00000000..a576c322 --- /dev/null +++ b/kubernetes/infrastructure-inhouse/bird/vip-rules.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# Keeps one routing rule per LoadBalancer pool block: from lookup 47. +# +# A reply from a VIP reaches the routing decision with the VIP as its source: +# Cilium reverses the service NAT before the packet enters the stack. The pod +# rules (1000/1001, talos/patches/site-homelab.yaml) do not match it, so +# without these rules it takes main's default over mgmt0 while its request +# arrived on ingress0. The blocks are read from the pools, so the prefix is +# written only there. +# +# Rules are left in place on exit: if bird is gone, table 47 is empty and the +# lookup falls through to main, as it does for the pod rules. +set -eu + +PRIO=1002 +TABLE=47 + +family_of() { + case $1 in *:*) echo 6 ;; *) echo 4 ;; esac +} + +reconcile() { + # An API error skips this round; the rules in place stay. + if ! blocks=$(kubectl get ciliumloadbalancerippools.cilium.io \ + -o go-template-file=/config/vip-rules.tmpl); then + echo "reading the pools failed; retrying" >&2 + return 0 + fi + for b in $blocks; do + case $b in + range:*) + echo "pool block ${b#range:} is a range; only cidr blocks map to a rule" >&2 + return 1 + ;; + esac + done + + for fam in 4 6; do + want=$(for b in $blocks; do + if [ "$(family_of "$b")" = "$fam" ]; then echo "$b"; fi + done | sort -u) + have=$(ip -"$fam" rule show | + awk -v p="$PRIO:" -v t="$TABLE" '$1 == p && $2 == "from" && $4 == "lookup" && $5 == t { print $3 }' | + sort -u) + + for b in $have; do + if ! echo "$want" | grep -qxF "$b"; then + ip -"$fam" rule del priority "$PRIO" from "$b" table "$TABLE" + echo "removed: priority $PRIO from $b table $TABLE" + fi + done + for b in $want; do + if ! echo "$have" | grep -qxF "$b"; then + ip -"$fam" rule add priority "$PRIO" from "$b" table "$TABLE" + echo "added: priority $PRIO from $b table $TABLE" + fi + done + done +} + +trap 'exit 0' TERM INT +while :; do + reconcile + sleep 60 & + wait $! +done diff --git a/kubernetes/infrastructure-inhouse/bird/vip-rules.tmpl b/kubernetes/infrastructure-inhouse/bird/vip-rules.tmpl new file mode 100644 index 00000000..818956b3 --- /dev/null +++ b/kubernetes/infrastructure-inhouse/bird/vip-rules.tmpl @@ -0,0 +1,5 @@ +{{- /* One line per block of every enabled CiliumLoadBalancerIPPool. A start/stop + block renders as range:-, which vip-rules.sh refuses. */ -}} +{{- range .items }}{{ if not .spec.disabled }}{{ range .spec.blocks }} +{{- if .cidr }}{{ .cidr }}{{ else }}range:{{ .start }}-{{ .stop }}{{ end }} +{{ end }}{{ end }}{{ end -}} diff --git a/kubernetes/infrastructure/post-crds/cilium-bgp.yaml b/kubernetes/infrastructure/post-crds/cilium-bgp.yaml index 7ed8d510..e323a924 100644 --- a/kubernetes/infrastructure/post-crds/cilium-bgp.yaml +++ b/kubernetes/infrastructure/post-crds/cilium-bgp.yaml @@ -10,6 +10,9 @@ # # IPv4 + IPv6 ranges keep matching last bytes (10.2.99.x ↔ fd99::x) # so each LB has the same "tail" in both families. +# +# Blocks are CIDRs: bird's vip-rules turns each into a routing rule into +# table 47, so VIP replies leave on ingress0 (infrastructure-inhouse/bird). apiVersion: cilium.io/v2alpha1 kind: CiliumLoadBalancerIPPool metadata: @@ -19,10 +22,8 @@ spec: matchLabels: lb-pool: bgp blocks: - - start: "10.2.99.0" - stop: "10.2.99.255" - - start: "fd99::" - stop: "fd99::ff" + - cidr: "10.2.99.0/24" + - cidr: "fd99::/120" --- # Cilium BGPv2 control-plane config peering each node with the UDM over the # ingress VLAN 47 — never over v30iot, which is the untrusted IoT segment. diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index f347f3ce..0b7a07a5 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -102,7 +102,8 @@ cluster: # # Priorities are below main (32766) and clear of the rule Cilium installs at 9. # The two families are separate rule tables, so the priorities need not match; -# the document name is the priority and must be unique. +# the document name is the priority and must be unique. 1002 is taken by bird's +# vip-rules: replies from LoadBalancer VIPs, whose source is the VIP. apiVersion: v1alpha1 kind: RoutingRuleConfig name: "1000"