From 3a9ea5eefeced653d5a73cc338821a4d46ab1476 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Wed, 16 Sep 2026 10:43:42 +0200 Subject: [PATCH] feat(bird): iBGP mesh over pod0, routes into table 48 (#576) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing reads table 48 yet. The mesh fills it so the routing rules land on a table that is already correct, and so a mistake here shows as an empty table rather than as broken pod traffic. No address is written down twice. Each node publishes its pod0 addresses as Talos nodeAnnotations, beside the VLANConfig that assigns them; the init container reads the node list and renders from it. bird.conf holds no address at all, and the pod CIDRs are not restated either — the own-CIDR filter keys on the outgoing interface plus a maximum-length exclusion, which separates the node's pod route from cilium_host's own host address without naming a prefix. Rendering is go-template-file, not string building: kubectl renders the two templates, a field selector drops the node itself server-side, and in-cluster config removes the token and CA plumbing. kubectl comes from the Alpine package, so there is no version literal to drift out of sync and no download to trust; it only ever reads core/v1 Nodes. jq and curl are gone. The init container ends with bird -p, so BIRD's own parser gates startup: a node whose annotation is missing renders as and fails there rather than crashlooping the daemon or forming a mesh with a hole. learn all rather than learn, since plain learn skips proto kernel. direct is explicit because iBGP defaults to multihop, which would make the gateway recursive. strict bind keeps the listener off mgmt0 and ingress0. Verified against the live cluster with all three nodes annotated via --mode=try: the templates rendered the expected defines and four peer protocols, and bird -p parsed the result inside the image with exit 0 — which also confirms net.len and the include ordering. kubectl kustomize builds. Refs #576 Co-Authored-By: Claude Opus 5 (1M context) --- ci/images/bird/Containerfile | 12 +- kubernetes/infrastructure/bird/bird.conf | 89 ++++++++++++++ kubernetes/infrastructure/bird/daemonset.yaml | 113 ++++++++++++++++++ kubernetes/infrastructure/bird/defines.tmpl | 2 + .../infrastructure/bird/kustomization.yaml | 12 ++ kubernetes/infrastructure/bird/peers.tmpl | 7 ++ kubernetes/infrastructure/kustomization.yaml | 1 + kubernetes/talos/patches/k8s-1.yaml | 6 + kubernetes/talos/patches/k8s-2.yaml | 4 + kubernetes/talos/patches/k8s-3.yaml | 4 + 10 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 kubernetes/infrastructure/bird/bird.conf create mode 100644 kubernetes/infrastructure/bird/daemonset.yaml create mode 100644 kubernetes/infrastructure/bird/defines.tmpl create mode 100644 kubernetes/infrastructure/bird/kustomization.yaml create mode 100644 kubernetes/infrastructure/bird/peers.tmpl diff --git a/ci/images/bird/Containerfile b/ci/images/bird/Containerfile index 5c399494..487f133c 100644 --- a/ci/images/bird/Containerfile +++ b/ci/images/bird/Containerfile @@ -9,14 +9,14 @@ FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc # Config is read from /etc/bird/bird.conf, not the package default # /etc/bird.conf, so the mount cannot shadow what the package ships. # -# curl and jq are for the init container, which renders the per-node peer list -# from the Kubernetes API. busybox wget cannot verify against the API server's -# CA bundle, and the query must not run unverified. -RUN apk add --no-cache bird2 curl jq +# kubectl renders the init container's per-node config: go-template-file for +# the templating, a field selector for excluding the node itself, in-cluster +# config for the auth. From the distro, so there is no version literal to drift +# and no download to trust. It is only ever used to read core/v1 Nodes. +RUN apk add --no-cache bird2 kubectl # Fails the build rather than the first pod if a binary or its path is wrong. RUN /usr/sbin/bird --version \ - && curl --version > /dev/null \ - && jq --version > /dev/null + && kubectl version --client > /dev/null ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"] diff --git a/kubernetes/infrastructure/bird/bird.conf b/kubernetes/infrastructure/bird/bird.conf new file mode 100644 index 00000000..0586a15f --- /dev/null +++ b/kubernetes/infrastructure/bird/bird.conf @@ -0,0 +1,89 @@ +# Identical on every node and free of addresses. defines.conf and peers.conf +# are rendered per node by the init container; the first is included above the +# templates that reference it, the second below them. +include "/etc/bird/defines.conf"; + +router id from "pod0"; +log stderr all; + +# Mirrors of the kernel main table, filtered to this node's own pod CIDR. +# master4/master6 are implicit and hold what BGP exchanges. +ipv4 table kmain4; +ipv6 table kmain6; + +protocol device { scan time 10; } + +# Two conditions. cilium_host also carries its own host address, which the +# length check drops; and Cilium's routes for the OTHER nodes are protocol +# kernel too, which the interface check drops. ifname is set for the +# gatewayless IPv6 device route as well, so this works in both families. +filter own_pod4 { + if ifname !~ "cilium_host" then reject; + if net.len = 32 then reject; + accept; +} +filter own_pod6 { + if ifname !~ "cilium_host" then reject; + if net.len = 128 then reject; + accept; +} + +# learn all, not learn: plain learn skips routes marked proto kernel, which is +# what Cilium's are. +protocol kernel klearn4 { + learn all; + scan time 10; + ipv4 { table kmain4; import filter own_pod4; export none; }; +} +protocol kernel klearn6 { + learn all; + scan time 10; + ipv6 { table kmain6; import filter own_pod6; export none; }; +} + +protocol pipe own4 { table kmain4; peer table master4; export all; import none; } +protocol pipe own6 { table kmain6; peer table master6; export all; import none; } + +# Only BGP-learned routes reach table 48. The local CIDR stays in main, where +# cilium_host already serves it. +filter to_t48 { + if source != RTS_BGP then reject; + accept; +} + +protocol kernel k48v4 { + kernel table 48; + metric 0; + ipv4 { table master4; import none; export filter to_t48; }; +} +protocol kernel k48v6 { + kernel table 48; + metric 0; + ipv6 { table master6; import none; export filter to_t48; }; +} + +# iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM +# from ingress0 and is unrelated. +# +# No prefix filters: the only route this node offers is its own pod CIDR, which +# own_pod4/6 already selected, and the only peers are the other cluster nodes. +# +# direct is explicit because iBGP defaults to multihop, which makes the gateway +# recursive — BIRD would try to resolve the peer in the pod RIB. strict bind +# keeps the listener off mgmt0 and ingress0. +template bgp pod4 { + local as 64514; + source address MY4; + direct; + strict bind; + ipv4 { next hop self; import all; export all; }; +} +template bgp pod6 { + local as 64514; + source address MY6; + direct; + strict bind; + ipv6 { next hop self; import all; export all; }; +} + +include "/etc/bird/peers.conf"; diff --git a/kubernetes/infrastructure/bird/daemonset.yaml b/kubernetes/infrastructure/bird/daemonset.yaml new file mode 100644 index 00000000..75eace14 --- /dev/null +++ b/kubernetes/infrastructure/bird/daemonset.yaml @@ -0,0 +1,113 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bird + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: bird +rules: + # The peer list is read from the nodes' pod0 annotations, so no address is + # written into this repo twice. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bird +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: bird +subjects: + - kind: ServiceAccount + name: bird + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: bird + namespace: kube-system + labels: + tier: node + app: bird +spec: + selector: + matchLabels: + app: bird + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: bird + spec: + # Carries pod east-west once the routing rules land. Without it table 48 + # stays empty and the rules fall through to main, i.e. back to VLAN 40. + priorityClassName: system-node-critical + # Writes host routes and peers on host addresses. + hostNetwork: true + serviceAccountName: bird + # Every node here is a control plane node, so this is not optional. + tolerations: + - operator: Exists + effect: NoSchedule + initContainers: + - name: render-config + image: gitea.renner.uno/gitea_admin/bird:2026-09-16 + # The dated tag is rebuilt within the day it names. + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: + - | + set -eu + cp /config/bird.conf /etc/bird/bird.conf + kubectl get node "$NODE_NAME" \ + -o go-template-file=/config/defines.tmpl > /etc/bird/defines.conf + kubectl get nodes --field-selector "metadata.name!=$NODE_NAME" \ + -o go-template-file=/config/peers.tmpl > /etc/bird/peers.conf + # BIRD's own parser is the check: a node whose annotation is + # missing renders as and fails here rather than + # crashlooping the daemon or forming a mesh with a hole. + bird -p -c /etc/bird/bird.conf + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: config + mountPath: /config + - name: etc-bird + mountPath: /etc/bird + containers: + - name: bird + image: gitea.renner.uno/gitea_admin/bird:2026-09-16 + imagePullPolicy: Always + resources: + requests: + cpu: "50m" + memory: "32Mi" + limits: + cpu: "200m" + memory: "128Mi" + securityContext: + capabilities: + drop: ["ALL"] + # NET_ADMIN writes the routes; NET_BIND_SERVICE binds 179. + add: ["NET_ADMIN", "NET_BIND_SERVICE"] + volumeMounts: + - name: etc-bird + mountPath: /etc/bird + volumes: + - name: config + configMap: + name: bird-config + - name: etc-bird + emptyDir: {} diff --git a/kubernetes/infrastructure/bird/defines.tmpl b/kubernetes/infrastructure/bird/defines.tmpl new file mode 100644 index 00000000..db8d6fb4 --- /dev/null +++ b/kubernetes/infrastructure/bird/defines.tmpl @@ -0,0 +1,2 @@ +define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }}; +define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }}; diff --git a/kubernetes/infrastructure/bird/kustomization.yaml b/kubernetes/infrastructure/bird/kustomization.yaml new file mode 100644 index 00000000..447fbce2 --- /dev/null +++ b/kubernetes/infrastructure/bird/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - daemonset.yaml +configMapGenerator: + # Hashed name, so editing any of these rolls the DaemonSet. + - name: bird-config + namespace: kube-system + files: + - bird.conf + - defines.tmpl + - peers.tmpl diff --git a/kubernetes/infrastructure/bird/peers.tmpl b/kubernetes/infrastructure/bird/peers.tmpl new file mode 100644 index 00000000..05e150a5 --- /dev/null +++ b/kubernetes/infrastructure/bird/peers.tmpl @@ -0,0 +1,7 @@ +{{- range $i, $n := .items -}} +{{- $v6 := index $n.metadata.annotations "net.renner.uno/pod0-v6" -}} +{{- $v4 := index $n.metadata.annotations "net.renner.uno/pod0-v4" -}} +# {{ $n.metadata.name }} +protocol bgp peer{{ $i }}_v6 from pod6 { neighbor {{ $v6 }} as 64514; } +protocol bgp peer{{ $i }}_v4 from pod4 { neighbor {{ $v4 }} as 64514; } +{{ end -}} diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index 9644948e..90da26c5 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -7,6 +7,7 @@ resources: - cluster-settings/ - cilium/ - multus/ + - bird/ - cert-manager/ - snapshot-controller/ - metrics-server/ diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index b09369b3..cddd38c3 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -52,6 +52,12 @@ machine: # this label instead of listing hostnames, so which boxes carry storage is # stated once, here, by the machine that knows. storage.renner.uno/ceph-osd: "true" + # Publishes this node's pod0 addresses, declared below in its VLANConfig, so + # the bird DaemonSet reads its peers from the node list rather than from a + # copy. Annotations, not labels: a label value cannot contain a colon. + nodeAnnotations: + net.renner.uno/pod0-v6: fd02:48::11 + net.renner.uno/pod0-v4: 10.2.48.11 udev: # Renames every port this node uses to its role at boot, matched by # burned-in MAC — kernel names derive from PCI topology and shift when diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index 12c10cf1..cf3ad01c 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -33,6 +33,10 @@ machine: topology.kubernetes.io/zone: saturn # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # pod0 addresses for the bird peer list — see k8s-1.yaml. + nodeAnnotations: + net.renner.uno/pod0-v6: fd02:48::12 + net.renner.uno/pod0-v4: 10.2.48.12 udev: # Renames every port to its role at boot — see k8s-1.yaml. rules: diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index 491eaf8c..5d8673e0 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -29,6 +29,10 @@ machine: topology.kubernetes.io/zone: neptune # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # pod0 addresses for the bird peer list — see k8s-1.yaml. + nodeAnnotations: + net.renner.uno/pod0-v6: fd02:48::13 + net.renner.uno/pod0-v4: 10.2.48.13 udev: # Renames every port to its role at boot — see k8s-1.yaml. rules: