diff --git a/ci/images/bird/Containerfile b/ci/images/bird/Containerfile index 5c399494..487f133c 100644 --- a/ci/images/bird/Containerfile +++ b/ci/images/bird/Containerfile @@ -9,14 +9,14 @@ FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc # Config is read from /etc/bird/bird.conf, not the package default # /etc/bird.conf, so the mount cannot shadow what the package ships. # -# curl and jq are for the init container, which renders the per-node peer list -# from the Kubernetes API. busybox wget cannot verify against the API server's -# CA bundle, and the query must not run unverified. -RUN apk add --no-cache bird2 curl jq +# kubectl renders the init container's per-node config: go-template-file for +# the templating, a field selector for excluding the node itself, in-cluster +# config for the auth. From the distro, so there is no version literal to drift +# and no download to trust. It is only ever used to read core/v1 Nodes. +RUN apk add --no-cache bird2 kubectl # Fails the build rather than the first pod if a binary or its path is wrong. RUN /usr/sbin/bird --version \ - && curl --version > /dev/null \ - && jq --version > /dev/null + && kubectl version --client > /dev/null ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"] diff --git a/kubernetes/infrastructure/bird/bird.conf b/kubernetes/infrastructure/bird/bird.conf new file mode 100644 index 00000000..0586a15f --- /dev/null +++ b/kubernetes/infrastructure/bird/bird.conf @@ -0,0 +1,89 @@ +# Identical on every node and free of addresses. defines.conf and peers.conf +# are rendered per node by the init container; the first is included above the +# templates that reference it, the second below them. +include "/etc/bird/defines.conf"; + +router id from "pod0"; +log stderr all; + +# Mirrors of the kernel main table, filtered to this node's own pod CIDR. +# master4/master6 are implicit and hold what BGP exchanges. +ipv4 table kmain4; +ipv6 table kmain6; + +protocol device { scan time 10; } + +# Two conditions. cilium_host also carries its own host address, which the +# length check drops; and Cilium's routes for the OTHER nodes are protocol +# kernel too, which the interface check drops. ifname is set for the +# gatewayless IPv6 device route as well, so this works in both families. +filter own_pod4 { + if ifname !~ "cilium_host" then reject; + if net.len = 32 then reject; + accept; +} +filter own_pod6 { + if ifname !~ "cilium_host" then reject; + if net.len = 128 then reject; + accept; +} + +# learn all, not learn: plain learn skips routes marked proto kernel, which is +# what Cilium's are. +protocol kernel klearn4 { + learn all; + scan time 10; + ipv4 { table kmain4; import filter own_pod4; export none; }; +} +protocol kernel klearn6 { + learn all; + scan time 10; + ipv6 { table kmain6; import filter own_pod6; export none; }; +} + +protocol pipe own4 { table kmain4; peer table master4; export all; import none; } +protocol pipe own6 { table kmain6; peer table master6; export all; import none; } + +# Only BGP-learned routes reach table 48. The local CIDR stays in main, where +# cilium_host already serves it. +filter to_t48 { + if source != RTS_BGP then reject; + accept; +} + +protocol kernel k48v4 { + kernel table 48; + metric 0; + ipv4 { table master4; import none; export filter to_t48; }; +} +protocol kernel k48v6 { + kernel table 48; + metric 0; + ipv6 { table master6; import none; export filter to_t48; }; +} + +# iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM +# from ingress0 and is unrelated. +# +# No prefix filters: the only route this node offers is its own pod CIDR, which +# own_pod4/6 already selected, and the only peers are the other cluster nodes. +# +# direct is explicit because iBGP defaults to multihop, which makes the gateway +# recursive — BIRD would try to resolve the peer in the pod RIB. strict bind +# keeps the listener off mgmt0 and ingress0. +template bgp pod4 { + local as 64514; + source address MY4; + direct; + strict bind; + ipv4 { next hop self; import all; export all; }; +} +template bgp pod6 { + local as 64514; + source address MY6; + direct; + strict bind; + ipv6 { next hop self; import all; export all; }; +} + +include "/etc/bird/peers.conf"; diff --git a/kubernetes/infrastructure/bird/daemonset.yaml b/kubernetes/infrastructure/bird/daemonset.yaml new file mode 100644 index 00000000..75eace14 --- /dev/null +++ b/kubernetes/infrastructure/bird/daemonset.yaml @@ -0,0 +1,113 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bird + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: bird +rules: + # The peer list is read from the nodes' pod0 annotations, so no address is + # written into this repo twice. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bird +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: bird +subjects: + - kind: ServiceAccount + name: bird + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: bird + namespace: kube-system + labels: + tier: node + app: bird +spec: + selector: + matchLabels: + app: bird + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: bird + spec: + # Carries pod east-west once the routing rules land. Without it table 48 + # stays empty and the rules fall through to main, i.e. back to VLAN 40. + priorityClassName: system-node-critical + # Writes host routes and peers on host addresses. + hostNetwork: true + serviceAccountName: bird + # Every node here is a control plane node, so this is not optional. + tolerations: + - operator: Exists + effect: NoSchedule + initContainers: + - name: render-config + image: gitea.renner.uno/gitea_admin/bird:2026-09-16 + # The dated tag is rebuilt within the day it names. + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: + - | + set -eu + cp /config/bird.conf /etc/bird/bird.conf + kubectl get node "$NODE_NAME" \ + -o go-template-file=/config/defines.tmpl > /etc/bird/defines.conf + kubectl get nodes --field-selector "metadata.name!=$NODE_NAME" \ + -o go-template-file=/config/peers.tmpl > /etc/bird/peers.conf + # BIRD's own parser is the check: a node whose annotation is + # missing renders as and fails here rather than + # crashlooping the daemon or forming a mesh with a hole. + bird -p -c /etc/bird/bird.conf + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: config + mountPath: /config + - name: etc-bird + mountPath: /etc/bird + containers: + - name: bird + image: gitea.renner.uno/gitea_admin/bird:2026-09-16 + imagePullPolicy: Always + resources: + requests: + cpu: "50m" + memory: "32Mi" + limits: + cpu: "200m" + memory: "128Mi" + securityContext: + capabilities: + drop: ["ALL"] + # NET_ADMIN writes the routes; NET_BIND_SERVICE binds 179. + add: ["NET_ADMIN", "NET_BIND_SERVICE"] + volumeMounts: + - name: etc-bird + mountPath: /etc/bird + volumes: + - name: config + configMap: + name: bird-config + - name: etc-bird + emptyDir: {} diff --git a/kubernetes/infrastructure/bird/defines.tmpl b/kubernetes/infrastructure/bird/defines.tmpl new file mode 100644 index 00000000..db8d6fb4 --- /dev/null +++ b/kubernetes/infrastructure/bird/defines.tmpl @@ -0,0 +1,2 @@ +define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }}; +define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }}; diff --git a/kubernetes/infrastructure/bird/kustomization.yaml b/kubernetes/infrastructure/bird/kustomization.yaml new file mode 100644 index 00000000..447fbce2 --- /dev/null +++ b/kubernetes/infrastructure/bird/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - daemonset.yaml +configMapGenerator: + # Hashed name, so editing any of these rolls the DaemonSet. + - name: bird-config + namespace: kube-system + files: + - bird.conf + - defines.tmpl + - peers.tmpl diff --git a/kubernetes/infrastructure/bird/peers.tmpl b/kubernetes/infrastructure/bird/peers.tmpl new file mode 100644 index 00000000..05e150a5 --- /dev/null +++ b/kubernetes/infrastructure/bird/peers.tmpl @@ -0,0 +1,7 @@ +{{- range $i, $n := .items -}} +{{- $v6 := index $n.metadata.annotations "net.renner.uno/pod0-v6" -}} +{{- $v4 := index $n.metadata.annotations "net.renner.uno/pod0-v4" -}} +# {{ $n.metadata.name }} +protocol bgp peer{{ $i }}_v6 from pod6 { neighbor {{ $v6 }} as 64514; } +protocol bgp peer{{ $i }}_v4 from pod4 { neighbor {{ $v4 }} as 64514; } +{{ end -}} diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index 9644948e..90da26c5 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -7,6 +7,7 @@ resources: - cluster-settings/ - cilium/ - multus/ + - bird/ - cert-manager/ - snapshot-controller/ - metrics-server/ diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index b09369b3..cddd38c3 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -52,6 +52,12 @@ machine: # this label instead of listing hostnames, so which boxes carry storage is # stated once, here, by the machine that knows. storage.renner.uno/ceph-osd: "true" + # Publishes this node's pod0 addresses, declared below in its VLANConfig, so + # the bird DaemonSet reads its peers from the node list rather than from a + # copy. Annotations, not labels: a label value cannot contain a colon. + nodeAnnotations: + net.renner.uno/pod0-v6: fd02:48::11 + net.renner.uno/pod0-v4: 10.2.48.11 udev: # Renames every port this node uses to its role at boot, matched by # burned-in MAC — kernel names derive from PCI topology and shift when diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index 12c10cf1..cf3ad01c 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -33,6 +33,10 @@ machine: topology.kubernetes.io/zone: saturn # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # pod0 addresses for the bird peer list — see k8s-1.yaml. + nodeAnnotations: + net.renner.uno/pod0-v6: fd02:48::12 + net.renner.uno/pod0-v4: 10.2.48.12 udev: # Renames every port to its role at boot — see k8s-1.yaml. rules: diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index 491eaf8c..5d8673e0 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -29,6 +29,10 @@ machine: topology.kubernetes.io/zone: neptune # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # pod0 addresses for the bird peer list — see k8s-1.yaml. + nodeAnnotations: + net.renner.uno/pod0-v6: fd02:48::13 + net.renner.uno/pod0-v4: 10.2.48.13 udev: # Renames every port to its role at boot — see k8s-1.yaml. rules: