feat(bird): iBGP mesh over pod0, routes into table 48 (2/3) (#576)

Nothing reads table 48 yet. The DaemonSet fills it so the routing rules in 3/3 land on a table that is already correct, and so a mistake here shows up as an empty table rather than as broken pod traffic.

Each node advertises only its own pod CIDR and learns the other two. The own-CIDR filter keys on ifname rather than the prefix, because Cilium's routes for the OTHER nodes are protocol kernel as well — local is cilium_host, learned is mgmt0 — and ifname is set for the gatewayless IPv6 device route too, which is what FRR's route-maps could not match. learn all rather than learn, since plain learn skips routes marked proto kernel.

Only BGP-learned routes are exported to table 48; the local CIDR stays in main, where cilium_host already serves it. direct is explicit because iBGP defaults to multihop and would make the gateway recursive. strict bind keeps the listener off mgmt0 and ingress0 — the option FRR lacks, since its --listenon implies --no_kernel.

Per-node config is one ConfigMap key per node, copied by an init container from spec.nodeName. The addresses duplicate the pod0 statics in kubernetes/talos/patches/k8s-{1,2,3}.yaml; reading them off the interface instead would need iproute2 in the image, which Alpine does not put in busybox.

Verified: kubectl kustomize builds, and bird -p parses the rendered config inside the 2.17.2 image with exit 0. A deliberately broken copy exits non-zero with a syntax error, so the check distinguishes the two. Peering, route exchange and the table-48 writes are runtime behaviour and are proven by the rollout, not by this.

Refs #576

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

ci/images/bird/Containerfile

@@ -8,15 +8,9 @@ FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc
 #
 # Config is read from /etc/bird/bird.conf, not the package default
 # /etc/bird.conf, so the mount cannot shadow what the package ships.
-#
-# kubectl renders the init container's per-node config: go-template-file for
-# the templating, a field selector for excluding the node itself, in-cluster
-# config for the auth. From the distro, so there is no version literal to drift
-# and no download to trust. It is only ever used to read core/v1 Nodes.
-RUN apk add --no-cache bird2 kubectl
+RUN apk add --no-cache bird2
 
-# Fails the build rather than the first pod if a binary or its path is wrong.
-RUN /usr/sbin/bird --version \
- && kubectl version --client > /dev/null
+# Fails the build rather than the first pod if the binary or its path is wrong.
+RUN /usr/sbin/bird --version
 
 ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]

kubernetes/infrastructure/bird/bird.conf

@@ -1,67 +0,0 @@
-# Identical on every node and free of addresses. defines.conf and peers.conf
-# are rendered per node by the init container; the first is included above the
-# templates that reference it, the second below them.
-include "/etc/bird/defines.conf";
-
-router id from "pod0";
-log stderr all;
-
-protocol device { scan time 10; }
-
-# This node's own pod CIDR, originated rather than read back out of the kernel.
-# Reading it back made BIRD's view depend on the shape of Cilium's route, and
-# the IPv4 one is rejected outright: its next hop sits inside the prefix it
-# describes, which KRT refuses with "strange next-hop". The prefix comes from
-# the Node object either way, so there is nothing to learn.
-#
-# blackhole because this route is only ever advertised. Nothing installs it:
-# the only kernel protocol exporting from these tables takes BGP routes.
-protocol static own4 {
-  ipv4 { table master4; };
-  route OWN_POD4 blackhole;
-}
-protocol static own6 {
-  ipv6 { table master6; };
-  route OWN_POD6 blackhole;
-}
-
-# Only BGP-learned routes reach table 48 — not the static above, and not
-# anything else that may end up in the master tables.
-filter to_t48 {
-  if source != RTS_BGP then reject;
-  accept;
-}
-
-protocol kernel k48v4 {
-  kernel table 48;
-  metric 0;
-  ipv4 { table master4; import none; export filter to_t48; };
-}
-protocol kernel k48v6 {
-  kernel table 48;
-  metric 0;
-  ipv6 { table master6; import none; export filter to_t48; };
-}
-
-# iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM
-# from ingress0 and is unrelated.
-#
-# direct is explicit because iBGP defaults to multihop, which makes the gateway
-# recursive — BIRD would try to resolve the peer in the pod RIB. strict bind
-# keeps the listener off mgmt0 and ingress0.
-template bgp pod4 {
-  local as 64514;
-  source address MY4;
-  direct;
-  strict bind;
-  ipv4 { next hop self; import all; export all; };
-}
-template bgp pod6 {
-  local as 64514;
-  source address MY6;
-  direct;
-  strict bind;
-  ipv6 { next hop self; import all; export all; };
-}
-
-include "/etc/bird/peers.conf";

kubernetes/infrastructure/bird/daemonset.yaml

@@ -1,32 +1,136 @@
 apiVersion: v1
-kind: ServiceAccount
+kind: ConfigMap
 metadata:
-  name: bird
+  name: bird-config
   namespace: kube-system
+data:
+  bird.conf: |
+    # Per-node addresses, written by the init container from bird-nodes.
+    include "/etc/bird/node.conf";
+
+    router id from "pod0";
+    log stderr all;
+
+    # Mirrors of the kernel main table, filtered down to this node's own pod
+    # CIDR. master4/master6 are implicit and hold what BGP exchanges.
+    ipv4 table kmain4;
+    ipv6 table kmain6;
+
+    protocol device { scan time 10; }
+
+    # Cilium's routes for the OTHER nodes' pod CIDRs are protocol kernel too,
+    # so the prefix alone does not identify the local one. The outgoing
+    # interface does: local is cilium_host, learned is mgmt0. ifname is set for
+    # the gatewayless IPv6 device route as well.
+    filter own_pod4 {
+      if ! (net ~ [ 10.244.0.0/16{17,32} ]) then reject;
+      if ifname !~ "cilium_host" then reject;
+      accept;
+    }
+    filter own_pod6 {
+      if ! (net ~ [ fd10:244::/48{49,128} ]) then reject;
+      if ifname !~ "cilium_host" then reject;
+      accept;
+    }
+
+    # learn all, not learn: plain learn skips routes marked proto kernel, which
+    # is what Cilium's are.
+    protocol kernel klearn4 {
+      learn all;
+      scan time 10;
+      ipv4 { table kmain4; import filter own_pod4; export none; };
+    }
+    protocol kernel klearn6 {
+      learn all;
+      scan time 10;
+      ipv6 { table kmain6; import filter own_pod6; export none; };
+    }
+
+    protocol pipe own4 { table kmain4; peer table master4; export all; import none; }
+    protocol pipe own6 { table kmain6; peer table master6; export all; import none; }
+
+    # Only BGP-learned routes reach table 48. The local CIDR stays out of it —
+    # it is already reachable via cilium_host in main.
+    filter to_t48 {
+      if source != RTS_BGP then reject;
+      accept;
+    }
+
+    protocol kernel k48v4 {
+      kernel table 48;
+      metric 0;
+      ipv4 { table master4; import none; export filter to_t48; };
+    }
+    protocol kernel k48v6 {
+      kernel table 48;
+      metric 0;
+      ipv6 { table master6; import none; export filter to_t48; };
+    }
+
+    # iBGP full mesh over pod0. 64514 keeps clear of Cilium's 64513, which
+    # peers with the UDM from ingress0 and is unrelated to this.
+    #
+    # direct is explicit because iBGP defaults to multihop, which makes the
+    # gateway recursive — BIRD would try to resolve the peer in the pod RIB.
+    # strict bind keeps the listener off mgmt0 and ingress0.
+    template bgp pod4 {
+      local as 64514;
+      source address MY4;
+      direct;
+      strict bind;
+      ipv4 {
+        next hop self;
+        import filter { if net ~ [ 10.244.0.0/16{17,32} ] then accept; reject; };
+        export filter { if net ~ [ 10.244.0.0/16{17,32} ] then accept; reject; };
+      };
+    }
+    template bgp pod6 {
+      local as 64514;
+      source address MY6;
+      direct;
+      strict bind;
+      ipv6 {
+        next hop self;
+        import filter { if net ~ [ fd10:244::/48{49,128} ] then accept; reject; };
+        export filter { if net ~ [ fd10:244::/48{49,128} ] then accept; reject; };
+      };
+    }
+
+    protocol bgp p1v4 from pod4 { neighbor P1_4 as 64514; }
+    protocol bgp p2v4 from pod4 { neighbor P2_4 as 64514; }
+    protocol bgp p1v6 from pod6 { neighbor P1_6 as 64514; }
+    protocol bgp p2v6 from pod6 { neighbor P2_6 as 64514; }
 ---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+apiVersion: v1
+kind: ConfigMap
 metadata:
-  name: bird
-rules:
-  # The peer list is read from the nodes' pod0 annotations, so no address is
-  # written into this repo twice.
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: bird
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: bird
-subjects:
-  - kind: ServiceAccount
-    name: bird
-    namespace: kube-system
+  name: bird-nodes
+  namespace: kube-system
+data:
+  # One key per node, selected by name. The addresses are the pod0 statics from
+  # kubernetes/talos/patches/k8s-{1,2,3}.yaml; a node added there needs a key
+  # here and a peer line in the other two.
+  k8s-1: |
+    define MY4  = 10.2.48.11;
+    define MY6  = fd02:48::11;
+    define P1_4 = 10.2.48.12;
+    define P1_6 = fd02:48::12;
+    define P2_4 = 10.2.48.13;
+    define P2_6 = fd02:48::13;
+  k8s-2: |
+    define MY4  = 10.2.48.12;
+    define MY6  = fd02:48::12;
+    define P1_4 = 10.2.48.11;
+    define P1_6 = fd02:48::11;
+    define P2_4 = 10.2.48.13;
+    define P2_6 = fd02:48::13;
+  k8s-3: |
+    define MY4  = 10.2.48.13;
+    define MY6  = fd02:48::13;
+    define P1_4 = 10.2.48.11;
+    define P1_6 = fd02:48::11;
+    define P2_4 = 10.2.48.12;
+    define P2_6 = fd02:48::12;
 ---
 apiVersion: apps/v1
 kind: DaemonSet
@@ -48,37 +152,22 @@ spec:
         tier: node
         app: bird
     spec:
-      # Carries pod east-west once the routing rules land. Without it table 48
-      # stays empty and the rules fall through to main, i.e. back to VLAN 40.
+      # Carries pod east-west once the routing rules land, so a node without it
+      # falls back to VLAN 40 rather than losing connectivity.
       priorityClassName: system-node-critical
       # Writes host routes and peers on host addresses.
       hostNetwork: true
-      serviceAccountName: bird
       # Every node here is a control plane node, so this is not optional.
       tolerations:
         - operator: Exists
           effect: NoSchedule
       initContainers:
+        # bird.conf is identical everywhere; node.conf is this node's key.
         - name: render-config
-          # By digest, not by the dated tag: the tag is rebuilt within the day
-          # it names, and the cluster's image path keeps resolving it to the
-          # first build of that day — imagePullPolicy: Always does not help,
-          # because the tag itself resolves stale. Update this when the image
-          # is rebuilt.
-          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
+          image: gitea.renner.uno/gitea_admin/bird:2026-09-16
           command: ["/bin/sh", "-c"]
           args:
-            - |
-              set -eu
-              cp /config/bird.conf /etc/bird/bird.conf
-              kubectl get node "$NODE_NAME" \
-                -o go-template-file=/config/defines.tmpl > /etc/bird/defines.conf
-              kubectl get nodes --field-selector "metadata.name!=$NODE_NAME" \
-                -o go-template-file=/config/peers.tmpl > /etc/bird/peers.conf
-              # BIRD's own parser is the check: a node whose annotation is
-              # missing renders as <no value> and fails here rather than
-              # crashlooping the daemon or forming a mesh with a hole.
-              bird -p -c /etc/bird/bird.conf
+            - cp /config/bird.conf /etc/bird/bird.conf && cp /nodes/"$NODE_NAME" /etc/bird/node.conf
           env:
             - name: NODE_NAME
               valueFrom:
@@ -87,11 +176,13 @@ spec:
           volumeMounts:
             - name: config
               mountPath: /config
+            - name: nodes
+              mountPath: /nodes
             - name: etc-bird
               mountPath: /etc/bird
       containers:
         - name: bird
-          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
+          image: gitea.renner.uno/gitea_admin/bird:2026-09-16
           resources:
             requests:
               cpu: "50m"
@@ -111,5 +202,8 @@ spec:
         - name: config
           configMap:
             name: bird-config
+        - name: nodes
+          configMap:
+            name: bird-nodes
         - name: etc-bird
           emptyDir: {}

kubernetes/infrastructure/bird/defines.tmpl

@@ -1,7 +0,0 @@
-define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }};
-define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }};
-{{/* podCIDRs follows the cluster's podSubnets order, v4 first. A flip puts an
-     IPv6 prefix into an ipv4 protocol, which fails the parse in the init
-     container rather than passing silently. */}}
-define OWN_POD4 = {{ index .spec.podCIDRs 0 }};
-define OWN_POD6 = {{ index .spec.podCIDRs 1 }};

kubernetes/infrastructure/bird/kustomization.yaml

@@ -2,11 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - daemonset.yaml
-configMapGenerator:
-  # Hashed name, so editing any of these rolls the DaemonSet.
-  - name: bird-config
-    namespace: kube-system
-    files:
-      - bird.conf
-      - defines.tmpl
-      - peers.tmpl

kubernetes/infrastructure/bird/peers.tmpl

@@ -1,7 +0,0 @@
-{{- range $i, $n := .items -}}
-{{- $v6 := index $n.metadata.annotations "net.renner.uno/pod0-v6" -}}
-{{- $v4 := index $n.metadata.annotations "net.renner.uno/pod0-v4" -}}
-# {{ $n.metadata.name }}
-protocol bgp peer{{ $i }}_v6 from pod6 { neighbor {{ $v6 }} as 64514; }
-protocol bgp peer{{ $i }}_v4 from pod4 { neighbor {{ $v4 }} as 64514; }
-{{ end -}}

kubernetes/talos/patches/k8s-1.yaml

@@ -52,12 +52,6 @@ machine:
     # this label instead of listing hostnames, so which boxes carry storage is
     # stated once, here, by the machine that knows.
     storage.renner.uno/ceph-osd: "true"
-  # Publishes this node's pod0 addresses, declared below in its VLANConfig, so
-  # the bird DaemonSet reads its peers from the node list rather than from a
-  # copy. Annotations, not labels: a label value cannot contain a colon.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::11
-    net.renner.uno/pod0-v4: 10.2.48.11
   udev:
     # Renames every port this node uses to its role at boot, matched by
     # burned-in MAC — kernel names derive from PCI topology and shift when

kubernetes/talos/patches/k8s-2.yaml

@@ -33,10 +33,6 @@ machine:
     topology.kubernetes.io/zone: saturn
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
-  # pod0 addresses for the bird peer list — see k8s-1.yaml.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::12
-    net.renner.uno/pod0-v4: 10.2.48.12
   udev:
     # Renames every port to its role at boot — see k8s-1.yaml.
     rules:

kubernetes/talos/patches/k8s-3.yaml

@@ -29,10 +29,6 @@ machine:
     topology.kubernetes.io/zone: neptune
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
-  # pod0 addresses for the bird peer list — see k8s-1.yaml.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::13
-    net.renner.uno/pod0-v4: 10.2.48.13
   udev:
     # Renames every port to its role at boot — see k8s-1.yaml.
     rules: