fix(bird): reference the image by digest, the dated tag resolves stale

The DaemonSet came up with Init:Error, kubectl: not found. The build had pushed sha256:3bc85006 for bird:2026-09-16, but every pod resolved that tag to sha256:75e19214 — the first build of the same day, from before kubectl was added.

imagePullPolicy: Always does not help: it re-pulls, but the tag itself resolves to the stale digest. Deleting the pods changed nothing; three fresh pods on three nodes all resolved to the old digest. A one-off pod pulling the digest directly ran kubectl 1.33.1 and BIRD 2.17.2, so the registry holds the right image and only tag resolution is stale.

The dated tag cannot carry a same-day rebuild, and this image was rebuilt three times today. Referencing the digest makes the deployment deterministic. The cost is that the reference has to be updated whenever the image is rebuilt; the alternative, tagging by commit, does not work for a manifest living in the same commit.

Refs #576

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.woodpecker/build-ci-images.yml

@@ -376,3 +376,29 @@ steps:
           --dockerfile Containerfile \
           --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
           --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
+
+  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
+  # artifact like heating-can; same registry so the audit tracks it.
+  - name: build-bird
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: ci/images/bird
+    depends_on: []
+    when:
+      - event: deployment
+      - event: push
+        path: ["ci/images/bird/**", ".woodpecker/build-ci-images.yml"]
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=$(date +%Y-%m-%d)
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Containerfile \
+          --destination gitea.renner.uno/gitea_admin/bird:$TAG \
+          --destination gitea.renner.uno/gitea_admin/bird:latest

ci/images/bird/Containerfile

@@ -0,0 +1,22 @@
+FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
+
+# bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
+# step — the image is the binary plus its runtime.
+#
+# 3.22 rather than the 3.21 the other images here pin: bird2 first appears in
+# 3.22/community.
+#
+# Config is read from /etc/bird/bird.conf, not the package default
+# /etc/bird.conf, so the mount cannot shadow what the package ships.
+#
+# kubectl renders the init container's per-node config: go-template-file for
+# the templating, a field selector for excluding the node itself, in-cluster
+# config for the auth. From the distro, so there is no version literal to drift
+# and no download to trust. It is only ever used to read core/v1 Nodes.
+RUN apk add --no-cache bird2 kubectl
+
+# Fails the build rather than the first pod if a binary or its path is wrong.
+RUN /usr/sbin/bird --version \
+ && kubectl version --client > /dev/null
+
+ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]

kubernetes/apps/home-assistant/deployment.yaml

@@ -95,7 +95,7 @@ spec:
               memory: 32Mi
       containers:
         - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.1 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.9.2 # {"$imagepolicy": "flux-system:home-assistant"}
           env:
             - name: TZ
               value: Europe/Berlin

kubernetes/apps/immich/machine-learning.yaml

@@ -32,7 +32,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.1.0 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
           ports:
             - name: http
               containerPort: 3003

kubernetes/apps/immich/server.yaml

@@ -87,7 +87,7 @@ spec:
               mountPath: /ca
       containers:
         - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.1.0 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
           ports:
             - name: http
               containerPort: 3001

kubernetes/apps/music-assistant/deployment.yaml

@@ -55,7 +55,7 @@ spec:
         # stdlib only; Flux keeps the tag in lockstep via the shared image
         # policy) — no extra dependency.
         - name: reconcile-publish-ip
-          image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
           command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
           volumeMounts:
             - name: data
@@ -65,7 +65,7 @@ spec:
               readOnly: true
       containers:
         - name: music-assistant
-          image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
+          image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
           env:
             - name: LOG_LEVEL
               value: info

kubernetes/apps/paperless/gotenberg.yaml

@@ -22,7 +22,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: gotenberg
-          image: gotenberg/gotenberg:8.36.0 # {"$imagepolicy": "flux-system:gotenberg"}
+          image: gotenberg/gotenberg:8.37.0 # {"$imagepolicy": "flux-system:gotenberg"}
           args:
             - "gotenberg"
             - "--chromium-disable-javascript=true"

kubernetes/apps/trmnl/terminus.yaml

@@ -134,7 +134,7 @@ spec:
         # is a device-bricking class of risk. Bumps are a reviewed, manual
         # act — see kubernetes/infrastructure/image-automation/README.md.
         - name: terminus
-          image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
+          image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
           ports:
             - name: http
               containerPort: 2345
@@ -197,7 +197,7 @@ spec:
             capabilities:
               drop: ["ALL"]
         - name: sidekiq
-          image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
+          image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
           command: ["bundle", "exec", "sidekiq", "-r", "./config/sidekiq.rb"]
           envFrom:
             - configMapRef:

kubernetes/apps/whisper/deployment.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: whisper
-          image: rhasspy/wyoming-whisper:3.7.0 # {"$imagepolicy": "flux-system:wyoming-whisper"}
+          image: rhasspy/wyoming-whisper:3.8.1 # {"$imagepolicy": "flux-system:wyoming-whisper"}
           args:
             - --model=base-int8
             - --language=de

kubernetes/apps/woodpecker/agent.yaml

@@ -34,7 +34,7 @@ spec:
           # Image automation, range 3.x (see image-automation/woodpecker.yaml).
           # MUST stay version-matched with woodpecker-server (gRPC parity) — both
           # track the same range/filter so they bump together on each release.
-          image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
+          image: woodpeckerci/woodpecker-agent:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-agent"}
           env:
             - name: WOODPECKER_SERVER
               value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000

kubernetes/apps/woodpecker/server.yaml

@@ -40,7 +40,7 @@ spec:
           # minor + patch auto-update within 3.x; only the 4.0 major is held for
           # a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
           # -bump hazard is gone). Stays in lockstep with the agent.
-          image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
+          image: woodpeckerci/woodpecker-server:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-server"}
           env:
             - name: WOODPECKER_HOST
               value: https://ci.renner.uno

kubernetes/bootstrap.sh

@@ -1303,32 +1303,6 @@ EOF
     done
     ok
 
-    # CNI=antrea swaps the CNI for a throwaway evaluation cluster only — see
-    # kubernetes/cni-eval/README.md. Unset, nothing below changes. Refused on
-    # the production nodes: this installs a CNI Flux does not manage, so a real
-    # cluster would come up with a network no HelmRelease owns.
-    if [[ "${CNI:-cilium}" == "antrea" ]]; then
-        [[ "$SITE" != "homelab" ]] \
-            || fail "CNI=antrea is for throwaway clusters (vmtest/hcloudtest), not the homelab site"
-        step 0 "Installing Antrea (CNI) — evaluation override..."
-        ANTREA_VALUES="$REPO_ROOT/kubernetes/cni-eval/antrea-values.yaml"
-        [[ -f "$ANTREA_VALUES" ]] || fail "antrea values not found at $ANTREA_VALUES"
-        # >=2.6 is a hard floor, not a preference: earlier agents fail to start
-        # on Talos >=1.7 after its iptables-legacy to iptables-nft switch.
-        helm repo add antrea https://charts.antrea.io >/dev/null
-        helm repo update antrea >/dev/null
-        helm upgrade --install antrea antrea/antrea \
-            --version "${ANTREA_VERSION:->=2.6.0}" \
-            --namespace kube-system \
-            -f "$ANTREA_VALUES" \
-            --wait --timeout 10m
-        ok
-        # Skips the Cilium install below without duplicating the node-Ready
-        # wait that follows it.
-        CNI_INSTALLED=1
-    fi
-
-    if [[ "${CNI_INSTALLED:-0}" != "1" ]]; then
     step 0 "Installing Cilium (CNI)..."
     # Must run before the node-Ready wait below: with `cni: none` the kubelet
     # reports NotReady until a CNI is present, so that loop would time out.
@@ -1357,7 +1331,6 @@ EOF
         --wait --timeout 10m
     rm -f "$CILIUM_VALUES"
     ok
-    fi
 
     step 0 "Waiting for K8s nodes..."
     expected=${#ALL_NODES[@]}
@@ -1419,16 +1392,6 @@ EOF
             echo -e "  $vm_node: partition and label ${GREEN}(OK)${NC}"
         done
 
-        # The CNI evaluation runs here, inside the same process: KUBECONFIG
-        # points at the throwaway cluster only for the life of this phase, and
-        # the VMs are destroyed the moment it ends.
-        if [[ "${CNI:-cilium}" == "antrea" ]]; then
-            step 0 "Running the Antrea BGP evaluation..."
-            "$REPO_ROOT/kubernetes/cni-eval/run-bgp-test.sh" \
-                || fail "Antrea BGP evaluation failed — see above"
-            ok
-        fi
-
         echo -e "${GREEN}VM rebuild test passed.${NC} ${#ALL_NODES[@]} nodes Ready on the repo's own config."
     else
         mkdir -p "$ACCESS_STAGING"

kubernetes/cni-eval/README.md

@@ -1,139 +0,0 @@
-# CNI evaluation — Antrea
-
-Throwaway-cluster harness for deciding whether Antrea can replace Cilium.
-Nothing here is reconciled by Flux and nothing here runs against the homelab.
-
-## Why this exists
-
-Cilium ties pod east-west traffic to the kubelet's Kubernetes node IP and gives
-no way to separate them. Three attempts on the live cluster, 2026-09-08:
-
-| Attempt | Result |
-|---|---|
-| `--ipv4-node`/`--ipv6-node` via `CiliumNodeConfig` | agent logged both flags at startup, still published its `mgmt0` address |
-| `directRoutingDevice: pod0` | agent logged `Direct routing device detected direct-routing-device=pod0`, still installed `10.244.2.0/24 via 10.2.40.138 dev mgmt0` |
-| tunnel and native mode, v4 and v6 | routes and tunnel endpoints always keyed to the peer's `InternalIP` |
-
-That blocks issue #576: pod traffic cannot leave the management VLAN while the
-apiserver endpoints have to stay on it, because both follow the same address.
-
-Antrea documents one setting covering both paths — *"The IP address used for
-tunneling **or** routing the traffic across Nodes is decided in the following
-order of preference: 1. TransportInterface 2. TransportInterfaceCIDRs 3. The
-Node IP."* Calico can do it too. Antrea fits the rest of the requirements
-better, with one exception: **its BGP is alpha**, and BGP is load-bearing here —
-every LoadBalancer VIP reaches the LAN through it.
-
-So the question this harness answers is narrow: **does Antrea's alpha
-`BGPPolicy` actually hold a session with FRR and advertise what the homelab
-needs?** If it does not, the answer is Calico and this goes no further.
-
-## Stage 1 — free, local
-
-The QEMU provisioner gives one flat NIC per VM (`site-qemu.yaml`: *"One NIC, no
-VLANs"*), so this stage cannot test interface pinning. It tests everything else,
-at no cost:
-
-Runs from the `feat/antrea-eval` branch, which is never merged:
-`bootstrap.sh` on master is the production rebuild path.
-
-**In the cluster — the way to run it.** The VMs want ~40 GiB, and
-`kubernetes/vmtest-job.yaml` records that running this on the workstation
-OOM-killed the desktop session. The eval Job is that same Job with three values
-changed: the branch, `CNI=antrea`, and the clone source — the GitHub mirror is
-private and the stock Job clones it anonymously, so this one clones the
-anonymously-readable Gitea repo instead.
-
-```bash
-yq -y '(select(.kind == "Job") | .spec.template.spec.containers[0].env[] | select(.name == "BRANCH") | .value) = "feat/antrea-eval"
-  | (select(.kind == "Job") | .spec.template.spec.containers[0].env) += [{"name": "CNI", "value": "antrea"}]
-  | (select(.kind == "Job") | .spec.template.spec.containers[0].args[0]) |= sub("https://github.com/konairius/Homelab.git"; "https://gitea.renner.uno/k.renner/homelab.git")' \
-  kubernetes/vmtest-job.yaml | kubectl apply -f -
-
-kubectl -n vmtest logs -f job/vmtest
-```
-
-`-y` is python-yq; go-yq emits YAML without it. The Talos-only phase needs one
-secret, `ghcr-pull`; the log prints the exact command to stream it in. The BGP
-evaluation runs inside the Job once the nodes are Ready, so PASS/FAIL is in the
-log and the VMs are destroyed when it ends. The Job cannot reach this cluster:
-it mounts no service account token. Clean up with `kubectl delete namespace vmtest`.
-
-**On a workstation — only with ~40 GiB to spare.** `vmtest` destroys its VMs on
-exit and exports its isolated `KUBECONFIG` only inside its own process, so hold
-them open and point the test at them explicitly:
-
-```bash
-# terminal 1 — needs sudo; holds the VMs until stopped
-VMTEST_KEEP=1 CNI=antrea ./kubernetes/bootstrap.sh vmtest
-# it prints: talosctl --talosconfig /tmp/tmp.XXXXXX/talosconfig ...
-
-# terminal 2 — the kubeconfig sits next to that talosconfig
-KUBECONFIG=/tmp/tmp.XXXXXX/kubeconfig ./kubernetes/cni-eval/run-bgp-test.sh
-
-# teardown
-sudo talosctl cluster destroy --name bootstrap-vmtest --state /tmp/bootstrap-vmtest
-sudo rm -rf /tmp/bootstrap-vmtest
-```
-
-`run-bgp-test.sh` refuses to run without an explicit `KUBECONFIG`, and refuses
-any cluster whose API is `k8s.renner.uno` or on 10.2.40.0/24, or whose nodes are
-named `k8s-1..3`.
-
-What passing means:
-
-- Antrea installs and runs on Talos at all (needs >= 2.6 — earlier agents fail
-  to start on Talos >= 1.7 after its iptables-nft switch).
-- The cluster reaches Ready with `cluster.proxy.disabled: true`, i.e. AntreaProxy
-  genuinely replaces kube-proxy, reaching the API through KubePrism.
-- `ServiceExternalIP` allocates a LoadBalancer IP.
-- `BGPPolicy` establishes with FRR from every node and advertises the pod CIDRs
-  and the LoadBalancer IP — verified by reading FRR's table, not Antrea's.
-
-`run-bgp-test.sh` deploys FRR as an ordinary pod standing in for the UDM: same
-ASNs as `unifi/bgp.conf` (64512/64513), dynamic neighbours via a listen range
-rather than named node addresses. Its peer address is read from the running pod
-at apply time rather than written into the manifest.
-
-## Stage 2 — billed, only if stage 1 passes
-
-Hetzner nodes have two interfaces and no shared L2, so this is the only place
-the two remaining requirements can be tested:
-
-```bash
-CNI=antrea ./kubernetes/bootstrap.sh hcloudtest
-```
-
-Uncomment `transportInterface` and `trafficEncapMode: hybrid` in
-`antrea-values.yaml` first, pointing the former at the private-network NIC while
-`site-hcloud.yaml` keeps the kubelet node IP where it is. Then confirm:
-
-- pod east-west leaves on the nominated interface while the node IP does not
-  move — the property Cilium cannot express;
-- `hybrid` encapsulates across a routed network, which is what the Hetzner
-  cluster needs and the homelab does not.
-
-`hcloudtest` bills by the hour on three cx43s and tears down from its EXIT trap.
-If a run is killed before the trap fires, `./kubernetes/bootstrap.sh hcloudclean`
-deletes by label.
-
-## What this harness does not cover
-
-- **Policy migration.** 59 `CiliumNetworkPolicy` objects across 34 files, using
-  `fromEntities`/`toEntities` in 15, typed ICMP in 24, and `toFQDNs` in
-  `trmnl-ns.yaml`. Antrea has equivalents including FQDN egress in OSS, but the
-  port is real work and is not tested here.
-- **Multus.** Required either way — Antrea's native secondary networks are VLAN
-  and SR-IOV only, and all six NADs are macvlan or host-device. Unchanged by a
-  CNI swap.
-- **In-place migration.** There isn't one. Talos documents CNI choice as
-  bootstrap-time only, and a Sidero maintainer's guidance is *"much easier to
-  redeploy the cluster from scratch"*. Adopting Antrea means a cold rebuild.
-
-## Cleanup
-
-```bash
-kubectl delete namespace cni-eval
-kubectl delete bgppolicy cni-eval-peer
-kubectl delete externalippool cni-eval-lb
-```

kubernetes/cni-eval/antrea-values.yaml

@@ -1,33 +0,0 @@
-# Antrea Helm values for the CNI evaluation. Not reconciled by Flux; used only
-# by `CNI=antrea ./bootstrap.sh vmtest` and `hcloudtest`.
-
-# Talos blocks CAP_SYS_MODULE for workloads and builds openvswitch into its
-# kernel, so the modprobe init step is skipped rather than granted.
-agent:
-  dontLoadKernelModules: true
-  installCNI:
-    securityContext:
-      capabilities: []
-
-# talos/patches/cluster.yaml sets cluster.proxy.disabled, so there is no
-# kube-proxy: AntreaProxy has to serve every Service type, and the agent has to
-# reach the API server through KubePrism rather than a Service IP.
-antreaProxy:
-  proxyAll: true
-kubeAPIServerOverride: "localhost:7445"
-
-# BGPPolicy is alpha and off by default. ServiceExternalIP is the LoadBalancer
-# IPAM; on by default since 2.3, named here so the manifest states it.
-featureGates:
-  BGPPolicy: true
-  ServiceExternalIP: true
-
-# Matches --service-cluster-ip-range in talos/patches/cluster.yaml.
-serviceCIDR: 10.96.0.0/12
-serviceCIDRv6: fd10:96::/112
-
-# Stage 2 only (hcloudtest, two NICs). Pins pod east-west to a named interface
-# while the kubelet node IP stays where it is — the property Cilium cannot
-# express. Left unset here because the QEMU provisioner gives one flat NIC.
-# transportInterface: ""
-# trafficEncapMode: hybrid

kubernetes/cni-eval/bgp-test.yaml

@@ -1,48 +0,0 @@
-# What the test asks Antrea to advertise. Applied after frr-peer.yaml.
-# __FRR_PEER_IP__ is substituted by run-bgp-test.sh from the running FRR pod —
-# not written here, because it is a lease, not an identity.
----
-apiVersion: crd.antrea.io/v1beta1
-kind: ExternalIPPool
-metadata:
-  name: cni-eval-lb
-spec:
-  ipRanges:
-    - start: 10.99.99.10
-      end: 10.99.99.20
-  nodeSelector: {}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: lb-probe
-  namespace: cni-eval
-  annotations:
-    service.antrea.io/external-ip-pool: cni-eval-lb
-spec:
-  type: LoadBalancer
-  selector:
-    app: frr
-  ports:
-    - port: 179
-      targetPort: 179
----
-apiVersion: crd.antrea.io/v1alpha1
-kind: BGPPolicy
-metadata:
-  name: cni-eval-peer
-spec:
-  nodeSelector:
-    matchLabels:
-      kubernetes.io/os: linux
-  localASN: 64513
-  listenPort: 179
-  advertisements:
-    pod: {}
-    service:
-      ipTypes:
-        - LoadBalancerIP
-  bgpPeers:
-    - address: __FRR_PEER_IP__
-      asn: 64512
-      port: 179

kubernetes/cni-eval/frr-peer.yaml

@@ -1,106 +0,0 @@
-# FRR standing in for the UDM, so Antrea's BGP can be tested without one.
-# Mirrors unifi/bgp.conf where it matters: same ASNs, dynamic neighbours via a
-# listen range rather than named node addresses.
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: cni-eval
-  labels:
-    # Talos enforces baseline, which rejects the NET_ADMIN and NET_RAW zebra
-    # needs. Throwaway cluster only.
-    pod-security.kubernetes.io/enforce: privileged
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: frr-config
-  namespace: cni-eval
-data:
-  daemons: |
-    bgpd=yes
-    zebra=yes
-    vtysh_enable=yes
-    bgpd_options="-A 127.0.0.1"
-    zebra_options="-A 127.0.0.1 -s 90000000"
-  frr.conf: |
-    frr defaults traditional
-    hostname frr-eval
-    log stdout informational
-    !
-    router bgp 64512
-     bgp router-id 172.31.255.1
-     no bgp ebgp-requires-policy
-     bgp listen range 0.0.0.0/0 peer-group K8S
-     bgp listen range ::/0 peer-group K8S
-     neighbor K8S peer-group
-     neighbor K8S remote-as 64513
-     neighbor K8S timers 30 90
-     !
-     address-family ipv4 unicast
-      neighbor K8S activate
-      neighbor K8S soft-reconfiguration inbound
-     exit-address-family
-     !
-     address-family ipv6 unicast
-      neighbor K8S activate
-      neighbor K8S soft-reconfiguration inbound
-     exit-address-family
-    !
-    line vty
-    !
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: frr
-  namespace: cni-eval
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: frr
-  template:
-    metadata:
-      labels:
-        app: frr
-    spec:
-      # /etc/frr must be writable for the daemons and vtysh; a ConfigMap mount
-      # is not. Seeded from the image's own defaults plus the ConfigMap.
-      initContainers:
-        - name: seed-config
-          image: quay.io/frrouting/frr:10.2.1
-          command:
-            - sh
-            - -c
-            - cp -a /etc/frr/. /frr-etc/ && cp /config/frr.conf /config/daemons /frr-etc/
-          volumeMounts:
-            - name: config
-              mountPath: /config
-            - name: etc
-              mountPath: /frr-etc
-      containers:
-        - name: frr
-          image: quay.io/frrouting/frr:10.2.1
-          securityContext:
-            capabilities:
-              add:
-                - NET_ADMIN
-                - NET_BIND_SERVICE
-                - NET_RAW
-          ports:
-            - containerPort: 179
-              name: bgp
-          volumeMounts:
-            - name: etc
-              mountPath: /etc/frr
-            - name: run
-              mountPath: /var/run/frr
-      volumes:
-        - name: config
-          configMap:
-            name: frr-config
-        - name: etc
-          emptyDir: {}
-        - name: run
-          emptyDir: {}

kubernetes/cni-eval/run-bgp-test.sh

@@ -1,118 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Verifies Antrea's alpha BGPPolicy against an FRR peer. Answers exactly one
-# question: does the session establish and carry the routes the homelab needs
-# (pod CIDRs and LoadBalancer IPs). Everything it asserts is read back from
-# FRR, not from Antrea's own view of what it advertised.
-#
-#   kubernetes/cni-eval/run-bgp-test.sh
-#
-# Expects a cluster already built with CNI=antrea. Leaves the cni-eval
-# namespace behind for inspection; the cleanup line is printed at the end.
-
-DIR="$(cd "$(dirname "$0")" && pwd)"
-RED=$'\033[0;31m'; GREEN=$'\033[0;32m'; YELLOW=$'\033[1;33m'; NC=$'\033[0m'
-fail() { echo "${RED}FAIL${NC} $*" >&2; exit 1; }
-pass() { echo "${GREEN}PASS${NC} $*"; }
-info() { echo "${YELLOW}==>${NC} $*"; }
-
-# vmtest and hcloudtest export their isolated KUBECONFIG only inside their own
-# process. A shell that runs this afterwards gets the default kubeconfig, which
-# is production. Refuse unless pointed at a throwaway cluster explicitly.
-[[ -n "${KUBECONFIG:-}" ]] \
-    || fail "set KUBECONFIG to the throwaway cluster's kubeconfig — the default one is production"
-server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
-case "$server" in
-    *k8s.renner.uno*|*10.2.40.*) fail "KUBECONFIG points at production ($server) — refusing" ;;
-esac
-node_names=$(kubectl get nodes -o name)
-if grep -qx 'node/k8s-[123]' <<<"$node_names"; then
-    fail "this cluster has production node names — refusing"
-fi
-pass "target is a throwaway cluster ($server)"
-
-info "Checking Antrea is the running CNI..."
-kubectl -n kube-system get daemonset antrea-agent >/dev/null \
-    || fail "antrea-agent DaemonSet not found — was the cluster built with CNI=antrea?"
-
-info "Confirming the BGPPolicy feature gate is on..."
-gates=$(kubectl -n kube-system get configmap antrea-config -o jsonpath='{.data.antrea-agent\.conf}')
-grep -q 'BGPPolicy: *true' <<<"$gates" \
-    || fail "BGPPolicy feature gate is not enabled in antrea-config"
-pass "BGPPolicy enabled"
-
-info "Deploying the FRR peer..."
-kubectl apply -f "$DIR/frr-peer.yaml"
-kubectl -n cni-eval rollout status deployment/frr --timeout=180s \
-    || fail "FRR did not become ready"
-
-# Read the address rather than assign one: the peer is a pod, and its address
-# is whatever the CNI under test gave it.
-frr_ip=$(kubectl -n cni-eval get pod -l app=frr -o jsonpath='{.items[0].status.podIP}')
-[[ -n "$frr_ip" ]] || fail "FRR pod has no IP"
-pass "FRR peer at $frr_ip"
-
-info "Applying ExternalIPPool, LoadBalancer Service and BGPPolicy..."
-sed "s|__FRR_PEER_IP__|$frr_ip|" "$DIR/bgp-test.yaml" | kubectl apply -f -
-
-info "Waiting for a LoadBalancer IP (ServiceExternalIP)..."
-lb=""
-for _ in $(seq 1 30); do
-    lb=$(kubectl -n cni-eval get svc lb-probe -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
-    [[ -n "$lb" ]] && break
-    sleep 5
-done
-[[ -n "$lb" ]] || fail "no LoadBalancer IP assigned after 150s — ServiceExternalIP did not allocate"
-pass "LoadBalancer IP $lb"
-
-frr_pod=$(kubectl -n cni-eval get pod -l app=frr -o jsonpath='{.items[0].metadata.name}')
-
-info "Waiting for BGP sessions to establish..."
-nodes=$(kubectl get nodes -o name | wc -l)
-established=0
-for _ in $(seq 1 36); do
-    # The text summary shows an established peer as a prefix count, not a state
-    # name; the per-neighbour JSON carries bgpState explicitly.
-    neighbors=$(kubectl -n cni-eval exec "$frr_pod" -- vtysh -c 'show ip bgp neighbor json' || true)
-    established=$(jq '[.[] | select(type == "object" and .bgpState == "Established")] | length' <<<"$neighbors" || echo 0)
-    established=${established:-0}
-    [[ "$established" -ge "$nodes" ]] && break
-    sleep 5
-done
-kubectl -n cni-eval exec "$frr_pod" -- vtysh -c 'show ip bgp summary'
-[[ "$established" -ge "$nodes" ]] \
-    || fail "only $established of $nodes sessions established after 180s"
-pass "$established/$nodes sessions established"
-
-info "Checking the advertised routes as FRR sees them..."
-kubectl -n cni-eval exec "$frr_pod" -- vtysh -c 'show ip bgp'
-routes=$(kubectl -n cni-eval exec "$frr_pod" -- vtysh -c 'show ip bgp json')
-
-# Exact prefix keys rather than substring matches.
-advertised() { jq -e --arg p "$1" '.routes | has($p)' <<<"$routes" >/dev/null; }
-
-pod_cidrs=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.podCIDR}{"\n"}{end}')
-missing=0
-while read -r cidr; do
-    [[ -z "$cidr" ]] && continue
-    if advertised "$cidr"; then
-        pass "pod CIDR $cidr advertised"
-    else
-        echo "${RED}FAIL${NC} pod CIDR $cidr NOT advertised"
-        missing=1
-    fi
-done <<<"$pod_cidrs"
-
-if grep -qF "$lb" <<<"$routes"; then
-    pass "LoadBalancer IP $lb advertised"
-else
-    echo "${RED}FAIL${NC} LoadBalancer IP $lb NOT advertised"
-    missing=1
-fi
-
-[[ "$missing" -eq 0 ]] || fail "one or more expected prefixes were not advertised"
-
-echo
-pass "Antrea BGPPolicy carried pod CIDRs and LoadBalancer IPs to FRR."
-echo "Clean up: kubectl delete namespace cni-eval && kubectl delete bgppolicy cni-eval-peer && kubectl delete externalippool cni-eval-lb"

kubernetes/infrastructure/bird/bird.conf

@@ -0,0 +1,89 @@
+# Identical on every node and free of addresses. defines.conf and peers.conf
+# are rendered per node by the init container; the first is included above the
+# templates that reference it, the second below them.
+include "/etc/bird/defines.conf";
+
+router id from "pod0";
+log stderr all;
+
+# Mirrors of the kernel main table, filtered to this node's own pod CIDR.
+# master4/master6 are implicit and hold what BGP exchanges.
+ipv4 table kmain4;
+ipv6 table kmain6;
+
+protocol device { scan time 10; }
+
+# Two conditions. cilium_host also carries its own host address, which the
+# length check drops; and Cilium's routes for the OTHER nodes are protocol
+# kernel too, which the interface check drops. ifname is set for the
+# gatewayless IPv6 device route as well, so this works in both families.
+filter own_pod4 {
+  if ifname !~ "cilium_host" then reject;
+  if net.len = 32 then reject;
+  accept;
+}
+filter own_pod6 {
+  if ifname !~ "cilium_host" then reject;
+  if net.len = 128 then reject;
+  accept;
+}
+
+# learn all, not learn: plain learn skips routes marked proto kernel, which is
+# what Cilium's are.
+protocol kernel klearn4 {
+  learn all;
+  scan time 10;
+  ipv4 { table kmain4; import filter own_pod4; export none; };
+}
+protocol kernel klearn6 {
+  learn all;
+  scan time 10;
+  ipv6 { table kmain6; import filter own_pod6; export none; };
+}
+
+protocol pipe own4 { table kmain4; peer table master4; export all; import none; }
+protocol pipe own6 { table kmain6; peer table master6; export all; import none; }
+
+# Only BGP-learned routes reach table 48. The local CIDR stays in main, where
+# cilium_host already serves it.
+filter to_t48 {
+  if source != RTS_BGP then reject;
+  accept;
+}
+
+protocol kernel k48v4 {
+  kernel table 48;
+  metric 0;
+  ipv4 { table master4; import none; export filter to_t48; };
+}
+protocol kernel k48v6 {
+  kernel table 48;
+  metric 0;
+  ipv6 { table master6; import none; export filter to_t48; };
+}
+
+# iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM
+# from ingress0 and is unrelated.
+#
+# No prefix filters: the only route this node offers is its own pod CIDR, which
+# own_pod4/6 already selected, and the only peers are the other cluster nodes.
+#
+# direct is explicit because iBGP defaults to multihop, which makes the gateway
+# recursive — BIRD would try to resolve the peer in the pod RIB. strict bind
+# keeps the listener off mgmt0 and ingress0.
+template bgp pod4 {
+  local as 64514;
+  source address MY4;
+  direct;
+  strict bind;
+  ipv4 { next hop self; import all; export all; };
+}
+template bgp pod6 {
+  local as 64514;
+  source address MY6;
+  direct;
+  strict bind;
+  ipv6 { next hop self; import all; export all; };
+}
+
+include "/etc/bird/peers.conf";

kubernetes/infrastructure/bird/daemonset.yaml

@@ -0,0 +1,115 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: bird
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: bird
+rules:
+  # The peer list is read from the nodes' pod0 annotations, so no address is
+  # written into this repo twice.
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bird
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: bird
+subjects:
+  - kind: ServiceAccount
+    name: bird
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: bird
+  namespace: kube-system
+  labels:
+    tier: node
+    app: bird
+spec:
+  selector:
+    matchLabels:
+      app: bird
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: bird
+    spec:
+      # Carries pod east-west once the routing rules land. Without it table 48
+      # stays empty and the rules fall through to main, i.e. back to VLAN 40.
+      priorityClassName: system-node-critical
+      # Writes host routes and peers on host addresses.
+      hostNetwork: true
+      serviceAccountName: bird
+      # Every node here is a control plane node, so this is not optional.
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      initContainers:
+        - name: render-config
+          # By digest, not by the dated tag: the tag is rebuilt within the day
+          # it names, and the cluster's image path keeps resolving it to the
+          # first build of that day — imagePullPolicy: Always does not help,
+          # because the tag itself resolves stale. Update this when the image
+          # is rebuilt.
+          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -eu
+              cp /config/bird.conf /etc/bird/bird.conf
+              kubectl get node "$NODE_NAME" \
+                -o go-template-file=/config/defines.tmpl > /etc/bird/defines.conf
+              kubectl get nodes --field-selector "metadata.name!=$NODE_NAME" \
+                -o go-template-file=/config/peers.tmpl > /etc/bird/peers.conf
+              # BIRD's own parser is the check: a node whose annotation is
+              # missing renders as <no value> and fails here rather than
+              # crashlooping the daemon or forming a mesh with a hole.
+              bird -p -c /etc/bird/bird.conf
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: etc-bird
+              mountPath: /etc/bird
+      containers:
+        - name: bird
+          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
+          resources:
+            requests:
+              cpu: "50m"
+              memory: "32Mi"
+            limits:
+              cpu: "200m"
+              memory: "128Mi"
+          securityContext:
+            capabilities:
+              drop: ["ALL"]
+              # NET_ADMIN writes the routes; NET_BIND_SERVICE binds 179.
+              add: ["NET_ADMIN", "NET_BIND_SERVICE"]
+          volumeMounts:
+            - name: etc-bird
+              mountPath: /etc/bird
+      volumes:
+        - name: config
+          configMap:
+            name: bird-config
+        - name: etc-bird
+          emptyDir: {}

kubernetes/infrastructure/bird/defines.tmpl

@@ -0,0 +1,2 @@
+define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }};
+define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }};

kubernetes/infrastructure/bird/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - daemonset.yaml
+configMapGenerator:
+  # Hashed name, so editing any of these rolls the DaemonSet.
+  - name: bird-config
+    namespace: kube-system
+    files:
+      - bird.conf
+      - defines.tmpl
+      - peers.tmpl

kubernetes/infrastructure/bird/peers.tmpl

@@ -0,0 +1,7 @@
+{{- range $i, $n := .items -}}
+{{- $v6 := index $n.metadata.annotations "net.renner.uno/pod0-v6" -}}
+{{- $v4 := index $n.metadata.annotations "net.renner.uno/pod0-v4" -}}
+# {{ $n.metadata.name }}
+protocol bgp peer{{ $i }}_v6 from pod6 { neighbor {{ $v6 }} as 64514; }
+protocol bgp peer{{ $i }}_v4 from pod4 { neighbor {{ $v4 }} as 64514; }
+{{ end -}}

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -180,3 +180,17 @@ spec:
     # the supernets here cover the LAN as well, which is the point.
     ipv4NativeRoutingCIDR: 10.0.0.0/8
     ipv6NativeRoutingCIDR: fd00::/8
+    # Pod east-west leaves the node on pod0 (VLAN 48) rather than the
+    # management VLAN. Native routing hands forwarding to the kernel;
+    # autoDirectNodeRoutes installs the remote PodCIDR routes, which is
+    # allowed because all three nodes share pod0's L2 segment.
+    #
+    # directRoutingDevice is explicit on purpose: with several addressed
+    # devices Cilium falls back to the one carrying the Kubernetes InternalIP,
+    # which is mgmt0.
+    #
+    # Encapsulation is gone, so the datapath MTU goes 1450 -> 1500.
+    routingMode: native
+    autoDirectNodeRoutes: true
+    nodePort:
+      directRoutingDevice: pod0

kubernetes/infrastructure/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
   - cluster-settings/
   - cilium/
   - multus/
+  - bird/
   - cert-manager/
   - snapshot-controller/
   - metrics-server/

kubernetes/talos/patches/k8s-1.yaml

@@ -52,6 +52,12 @@ machine:
     # this label instead of listing hostnames, so which boxes carry storage is
     # stated once, here, by the machine that knows.
     storage.renner.uno/ceph-osd: "true"
+  # Publishes this node's pod0 addresses, declared below in its VLANConfig, so
+  # the bird DaemonSet reads its peers from the node list rather than from a
+  # copy. Annotations, not labels: a label value cannot contain a colon.
+  nodeAnnotations:
+    net.renner.uno/pod0-v6: fd02:48::11
+    net.renner.uno/pod0-v4: 10.2.48.11
   udev:
     # Renames every port this node uses to its role at boot, matched by
     # burned-in MAC — kernel names derive from PCI topology and shift when
@@ -182,8 +188,14 @@ addresses:
 # network's clients selectively, so the client goes rather than the record.
 # DHCP is off on VLAN 47 for the same reason.
 #
-# The metric loses to mgmt0's default 1024 deliberately: this pass only adds
-# the segment. The default route moves here in the change that flips it.
+# Both defaults live in table 47, not in main, so only the pod-source rules in
+# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
+# reply the node sends from its own addresses.
+#
+# The v6 gateway is the UDM's configured ULA on this segment, the same address
+# BGP peers with. The RA-learned link-local would be the alternative and is not
+# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
+# default would land in main, where it does not belong.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ingress0
@@ -193,8 +205,10 @@ up: true
 addresses:
   - address: 10.2.47.11/24
 routes:
+  - gateway: fd02:47::1
+    table: "47"
   - gateway: 10.2.47.1
-    metric: 2048
+    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication and nothing else. The udev rule in the machine

kubernetes/talos/patches/k8s-2.yaml

@@ -33,6 +33,10 @@ machine:
     topology.kubernetes.io/zone: saturn
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
+  # pod0 addresses for the bird peer list — see k8s-1.yaml.
+  nodeAnnotations:
+    net.renner.uno/pod0-v6: fd02:48::12
+    net.renner.uno/pod0-v4: 10.2.48.12
   udev:
     # Renames every port to its role at boot — see k8s-1.yaml.
     rules:
@@ -129,8 +133,10 @@ up: true
 addresses:
   - address: 10.2.47.12/24
 routes:
+  - gateway: fd02:47::1
+    table: "47"
   - gateway: 10.2.47.1
-    metric: 2048
+    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.

kubernetes/talos/patches/k8s-3.yaml

@@ -29,6 +29,10 @@ machine:
     topology.kubernetes.io/zone: neptune
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
+  # pod0 addresses for the bird peer list — see k8s-1.yaml.
+  nodeAnnotations:
+    net.renner.uno/pod0-v6: fd02:48::13
+    net.renner.uno/pod0-v4: 10.2.48.13
   udev:
     # Renames every port to its role at boot — see k8s-1.yaml.
     rules:
@@ -116,8 +120,10 @@ up: true
 addresses:
   - address: 10.2.47.13/24
 routes:
+  - gateway: fd02:47::1
+    table: "47"
   - gateway: 10.2.47.1
-    metric: 2048
+    table: "47"
 ---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
 # here on purpose; the link is handed whole to the OSD pod (host-device).

kubernetes/talos/patches/site-homelab.yaml

@@ -20,3 +20,47 @@ machine:
       validSubnets:
         - 10.2.40.0/24
         - fd40::/64
+---
+# Pod egress leaves over ingress0. Matching on the pod source prefix is what
+# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
+# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
+# source onto VLAN 47.
+#
+# The routing decision runs before Cilium's masquerade hook, so the rule sees
+# the pod address and the masquerade then rewrites it to ingress0's.
+#
+# Table 47 holds nothing but the two default routes, so the destination rules
+# below have to run first: without them a pod's packet to a service or to a pod
+# on another node also matches the source rule, finds only the default, and
+# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
+#
+# They send everything inside the cluster and the local networks back to main.
+# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
+# is not foreign traffic.
+#
+# Priorities are below main (32766) and clear of the rule Cilium installs at 9.
+# The two families are separate rule tables, so the priorities need not match;
+# the document name is the priority and must be unique.
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "990"
+dst: fd00::/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "991"
+dst: 10.0.0.0/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "1000"
+src: fd10:244::/48
+table: "47"
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "1001"
+src: 10.244.0.0/16
+table: "47"