feat: one pod routing table, decided by longest prefix (#576)

The rule stack grew a carve-out layer because table 47 held nothing but a default route: it matched everything, so every cluster-internal destination had to be excluded by a rule naming its prefix. Those exclusions were wrong twice over — too broad, since fd00::/8 and 10.0.0.0/8 also caught every LAN client and sent its reply back over mgmt0, and unnecessary, since the networks in question are directly attached and a connected route already beats a default.

So the table gets the connected routes and decides on its own. bird's direct protocol generates them from the interfaces rather than from a list, alongside the peers' pod CIDRs it already learns and the egress default it now originates. One rule per family remains, on the pod source prefix; 980/981, 990/991 and table 48 are gone.

bird owns the table outright, including the default. That is what keeps the failure benign: if the daemon stops the table empties, the rule finds nothing and falls through to main, so pod traffic returns to VLAN 40 unseparated rather than misrouted. A default declared in the Talos config would have outlived the specific routes and sent pod traffic to the UDM.

The ingress gateway moves from a route in each node patch to one node annotation in the site patch, which is now the only place it is stated.

Verified: bird -p parses the rendered config inside the image, exit 0; the templates render GW6 fd02:47::1 and GW4 10.2.47.1 from the live k8s-1 object; kubectl kustomize builds.

Refs #576

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

kubernetes/infrastructure/bird/bird.conf

@@ -8,14 +8,43 @@ log stderr all;
 
 protocol device { scan time 10; }
 
+# Everything a pod may need to reach, in one table, resolved by longest prefix:
+# the node's attached networks, the peers' pod CIDRs, and a default for the
+# rest. No rule has to carve out exceptions, because the specific routes win on
+# their own.
+#
+# The table is BIRD's alone. If this daemon stops, it empties, the rule finds
+# nothing and falls through to main — pod traffic returns to VLAN 40,
+# unseparated rather than misrouted. That is why the default route lives here
+# and not in the Talos config: a default that outlived the specific routes
+# would send pod traffic to the UDM.
+
+# The node's attached networks, taken from the interfaces rather than written
+# down. The usual advice is not to export these to the kernel because it
+# maintains them itself — true of main, not of a table the kernel never fills.
+protocol direct attached {
+  interface "mgmt0", "ingress0", "pod0";
+  ipv4 { table master4; };
+  ipv6 { table master6; };
+}
+
+# Pod egress. GW4/GW6 are the UDM on the ingress segment, published as node
+# annotations beside the address that defines them.
+protocol static egress4 {
+  ipv4 { table master4; };
+  route 0.0.0.0/0 via GW4;
+}
+protocol static egress6 {
+  ipv6 { table master6; };
+  route ::/0 via GW6;
+}
+
 # This node's own pod CIDR, originated rather than read back out of the kernel.
 # Reading it back made BIRD's view depend on the shape of Cilium's route, and
 # the IPv4 one is rejected outright: its next hop sits inside the prefix it
-# describes, which KRT refuses with "strange next-hop". The prefix comes from
-# the Node object either way, so there is nothing to learn.
+# describes, which KRT refuses with "strange next-hop".
 #
-# blackhole because this route is only ever advertised. Nothing installs it:
-# the only kernel protocol exporting from these tables takes BGP routes.
+# blackhole because it is only ever advertised — it is filtered out below.
 protocol static own4 {
   ipv4 { table master4; };
   route OWN_POD4 blackhole;
@@ -25,22 +54,22 @@ protocol static own6 {
   route OWN_POD6 blackhole;
 }
 
-# Only BGP-learned routes reach table 48 — not the static above, and not
-# anything else that may end up in the master tables.
-filter to_t48 {
-  if source != RTS_BGP then reject;
+# Everything except this node's own CIDR, which cilium_host already serves in
+# main and which is a blackhole here.
+filter to_pod_table {
+  if source = RTS_STATIC && dest = RTD_BLACKHOLE then reject;
   accept;
 }
 
-protocol kernel k48v4 {
-  kernel table 48;
+protocol kernel k4 {
+  kernel table 47;
   metric 0;
-  ipv4 { table master4; import none; export filter to_t48; };
+  ipv4 { table master4; import none; export filter to_pod_table; };
 }
-protocol kernel k48v6 {
-  kernel table 48;
+protocol kernel k6 {
+  kernel table 47;
   metric 0;
-  ipv6 { table master6; import none; export filter to_t48; };
+  ipv6 { table master6; import none; export filter to_pod_table; };
 }
 
 # iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM
@@ -49,19 +78,30 @@ protocol kernel k48v6 {
 # direct is explicit because iBGP defaults to multihop, which makes the gateway
 # recursive — BIRD would try to resolve the peer in the pod RIB. strict bind
 # keeps the listener off mgmt0 and ingress0.
+#
+# Only the node's own CIDR is offered: the export filter drops everything the
+# static protocols and the direct protocol produce.
 template bgp pod4 {
   local as 64514;
   source address MY4;
   direct;
   strict bind;
-  ipv4 { next hop self; import all; export all; };
+  ipv4 {
+    next hop self;
+    import all;
+    export filter { if net = OWN_POD4 then accept; reject; };
+  };
 }
 template bgp pod6 {
   local as 64514;
   source address MY6;
   direct;
   strict bind;
-  ipv6 { next hop self; import all; export all; };
+  ipv6 {
+    next hop self;
+    import all;
+    export filter { if net = OWN_POD6 then accept; reject; };
+  };
 }
 
 include "/etc/bird/peers.conf";

kubernetes/infrastructure/bird/defines.tmpl

@@ -1,5 +1,7 @@
 define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }};
 define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }};
+define GW6 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v6" }};
+define GW4 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v4" }};
 {{/* podCIDRs follows the cluster's podSubnets order, v4 first. A flip puts an
      IPv6 prefix into an ipv4 protocol, which fails the parse in the init
      container rather than passing silently. */}}

kubernetes/talos/patches/k8s-1.yaml

@@ -188,14 +188,13 @@ addresses:
 # network's clients selectively, so the client goes rather than the record.
 # DHCP is off on VLAN 47 for the same reason.
 #
-# Both defaults live in table 47, not in main, so only the pod-source rules in
-# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
-# reply the node sends from its own addresses.
+# No route here. Table 47 belongs to bird, which fills it with this node's
+# attached networks, the peers' pod CIDRs and the egress default, and empties it
+# if it stops — leaving the rule to fall through to main. A default declared
+# here would outlive the specific routes and send pod traffic to the UDM.
 #
-# The v6 gateway is the UDM's configured ULA on this segment, the same address
-# BGP peers with. The RA-learned link-local would be the alternative and is not
-# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
-# default would land in main, where it does not belong.
+# accept_ra_defrtr stays 0 on this link so the RA default does not land in main,
+# where mgmt0 owns it.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ingress0
@@ -204,11 +203,6 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.11/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication and nothing else. The udev rule in the machine

kubernetes/talos/patches/k8s-2.yaml

@@ -132,11 +132,6 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.12/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.

kubernetes/talos/patches/k8s-3.yaml

@@ -119,11 +119,6 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.13/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
 ---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
 # here on purpose; the link is handed whole to the OSD pod (host-device).

kubernetes/talos/patches/site-homelab.yaml

@@ -20,62 +20,30 @@ machine:
       validSubnets:
         - 10.2.40.0/24
         - fd40::/64
+  # The UDM on the ingress segment. Published here rather than as a route
+  # because bird owns table 47 and originates the default from it; this is the
+  # one place the address is stated.
+  nodeAnnotations:
+    net.renner.uno/egress-gw-v6: fd02:47::1
+    net.renner.uno/egress-gw-v4: 10.2.47.1
 ---
-# Pod egress leaves over ingress0. Matching on the pod source prefix is what
-# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
-# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
-# source onto VLAN 47.
+# All pod traffic goes to table 47, and the table decides — longest prefix
+# match, no exceptions to carve out. bird fills it with the node's attached
+# networks, the peers' pod CIDRs and the egress default, so a packet to a node
+# on the mgmt segment takes the connected route, a packet to a pod on another
+# node takes pod0, and anything else takes the default over ingress0.
 #
-# The routing decision runs before Cilium's masquerade hook, so the rule sees
-# the pod address and the masquerade then rewrites it to ingress0's.
+# Matching the source is what keeps the node's own traffic out: it answers on
+# mgmt0, and a reply leaving over ingress0 would carry an mgmt0 source onto
+# VLAN 47. The routing decision runs before Cilium's masquerade hook, so the
+# rule sees the pod address and the masquerade then rewrites it.
 #
-# Table 47 holds nothing but the two default routes, so the destination rules
-# below have to run first: without them a pod's packet to a service or to a pod
-# on another node also matches the source rule, finds only the default, and
-# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
-#
-# They send everything inside the cluster and the local networks back to main.
-# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
-# is not foreign traffic.
+# If bird stops, table 47 empties, the lookup misses and the rule falls through
+# to main — pod traffic returns to VLAN 40, unseparated rather than misrouted.
 #
 # Priorities are below main (32766) and clear of the rule Cilium installs at 9.
 # The two families are separate rule tables, so the priorities need not match;
 # the document name is the priority and must be unique.
-# Pod to pod between nodes goes over pod0, via table 48, which the bird mesh
-# fills with one route per peer pod CIDR and the peer's pod0 address as the
-# next hop. Ahead of 990/991 so it wins for that traffic, and matching source
-# AND destination so nothing else lands there: the node's own traffic to a pod
-# would otherwise leave with a pod0 source, which CiliumNode does not list, so
-# Cilium would classify it as world rather than host.
-#
-# Traffic to a pod on this node falls through — table 48 holds only the peers'
-# CIDRs, so the lookup misses and 990/991 send it to main and cilium_host.
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "980"
-src: fd10:244::/48
-dst: fd10:244::/48
-table: "48"
----
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "981"
-src: 10.244.0.0/16
-dst: 10.244.0.0/16
-table: "48"
----
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "990"
-dst: fd00::/8
-table: main
----
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "991"
-dst: 10.0.0.0/8
-table: main
----
 apiVersion: v1alpha1
 kind: RoutingRuleConfig
 name: "1000"