feat(ceph): serve reads from the local OSD (#576)

Measuring VLAN 40 for the traffic-class split turned up a share of the
management segment that is not management: the node's Ceph client
talking to OSDs on the other two nodes, sourced from the node address
on mgmt0.

Ceph sends every client read to the acting primary of the PG, which
with three OSDs is remote two times out of three. Read affinity has the
client read from the replica in its own CRUSH location instead.

The preconditions hold here. replicapool and cephfs/data0 are size 3
over failureDomain host across three nodes, so every object has a copy
on every node. The OSDs are created under zone buckets
(root=default host=k8s-N zone=<name>) and the nodes carry
topology.kubernetes.io/zone, which is in Rook's default
crushLocationLabels. Ceph is v20.2.4 and the kernel 6.18, against a
documented minimum of 5.8.

Writes are unaffected: they always go to the primary, and the
replication behind them already rides the VLAN 49 cluster network.

cephfs-ci is size 2, so a local replica is not guaranteed there and the
client falls back to the primary.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

kubernetes/apps/zone-configurator/deployment.yaml

@@ -21,7 +21,7 @@ spec:
       # Service; nothing on this path needs the host's network namespace.
       containers:
         - name: zone-configurator
-          image: everythingsmarthome/everything-presence-mmwave-configurator:2.2.4 # {"$imagepolicy": "flux-system:mmwave-configurator"}
+          image: everythingsmarthome/everything-presence-mmwave-configurator:2.2.5 # {"$imagepolicy": "flux-system:mmwave-configurator"}
           env:
             - name: HA_BASE_URL
               value: "http://home-assistant:80"

kubernetes/infrastructure/bird/bird.conf

@@ -8,39 +8,64 @@ log stderr all;
 
 protocol device { scan time 10; }
 
-# This node's own pod CIDR, originated rather than read back out of the kernel.
-# Reading it back made BIRD's view depend on the shape of Cilium's route, and
-# the IPv4 one is rejected outright: its next hop sits inside the prefix it
-# describes, which KRT refuses with "strange next-hop". The prefix comes from
-# the Node object either way, so there is nothing to learn.
+# Everything a pod may need to reach, in one table, resolved by longest prefix:
+# the node's attached networks, the peers' pod CIDRs, and a default for the
+# rest. No rule has to carve out exceptions, because the specific routes win on
+# their own.
 #
-# blackhole because this route is only ever advertised. Nothing installs it:
-# the only kernel protocol exporting from these tables takes BGP routes.
+# The table is BIRD's alone. If this daemon stops, it empties, the rule finds
+# nothing and falls through to main — pod traffic returns to VLAN 40,
+# unseparated rather than misrouted. That is why the default route lives here
+# and not in the Talos config: a default that outlived the specific routes
+# would send pod traffic to the UDM.
+
+# The node's attached networks, taken from the interfaces rather than written
+# down. The usual advice is not to export these to the kernel because it
+# maintains them itself — true of main, not of a table the kernel never fills.
+protocol direct attached {
+  interface "mgmt0", "ingress0", "pod0";
+  ipv4 { table master4; };
+  ipv6 { table master6; };
+}
+
+# Pod egress. GW4/GW6 are the UDM on the ingress segment, published as node
+# annotations beside the address that defines them.
+protocol static egress4 {
+  ipv4 { table master4; };
+  route 0.0.0.0/0 via GW4;
+}
+protocol static egress6 {
+  ipv6 { table master6; };
+  route ::/0 via GW6;
+}
+
+# This node's own pod CIDR, via the interface Cilium serves it on. Originated
+# rather than read back out of the kernel: reading it back made BIRD's view
+# depend on the shape of Cilium's route, and the IPv4 one is rejected outright,
+# its next hop sitting inside the prefix it describes.
+#
+# It belongs in the table, not just in the BGP announcement. A reply from a pod
+# on another node is forwarded traffic with a pod source, so the rule matches it
+# too; without this route it would find only the default and leave for the UDM.
+# Measured — cross-node pod traffic, and with it cluster DNS, stopped.
 protocol static own4 {
   ipv4 { table master4; };
-  route OWN_POD4 blackhole;
+  route OWN_POD4 via "cilium_host";
 }
 protocol static own6 {
   ipv6 { table master6; };
-  route OWN_POD6 blackhole;
+  route OWN_POD6 via "cilium_host";
 }
 
-# Only BGP-learned routes reach table 48 — not the static above, and not
-# anything else that may end up in the master tables.
-filter to_t48 {
-  if source != RTS_BGP then reject;
-  accept;
-}
-
-protocol kernel k48v4 {
-  kernel table 48;
+protocol kernel k4 {
+  kernel table 47;
   metric 0;
-  ipv4 { table master4; import none; export filter to_t48; };
+  ipv4 { table master4; import none; export all; };
 }
-protocol kernel k48v6 {
-  kernel table 48;
+protocol kernel k6 {
+  kernel table 47;
   metric 0;
-  ipv6 { table master6; import none; export filter to_t48; };
+  ipv6 { table master6; import none; export all; };
 }
 
 # iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM
@@ -49,19 +74,30 @@ protocol kernel k48v6 {
 # direct is explicit because iBGP defaults to multihop, which makes the gateway
 # recursive — BIRD would try to resolve the peer in the pod RIB. strict bind
 # keeps the listener off mgmt0 and ingress0.
+#
+# Only the node's own CIDR is offered: the export filter drops everything the
+# static protocols and the direct protocol produce.
 template bgp pod4 {
   local as 64514;
   source address MY4;
   direct;
   strict bind;
-  ipv4 { next hop self; import all; export all; };
+  ipv4 {
+    next hop self;
+    import all;
+    export filter { if net = OWN_POD4 then accept; reject; };
+  };
 }
 template bgp pod6 {
   local as 64514;
   source address MY6;
   direct;
   strict bind;
-  ipv6 { next hop self; import all; export all; };
+  ipv6 {
+    next hop self;
+    import all;
+    export filter { if net = OWN_POD6 then accept; reject; };
+  };
 }
 
 include "/etc/bird/peers.conf";

kubernetes/infrastructure/bird/defines.tmpl

@@ -1,5 +1,7 @@
 define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }};
 define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }};
+define GW6 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v6" }};
+define GW4 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v4" }};
 {{/* podCIDRs follows the cluster's podSubnets order, v4 first. A flip puts an
      IPv6 prefix into an ipv4 protocol, which fails the parse in the init
      container rather than passing silently. */}}

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -126,6 +126,16 @@ spec:
     addressRanges:
       cluster:
         - 10.99.0.0/29
+  # Reads go to the OSD in the client's own CRUSH location. crushLocationLabels
+  # is unset, so Rook's default list applies; of those the nodes carry
+  # topology.kubernetes.io/zone, one zone per node, matching the zone buckets
+  # the OSDs are created under.
+  #
+  # Passed as a krbd/mount option at map time, so a volume keeps its current
+  # behaviour until it is remounted.
+  csi:
+    readAffinity:
+      enabled: true
   # Nodes listed explicitly, not useAllNodes: under useAllNodes the operator
   # re-provisions an OSD on any node whose device still carries a BlueStore
   # signature, so purging an OSD before pulling its drive is impossible — it

kubernetes/talos/patches/k8s-1.yaml

@@ -188,14 +188,13 @@ addresses:
 # network's clients selectively, so the client goes rather than the record.
 # DHCP is off on VLAN 47 for the same reason.
 #
-# Both defaults live in table 47, not in main, so only the pod-source rules in
-# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
-# reply the node sends from its own addresses.
+# No route here. Table 47 belongs to bird, which fills it with this node's
+# attached networks, the peers' pod CIDRs and the egress default, and empties it
+# if it stops — leaving the rule to fall through to main. A default declared
+# here would outlive the specific routes and send pod traffic to the UDM.
 #
-# The v6 gateway is the UDM's configured ULA on this segment, the same address
-# BGP peers with. The RA-learned link-local would be the alternative and is not
-# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
-# default would land in main, where it does not belong.
+# accept_ra_defrtr stays 0 on this link so the RA default does not land in main,
+# where mgmt0 owns it.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ingress0
@@ -204,11 +203,6 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.11/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication and nothing else. The udev rule in the machine

kubernetes/talos/patches/k8s-2.yaml

@@ -132,11 +132,6 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.12/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.

kubernetes/talos/patches/k8s-3.yaml

@@ -119,11 +119,6 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.13/24
-routes:
-  - gateway: fd02:47::1
-    table: "47"
-  - gateway: 10.2.47.1
-    table: "47"
 ---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
 # here on purpose; the link is handed whole to the OSD pod (host-device).

kubernetes/talos/patches/site-homelab.yaml

@@ -20,62 +20,30 @@ machine:
       validSubnets:
         - 10.2.40.0/24
         - fd40::/64
+  # The UDM on the ingress segment. Published here rather than as a route
+  # because bird owns table 47 and originates the default from it; this is the
+  # one place the address is stated.
+  nodeAnnotations:
+    net.renner.uno/egress-gw-v6: fd02:47::1
+    net.renner.uno/egress-gw-v4: 10.2.47.1
 ---
-# Pod egress leaves over ingress0. Matching on the pod source prefix is what
-# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
-# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
-# source onto VLAN 47.
+# All pod traffic goes to table 47, and the table decides — longest prefix
+# match, no exceptions to carve out. bird fills it with the node's attached
+# networks, the peers' pod CIDRs and the egress default, so a packet to a node
+# on the mgmt segment takes the connected route, a packet to a pod on another
+# node takes pod0, and anything else takes the default over ingress0.
 #
-# The routing decision runs before Cilium's masquerade hook, so the rule sees
-# the pod address and the masquerade then rewrites it to ingress0's.
+# Matching the source is what keeps the node's own traffic out: it answers on
+# mgmt0, and a reply leaving over ingress0 would carry an mgmt0 source onto
+# VLAN 47. The routing decision runs before Cilium's masquerade hook, so the
+# rule sees the pod address and the masquerade then rewrites it.
 #
-# Table 47 holds nothing but the two default routes, so the destination rules
-# below have to run first: without them a pod's packet to a service or to a pod
-# on another node also matches the source rule, finds only the default, and
-# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
-#
-# They send everything inside the cluster and the local networks back to main.
-# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
-# is not foreign traffic.
+# If bird stops, table 47 empties, the lookup misses and the rule falls through
+# to main — pod traffic returns to VLAN 40, unseparated rather than misrouted.
 #
 # Priorities are below main (32766) and clear of the rule Cilium installs at 9.
 # The two families are separate rule tables, so the priorities need not match;
 # the document name is the priority and must be unique.
-# Pod to pod between nodes goes over pod0, via table 48, which the bird mesh
-# fills with one route per peer pod CIDR and the peer's pod0 address as the
-# next hop. Ahead of 990/991 so it wins for that traffic, and matching source
-# AND destination so nothing else lands there: the node's own traffic to a pod
-# would otherwise leave with a pod0 source, which CiliumNode does not list, so
-# Cilium would classify it as world rather than host.
-#
-# Traffic to a pod on this node falls through — table 48 holds only the peers'
-# CIDRs, so the lookup misses and 990/991 send it to main and cilium_host.
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "980"
-src: fd10:244::/48
-dst: fd10:244::/48
-table: "48"
----
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "981"
-src: 10.244.0.0/16
-dst: 10.244.0.0/16
-table: "48"
----
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "990"
-dst: fd00::/8
-table: main
----
-apiVersion: v1alpha1
-kind: RoutingRuleConfig
-name: "991"
-dst: 10.0.0.0/8
-table: main
----
 apiVersion: v1alpha1
 kind: RoutingRuleConfig
 name: "1000"