fix: [alert] LokiCanaryMissingEntries (cluster) (#844)

.woodpecker/build-ci-images.yml

@@ -376,29 +376,3 @@ steps:
           --dockerfile Containerfile \
           --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
           --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
-
-  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
-  # artifact like heating-can; same registry so the audit tracks it.
-  - name: build-bird
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
-    directory: ci/images/bird
-    depends_on: []
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/bird/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/bird:$TAG \
-          --destination gitea.renner.uno/gitea_admin/bird:latest

ci/images/bird/Containerfile

@@ -1,22 +0,0 @@
-FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
-
-# bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
-# step — the image is the binary plus its runtime.
-#
-# 3.22 rather than the 3.21 the other images here pin: bird2 first appears in
-# 3.22/community.
-#
-# Config is read from /etc/bird/bird.conf, not the package default
-# /etc/bird.conf, so the mount cannot shadow what the package ships.
-#
-# kubectl renders the init container's per-node config: go-template-file for
-# the templating, a field selector for excluding the node itself, in-cluster
-# config for the auth. From the distro, so there is no version literal to drift
-# and no download to trust. It is only ever used to read core/v1 Nodes.
-RUN apk add --no-cache bird2 kubectl
-
-# Fails the build rather than the first pod if a binary or its path is wrong.
-RUN /usr/sbin/bird --version \
- && kubectl version --client > /dev/null
-
-ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]

kubernetes/apps/monitoring-rules/prometheusrule-loki.yaml

@@ -47,10 +47,21 @@ spec:
         - alert: LokiCanaryMissingEntries
           # loki-canary writes test logs and reads them back. Missing
           # entries mean the ingest → store → query path lost data.
-          expr: sum(increase(loki_canary_missing_entries_total[10m])) > 0
+          #
+          # Aggregated per pod, not cluster-wide: the canary is a DaemonSet
+          # and pushes directly to Loki, so a single wedged node stalls only
+          # its own canary. A bare sum() turned that into an unlabelled
+          # "log data loss" alert that named neither the node nor Loki.
+          # Grouping by pod keeps genuine Loki data loss loud (every canary
+          # fires) while a single-node stall is self-describing.
+          expr: sum by (pod) (increase(loki_canary_missing_entries_total[10m])) > 0
           for: 10m
           labels:
             severity: warning
           annotations:
-            summary: "Loki canary detected missing log entries"
-            description: "loki-canary reported entries it wrote were not returned by the read path in the last 10m — log data loss."
+            summary: "Loki canary {{ $labels.pod }} detected missing log entries"
+            description: >-
+              loki-canary {{ $labels.pod }} reported entries it wrote were not
+              returned by the read path in the last 10m. If only this canary is
+              affected, suspect its node (blocked I/O, resource starvation)
+              rather than Loki; if all canaries are affected, suspect Loki.

kubernetes/infrastructure/bird/bird.conf

@@ -1,107 +0,0 @@
-# Identical on every node and free of addresses. defines.conf and peers.conf
-# are rendered per node by the init container; the first is included above the
-# templates that reference it, the second below them.
-include "/etc/bird/defines.conf";
-
-router id from "pod0";
-log stderr all;
-
-protocol device { scan time 10; }
-
-# Everything a pod may need to reach, in one table, resolved by longest prefix:
-# the node's attached networks, the peers' pod CIDRs, and a default for the
-# rest. No rule has to carve out exceptions, because the specific routes win on
-# their own.
-#
-# The table is BIRD's alone. If this daemon stops, it empties, the rule finds
-# nothing and falls through to main — pod traffic returns to VLAN 40,
-# unseparated rather than misrouted. That is why the default route lives here
-# and not in the Talos config: a default that outlived the specific routes
-# would send pod traffic to the UDM.
-
-# The node's attached networks, taken from the interfaces rather than written
-# down. The usual advice is not to export these to the kernel because it
-# maintains them itself — true of main, not of a table the kernel never fills.
-protocol direct attached {
-  interface "mgmt0", "ingress0", "pod0";
-  ipv4 { table master4; };
-  ipv6 { table master6; };
-}
-
-# Pod egress. GW4/GW6 are the UDM on the ingress segment, published as node
-# annotations beside the address that defines them.
-protocol static egress4 {
-  ipv4 { table master4; };
-  route 0.0.0.0/0 via GW4;
-}
-protocol static egress6 {
-  ipv6 { table master6; };
-  route ::/0 via GW6;
-}
-
-# This node's own pod CIDR, originated rather than read back out of the kernel.
-# Reading it back made BIRD's view depend on the shape of Cilium's route, and
-# the IPv4 one is rejected outright: its next hop sits inside the prefix it
-# describes, which KRT refuses with "strange next-hop".
-#
-# blackhole because it is only ever advertised — it is filtered out below.
-protocol static own4 {
-  ipv4 { table master4; };
-  route OWN_POD4 blackhole;
-}
-protocol static own6 {
-  ipv6 { table master6; };
-  route OWN_POD6 blackhole;
-}
-
-# Everything except this node's own CIDR, which cilium_host already serves in
-# main and which is a blackhole here.
-filter to_pod_table {
-  if source = RTS_STATIC && dest = RTD_BLACKHOLE then reject;
-  accept;
-}
-
-protocol kernel k4 {
-  kernel table 47;
-  metric 0;
-  ipv4 { table master4; import none; export filter to_pod_table; };
-}
-protocol kernel k6 {
-  kernel table 47;
-  metric 0;
-  ipv6 { table master6; import none; export filter to_pod_table; };
-}
-
-# iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM
-# from ingress0 and is unrelated.
-#
-# direct is explicit because iBGP defaults to multihop, which makes the gateway
-# recursive — BIRD would try to resolve the peer in the pod RIB. strict bind
-# keeps the listener off mgmt0 and ingress0.
-#
-# Only the node's own CIDR is offered: the export filter drops everything the
-# static protocols and the direct protocol produce.
-template bgp pod4 {
-  local as 64514;
-  source address MY4;
-  direct;
-  strict bind;
-  ipv4 {
-    next hop self;
-    import all;
-    export filter { if net = OWN_POD4 then accept; reject; };
-  };
-}
-template bgp pod6 {
-  local as 64514;
-  source address MY6;
-  direct;
-  strict bind;
-  ipv6 {
-    next hop self;
-    import all;
-    export filter { if net = OWN_POD6 then accept; reject; };
-  };
-}
-
-include "/etc/bird/peers.conf";

kubernetes/infrastructure/bird/daemonset.yaml

@@ -1,115 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: bird
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: bird
-rules:
-  # The peer list is read from the nodes' pod0 annotations, so no address is
-  # written into this repo twice.
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: bird
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: bird
-subjects:
-  - kind: ServiceAccount
-    name: bird
-    namespace: kube-system
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: bird
-  namespace: kube-system
-  labels:
-    tier: node
-    app: bird
-spec:
-  selector:
-    matchLabels:
-      app: bird
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: bird
-    spec:
-      # Carries pod east-west once the routing rules land. Without it table 48
-      # stays empty and the rules fall through to main, i.e. back to VLAN 40.
-      priorityClassName: system-node-critical
-      # Writes host routes and peers on host addresses.
-      hostNetwork: true
-      serviceAccountName: bird
-      # Every node here is a control plane node, so this is not optional.
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-      initContainers:
-        - name: render-config
-          # By digest, not by the dated tag: the tag is rebuilt within the day
-          # it names, and the cluster's image path keeps resolving it to the
-          # first build of that day — imagePullPolicy: Always does not help,
-          # because the tag itself resolves stale. Update this when the image
-          # is rebuilt.
-          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              set -eu
-              cp /config/bird.conf /etc/bird/bird.conf
-              kubectl get node "$NODE_NAME" \
-                -o go-template-file=/config/defines.tmpl > /etc/bird/defines.conf
-              kubectl get nodes --field-selector "metadata.name!=$NODE_NAME" \
-                -o go-template-file=/config/peers.tmpl > /etc/bird/peers.conf
-              # BIRD's own parser is the check: a node whose annotation is
-              # missing renders as <no value> and fails here rather than
-              # crashlooping the daemon or forming a mesh with a hole.
-              bird -p -c /etc/bird/bird.conf
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          volumeMounts:
-            - name: config
-              mountPath: /config
-            - name: etc-bird
-              mountPath: /etc/bird
-      containers:
-        - name: bird
-          image: gitea.renner.uno/gitea_admin/bird@sha256:3bc8500613eeb45b2d8858b5445198bae3990ecf64f6d891aa94797f5a09dc1d
-          resources:
-            requests:
-              cpu: "50m"
-              memory: "32Mi"
-            limits:
-              cpu: "200m"
-              memory: "128Mi"
-          securityContext:
-            capabilities:
-              drop: ["ALL"]
-              # NET_ADMIN writes the routes; NET_BIND_SERVICE binds 179.
-              add: ["NET_ADMIN", "NET_BIND_SERVICE"]
-          volumeMounts:
-            - name: etc-bird
-              mountPath: /etc/bird
-      volumes:
-        - name: config
-          configMap:
-            name: bird-config
-        - name: etc-bird
-          emptyDir: {}

kubernetes/infrastructure/bird/defines.tmpl

@@ -1,9 +0,0 @@
-define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }};
-define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }};
-define GW6 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v6" }};
-define GW4 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v4" }};
-{{/* podCIDRs follows the cluster's podSubnets order, v4 first. A flip puts an
-     IPv6 prefix into an ipv4 protocol, which fails the parse in the init
-     container rather than passing silently. */}}
-define OWN_POD4 = {{ index .spec.podCIDRs 0 }};
-define OWN_POD6 = {{ index .spec.podCIDRs 1 }};

kubernetes/infrastructure/bird/kustomization.yaml

@@ -1,12 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - daemonset.yaml
-configMapGenerator:
-  # Hashed name, so editing any of these rolls the DaemonSet.
-  - name: bird-config
-    namespace: kube-system
-    files:
-      - bird.conf
-      - defines.tmpl
-      - peers.tmpl

kubernetes/infrastructure/bird/peers.tmpl

@@ -1,7 +0,0 @@
-{{- range $i, $n := .items -}}
-{{- $v6 := index $n.metadata.annotations "net.renner.uno/pod0-v6" -}}
-{{- $v4 := index $n.metadata.annotations "net.renner.uno/pod0-v4" -}}
-# {{ $n.metadata.name }}
-protocol bgp peer{{ $i }}_v6 from pod6 { neighbor {{ $v6 }} as 64514; }
-protocol bgp peer{{ $i }}_v4 from pod4 { neighbor {{ $v4 }} as 64514; }
-{{ end -}}

kubernetes/infrastructure/kustomization.yaml

@@ -7,7 +7,6 @@ resources:
   - cluster-settings/
   - cilium/
   - multus/
-  - bird/
   - cert-manager/
   - snapshot-controller/
   - metrics-server/

kubernetes/talos/patches/k8s-1.yaml

@@ -52,12 +52,6 @@ machine:
     # this label instead of listing hostnames, so which boxes carry storage is
     # stated once, here, by the machine that knows.
     storage.renner.uno/ceph-osd: "true"
-  # Publishes this node's pod0 addresses, declared below in its VLANConfig, so
-  # the bird DaemonSet reads its peers from the node list rather than from a
-  # copy. Annotations, not labels: a label value cannot contain a colon.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::11
-    net.renner.uno/pod0-v4: 10.2.48.11
   udev:
     # Renames every port this node uses to its role at boot, matched by
     # burned-in MAC — kernel names derive from PCI topology and shift when
@@ -188,13 +182,14 @@ addresses:
 # network's clients selectively, so the client goes rather than the record.
 # DHCP is off on VLAN 47 for the same reason.
 #
-# No route here. Table 47 belongs to bird, which fills it with this node's
-# attached networks, the peers' pod CIDRs and the egress default, and empties it
-# if it stops — leaving the rule to fall through to main. A default declared
-# here would outlive the specific routes and send pod traffic to the UDM.
+# Both defaults live in table 47, not in main, so only the pod-source rules in
+# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
+# reply the node sends from its own addresses.
 #
-# accept_ra_defrtr stays 0 on this link so the RA default does not land in main,
-# where mgmt0 owns it.
+# The v6 gateway is the UDM's configured ULA on this segment, the same address
+# BGP peers with. The RA-learned link-local would be the alternative and is not
+# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
+# default would land in main, where it does not belong.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ingress0
@@ -203,6 +198,11 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.11/24
+routes:
+  - gateway: fd02:47::1
+    table: "47"
+  - gateway: 10.2.47.1
+    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication and nothing else. The udev rule in the machine

kubernetes/talos/patches/k8s-2.yaml

@@ -33,10 +33,6 @@ machine:
     topology.kubernetes.io/zone: saturn
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
-  # pod0 addresses for the bird peer list — see k8s-1.yaml.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::12
-    net.renner.uno/pod0-v4: 10.2.48.12
   udev:
     # Renames every port to its role at boot — see k8s-1.yaml.
     rules:
@@ -132,6 +128,11 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.12/24
+routes:
+  - gateway: fd02:47::1
+    table: "47"
+  - gateway: 10.2.47.1
+    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.

kubernetes/talos/patches/k8s-3.yaml

@@ -29,10 +29,6 @@ machine:
     topology.kubernetes.io/zone: neptune
     # Holds a Ceph OSD drive — see k8s-1.yaml.
     storage.renner.uno/ceph-osd: "true"
-  # pod0 addresses for the bird peer list — see k8s-1.yaml.
-  nodeAnnotations:
-    net.renner.uno/pod0-v6: fd02:48::13
-    net.renner.uno/pod0-v4: 10.2.48.13
   udev:
     # Renames every port to its role at boot — see k8s-1.yaml.
     rules:
@@ -119,6 +115,11 @@ parent: uplink
 up: true
 addresses:
   - address: 10.2.47.13/24
+routes:
+  - gateway: fd02:47::1
+    table: "47"
+  - gateway: 10.2.47.1
+    table: "47"
 ---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
 # here on purpose; the link is handed whole to the OSD pod (host-device).

kubernetes/talos/patches/site-homelab.yaml

@@ -20,32 +20,41 @@ machine:
       validSubnets:
         - 10.2.40.0/24
         - fd40::/64
-  # The UDM on the ingress segment. Published here rather than as a route
-  # because bird owns table 47 and originates the default from it; this is the
-  # one place the address is stated.
-  nodeAnnotations:
-    net.renner.uno/egress-gw-v6: fd02:47::1
-    net.renner.uno/egress-gw-v4: 10.2.47.1
 ---
-# All pod traffic goes to table 47, and the table decides — longest prefix
-# match, no exceptions to carve out. bird fills it with the node's attached
-# networks, the peers' pod CIDRs and the egress default, so a packet to a node
-# on the mgmt segment takes the connected route, a packet to a pod on another
-# node takes pod0, and anything else takes the default over ingress0.
+# Pod egress leaves over ingress0. Matching on the pod source prefix is what
+# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
+# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
+# source onto VLAN 47.
 #
-# Matching the source is what keeps the node's own traffic out: it answers on
-# mgmt0, and a reply leaving over ingress0 would carry an mgmt0 source onto
-# VLAN 47. The routing decision runs before Cilium's masquerade hook, so the
-# rule sees the pod address and the masquerade then rewrites it.
+# The routing decision runs before Cilium's masquerade hook, so the rule sees
+# the pod address and the masquerade then rewrites it to ingress0's.
 #
-# If bird stops, table 47 empties, the lookup misses and the rule falls through
-# to main — pod traffic returns to VLAN 40, unseparated rather than misrouted.
+# Table 47 holds nothing but the two default routes, so the destination rules
+# below have to run first: without them a pod's packet to a service or to a pod
+# on another node also matches the source rule, finds only the default, and
+# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
+#
+# They send everything inside the cluster and the local networks back to main.
+# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
+# is not foreign traffic.
 #
 # Priorities are below main (32766) and clear of the rule Cilium installs at 9.
 # The two families are separate rule tables, so the priorities need not match;
 # the document name is the priority and must be unique.
 apiVersion: v1alpha1
 kind: RoutingRuleConfig
+name: "990"
+dst: fd00::/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "991"
+dst: 10.0.0.0/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
 name: "1000"
 src: fd10:244::/48
 table: "47"