fix: [alert] LokiCanaryMissingEntries (cluster) (#844 )

2026-09-16 07:45:29 +00:00
3 changed files with 14 additions and 45 deletions
@@ -376,29 +376,3 @@ steps:
          --dockerfile Containerfile \
          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
          --destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
-
-  # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
-  # artifact like heating-can; same registry so the audit tracks it.
-  - name: build-bird
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
-    directory: ci/images/bird
-    depends_on: []
-    when:
-      - event: deployment
-      - event: push
-        path: ["ci/images/bird/**", ".woodpecker/build-ci-images.yml"]
-    environment:
-      GITEA_USER: gitea_admin
-      GITEA_TOKEN:
-        from_secret: ci_images_token
-    commands:
-      - mkdir -p /kaniko/.docker
-      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
-      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
-      - TAG=$(date +%Y-%m-%d)
-      - |
-        /kaniko/executor \
-          --context . \
-          --dockerfile Containerfile \
-          --destination gitea.renner.uno/gitea_admin/bird:$TAG \
-          --destination gitea.renner.uno/gitea_admin/bird:latest
@@ -1,16 +0,0 @@
-FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
-
-# bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
-# step — the image is the binary plus its runtime.
-#
-# 3.22 rather than the 3.21 the other images here pin: bird2 first appears in
-# 3.22/community.
-#
-# Config is read from /etc/bird/bird.conf, not the package default
-# /etc/bird.conf, so the mount cannot shadow what the package ships.
-RUN apk add --no-cache bird2
-
-# Fails the build rather than the first pod if the binary or its path is wrong.
-RUN /usr/sbin/bird --version
-
-ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]
@@ -47,10 +47,21 @@ spec:
        - alert: LokiCanaryMissingEntries
          # loki-canary writes test logs and reads them back. Missing
          # entries mean the ingest → store → query path lost data.
-          expr: sum(increase(loki_canary_missing_entries_total[10m])) > 0
+          #
+          # Aggregated per pod, not cluster-wide: the canary is a DaemonSet
+          # and pushes directly to Loki, so a single wedged node stalls only
+          # its own canary. A bare sum() turned that into an unlabelled
+          # "log data loss" alert that named neither the node nor Loki.
+          # Grouping by pod keeps genuine Loki data loss loud (every canary
+          # fires) while a single-node stall is self-describing.
+          expr: sum by (pod) (increase(loki_canary_missing_entries_total[10m])) > 0
          for: 10m
          labels:
            severity: warning
          annotations:
-            summary: "Loki canary detected missing log entries"
-            description: "loki-canary reported entries it wrote were not returned by the read path in the last 10m — log data loss."
+            summary: "Loki canary {{ $labels.pod }} detected missing log entries"
+            description: >-
+              loki-canary {{ $labels.pod }} reported entries it wrote were not
+              returned by the read path in the last 10m. If only this canary is
+              affected, suspect its node (blocked I/O, resource starvation)
+              rather than Loki; if all canaries are affected, suspect Loki.