kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -6,6 +6,7 @@ kind: Kustomization
 resources:
   - prometheusrule-hetzner-backup.yaml
   - prometheusrule-loki.yaml
+  - prometheusrule-netbird-ingress.yaml
   - prometheusrule-velero.yaml
   - prometheusrule-woodpecker-rotation.yaml
   - alloy-servicemonitor.yaml

kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml

@@ -0,0 +1,39 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: netbird-ingress
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: netbird-ingress.rules
+      rules:
+        - alert: NetbirdIngressDegraded
+          # One or more (but not all) routing peers NotReady: the HA route
+          # group is running with reduced redundancy. Seen 2026-07-24 when
+          # ephemeral peers were reaped after a WAN outage and the pods sat
+          # NotReady in NeedsLogin for ~20 h with no signal.
+          expr: |
+            kube_daemonset_status_number_ready{namespace="ingress", daemonset="netbird-ingress"}
+              < kube_daemonset_status_desired_number_scheduled{namespace="ingress", daemonset="netbird-ingress"}
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "netbird-ingress running degraded ({{ $value }} peers Ready)"
+            description: "The netbird-ingress DaemonSet has fewer Ready pods than desired for 10m+. The mesh ingress route is losing redundancy. Check `kubectl -n ingress get pods -l app=netbird-ingress` and the pods' readiness events (NeedsLogin means the peer was deregistered; the liveness probe should restart it)."
+
+        - alert: NetbirdIngressDown
+          # ALL routing peers NotReady: the mesh-side ingress path (public
+          # WAN edge via netbird.renner.uno) is down.
+          expr: |
+            kube_daemonset_status_number_ready{namespace="ingress", daemonset="netbird-ingress"} == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "netbird-ingress has 0 Ready peers — mesh ingress is down"
+            description: "No netbird-ingress routing peer is Ready; nothing serves the k8s-ingress-router route group, so mesh/public ingress into the cluster is down. Check pod readiness events in `kubectl -n ingress describe ds netbird-ingress` and connectivity to netbird.renner.uno."

kubernetes/infrastructure/netbird-ingress/daemonset.yaml

@@ -112,9 +112,25 @@ spec:
             preStop:
               exec:
                 command: ["netbird", "down"]
+          # Restart on dead auth states, and ONLY on those. The setup key mints
+          # ephemeral peers (reaped by management after >10 min offline), and the
+          # client treats the resulting "peer is not registered" as unrecoverable:
+          # it exits its retry loop and sits in NeedsLogin — the daemon cannot
+          # re-enroll because NB_SETUP_KEY is only consumed by `netbird up` at
+          # container start. So NeedsLogin (and its siblings LoginFailed /
+          # SessionExpired) is always fatal here and always fixed by a restart
+          # (observed 2026-07-24: a 24-min WAN outage to netbird.renner.uno left
+          # all 3 peers NotReady for ~20 h). A plain `--check ready` liveness
+          # would be wrong: it also fails during transient management outages,
+          # where the established WireGuard data plane still works and a restart
+          # would kill it. Transient states (Disconnected/Connecting) don't match
+          # the pattern, so they keep retrying silently as before.
           livenessProbe:
             exec:
-              command: ["netbird", "status", "--check", "live"]
+              command:
+                - sh
+                - -c
+                - netbird status --check ready 2>&1 | grep -qE 'NeedsLogin|LoginFailed|SessionExpired' && exit 1 || exit 0
             initialDelaySeconds: 5
             periodSeconds: 10
             timeoutSeconds: 5