From c59ad31db9315b6d9a84010c546138a0fd06e60d Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sat, 25 Jul 2026 09:06:01 +0200 Subject: [PATCH] fix(netbird-ingress): restart peers stuck in NeedsLogin, alert on NotReady A 24-min WAN outage to netbird.renner.uno on 2026-07-24 exceeded the 10-min ephemeral-peer lifetime, so management deregistered all three routing peers. The client treats 'peer is not registered' as unrecoverable and parks in NeedsLogin; that fails readiness but not the '--check live' liveness probe, so the pods sat Running-but-NotReady for ~20 h until they were deleted by hand. - liveness probe now fails on the dead auth states (NeedsLogin / LoginFailed / SessionExpired) and only on those: a restart re-runs the entrypoint, which re-enrolls with NB_SETUP_KEY. Transient Disconnected/Connecting states still retry silently, so a management outage does not kill the working WireGuard data plane. - new PrometheusRule: warning when the DaemonSet runs degraded >10m, critical when 0 peers are Ready >5m. Co-Authored-By: Claude Fable 5 --- .../apps/monitoring-rules/kustomization.yaml | 1 + .../prometheusrule-netbird-ingress.yaml | 39 +++++++++++++++++++ .../netbird-ingress/daemonset.yaml | 18 ++++++++- 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index e30219b0..b7c9d574 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization resources: - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml + - prometheusrule-netbird-ingress.yaml - prometheusrule-velero.yaml - prometheusrule-woodpecker-rotation.yaml - alloy-servicemonitor.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml new file mode 100644 index 00000000..e715ca37 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml @@ -0,0 +1,39 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: netbird-ingress + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus resource selects on this label; + # without it the rule is created but never loaded into Prometheus. + release: kube-prometheus-stack +spec: + groups: + - name: netbird-ingress.rules + rules: + - alert: NetbirdIngressDegraded + # One or more (but not all) routing peers NotReady: the HA route + # group is running with reduced redundancy. Seen 2026-07-24 when + # ephemeral peers were reaped after a WAN outage and the pods sat + # NotReady in NeedsLogin for ~20 h with no signal. + expr: | + kube_daemonset_status_number_ready{namespace="ingress", daemonset="netbird-ingress"} + < kube_daemonset_status_desired_number_scheduled{namespace="ingress", daemonset="netbird-ingress"} + for: 10m + labels: + severity: warning + annotations: + summary: "netbird-ingress running degraded ({{ $value }} peers Ready)" + description: "The netbird-ingress DaemonSet has fewer Ready pods than desired for 10m+. The mesh ingress route is losing redundancy. Check `kubectl -n ingress get pods -l app=netbird-ingress` and the pods' readiness events (NeedsLogin means the peer was deregistered; the liveness probe should restart it)." + + - alert: NetbirdIngressDown + # ALL routing peers NotReady: the mesh-side ingress path (public + # WAN edge via netbird.renner.uno) is down. + expr: | + kube_daemonset_status_number_ready{namespace="ingress", daemonset="netbird-ingress"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "netbird-ingress has 0 Ready peers — mesh ingress is down" + description: "No netbird-ingress routing peer is Ready; nothing serves the k8s-ingress-router route group, so mesh/public ingress into the cluster is down. Check pod readiness events in `kubectl -n ingress describe ds netbird-ingress` and connectivity to netbird.renner.uno." diff --git a/kubernetes/infrastructure/netbird-ingress/daemonset.yaml b/kubernetes/infrastructure/netbird-ingress/daemonset.yaml index 345f6241..5cded1d4 100644 --- a/kubernetes/infrastructure/netbird-ingress/daemonset.yaml +++ b/kubernetes/infrastructure/netbird-ingress/daemonset.yaml @@ -112,9 +112,25 @@ spec: preStop: exec: command: ["netbird", "down"] + # Restart on dead auth states, and ONLY on those. The setup key mints + # ephemeral peers (reaped by management after >10 min offline), and the + # client treats the resulting "peer is not registered" as unrecoverable: + # it exits its retry loop and sits in NeedsLogin — the daemon cannot + # re-enroll because NB_SETUP_KEY is only consumed by `netbird up` at + # container start. So NeedsLogin (and its siblings LoginFailed / + # SessionExpired) is always fatal here and always fixed by a restart + # (observed 2026-07-24: a 24-min WAN outage to netbird.renner.uno left + # all 3 peers NotReady for ~20 h). A plain `--check ready` liveness + # would be wrong: it also fails during transient management outages, + # where the established WireGuard data plane still works and a restart + # would kill it. Transient states (Disconnected/Connecting) don't match + # the pattern, so they keep retrying silently as before. livenessProbe: exec: - command: ["netbird", "status", "--check", "live"] + command: + - sh + - -c + - netbird status --check ready 2>&1 | grep -qE 'NeedsLogin|LoginFailed|SessionExpired' && exit 1 || exit 0 initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 5 -- 2.54.0