diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index e30219b0..b7c9d574 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization resources: - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml + - prometheusrule-netbird-ingress.yaml - prometheusrule-velero.yaml - prometheusrule-woodpecker-rotation.yaml - alloy-servicemonitor.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml new file mode 100644 index 00000000..e715ca37 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-ingress.yaml @@ -0,0 +1,39 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: netbird-ingress + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus resource selects on this label; + # without it the rule is created but never loaded into Prometheus. + release: kube-prometheus-stack +spec: + groups: + - name: netbird-ingress.rules + rules: + - alert: NetbirdIngressDegraded + # One or more (but not all) routing peers NotReady: the HA route + # group is running with reduced redundancy. Seen 2026-07-24 when + # ephemeral peers were reaped after a WAN outage and the pods sat + # NotReady in NeedsLogin for ~20 h with no signal. + expr: | + kube_daemonset_status_number_ready{namespace="ingress", daemonset="netbird-ingress"} + < kube_daemonset_status_desired_number_scheduled{namespace="ingress", daemonset="netbird-ingress"} + for: 10m + labels: + severity: warning + annotations: + summary: "netbird-ingress running degraded ({{ $value }} peers Ready)" + description: "The netbird-ingress DaemonSet has fewer Ready pods than desired for 10m+. The mesh ingress route is losing redundancy. Check `kubectl -n ingress get pods -l app=netbird-ingress` and the pods' readiness events (NeedsLogin means the peer was deregistered; the liveness probe should restart it)." + + - alert: NetbirdIngressDown + # ALL routing peers NotReady: the mesh-side ingress path (public + # WAN edge via netbird.renner.uno) is down. + expr: | + kube_daemonset_status_number_ready{namespace="ingress", daemonset="netbird-ingress"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "netbird-ingress has 0 Ready peers — mesh ingress is down" + description: "No netbird-ingress routing peer is Ready; nothing serves the k8s-ingress-router route group, so mesh/public ingress into the cluster is down. Check pod readiness events in `kubectl -n ingress describe ds netbird-ingress` and connectivity to netbird.renner.uno." diff --git a/kubernetes/infrastructure/netbird-ingress/daemonset.yaml b/kubernetes/infrastructure/netbird-ingress/daemonset.yaml index 345f6241..5cded1d4 100644 --- a/kubernetes/infrastructure/netbird-ingress/daemonset.yaml +++ b/kubernetes/infrastructure/netbird-ingress/daemonset.yaml @@ -112,9 +112,25 @@ spec: preStop: exec: command: ["netbird", "down"] + # Restart on dead auth states, and ONLY on those. The setup key mints + # ephemeral peers (reaped by management after >10 min offline), and the + # client treats the resulting "peer is not registered" as unrecoverable: + # it exits its retry loop and sits in NeedsLogin — the daemon cannot + # re-enroll because NB_SETUP_KEY is only consumed by `netbird up` at + # container start. So NeedsLogin (and its siblings LoginFailed / + # SessionExpired) is always fatal here and always fixed by a restart + # (observed 2026-07-24: a 24-min WAN outage to netbird.renner.uno left + # all 3 peers NotReady for ~20 h). A plain `--check ready` liveness + # would be wrong: it also fails during transient management outages, + # where the established WireGuard data plane still works and a restart + # would kill it. Transient states (Disconnected/Connecting) don't match + # the pattern, so they keep retrying silently as before. livenessProbe: exec: - command: ["netbird", "status", "--check", "live"] + command: + - sh + - -c + - netbird status --check ready 2>&1 | grep -qE 'NeedsLogin|LoginFailed|SessionExpired' && exit 1 || exit 0 initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 5