From 8b1480f429855f86ece1bc792de4d4e9f53cb3f3 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 26 Jul 2026 08:57:01 +0200 Subject: [PATCH 1/3] feat(monitoring): WAN-path probes for the NetBird edge, controller, relays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the vantage-point gap that let the 2026-07-24 controller outage run ~20 h unnoticed: every existing probe of the public names resolves in-cluster (CoreDNS/k8s_gateway) or gets hairpinned, so nothing measured the actual WAN path. - blackbox module http_2xx_wan_edge: probes the edge by proxy IP literal with SNI/Host pinned to home.renner.uno. The Server/Ingress VLANs are deliberately absent from the UniFi hairpin, so IP-literal probes from the in-cluster exporter genuinely leave via the WAN and come back through the netcup edge (verified live from a pod: v4 returns 200; v6 currently fails against proxy-1 — real breakage, fixed separately by the resolv.conf PR). - New probes: WAN edge per member/family (proxy-2 v4, proxy-1 v6), https://netbird.renner.uno (controller), relay :9000/health on both netcup members, tcp :33080 relay listeners. - New PrometheusRule netbird-availability: controller unreachable (critical 10m), whole edge down from WAN (critical 5m), single edge member down (warning 15m), relay member down (warning 15m). Co-Authored-By: Claude Fable 5 --- .../apps/monitoring-rules/kustomization.yaml | 1 + .../monitoring-rules/probes-blackbox.yaml | 68 +++++++++++++++++-- .../prometheusrule-netbird-availability.yaml | 67 ++++++++++++++++++ .../helmrelease-blackbox-exporter.yaml | 24 ++++++- 4 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index b7c9d574..ffd75801 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization resources: - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml + - prometheusrule-netbird-availability.yaml - prometheusrule-netbird-ingress.yaml - prometheusrule-velero.yaml - prometheusrule-woodpecker-rotation.yaml diff --git a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml index 4c496607..02816f40 100644 --- a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml +++ b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml @@ -35,6 +35,16 @@ spec: # returns 200. CNPG WAL archiving + Velero both depend on it; # this probe replaces the old GarageDown signal on nas-1. - https://fsn1.your-objectstorage.com + # NetBird controller VPS (Hetzner) — management/signal/relay/ + # dashboard. Not hairpinned and not split-horizon, so this + # traverses the WAN and detects a controller/DC outage (the + # 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200. + - https://netbird.renner.uno + # NetBird relay health endpoints on the netcup members (relay + # binary serves :9000/health, verified on 0.73.2). Plain http, + # ports outside the hairpin's 80/443 — real WAN path. + - http://proxy-1.renner.uno:9000/health + - http://proxy-2.renner.uno:9000/health # APP-HEALTH probes for the gated monitoring UIs, from the correct # in-cluster vantage (monitoring-vantage-point-rule): the gate is # only on the ingress, so the ClusterIP /-/healthy endpoints are @@ -47,7 +57,57 @@ spec: # external reachability probe above; it is not the health vantage.) - http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/-/healthy - http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090/-/healthy -# blackbox-tcp probe deleted 2026-05-20 — only target was the legacy -# postgres-ha keepalived VIP at 10.2.40.135:5432, decommissioned after -# the authentik DB moved to in-cluster CNPG (ak-pg). Re-introduce a -# fresh Probe here when there's a new TCP target to monitor. +# WAN-path probes of the public edge BY PROXY IP. The name-based +# home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway) +# and never leaves the LAN — the vantage-point blind spot that let the +# 2026-07-24 outage run silent. IP-literal targets dodge both the +# split-horizon DNS and the UniFi hairpin (absent on the Server/Ingress +# VLANs the nodes use), so these genuinely go WAN → netcup proxy → +# mesh → cluster. One target per edge member/family; keep the +# addresses in sync with proxy DNS + opentofu/unifi_nat_hairpin.tf. +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: blackbox-edge-wan + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + jobName: blackbox-edge-wan + prober: + url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115 + module: http_2xx_wan_edge + interval: 60s + scrapeTimeout: 15s + targets: + staticConfig: + static: + # proxy-2 v4 + - https://45.157.177.26 + # proxy-1 v6 (v6-only member) + - https://[2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29] +--- +# TCP reachability of the NetBird relay listeners on the netcup members +# (rel:// WebSocket on :33080). Complements the :9000/health probes in +# blackbox-http: health says the process is alive, this says the actual +# relay port is reachable from outside. +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: blackbox-tcp-relays + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + jobName: blackbox-tcp-relays + prober: + url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115 + module: tcp_connect + interval: 60s + scrapeTimeout: 10s + targets: + staticConfig: + static: + - proxy-1.renner.uno:33080 + - proxy-2.renner.uno:33080 diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml new file mode 100644 index 00000000..548e4c80 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml @@ -0,0 +1,67 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: netbird-availability + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus selects on this label. + release: kube-prometheus-stack +spec: + groups: + - name: netbird-availability.rules + rules: + - alert: NetbirdControllerUnreachable + # The Hetzner controller VPS (management/signal/TURN + VPS + # relay). Since the P2P fix + netcup relays, an outage costs + # control plane only (existing tunnels and the public edge + # survive) — but peer logins, route changes, the dashboard and + # emergency access depend on it, and the 2026-07-24 DC outage + # ran ~20 h with no signal. + expr: probe_success{instance="https://netbird.renner.uno"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "NetBird controller (netbird.renner.uno) unreachable for 10m+" + description: "The WAN-path blackbox probe of the NetBird controller VPS has been failing for 10 minutes. Peer logins, route changes and the dashboard are unavailable; existing tunnels and the public edge keep working (P2P + netcup relays). If it stays down >10 min, ephemeral peers get reaped — the ingress DaemonSet self-heals via its NeedsLogin liveness probe on recovery." + + - alert: NetbirdPublicEdgeWanDown + # BOTH IP-pinned WAN-path probes of the netcup edge members + # failing: the public edge is unreachable from the internet, + # whatever the internal probes say. This is the alert the + # hairpinned home.renner.uno probe can never raise. + expr: max(probe_success{job="blackbox-edge-wan"}) == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Public edge unreachable from the WAN (all members)" + description: "Both IP-pinned WAN-path probes of the reverse-proxy edge (proxy-2 v4, proxy-1 v6) have failed for 5 minutes — external clients cannot reach home.renner.uno and the other published apps. Check the netcup proxies (netbird-proxy containers), the mesh path to the cluster, and cluster Traefik." + + - alert: NetbirdPublicEdgeMemberDown + # A single edge member failing WAN reachability: DNS round-robin + # still serves clients via the other member, but redundancy is + # gone and v4-only or v6-only clients may be affected depending + # on which member died. + expr: probe_success{job="blackbox-edge-wan"} == 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Public edge member {{ $labels.instance }} unreachable from WAN" + description: "One reverse-proxy edge member has failed its WAN-path probe for 15 minutes. proxy-2 (45.157.177.26) carries all v4-only clients; proxy-1 is the v6-only member. Check the container and host." + + - alert: NetbirdRelayMemberDown + # A netcup relay member down (WebSocket :33080 unreachable or + # :9000/health failing): clients fail over to the remaining + # relays, so this is redundancy loss, not an outage. + expr: | + probe_success{job="blackbox-tcp-relays"} == 0 + or + probe_success{job="blackbox-http", instance=~"http://proxy-[12].renner.uno:9000/health"} == 0 + for: 15m + labels: + severity: warning + annotations: + summary: "NetBird relay member unhealthy ({{ $labels.instance }})" + description: "A netcup relay member has failed its probe for 15 minutes. Mesh clients fail over to the remaining relays (VPS + the other netcup member), so this is lost redundancy rather than an outage. Check the netbird-relay container on the affected host." diff --git a/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml b/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml index c060235c..e7273484 100644 --- a/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml +++ b/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml @@ -29,5 +29,25 @@ spec: release: kube-prometheus-stack interval: 30s # Default modules cover http_2xx, http_post_2xx, tcp_connect, - # icmp, dns. The chart's stock config is fine; Probe CRs in - # monitoring-rules/ point at it. + # icmp, dns. Helm deep-merges maps, so adding module keys under + # config.modules keeps the chart's stock modules intact. + config: + modules: + # WAN-path probe of the public edge by proxy IP. Probing the NAME + # from in-cluster is useless twice over: CoreDNS/k8s_gateway + # resolves public names straight to the cluster VIP, and the LAN + # hairpin DNAT rewrites the proxy IPs on most VLANs — but NOT on + # the Server/Ingress VLANs the nodes sit on (deliberately absent + # from unifi_nat_hairpin.tf), so an IP-literal probe from this + # exporter genuinely leaves via the WAN and comes back through + # the netcup reverse-proxy edge. SNI + Host are pinned to the + # served domain because the target is an IP literal. + http_2xx_wan_edge: + prober: http + timeout: 10s + http: + fail_if_not_ssl: true + tls_config: + server_name: home.renner.uno + headers: + Host: home.renner.uno -- 2.54.0 From e3bc00916dc1d4dec142014cb80caba716b735c9 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 26 Jul 2026 09:15:15 +0200 Subject: [PATCH 2/3] review: full per-member/family edge coverage; DNS is the member truth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review flagged the v6 target as 'proxy-2 per unifi_nat_hairpin.tf'. The tf label keys are the swapped ones: DNS (dig) and the live hosts (the v6-only member with the nameserver-less dhcpcd lease answers as proxy-1.renner.uno = 2a03:…:28:f6e…; the dual-stack member answers as proxy-2.renner.uno = 45.157.177.26 + 2a03:…:4b:c1e…) both agree with the probe's original attribution, as does the netbird-proxy role's 'proxy-1 is v6-only' doc. Keep DNS attribution, document the tf mislabel (tracked for a separate relabel — renaming those tf map keys churns NAT rule resources, so it doesn't belong in this PR), and: - add proxy-2's v6 address as a third target (previously its v6 path was unprobed; verified live from a pod, returns 200) - pin job="blackbox-http" on NetbirdControllerUnreachable for consistency with the other rules Co-Authored-By: Claude Fable 5 --- .../apps/monitoring-rules/probes-blackbox.yaml | 12 +++++++++++- .../prometheusrule-netbird-availability.yaml | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml index 02816f40..9f2bd209 100644 --- a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml +++ b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml @@ -83,8 +83,18 @@ spec: targets: staticConfig: static: - # proxy-2 v4 + # Member attribution follows DNS, which is the operational truth + # (verified 2026-07-26: dig + live ssh — the v6-only host with + # the dhcpcd/no-IPv4 lease is proxy-1.renner.uno = 2a03:…:28:f6e…; + # proxy-2.renner.uno is the dual-stack member). NOTE: the label + # keys in opentofu/unifi_nat_hairpin.tf have the two v6 + # addresses attributed to the OPPOSITE members — tracked for a + # separate relabel; the addresses themselves are hairpinned + # identically, so nothing was functionally wrong there. + # proxy-2 v4 (dual-stack member) - https://45.157.177.26 + # proxy-2 v6 + - https://[2a03:4000:4b:c1e:4474:43ff:fedf:8f85] # proxy-1 v6 (v6-only member) - https://[2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29] --- diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml index 548e4c80..0a9b47d0 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml @@ -17,7 +17,7 @@ spec: # survive) — but peer logins, route changes, the dashboard and # emergency access depend on it, and the 2026-07-24 DC outage # ran ~20 h with no signal. - expr: probe_success{instance="https://netbird.renner.uno"} == 0 + expr: probe_success{job="blackbox-http", instance="https://netbird.renner.uno"} == 0 for: 10m labels: severity: critical @@ -36,7 +36,7 @@ spec: severity: critical annotations: summary: "Public edge unreachable from the WAN (all members)" - description: "Both IP-pinned WAN-path probes of the reverse-proxy edge (proxy-2 v4, proxy-1 v6) have failed for 5 minutes — external clients cannot reach home.renner.uno and the other published apps. Check the netcup proxies (netbird-proxy containers), the mesh path to the cluster, and cluster Traefik." + description: "All IP-pinned WAN-path probes of the reverse-proxy edge (proxy-2 v4+v6, proxy-1 v6) have failed for 5 minutes — external clients cannot reach home.renner.uno and the other published apps. Check the netcup proxies (netbird-proxy containers), the mesh path to the cluster, and cluster Traefik." - alert: NetbirdPublicEdgeMemberDown # A single edge member failing WAN reachability: DNS round-robin -- 2.54.0 From 2bd54ef3781f43b781767d2ae73d4c8d5d3635c7 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 26 Jul 2026 09:24:38 +0200 Subject: [PATCH 3/3] review: escape dots in relay health instance regex Co-Authored-By: Claude Fable 5 --- .../monitoring-rules/prometheusrule-netbird-availability.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml index 0a9b47d0..4533a044 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml @@ -58,7 +58,7 @@ spec: expr: | probe_success{job="blackbox-tcp-relays"} == 0 or - probe_success{job="blackbox-http", instance=~"http://proxy-[12].renner.uno:9000/health"} == 0 + probe_success{job="blackbox-http", instance=~"http://proxy-[12]\\.renner\\.uno:9000/health"} == 0 for: 15m labels: severity: warning -- 2.54.0