2026-07-26 07:29:35 +00:00
4 changed files with 164 additions and 6 deletions
@@ -6,6 +6,7 @@ kind: Kustomization
 resources:
  - prometheusrule-hetzner-backup.yaml
  - prometheusrule-loki.yaml
+  - prometheusrule-netbird-availability.yaml
  - prometheusrule-netbird-ingress.yaml
  - prometheusrule-velero.yaml
  - prometheusrule-woodpecker-rotation.yaml
@@ -35,6 +35,16 @@ spec:
        # returns 200. CNPG WAL archiving + Velero both depend on it;
        # this probe replaces the old GarageDown signal on nas-1.
        - https://fsn1.your-objectstorage.com
+        # NetBird controller VPS (Hetzner) — management/signal/relay/
+        # dashboard. Not hairpinned and not split-horizon, so this
+        # traverses the WAN and detects a controller/DC outage (the
+        # 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200.
+        - https://netbird.renner.uno
+        # NetBird relay health endpoints on the netcup members (relay
+        # binary serves :9000/health, verified on 0.73.2). Plain http,
+        # ports outside the hairpin's 80/443 — real WAN path.
+        - http://proxy-1.renner.uno:9000/health
+        - http://proxy-2.renner.uno:9000/health
        # APP-HEALTH probes for the gated monitoring UIs, from the correct
        # in-cluster vantage (monitoring-vantage-point-rule): the gate is
        # only on the ingress, so the ClusterIP /-/healthy endpoints are
@@ -47,7 +57,67 @@ spec:
        # external reachability probe above; it is not the health vantage.)
        - http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/-/healthy
        - http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090/-/healthy
-# blackbox-tcp probe deleted 2026-05-20 — only target was the legacy
-# postgres-ha keepalived VIP at 10.2.40.135:5432, decommissioned after
-# the authentik DB moved to in-cluster CNPG (ak-pg). Re-introduce a
-# fresh Probe here when there's a new TCP target to monitor.
+# WAN-path probes of the public edge BY PROXY IP. The name-based
+# home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway)
+# and never leaves the LAN — the vantage-point blind spot that let the
+# 2026-07-24 outage run silent. IP-literal targets dodge both the
+# split-horizon DNS and the UniFi hairpin (absent on the Server/Ingress
+# VLANs the nodes use), so these genuinely go WAN → netcup proxy →
+# mesh → cluster. One target per edge member/family; keep the
+# addresses in sync with proxy DNS + opentofu/unifi_nat_hairpin.tf.
+---
+apiVersion: monitoring.coreos.com/v1
+kind: Probe
+metadata:
+  name: blackbox-edge-wan
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  jobName: blackbox-edge-wan
+  prober:
+    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
+  module: http_2xx_wan_edge
+  interval: 60s
+  scrapeTimeout: 15s
+  targets:
+    staticConfig:
+      static:
+        # Member attribution follows DNS, which is the operational truth
+        # (verified 2026-07-26: dig + live ssh — the v6-only host with
+        # the dhcpcd/no-IPv4 lease is proxy-1.renner.uno = 2a03:…:28:f6e…;
+        # proxy-2.renner.uno is the dual-stack member). NOTE: the label
+        # keys in opentofu/unifi_nat_hairpin.tf have the two v6
+        # addresses attributed to the OPPOSITE members — tracked for a
+        # separate relabel; the addresses themselves are hairpinned
+        # identically, so nothing was functionally wrong there.
+        # proxy-2 v4 (dual-stack member)
+        - https://45.157.177.26
+        # proxy-2 v6
+        - https://[2a03:4000:4b:c1e:4474:43ff:fedf:8f85]
+        # proxy-1 v6 (v6-only member)
+        - https://[2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29]
+---
+# TCP reachability of the NetBird relay listeners on the netcup members
+# (rel:// WebSocket on :33080). Complements the :9000/health probes in
+# blackbox-http: health says the process is alive, this says the actual
+# relay port is reachable from outside.
+apiVersion: monitoring.coreos.com/v1
+kind: Probe
+metadata:
+  name: blackbox-tcp-relays
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  jobName: blackbox-tcp-relays
+  prober:
+    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
+  module: tcp_connect
+  interval: 60s
+  scrapeTimeout: 10s
+  targets:
+    staticConfig:
+      static:
+        - proxy-1.renner.uno:33080
+        - proxy-2.renner.uno:33080
@@ -0,0 +1,67 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: netbird-availability
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: netbird-availability.rules
+      rules:
+        - alert: NetbirdControllerUnreachable
+          # The Hetzner controller VPS (management/signal/TURN + VPS
+          # relay). Since the P2P fix + netcup relays, an outage costs
+          # control plane only (existing tunnels and the public edge
+          # survive) — but peer logins, route changes, the dashboard and
+          # emergency access depend on it, and the 2026-07-24 DC outage
+          # ran ~20 h with no signal.
+          expr: probe_success{job="blackbox-http", instance="https://netbird.renner.uno"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "NetBird controller (netbird.renner.uno) unreachable for 10m+"
+            description: "The WAN-path blackbox probe of the NetBird controller VPS has been failing for 10 minutes. Peer logins, route changes and the dashboard are unavailable; existing tunnels and the public edge keep working (P2P + netcup relays). If it stays down >10 min, ephemeral peers get reaped — the ingress DaemonSet self-heals via its NeedsLogin liveness probe on recovery."
+
+        - alert: NetbirdPublicEdgeWanDown
+          # BOTH IP-pinned WAN-path probes of the netcup edge members
+          # failing: the public edge is unreachable from the internet,
+          # whatever the internal probes say. This is the alert the
+          # hairpinned home.renner.uno probe can never raise.
+          expr: max(probe_success{job="blackbox-edge-wan"}) == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Public edge unreachable from the WAN (all members)"
+            description: "All IP-pinned WAN-path probes of the reverse-proxy edge (proxy-2 v4+v6, proxy-1 v6) have failed for 5 minutes — external clients cannot reach home.renner.uno and the other published apps. Check the netcup proxies (netbird-proxy containers), the mesh path to the cluster, and cluster Traefik."
+
+        - alert: NetbirdPublicEdgeMemberDown
+          # A single edge member failing WAN reachability: DNS round-robin
+          # still serves clients via the other member, but redundancy is
+          # gone and v4-only or v6-only clients may be affected depending
+          # on which member died.
+          expr: probe_success{job="blackbox-edge-wan"} == 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Public edge member {{ $labels.instance }} unreachable from WAN"
+            description: "One reverse-proxy edge member has failed its WAN-path probe for 15 minutes. proxy-2 (45.157.177.26) carries all v4-only clients; proxy-1 is the v6-only member. Check the container and host."
+
+        - alert: NetbirdRelayMemberDown
+          # A netcup relay member down (WebSocket :33080 unreachable or
+          # :9000/health failing): clients fail over to the remaining
+          # relays, so this is redundancy loss, not an outage.
+          expr: |
+            probe_success{job="blackbox-tcp-relays"} == 0
+            or
+            probe_success{job="blackbox-http", instance=~"http://proxy-[12]\\.renner\\.uno:9000/health"} == 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "NetBird relay member unhealthy ({{ $labels.instance }})"
+            description: "A netcup relay member has failed its probe for 15 minutes. Mesh clients fail over to the remaining relays (VPS + the other netcup member), so this is lost redundancy rather than an outage. Check the netbird-relay container on the affected host."
@@ -29,5 +29,25 @@ spec:
          release: kube-prometheus-stack
        interval: 30s
    # Default modules cover http_2xx, http_post_2xx, tcp_connect,
-    # icmp, dns. The chart's stock config is fine; Probe CRs in
-    # monitoring-rules/ point at it.
+    # icmp, dns. Helm deep-merges maps, so adding module keys under
+    # config.modules keeps the chart's stock modules intact.
+    config:
+      modules:
+        # WAN-path probe of the public edge by proxy IP. Probing the NAME
+        # from in-cluster is useless twice over: CoreDNS/k8s_gateway
+        # resolves public names straight to the cluster VIP, and the LAN
+        # hairpin DNAT rewrites the proxy IPs on most VLANs — but NOT on
+        # the Server/Ingress VLANs the nodes sit on (deliberately absent
+        # from unifi_nat_hairpin.tf), so an IP-literal probe from this
+        # exporter genuinely leaves via the WAN and comes back through
+        # the netcup reverse-proxy edge. SNI + Host are pinned to the
+        # served domain because the target is an IP literal.
+        http_2xx_wan_edge:
+          prober: http
+          timeout: 10s
+          http:
+            fail_if_not_ssl: true
+            tls_config:
+              server_name: home.renner.uno
+            headers:
+              Host: home.renner.uno