diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index b7c9d574..ffd75801 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization resources: - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml + - prometheusrule-netbird-availability.yaml - prometheusrule-netbird-ingress.yaml - prometheusrule-velero.yaml - prometheusrule-woodpecker-rotation.yaml diff --git a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml index 4c496607..9f2bd209 100644 --- a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml +++ b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml @@ -35,6 +35,16 @@ spec: # returns 200. CNPG WAL archiving + Velero both depend on it; # this probe replaces the old GarageDown signal on nas-1. - https://fsn1.your-objectstorage.com + # NetBird controller VPS (Hetzner) — management/signal/relay/ + # dashboard. Not hairpinned and not split-horizon, so this + # traverses the WAN and detects a controller/DC outage (the + # 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200. + - https://netbird.renner.uno + # NetBird relay health endpoints on the netcup members (relay + # binary serves :9000/health, verified on 0.73.2). Plain http, + # ports outside the hairpin's 80/443 — real WAN path. + - http://proxy-1.renner.uno:9000/health + - http://proxy-2.renner.uno:9000/health # APP-HEALTH probes for the gated monitoring UIs, from the correct # in-cluster vantage (monitoring-vantage-point-rule): the gate is # only on the ingress, so the ClusterIP /-/healthy endpoints are @@ -47,7 +57,67 @@ spec: # external reachability probe above; it is not the health vantage.) - http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/-/healthy - http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090/-/healthy -# blackbox-tcp probe deleted 2026-05-20 — only target was the legacy -# postgres-ha keepalived VIP at 10.2.40.135:5432, decommissioned after -# the authentik DB moved to in-cluster CNPG (ak-pg). Re-introduce a -# fresh Probe here when there's a new TCP target to monitor. +# WAN-path probes of the public edge BY PROXY IP. The name-based +# home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway) +# and never leaves the LAN — the vantage-point blind spot that let the +# 2026-07-24 outage run silent. IP-literal targets dodge both the +# split-horizon DNS and the UniFi hairpin (absent on the Server/Ingress +# VLANs the nodes use), so these genuinely go WAN → netcup proxy → +# mesh → cluster. One target per edge member/family; keep the +# addresses in sync with proxy DNS + opentofu/unifi_nat_hairpin.tf. +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: blackbox-edge-wan + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + jobName: blackbox-edge-wan + prober: + url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115 + module: http_2xx_wan_edge + interval: 60s + scrapeTimeout: 15s + targets: + staticConfig: + static: + # Member attribution follows DNS, which is the operational truth + # (verified 2026-07-26: dig + live ssh — the v6-only host with + # the dhcpcd/no-IPv4 lease is proxy-1.renner.uno = 2a03:…:28:f6e…; + # proxy-2.renner.uno is the dual-stack member). NOTE: the label + # keys in opentofu/unifi_nat_hairpin.tf have the two v6 + # addresses attributed to the OPPOSITE members — tracked for a + # separate relabel; the addresses themselves are hairpinned + # identically, so nothing was functionally wrong there. + # proxy-2 v4 (dual-stack member) + - https://45.157.177.26 + # proxy-2 v6 + - https://[2a03:4000:4b:c1e:4474:43ff:fedf:8f85] + # proxy-1 v6 (v6-only member) + - https://[2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29] +--- +# TCP reachability of the NetBird relay listeners on the netcup members +# (rel:// WebSocket on :33080). Complements the :9000/health probes in +# blackbox-http: health says the process is alive, this says the actual +# relay port is reachable from outside. +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: blackbox-tcp-relays + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + jobName: blackbox-tcp-relays + prober: + url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115 + module: tcp_connect + interval: 60s + scrapeTimeout: 10s + targets: + staticConfig: + static: + - proxy-1.renner.uno:33080 + - proxy-2.renner.uno:33080 diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml new file mode 100644 index 00000000..4533a044 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-availability.yaml @@ -0,0 +1,67 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: netbird-availability + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus selects on this label. + release: kube-prometheus-stack +spec: + groups: + - name: netbird-availability.rules + rules: + - alert: NetbirdControllerUnreachable + # The Hetzner controller VPS (management/signal/TURN + VPS + # relay). Since the P2P fix + netcup relays, an outage costs + # control plane only (existing tunnels and the public edge + # survive) — but peer logins, route changes, the dashboard and + # emergency access depend on it, and the 2026-07-24 DC outage + # ran ~20 h with no signal. + expr: probe_success{job="blackbox-http", instance="https://netbird.renner.uno"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "NetBird controller (netbird.renner.uno) unreachable for 10m+" + description: "The WAN-path blackbox probe of the NetBird controller VPS has been failing for 10 minutes. Peer logins, route changes and the dashboard are unavailable; existing tunnels and the public edge keep working (P2P + netcup relays). If it stays down >10 min, ephemeral peers get reaped — the ingress DaemonSet self-heals via its NeedsLogin liveness probe on recovery." + + - alert: NetbirdPublicEdgeWanDown + # BOTH IP-pinned WAN-path probes of the netcup edge members + # failing: the public edge is unreachable from the internet, + # whatever the internal probes say. This is the alert the + # hairpinned home.renner.uno probe can never raise. + expr: max(probe_success{job="blackbox-edge-wan"}) == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Public edge unreachable from the WAN (all members)" + description: "All IP-pinned WAN-path probes of the reverse-proxy edge (proxy-2 v4+v6, proxy-1 v6) have failed for 5 minutes — external clients cannot reach home.renner.uno and the other published apps. Check the netcup proxies (netbird-proxy containers), the mesh path to the cluster, and cluster Traefik." + + - alert: NetbirdPublicEdgeMemberDown + # A single edge member failing WAN reachability: DNS round-robin + # still serves clients via the other member, but redundancy is + # gone and v4-only or v6-only clients may be affected depending + # on which member died. + expr: probe_success{job="blackbox-edge-wan"} == 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Public edge member {{ $labels.instance }} unreachable from WAN" + description: "One reverse-proxy edge member has failed its WAN-path probe for 15 minutes. proxy-2 (45.157.177.26) carries all v4-only clients; proxy-1 is the v6-only member. Check the container and host." + + - alert: NetbirdRelayMemberDown + # A netcup relay member down (WebSocket :33080 unreachable or + # :9000/health failing): clients fail over to the remaining + # relays, so this is redundancy loss, not an outage. + expr: | + probe_success{job="blackbox-tcp-relays"} == 0 + or + probe_success{job="blackbox-http", instance=~"http://proxy-[12]\\.renner\\.uno:9000/health"} == 0 + for: 15m + labels: + severity: warning + annotations: + summary: "NetBird relay member unhealthy ({{ $labels.instance }})" + description: "A netcup relay member has failed its probe for 15 minutes. Mesh clients fail over to the remaining relays (VPS + the other netcup member), so this is lost redundancy rather than an outage. Check the netbird-relay container on the affected host." diff --git a/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml b/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml index c060235c..e7273484 100644 --- a/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml +++ b/kubernetes/apps/monitoring/helmrelease-blackbox-exporter.yaml @@ -29,5 +29,25 @@ spec: release: kube-prometheus-stack interval: 30s # Default modules cover http_2xx, http_post_2xx, tcp_connect, - # icmp, dns. The chart's stock config is fine; Probe CRs in - # monitoring-rules/ point at it. + # icmp, dns. Helm deep-merges maps, so adding module keys under + # config.modules keeps the chart's stock modules intact. + config: + modules: + # WAN-path probe of the public edge by proxy IP. Probing the NAME + # from in-cluster is useless twice over: CoreDNS/k8s_gateway + # resolves public names straight to the cluster VIP, and the LAN + # hairpin DNAT rewrites the proxy IPs on most VLANs — but NOT on + # the Server/Ingress VLANs the nodes sit on (deliberately absent + # from unifi_nat_hairpin.tf), so an IP-literal probe from this + # exporter genuinely leaves via the WAN and comes back through + # the netcup reverse-proxy edge. SNI + Host are pinned to the + # served domain because the target is an IP literal. + http_2xx_wan_edge: + prober: http + timeout: 10s + http: + fail_if_not_ssl: true + tls_config: + server_name: home.renner.uno + headers: + Host: home.renner.uno