diff --git a/ansible/plays/install-netbird-proxy.yml b/ansible/plays/install-netbird-proxy.yml index 17304dc3..5e9d64c8 100644 --- a/ansible/plays/install-netbird-proxy.yml +++ b/ansible/plays/install-netbird-proxy.yml @@ -21,3 +21,4 @@ - common - docker-install - netbird-proxy + - netbird-proxy-watchdog diff --git a/ansible/plays/install-netbird-server.yml b/ansible/plays/install-netbird-server.yml index dcc99ea4..bfd32745 100644 --- a/ansible/plays/install-netbird-server.yml +++ b/ansible/plays/install-netbird-server.yml @@ -18,3 +18,4 @@ - common - docker-install - netbird-server + - netbird-proxy-watchdog diff --git a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml new file mode 100644 index 00000000..10ad91ab --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml @@ -0,0 +1,35 @@ +--- +# Self-healing watchdog for the NetBird reverse proxy (VPS-1 and proxy-1). +# +# Why it exists: the proxy's embedded NetBird client pins each proxied +# domain's route to a specific routing-peer identity. The netbird-ingress +# DaemonSet deliberately mints fresh peer identities on every rollout, and +# upstream's dns-interceptor never rebinds a domain route whose owning peer +# is gone (logs "already routed by peer [...]. HA routing disabled"; +# client/internal/routemanager/dnsinterceptor/handler.go, present through +# current main as of 2026-07). A proxy in that state serves only 502s and +# timeouts until its container restarts — which is exactly what this +# watchdog does. Incident: 2026-07-02 public-edge outage. + +# Domains probed end-to-end (proxy -> mesh -> Traefik). Must be +# cluster-hosted domains the proxy serves; don't list domains slated to +# move off-cluster. A restart fires only when ALL of them fail — a single +# app being down (Traefik's 502 passed through) must not restart the +# proxy, while the dead-route state kills every domain at once. +netbird_watchdog_probe_hosts: + - home.renner.uno + - immich.renner.uno + +# Address the probe connects to. 127.0.0.1 reaches the local proxy on both +# members: VPS-1 routes it via Traefik's HostSNI(*) TCP passthrough, proxy-1 +# binds :443 directly. +netbird_watchdog_probe_address: 127.0.0.1 + +netbird_watchdog_container: netbird-proxy + +# Consecutive hard failures (502 or no HTTP response) before restarting. +netbird_watchdog_fail_threshold: 3 + +# Minimum seconds between watchdog-issued restarts, so a genuinely broken +# backend can't put the proxy into a restart loop. +netbird_watchdog_cooldown_seconds: 600 diff --git a/ansible/roles/netbird-proxy-watchdog/tasks/main.yml b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml new file mode 100644 index 00000000..e199d584 --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml @@ -0,0 +1,32 @@ +--- +# Deploys the reverse-proxy watchdog (see defaults/main.yml for why). + +- name: Ensure curl is installed (probe dependency; proxy-1 is Arch, VPS-1 Debian) + ansible.builtin.package: + name: curl + state: present + +- name: Deploy the watchdog script + ansible.builtin.template: + src: netbird-proxy-watchdog.sh.j2 + dest: /usr/local/bin/netbird-proxy-watchdog + mode: "0755" + +- name: Deploy the watchdog systemd service + ansible.builtin.template: + src: netbird-proxy-watchdog.service.j2 + dest: /etc/systemd/system/netbird-proxy-watchdog.service + mode: "0644" + +- name: Deploy the watchdog systemd timer + ansible.builtin.template: + src: netbird-proxy-watchdog.timer.j2 + dest: /etc/systemd/system/netbird-proxy-watchdog.timer + mode: "0644" + +- name: Enable and start the watchdog timer + ansible.builtin.systemd: + name: netbird-proxy-watchdog.timer + enabled: true + state: started + daemon_reload: true diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2 new file mode 100644 index 00000000..33843cd4 --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=NetBird reverse-proxy watchdog (restart on sustained 502/timeout) +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/netbird-proxy-watchdog diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2 new file mode 100644 index 00000000..2d96101d --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2 @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Probe the local reverse proxy end-to-end and restart it after +# {{ netbird_watchdog_fail_threshold }} consecutive all-domains-down rounds. +# Only 502 and 000 (timeout / TLS or connect failure) count per domain, and +# a round fails only if EVERY probed domain fails — one dead backend shows +# up as a 502 on its own domain, while the proxy's dead-route state kills +# all domains at once. All output lands in the journal via the systemd unit. +set -u + +STATE_DIR=/run/netbird-proxy-watchdog +FAILS_FILE="$STATE_DIR/consecutive_failures" +LAST_RESTART_FILE="$STATE_DIR/last_restart_epoch" +mkdir -p "$STATE_DIR" + +failed=0 +total=0 +results="" +for host in {{ netbird_watchdog_probe_hosts | join(' ') }}; do + total=$((total + 1)) + code=$(curl -sk -m 10 -o /dev/null -w '%{http_code}' \ + --resolve "$host:443:{{ netbird_watchdog_probe_address }}" \ + "https://$host/" || true) + if [ "$code" = "502" ] || [ "$code" = "000" ]; then + failed=$((failed + 1)) + results="$results $host=$code(FAIL)" + else + results="$results $host=$code" + fi +done + +fails=$(cat "$FAILS_FILE" 2>/dev/null || echo 0) + +if [ "$failed" -lt "$total" ]; then + if [ "$fails" -gt 0 ]; then + echo "recovered after $fails failing round(s):$results" + else + echo "ok:$results" + fi + echo 0 > "$FAILS_FILE" + exit 0 +fi + +fails=$((fails + 1)) +echo "$fails" > "$FAILS_FILE" +echo "failure $fails/{{ netbird_watchdog_fail_threshold }} (all domains down):$results" + +if [ "$fails" -lt {{ netbird_watchdog_fail_threshold }} ]; then + exit 0 +fi + +now=$(date +%s) +last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0) +if [ $((now - last)) -lt {{ netbird_watchdog_cooldown_seconds }} ]; then + echo "restart suppressed: last restart $((now - last))s ago (cooldown {{ netbird_watchdog_cooldown_seconds }}s)" + exit 0 +fi + +echo "restarting {{ netbird_watchdog_container }} after $fails consecutive failing rounds" +if docker restart '{{ netbird_watchdog_container }}'; then + echo "$now" > "$LAST_RESTART_FILE" + echo 0 > "$FAILS_FILE" + echo "restart done" +else + echo "restart FAILED" + exit 1 +fi diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2 new file mode 100644 index 00000000..fe1aca0b --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Run the NetBird reverse-proxy watchdog every minute + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +AccuracySec=15s + +[Install] +WantedBy=timers.target