ansible/plays/install-netbird-proxy.yml

@@ -21,3 +21,4 @@
     - common
     - docker-install
     - netbird-proxy
+    - netbird-proxy-watchdog

ansible/plays/install-netbird-server.yml

@@ -18,3 +18,4 @@
     - common
     - docker-install
     - netbird-server
+    - netbird-proxy-watchdog

ansible/roles/netbird-proxy-watchdog/defaults/main.yml

@@ -0,0 +1,35 @@
+---
+# Self-healing watchdog for the NetBird reverse proxy (VPS-1 and proxy-1).
+#
+# Why it exists: the proxy's embedded NetBird client pins each proxied
+# domain's route to a specific routing-peer identity. The netbird-ingress
+# DaemonSet deliberately mints fresh peer identities on every rollout, and
+# upstream's dns-interceptor never rebinds a domain route whose owning peer
+# is gone (logs "already routed by peer [...]. HA routing disabled";
+# client/internal/routemanager/dnsinterceptor/handler.go, present through
+# current main as of 2026-07). A proxy in that state serves only 502s and
+# timeouts until its container restarts — which is exactly what this
+# watchdog does. Incident: 2026-07-02 public-edge outage.
+
+# Domains probed end-to-end (proxy -> mesh -> Traefik). Must be
+# cluster-hosted domains the proxy serves; don't list domains slated to
+# move off-cluster. A restart fires only when ALL of them fail — a single
+# app being down (Traefik's 502 passed through) must not restart the
+# proxy, while the dead-route state kills every domain at once.
+netbird_watchdog_probe_hosts:
+  - home.renner.uno
+  - immich.renner.uno
+
+# Address the probe connects to. 127.0.0.1 reaches the local proxy on both
+# members: VPS-1 routes it via Traefik's HostSNI(*) TCP passthrough, proxy-1
+# binds :443 directly.
+netbird_watchdog_probe_address: 127.0.0.1
+
+netbird_watchdog_container: netbird-proxy
+
+# Consecutive hard failures (502 or no HTTP response) before restarting.
+netbird_watchdog_fail_threshold: 3
+
+# Minimum seconds between watchdog-issued restarts, so a genuinely broken
+# backend can't put the proxy into a restart loop.
+netbird_watchdog_cooldown_seconds: 600

ansible/roles/netbird-proxy-watchdog/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+# Deploys the reverse-proxy watchdog (see defaults/main.yml for why).
+
+- name: Ensure curl is installed (probe dependency; proxy-1 is Arch, VPS-1 Debian)
+  ansible.builtin.package:
+    name: curl
+    state: present
+
+- name: Deploy the watchdog script
+  ansible.builtin.template:
+    src: netbird-proxy-watchdog.sh.j2
+    dest: /usr/local/bin/netbird-proxy-watchdog
+    mode: "0755"
+
+- name: Deploy the watchdog systemd service
+  ansible.builtin.template:
+    src: netbird-proxy-watchdog.service.j2
+    dest: /etc/systemd/system/netbird-proxy-watchdog.service
+    mode: "0644"
+
+- name: Deploy the watchdog systemd timer
+  ansible.builtin.template:
+    src: netbird-proxy-watchdog.timer.j2
+    dest: /etc/systemd/system/netbird-proxy-watchdog.timer
+    mode: "0644"
+
+- name: Enable and start the watchdog timer
+  ansible.builtin.systemd:
+    name: netbird-proxy-watchdog.timer
+    enabled: true
+    state: started
+    daemon_reload: true

ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=NetBird reverse-proxy watchdog (restart on sustained 502/timeout)
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/netbird-proxy-watchdog

ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Probe the local reverse proxy end-to-end and restart it after
+# {{ netbird_watchdog_fail_threshold }} consecutive all-domains-down rounds.
+# Only 502 and 000 (timeout / TLS or connect failure) count per domain, and
+# a round fails only if EVERY probed domain fails — one dead backend shows
+# up as a 502 on its own domain, while the proxy's dead-route state kills
+# all domains at once. All output lands in the journal via the systemd unit.
+set -u
+
+STATE_DIR=/run/netbird-proxy-watchdog
+FAILS_FILE="$STATE_DIR/consecutive_failures"
+LAST_RESTART_FILE="$STATE_DIR/last_restart_epoch"
+mkdir -p "$STATE_DIR"
+
+failed=0
+total=0
+results=""
+for host in {{ netbird_watchdog_probe_hosts | join(' ') }}; do
+  total=$((total + 1))
+  code=$(curl -sk -m 10 -o /dev/null -w '%{http_code}' \
+    --resolve "$host:443:{{ netbird_watchdog_probe_address }}" \
+    "https://$host/" || true)
+  if [ "$code" = "502" ] || [ "$code" = "000" ]; then
+    failed=$((failed + 1))
+    results="$results $host=$code(FAIL)"
+  else
+    results="$results $host=$code"
+  fi
+done
+
+fails=$(cat "$FAILS_FILE" 2>/dev/null || echo 0)
+
+if [ "$failed" -lt "$total" ]; then
+  if [ "$fails" -gt 0 ]; then
+    echo "recovered after $fails failing round(s):$results"
+  else
+    echo "ok:$results"
+  fi
+  echo 0 > "$FAILS_FILE"
+  exit 0
+fi
+
+fails=$((fails + 1))
+echo "$fails" > "$FAILS_FILE"
+echo "failure $fails/{{ netbird_watchdog_fail_threshold }} (all domains down):$results"
+
+if [ "$fails" -lt {{ netbird_watchdog_fail_threshold }} ]; then
+  exit 0
+fi
+
+now=$(date +%s)
+last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)
+if [ $((now - last)) -lt {{ netbird_watchdog_cooldown_seconds }} ]; then
+  echo "restart suppressed: last restart $((now - last))s ago (cooldown {{ netbird_watchdog_cooldown_seconds }}s)"
+  exit 0
+fi
+
+echo "restarting {{ netbird_watchdog_container }} after $fails consecutive failing rounds"
+if docker restart '{{ netbird_watchdog_container }}'; then
+  echo "$now" > "$LAST_RESTART_FILE"
+  echo 0 > "$FAILS_FILE"
+  echo "restart done"
+else
+  echo "restart FAILED"
+  exit 1
+fi

ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run the NetBird reverse-proxy watchdog every minute
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=1min
+AccuracySec=15s
+
+[Install]
+WantedBy=timers.target