From edf53620150fb2766727a1b39dfb7858b68286a3 Mon Sep 17 00:00:00 2001
From: Konstantin Renner <konstantin@renner.uno>
Date: Thu, 2 Jul 2026 07:07:43 +0200
Subject: [PATCH 1/2] feat(netbird): reverse-proxy watchdog (self-heal
 dead-route 502s)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The proxy's embedded client pins each proxied domain's route to a
routing-peer identity. netbird-ingress rollouts mint fresh identities by
design, and upstream's dns-interceptor never rebinds a route whose owning
peer is gone ('already routed by peer [...]. HA routing disabled') — the
proxy then serves only 502/timeouts until restarted (2026-07-02 outage).

Add a shared role deploying a systemd timer (1min) on both cluster
members (VPS-1, proxy-1): probe status.renner.uno through the local
proxy, restart the container after 3 consecutive 502/000 results, with a
10min cooldown against restart loops. Every probe outcome is journaled.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 ansible/plays/install-netbird-proxy.yml       |  1 +
 ansible/plays/install-netbird-server.yml      |  1 +
 .../netbird-proxy-watchdog/defaults/main.yml  | 30 +++++++++++
 .../netbird-proxy-watchdog/tasks/main.yml     | 32 +++++++++++
 .../netbird-proxy-watchdog.service.j2         |  8 +++
 .../templates/netbird-proxy-watchdog.sh.j2    | 53 +++++++++++++++++++
 .../templates/netbird-proxy-watchdog.timer.j2 | 10 ++++
 7 files changed, 135 insertions(+)
 create mode 100644 ansible/roles/netbird-proxy-watchdog/defaults/main.yml
 create mode 100644 ansible/roles/netbird-proxy-watchdog/tasks/main.yml
 create mode 100644 ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2
 create mode 100644 ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2
 create mode 100644 ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2

diff --git a/ansible/plays/install-netbird-proxy.yml b/ansible/plays/install-netbird-proxy.yml
index 17304dc3..5e9d64c8 100644
--- a/ansible/plays/install-netbird-proxy.yml
+++ b/ansible/plays/install-netbird-proxy.yml
@@ -21,3 +21,4 @@
     - common
     - docker-install
     - netbird-proxy
+    - netbird-proxy-watchdog
diff --git a/ansible/plays/install-netbird-server.yml b/ansible/plays/install-netbird-server.yml
index dcc99ea4..bfd32745 100644
--- a/ansible/plays/install-netbird-server.yml
+++ b/ansible/plays/install-netbird-server.yml
@@ -18,3 +18,4 @@
     - common
     - docker-install
     - netbird-server
+    - netbird-proxy-watchdog
diff --git a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml
new file mode 100644
index 00000000..faf0ff14
--- /dev/null
+++ b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml
@@ -0,0 +1,30 @@
+---
+# Self-healing watchdog for the NetBird reverse proxy (VPS-1 and proxy-1).
+#
+# Why it exists: the proxy's embedded NetBird client pins each proxied
+# domain's route to a specific routing-peer identity. The netbird-ingress
+# DaemonSet deliberately mints fresh peer identities on every rollout, and
+# upstream's dns-interceptor never rebinds a domain route whose owning peer
+# is gone (logs "already routed by peer [...]. HA routing disabled";
+# client/internal/routemanager/dnsinterceptor/handler.go, present through
+# current main as of 2026-07). A proxy in that state serves only 502s and
+# timeouts until its container restarts — which is exactly what this
+# watchdog does. Incident: 2026-07-02 public-edge outage.
+
+# Domain probed end-to-end (proxy -> mesh -> Traefik). Must be one of the
+# domains the proxy cluster serves.
+netbird_watchdog_probe_host: status.renner.uno
+
+# Address the probe connects to. 127.0.0.1 reaches the local proxy on both
+# members: VPS-1 routes it via Traefik's HostSNI(*) TCP passthrough, proxy-1
+# binds :443 directly.
+netbird_watchdog_probe_address: 127.0.0.1
+
+netbird_watchdog_container: netbird-proxy
+
+# Consecutive hard failures (502 or no HTTP response) before restarting.
+netbird_watchdog_fail_threshold: 3
+
+# Minimum seconds between watchdog-issued restarts, so a genuinely broken
+# backend can't put the proxy into a restart loop.
+netbird_watchdog_cooldown_seconds: 600
diff --git a/ansible/roles/netbird-proxy-watchdog/tasks/main.yml b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml
new file mode 100644
index 00000000..e199d584
--- /dev/null
+++ b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+# Deploys the reverse-proxy watchdog (see defaults/main.yml for why).
+
+- name: Ensure curl is installed (probe dependency; proxy-1 is Arch, VPS-1 Debian)
+  ansible.builtin.package:
+    name: curl
+    state: present
+
+- name: Deploy the watchdog script
+  ansible.builtin.template:
+    src: netbird-proxy-watchdog.sh.j2
+    dest: /usr/local/bin/netbird-proxy-watchdog
+    mode: "0755"
+
+- name: Deploy the watchdog systemd service
+  ansible.builtin.template:
+    src: netbird-proxy-watchdog.service.j2
+    dest: /etc/systemd/system/netbird-proxy-watchdog.service
+    mode: "0644"
+
+- name: Deploy the watchdog systemd timer
+  ansible.builtin.template:
+    src: netbird-proxy-watchdog.timer.j2
+    dest: /etc/systemd/system/netbird-proxy-watchdog.timer
+    mode: "0644"
+
+- name: Enable and start the watchdog timer
+  ansible.builtin.systemd:
+    name: netbird-proxy-watchdog.timer
+    enabled: true
+    state: started
+    daemon_reload: true
diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2
new file mode 100644
index 00000000..33843cd4
--- /dev/null
+++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.service.j2
@@ -0,0 +1,8 @@
+[Unit]
+Description=NetBird reverse-proxy watchdog (restart on sustained 502/timeout)
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/netbird-proxy-watchdog
diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2
new file mode 100644
index 00000000..2195a03a
--- /dev/null
+++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Probe the local reverse proxy end-to-end and restart it after
+# {{ netbird_watchdog_fail_threshold }} consecutive hard failures.
+# Only 502 and 000 (timeout / TLS or connect failure) count as failures —
+# those are the proxy's dead-route signatures; anything else, including
+# backend application errors, proves the proxy itself forwards fine.
+# All output lands in the journal via the systemd unit.
+set -u
+
+STATE_DIR=/run/netbird-proxy-watchdog
+FAILS_FILE="$STATE_DIR/consecutive_failures"
+LAST_RESTART_FILE="$STATE_DIR/last_restart_epoch"
+mkdir -p "$STATE_DIR"
+
+code=$(curl -sk -m 10 -o /dev/null -w '%{http_code}' \
+  --resolve '{{ netbird_watchdog_probe_host }}:443:{{ netbird_watchdog_probe_address }}' \
+  'https://{{ netbird_watchdog_probe_host }}/' || true)
+fails=$(cat "$FAILS_FILE" 2>/dev/null || echo 0)
+
+if [ "$code" != "502" ] && [ "$code" != "000" ]; then
+  if [ "$fails" -gt 0 ]; then
+    echo "recovered: probe={{ netbird_watchdog_probe_host }} status=$code after $fails failure(s)"
+  else
+    echo "ok: probe={{ netbird_watchdog_probe_host }} status=$code"
+  fi
+  echo 0 > "$FAILS_FILE"
+  exit 0
+fi
+
+fails=$((fails + 1))
+echo "$fails" > "$FAILS_FILE"
+echo "failure $fails/{{ netbird_watchdog_fail_threshold }}: probe={{ netbird_watchdog_probe_host }} status=$code"
+
+if [ "$fails" -lt {{ netbird_watchdog_fail_threshold }} ]; then
+  exit 0
+fi
+
+now=$(date +%s)
+last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)
+if [ $((now - last)) -lt {{ netbird_watchdog_cooldown_seconds }} ]; then
+  echo "restart suppressed: last restart $((now - last))s ago (cooldown {{ netbird_watchdog_cooldown_seconds }}s)"
+  exit 0
+fi
+
+echo "restarting {{ netbird_watchdog_container }} after $fails consecutive failures"
+if docker restart '{{ netbird_watchdog_container }}'; then
+  echo "$now" > "$LAST_RESTART_FILE"
+  echo 0 > "$FAILS_FILE"
+  echo "restart done"
+else
+  echo "restart FAILED"
+  exit 1
+fi
diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2
new file mode 100644
index 00000000..fe1aca0b
--- /dev/null
+++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.timer.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=Run the NetBird reverse-proxy watchdog every minute
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=1min
+AccuracySec=15s
+
+[Install]
+WantedBy=timers.target
-- 
2.54.0


From 8eca54577d644dd75b23a7369f6b44ba921ec811 Mon Sep 17 00:00:00 2001
From: Konstantin Renner <konstantin@renner.uno>
Date: Thu, 2 Jul 2026 09:25:46 +0200
Subject: [PATCH 2/2] fix(netbird-watchdog): probe home+immich, require all
 domains down
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

status.renner.uno will move off-cluster, so it's a bad canary. Probe two
permanently cluster-hosted domains instead and count a round as failed
only when EVERY domain fails: one dead backend 502s on its own domain,
while the dead-route state kills all domains at once — this keeps single
app outages from restarting the proxy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../netbird-proxy-watchdog/defaults/main.yml  | 11 ++++--
 .../templates/netbird-proxy-watchdog.sh.j2    | 39 ++++++++++++-------
 2 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml
index faf0ff14..10ad91ab 100644
--- a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml
+++ b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml
@@ -11,9 +11,14 @@
 # timeouts until its container restarts — which is exactly what this
 # watchdog does. Incident: 2026-07-02 public-edge outage.
 
-# Domain probed end-to-end (proxy -> mesh -> Traefik). Must be one of the
-# domains the proxy cluster serves.
-netbird_watchdog_probe_host: status.renner.uno
+# Domains probed end-to-end (proxy -> mesh -> Traefik). Must be
+# cluster-hosted domains the proxy serves; don't list domains slated to
+# move off-cluster. A restart fires only when ALL of them fail — a single
+# app being down (Traefik's 502 passed through) must not restart the
+# proxy, while the dead-route state kills every domain at once.
+netbird_watchdog_probe_hosts:
+  - home.renner.uno
+  - immich.renner.uno
 
 # Address the probe connects to. 127.0.0.1 reaches the local proxy on both
 # members: VPS-1 routes it via Traefik's HostSNI(*) TCP passthrough, proxy-1
diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2
index 2195a03a..2d96101d 100644
--- a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2
+++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-watchdog.sh.j2
@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 # Probe the local reverse proxy end-to-end and restart it after
-# {{ netbird_watchdog_fail_threshold }} consecutive hard failures.
-# Only 502 and 000 (timeout / TLS or connect failure) count as failures —
-# those are the proxy's dead-route signatures; anything else, including
-# backend application errors, proves the proxy itself forwards fine.
-# All output lands in the journal via the systemd unit.
+# {{ netbird_watchdog_fail_threshold }} consecutive all-domains-down rounds.
+# Only 502 and 000 (timeout / TLS or connect failure) count per domain, and
+# a round fails only if EVERY probed domain fails — one dead backend shows
+# up as a 502 on its own domain, while the proxy's dead-route state kills
+# all domains at once. All output lands in the journal via the systemd unit.
 set -u
 
 STATE_DIR=/run/netbird-proxy-watchdog
@@ -12,16 +12,29 @@ FAILS_FILE="$STATE_DIR/consecutive_failures"
 LAST_RESTART_FILE="$STATE_DIR/last_restart_epoch"
 mkdir -p "$STATE_DIR"
 
-code=$(curl -sk -m 10 -o /dev/null -w '%{http_code}' \
-  --resolve '{{ netbird_watchdog_probe_host }}:443:{{ netbird_watchdog_probe_address }}' \
-  'https://{{ netbird_watchdog_probe_host }}/' || true)
+failed=0
+total=0
+results=""
+for host in {{ netbird_watchdog_probe_hosts | join(' ') }}; do
+  total=$((total + 1))
+  code=$(curl -sk -m 10 -o /dev/null -w '%{http_code}' \
+    --resolve "$host:443:{{ netbird_watchdog_probe_address }}" \
+    "https://$host/" || true)
+  if [ "$code" = "502" ] || [ "$code" = "000" ]; then
+    failed=$((failed + 1))
+    results="$results $host=$code(FAIL)"
+  else
+    results="$results $host=$code"
+  fi
+done
+
 fails=$(cat "$FAILS_FILE" 2>/dev/null || echo 0)
 
-if [ "$code" != "502" ] && [ "$code" != "000" ]; then
+if [ "$failed" -lt "$total" ]; then
   if [ "$fails" -gt 0 ]; then
-    echo "recovered: probe={{ netbird_watchdog_probe_host }} status=$code after $fails failure(s)"
+    echo "recovered after $fails failing round(s):$results"
   else
-    echo "ok: probe={{ netbird_watchdog_probe_host }} status=$code"
+    echo "ok:$results"
   fi
   echo 0 > "$FAILS_FILE"
   exit 0
@@ -29,7 +42,7 @@ fi
 
 fails=$((fails + 1))
 echo "$fails" > "$FAILS_FILE"
-echo "failure $fails/{{ netbird_watchdog_fail_threshold }}: probe={{ netbird_watchdog_probe_host }} status=$code"
+echo "failure $fails/{{ netbird_watchdog_fail_threshold }} (all domains down):$results"
 
 if [ "$fails" -lt {{ netbird_watchdog_fail_threshold }} ]; then
   exit 0
@@ -42,7 +55,7 @@ if [ $((now - last)) -lt {{ netbird_watchdog_cooldown_seconds }} ]; then
   exit 0
 fi
 
-echo "restarting {{ netbird_watchdog_container }} after $fails consecutive failures"
+echo "restarting {{ netbird_watchdog_container }} after $fails consecutive failing rounds"
 if docker restart '{{ netbird_watchdog_container }}'; then
   echo "$now" > "$LAST_RESTART_FILE"
   echo 0 > "$FAILS_FILE"
-- 
2.54.0