ansible/roles/netbird-proxy-watchdog/defaults/main.yml

@@ -31,5 +31,13 @@ netbird_watchdog_container: netbird-proxy
 netbird_watchdog_fail_threshold: 3
 
 # Minimum seconds between watchdog-issued restarts, so a genuinely broken
-# backend can't put the proxy into a restart loop.
+# backend can't put the proxy into a restart loop. Shared by both the URL-probe
+# watchdog and the flood watchdog below.
 netbird_watchdog_cooldown_seconds: 600
+
+# Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain
+# substring in the proxy's docker logs by the flood watchdog. On this build the
+# full line is "... couldn't fetch peer state: <peer>"; "fetch peer state" is
+# the stable, apostrophe-free core. It only ever appears in bursts during a
+# wedge (verified over 14 days of logs), so a first-line match is safe.
+netbird_watchdog_flood_pattern: "fetch peer state"

ansible/roles/netbird-proxy-watchdog/tasks/main.yml

@@ -30,3 +30,27 @@
     enabled: true
     state: started
     daemon_reload: true
+
+# Flood watchdog: a persistent log follower that restarts the proxy the instant
+# the route-manager wedge ("couldn't fetch peer state" flood) appears, minutes
+# before the URL probe above would notice. See the script header for the why.
+- name: Deploy the flood watchdog script
+  ansible.builtin.template:
+    src: netbird-proxy-flood-watchdog.sh.j2
+    dest: /usr/local/bin/netbird-proxy-flood-watchdog
+    mode: "0755"
+  register: _flood_script
+
+- name: Deploy the flood watchdog systemd service
+  ansible.builtin.template:
+    src: netbird-proxy-flood-watchdog.service.j2
+    dest: /etc/systemd/system/netbird-proxy-flood-watchdog.service
+    mode: "0644"
+  register: _flood_unit
+
+- name: Enable and (re)start the flood watchdog
+  ansible.builtin.systemd:
+    name: netbird-proxy-flood-watchdog.service
+    enabled: true
+    state: "{{ 'restarted' if (_flood_script.changed or _flood_unit.changed) else 'started' }}"
+    daemon_reload: true

ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=NetBird reverse-proxy flood watchdog (instant restart on route-manager wedge)
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/netbird-proxy-flood-watchdog
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Instant recovery for the reverse-proxy route-manager wedge.
+#
+# When an ephemeral NetBird routing peer (a Kubernetes ingress pod) is DELETED
+# -- which happens on every ingress roll, since the pods re-enroll a fresh peer
+# identity on each restart -- the reverse-proxy's route manager can pin the now-
+# deleted peer and never re-select a healthy HA-group peer. It then serves
+# errors for every domain behind that route until the process is restarted,
+# flooding "couldn't fetch peer state: <peer>" in its own log. This is an
+# upstream bug, unfixed as of netbird v0.74.4 (cf. netbirdio/netbird#3976 and
+# the dnsinterceptor "HA routing disabled" + recalculateRoutes abort-on-remove
+# hazards); a process restart is the only reliable clear.
+#
+# That flood is the instant, unambiguous fingerprint: 14 days of proxy logs
+# show it ONLY in bursts of tens per minute, never as a benign trickle -- so we
+# restart on the FIRST matching line. ~1/6 of these bursts self-recover (the
+# re-select race sometimes wins) and get an unnecessary but near-instant proxy
+# restart: the deliberate trade for clearing a real wedge in seconds instead of
+# the 5-9 min the URL-probe watchdog took (its 127.0.0.1 probe stays warm during
+# the wedge and only trips minutes later, if at all on this member).
+#
+# The URL-probe watchdog (netbird-proxy-watchdog.timer) stays as the backstop
+# for other failure modes (e.g. a wedge with no peer-state log). Both share
+# LAST_RESTART_FILE, so the cooldown is global: neither can start a restart
+# loop, and the two can't double-restart each other.
+set -u
+
+CONTAINER='{{ netbird_watchdog_container }}'
+PATTERN='{{ netbird_watchdog_flood_pattern }}'
+COOLDOWN={{ netbird_watchdog_cooldown_seconds }}
+
+STATE_DIR=/run/netbird-proxy-watchdog
+LAST_RESTART_FILE="$STATE_DIR/last_restart_epoch"
+mkdir -p "$STATE_DIR"
+
+while true; do
+  # Follow only NEW log lines (--tail 0 so a container we just restarted, or
+  # old history, can't re-trigger us). grep -q exits on the first match, which
+  # closes the pipe and stops `docker logs -f`.
+  if docker logs --tail 0 -f "$CONTAINER" 2>&1 | grep -q -- "$PATTERN"; then
+    now=$(date +%s)
+    last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)
+    if [ $((now - last)) -lt "$COOLDOWN" ]; then
+      echo "flood ('$PATTERN') detected -- restart suppressed (last restart $((now - last))s ago, cooldown ${COOLDOWN}s)"
+    else
+      echo "flood ('$PATTERN') detected -- restarting $CONTAINER (route-manager wedge)"
+      if docker restart "$CONTAINER"; then
+        echo "$now" > "$LAST_RESTART_FILE"
+        echo "restart done"
+      else
+        echo "restart FAILED"
+      fi
+    fi
+  fi
+  # `docker logs -f` also exits when the container stops/restarts (ours, the
+  # probe watchdog's, or a deploy). Pause, then re-attach to the fresh stream.
+  sleep 2
+done