diff --git a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml index 10ad91ab..bf4eb442 100644 --- a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml +++ b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml @@ -31,5 +31,13 @@ netbird_watchdog_container: netbird-proxy netbird_watchdog_fail_threshold: 3 # Minimum seconds between watchdog-issued restarts, so a genuinely broken -# backend can't put the proxy into a restart loop. +# backend can't put the proxy into a restart loop. Shared by both the URL-probe +# watchdog and the flood watchdog below. netbird_watchdog_cooldown_seconds: 600 + +# Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain +# substring in the proxy's docker logs by the flood watchdog. On this build the +# full line is "... couldn't fetch peer state: "; "fetch peer state" is +# the stable, apostrophe-free core. It only ever appears in bursts during a +# wedge (verified over 14 days of logs), so a first-line match is safe. +netbird_watchdog_flood_pattern: "fetch peer state" diff --git a/ansible/roles/netbird-proxy-watchdog/tasks/main.yml b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml index e199d584..c07a65dc 100644 --- a/ansible/roles/netbird-proxy-watchdog/tasks/main.yml +++ b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml @@ -30,3 +30,27 @@ enabled: true state: started daemon_reload: true + +# Flood watchdog: a persistent log follower that restarts the proxy the instant +# the route-manager wedge ("couldn't fetch peer state" flood) appears, minutes +# before the URL probe above would notice. See the script header for the why. +- name: Deploy the flood watchdog script + ansible.builtin.template: + src: netbird-proxy-flood-watchdog.sh.j2 + dest: /usr/local/bin/netbird-proxy-flood-watchdog + mode: "0755" + register: _flood_script + +- name: Deploy the flood watchdog systemd service + ansible.builtin.template: + src: netbird-proxy-flood-watchdog.service.j2 + dest: /etc/systemd/system/netbird-proxy-flood-watchdog.service + mode: "0644" + register: _flood_unit + +- name: Enable and (re)start the flood watchdog + ansible.builtin.systemd: + name: netbird-proxy-flood-watchdog.service + enabled: true + state: "{{ 'restarted' if (_flood_script.changed or _flood_unit.changed) else 'started' }}" + daemon_reload: true diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2 new file mode 100644 index 00000000..47f63bdb --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=NetBird reverse-proxy flood watchdog (instant restart on route-manager wedge) +Requires=docker.service +After=docker.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/netbird-proxy-flood-watchdog +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2 new file mode 100644 index 00000000..b585669c --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2 @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Instant recovery for the reverse-proxy route-manager wedge. +# +# When an ephemeral NetBird routing peer (a Kubernetes ingress pod) is DELETED +# -- which happens on every ingress roll, since the pods re-enroll a fresh peer +# identity on each restart -- the reverse-proxy's route manager can pin the now- +# deleted peer and never re-select a healthy HA-group peer. It then serves +# errors for every domain behind that route until the process is restarted, +# flooding "couldn't fetch peer state: " in its own log. This is an +# upstream bug, unfixed as of netbird v0.74.4 (cf. netbirdio/netbird#3976 and +# the dnsinterceptor "HA routing disabled" + recalculateRoutes abort-on-remove +# hazards); a process restart is the only reliable clear. +# +# That flood is the instant, unambiguous fingerprint: 14 days of proxy logs +# show it ONLY in bursts of tens per minute, never as a benign trickle -- so we +# restart on the FIRST matching line. ~1/6 of these bursts self-recover (the +# re-select race sometimes wins) and get an unnecessary but near-instant proxy +# restart: the deliberate trade for clearing a real wedge in seconds instead of +# the 5-9 min the URL-probe watchdog took (its 127.0.0.1 probe stays warm during +# the wedge and only trips minutes later, if at all on this member). +# +# The URL-probe watchdog (netbird-proxy-watchdog.timer) stays as the backstop +# for other failure modes (e.g. a wedge with no peer-state log). Both share +# LAST_RESTART_FILE, so the cooldown is global: neither can start a restart +# loop, and the two can't double-restart each other. +set -u + +CONTAINER='{{ netbird_watchdog_container }}' +PATTERN='{{ netbird_watchdog_flood_pattern }}' +COOLDOWN={{ netbird_watchdog_cooldown_seconds }} + +STATE_DIR=/run/netbird-proxy-watchdog +LAST_RESTART_FILE="$STATE_DIR/last_restart_epoch" +mkdir -p "$STATE_DIR" + +while true; do + # Follow only NEW log lines (--tail 0 so a container we just restarted, or + # old history, can't re-trigger us). grep -q exits on the first match, which + # closes the pipe and stops `docker logs -f`. + if docker logs --tail 0 -f "$CONTAINER" 2>&1 | grep -q -- "$PATTERN"; then + now=$(date +%s) + last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0) + if [ $((now - last)) -lt "$COOLDOWN" ]; then + echo "flood ('$PATTERN') detected -- restart suppressed (last restart $((now - last))s ago, cooldown ${COOLDOWN}s)" + else + echo "flood ('$PATTERN') detected -- restarting $CONTAINER (route-manager wedge)" + if docker restart "$CONTAINER"; then + echo "$now" > "$LAST_RESTART_FILE" + echo "restart done" + else + echo "restart FAILED" + fi + fi + fi + # `docker logs -f` also exits when the container stops/restarts (ours, the + # probe watchdog's, or a deploy). Pause, then re-attach to the fresh stream. + sleep 2 +done