From 6c66ca3c42d848478949e661bf4963b313ac5268 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Sat, 11 Jul 2026 17:04:36 +0200 Subject: [PATCH] netbird-proxy-watchdog: add flood watchdog for instant route-manager-wedge recovery The reverse-proxy wedges when an ephemeral NetBird routing peer (a k8s ingress pod) is DELETED: its route manager pins the now-gone peer and never re-selects a healthy HA-group peer, serving errors for every domain behind that route until restarted. Upstream bug, unfixed as of netbird v0.74.4 (netbirdio/netbird #3976; dnsinterceptor disables HA for an already-routed IP + recalculateRoutes aborts on the dead peer's teardown error). A restart is the only reliable clear. The existing URL-probe watchdog does eventually catch it, but far too slowly: its 127.0.0.1 probe stays warm for ~3 min after the wedge starts, then needs 3 consecutive 60s rounds -- so it took 5-9 min on 2026-07-11, and both proxy members wedged so the public outage lasted until the slower one recovered. The wedge floods "couldn't fetch peer state" in the proxy's own log the instant it happens, minutes before the probe notices. That flood is an unambiguous fingerprint: 14 days of logs show it ONLY in bursts of tens per minute, never a benign trickle. So add a persistent follower that restarts the proxy on the first matching line. Trade, chosen deliberately: ~1/6 of these bursts self-recover (the re-select race sometimes wins) and get an unnecessary restart -- but a proxy restart is near-instant, so a rare spurious restart beats a multi-minute outage. - new netbird-proxy-flood-watchdog.sh (docker logs -f | grep, restart on match) - new netbird-proxy-flood-watchdog.service (Type=simple, Restart=always) - shares LAST_RESTART_FILE + cooldown with the probe watchdog (no loops, no double-restart); URL-probe watchdog kept unchanged as the backstop. Deploying only adds a systemd service; it does not touch the proxy or ingress, so it can't itself cause an outage. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../netbird-proxy-watchdog/defaults/main.yml | 10 +++- .../netbird-proxy-watchdog/tasks/main.yml | 24 ++++++++ .../netbird-proxy-flood-watchdog.service.j2 | 13 +++++ .../netbird-proxy-flood-watchdog.sh.j2 | 58 +++++++++++++++++++ 4 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2 create mode 100644 ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2 diff --git a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml index 10ad91ab..bf4eb442 100644 --- a/ansible/roles/netbird-proxy-watchdog/defaults/main.yml +++ b/ansible/roles/netbird-proxy-watchdog/defaults/main.yml @@ -31,5 +31,13 @@ netbird_watchdog_container: netbird-proxy netbird_watchdog_fail_threshold: 3 # Minimum seconds between watchdog-issued restarts, so a genuinely broken -# backend can't put the proxy into a restart loop. +# backend can't put the proxy into a restart loop. Shared by both the URL-probe +# watchdog and the flood watchdog below. netbird_watchdog_cooldown_seconds: 600 + +# Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain +# substring in the proxy's docker logs by the flood watchdog. On this build the +# full line is "... couldn't fetch peer state: "; "fetch peer state" is +# the stable, apostrophe-free core. It only ever appears in bursts during a +# wedge (verified over 14 days of logs), so a first-line match is safe. +netbird_watchdog_flood_pattern: "fetch peer state" diff --git a/ansible/roles/netbird-proxy-watchdog/tasks/main.yml b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml index e199d584..c07a65dc 100644 --- a/ansible/roles/netbird-proxy-watchdog/tasks/main.yml +++ b/ansible/roles/netbird-proxy-watchdog/tasks/main.yml @@ -30,3 +30,27 @@ enabled: true state: started daemon_reload: true + +# Flood watchdog: a persistent log follower that restarts the proxy the instant +# the route-manager wedge ("couldn't fetch peer state" flood) appears, minutes +# before the URL probe above would notice. See the script header for the why. +- name: Deploy the flood watchdog script + ansible.builtin.template: + src: netbird-proxy-flood-watchdog.sh.j2 + dest: /usr/local/bin/netbird-proxy-flood-watchdog + mode: "0755" + register: _flood_script + +- name: Deploy the flood watchdog systemd service + ansible.builtin.template: + src: netbird-proxy-flood-watchdog.service.j2 + dest: /etc/systemd/system/netbird-proxy-flood-watchdog.service + mode: "0644" + register: _flood_unit + +- name: Enable and (re)start the flood watchdog + ansible.builtin.systemd: + name: netbird-proxy-flood-watchdog.service + enabled: true + state: "{{ 'restarted' if (_flood_script.changed or _flood_unit.changed) else 'started' }}" + daemon_reload: true diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2 new file mode 100644 index 00000000..47f63bdb --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=NetBird reverse-proxy flood watchdog (instant restart on route-manager wedge) +Requires=docker.service +After=docker.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/netbird-proxy-flood-watchdog +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2 b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2 new file mode 100644 index 00000000..b585669c --- /dev/null +++ b/ansible/roles/netbird-proxy-watchdog/templates/netbird-proxy-flood-watchdog.sh.j2 @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Instant recovery for the reverse-proxy route-manager wedge. +# +# When an ephemeral NetBird routing peer (a Kubernetes ingress pod) is DELETED +# -- which happens on every ingress roll, since the pods re-enroll a fresh peer +# identity on each restart -- the reverse-proxy's route manager can pin the now- +# deleted peer and never re-select a healthy HA-group peer. It then serves +# errors for every domain behind that route until the process is restarted, +# flooding "couldn't fetch peer state: " in its own log. This is an +# upstream bug, unfixed as of netbird v0.74.4 (cf. netbirdio/netbird#3976 and +# the dnsinterceptor "HA routing disabled" + recalculateRoutes abort-on-remove +# hazards); a process restart is the only reliable clear. +# +# That flood is the instant, unambiguous fingerprint: 14 days of proxy logs +# show it ONLY in bursts of tens per minute, never as a benign trickle -- so we +# restart on the FIRST matching line. ~1/6 of these bursts self-recover (the +# re-select race sometimes wins) and get an unnecessary but near-instant proxy +# restart: the deliberate trade for clearing a real wedge in seconds instead of +# the 5-9 min the URL-probe watchdog took (its 127.0.0.1 probe stays warm during +# the wedge and only trips minutes later, if at all on this member). +# +# The URL-probe watchdog (netbird-proxy-watchdog.timer) stays as the backstop +# for other failure modes (e.g. a wedge with no peer-state log). Both share +# LAST_RESTART_FILE, so the cooldown is global: neither can start a restart +# loop, and the two can't double-restart each other. +set -u + +CONTAINER='{{ netbird_watchdog_container }}' +PATTERN='{{ netbird_watchdog_flood_pattern }}' +COOLDOWN={{ netbird_watchdog_cooldown_seconds }} + +STATE_DIR=/run/netbird-proxy-watchdog +LAST_RESTART_FILE="$STATE_DIR/last_restart_epoch" +mkdir -p "$STATE_DIR" + +while true; do + # Follow only NEW log lines (--tail 0 so a container we just restarted, or + # old history, can't re-trigger us). grep -q exits on the first match, which + # closes the pipe and stops `docker logs -f`. + if docker logs --tail 0 -f "$CONTAINER" 2>&1 | grep -q -- "$PATTERN"; then + now=$(date +%s) + last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0) + if [ $((now - last)) -lt "$COOLDOWN" ]; then + echo "flood ('$PATTERN') detected -- restart suppressed (last restart $((now - last))s ago, cooldown ${COOLDOWN}s)" + else + echo "flood ('$PATTERN') detected -- restarting $CONTAINER (route-manager wedge)" + if docker restart "$CONTAINER"; then + echo "$now" > "$LAST_RESTART_FILE" + echo "restart done" + else + echo "restart FAILED" + fi + fi + fi + # `docker logs -f` also exits when the container stops/restarts (ours, the + # probe watchdog's, or a deploy). Pause, then re-attach to the fresh stream. + sleep 2 +done -- 2.54.0