From 67353b8e29b53646abb6ff589e46fa1a6c9d6b38 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Sat, 11 Jul 2026 10:37:27 +0200 Subject: [PATCH] netbird-ingress: graceful drain + paced roll so rolls don't drop public ingress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every netbird image bump (image-automation, ~weekly) rolls this DaemonSet and takes public ingress (home.renner.uno, mesh-only via the HA route group) down for ~1 min — observed 2026-07-11 08:58 on the uptime monitor, correlated exactly with the 3 peers rolling at 06:56:18/39/58 UTC. It is NOT the IPS/firewall (the UDM was idle; the "intrusions" were the admin laptop hitting its own ingress) — it's the roll itself failing over slowly. Root cause: the redundant roll (maxUnavailable:1, 2/3 peers always up) should be seamless, but two gaps break the failover: - No preStop drain: a rolled peer is only SIGTERM'd, so the mesh keeps the dead peer in the route group until a keepalive timeout (~25-30s); traffic riding through it black-holes until then. Compounded by ephemeral enrollment (emptyDir) — the restarted pod enrolls a NEW peer and the old one lingers until reaped. - No minReadySeconds: the roll advances to the next node as soon as `netbird status --check ready` passes (seconds), faster than the mesh re-converges onto the freshly-enrolled peer — so the HA route is degraded for the whole ~1-min roll. Fix: - preStop `netbird down` — deregister the peer from management before the process dies, so the HA route fails over to the other per-node peers immediately instead of after a timeout. - minReadySeconds: 45 — hold each new peer Ready until the mesh has converged onto it before rolling the next, so >=2 peers are always actually serving. Keeps auto-updates on; turns the surprise ~1-min outage into a seamless roll. Validation: watch the uptime monitor across the next roll (natural image bump or a deliberate `kubectl rollout restart ds/netbird-ingress`). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../netbird-ingress/daemonset.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/kubernetes/infrastructure/netbird-ingress/daemonset.yaml b/kubernetes/infrastructure/netbird-ingress/daemonset.yaml index 54fd1c1e..2c18787c 100644 --- a/kubernetes/infrastructure/netbird-ingress/daemonset.yaml +++ b/kubernetes/infrastructure/netbird-ingress/daemonset.yaml @@ -38,6 +38,16 @@ spec: rollingUpdate: # roll one node's peer at a time; the others keep serving the route. maxUnavailable: 1 + # Pace the roll to NetBird's route re-convergence. A rolled peer is replaced + # by a freshly-enrolled peer (ephemeral state below) that must join the + # k8s-ingress-router route group and propagate to mesh clients before it + # actually forwards. Without this, the roll steps to the next node as soon as + # `netbird status --check ready` passes (seconds) — faster than the mesh + # re-converges — so the HA route runs degraded for the whole roll and public + # ingress (home.renner.uno) drops for ~1 min (observed 2026-07-11). Hold each + # new peer Ready this long before rolling the next, so >=2 peers are always + # actually serving the route. + minReadySeconds: 45 template: metadata: labels: @@ -86,6 +96,17 @@ spec: memory: 64Mi limits: memory: 256Mi + # Graceful drain: withdraw this peer from the mesh BEFORE the process + # is killed. Without a preStop the pod is only SIGTERM'd, so the mesh + # keeps the (now-dead) peer in the HA route group until a keepalive + # timeout (~25-30s) — traffic riding through it black-holes until + # then. `netbird down` disconnects the peer from the management server + # so it fails the route over to the other per-node peers immediately. + # Runs within terminationGracePeriodSeconds (default 30s). + lifecycle: + preStop: + exec: + command: ["netbird", "down"] livenessProbe: exec: command: ["netbird", "status", "--check", "live"] -- 2.54.0