fix: [alert] KubeJobFailed (netbird-backup) (#712 )

2026-08-29 01:45:13 +00:00
1 changed files with 23 additions and 5 deletions
@@ -1,10 +1,11 @@
 #!/bin/sh
 # Forced-command target for the netbird-backup SSH key (see the
 # "Authorize the cluster backup key" task): takes a CONSISTENT snapshot
-# of the management SQLite store via the online-backup API (safe against
-# a live writer, unlike copying store.db) and streams a gzipped tar to
-# stdout. The key's authorized_keys entry is restricted to exactly this
-# script — the in-cluster puller cannot run anything else.
+# of the management SQLite store via the online-backup API (which, unlike
+# copying store.db, stays consistent while netbird-management writes) and
+# streams a gzipped tar to stdout. The key's authorized_keys entry is
+# restricted to exactly this script — the in-cluster puller cannot run
+# anything else.
 #
 # Deliberately store.db ONLY: /opt/netbird/config is fully regenerable
 # from the netbird-server role's templates + vault, and including it
@@ -14,5 +15,22 @@
 set -eu
 tmp=$(mktemp -d)
 trap 'rm -rf "$tmp"' EXIT
-sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'"
+# Consistency is not the same as immunity to SQLITE_BUSY: the sqlite3
+# CLI defaults to a busy timeout of 0, so a single collision with the
+# management container holding the write lock aborts the dump outright
+# (2026-08-29: all three Job attempts died with "Error: database is
+# locked" within 51 s of each other). Wait a contending writer out, and
+# retry a few times on top. Worst case ~80 s per invocation, which stays
+# inside the Job's activeDeadlineSeconds: 600 budget even across its
+# three pod attempts.
+ok=0
+for attempt in 1 2 3; do
+    rm -f "$tmp"/store.db*
+    if sqlite3 -cmd ".timeout 20000" /opt/netbird/management/store.db ".backup '$tmp/store.db'"; then
+        ok=1
+        break
+    fi
+    [ "$attempt" = 3 ] || sleep 10
+done
+[ "$ok" = 1 ] || exit 1
 tar -C "$tmp" -czf - store.db