ansible/roles/netbird-server/files/netbird-backup-dump

@@ -0,0 +1,18 @@
+#!/bin/sh
+# Forced-command target for the netbird-backup SSH key (see the
+# "Authorize the cluster backup key" task): takes a CONSISTENT snapshot
+# of the management SQLite store via the online-backup API (safe against
+# a live writer, unlike copying store.db) and streams a gzipped tar to
+# stdout. The key's authorized_keys entry is restricted to exactly this
+# script — the in-cluster puller cannot run anything else.
+#
+# Deliberately store.db ONLY: /opt/netbird/config is fully regenerable
+# from the netbird-server role's templates + vault, and including it
+# would put the plaintext TURN secret (turnserver.conf) into every
+# nightly tarball. Restore = re-run the role (rebuilds config), then
+# untar this dump into /opt/netbird/management/.
+set -eu
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'"
+tar -C "$tmp" -czf - store.db

ansible/roles/netbird-server/tasks/main.yml

@@ -77,6 +77,30 @@
   register: docker_compose_result
   changed_when: "'Started' in docker_compose_result.stdout or 'Creating' in docker_compose_result.stdout"
 
+# Off-VPS backup of the management store (SPOF assessment 2026-07-25:
+# this SQLite DB — peers, setup keys, groups, routes — had zero backup
+# coverage; losing the VPS meant re-enrolling the entire mesh). The VPS
+# cannot reach any PBS (both PBS names resolve to the public proxy,
+# ports closed), so the CLUSTER pulls: a CronJob
+# (kubernetes/infrastructure/netbird-backup) ssh's in with a dedicated
+# key that is locked to the dump script below, and lands the tarball on
+# a Ceph PVC that Velero ships offsite. The dump is store.db ONLY —
+# /opt/netbird/config is regenerable from this role's templates + vault
+# and would otherwise leak the TURN secret into every tarball. Restore:
+# re-run this role (rebuilds config), then untar the dump into
+# /opt/netbird/management/.
+- name: Install the backup dump script
+  ansible.builtin.copy:
+    src: netbird-backup-dump
+    dest: /usr/local/bin/netbird-backup-dump
+    mode: "0755"
+
+- name: Authorize the cluster backup key (forced command, no shell)
+  ansible.posix.authorized_key:
+    user: root
+    key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeW/L9bHMTSa5/Qb0ClppCf4Z0hL39JwzI6tBmKd2FG netbird-backup@homelab-cluster"
+    key_options: 'restrict,command="/usr/local/bin/netbird-backup-dump"'
+
 - name: Wait for management container to be ready
   ansible.builtin.pause:
     seconds: 10

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
   - prometheusrule-hetzner-backup.yaml
   - prometheusrule-loki.yaml
   - prometheusrule-netbird-availability.yaml
+  - prometheusrule-netbird-backup.yaml
   - prometheusrule-netbird-ingress.yaml
   - prometheusrule-velero.yaml
   - prometheusrule-woodpecker-rotation.yaml

kubernetes/apps/monitoring-rules/prometheusrule-netbird-backup.yaml

@@ -0,0 +1,44 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: netbird-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: netbird-backup.rules
+      rules:
+        - alert: NetbirdVpsBackupStale
+          # The nightly pull of the controller VPS's management store
+          # hasn't succeeded in over 26 h (one missed cycle + slack).
+          # This dump is the only copy of the mesh state (peers, setup
+          # keys, routes) that survives losing the Hetzner VPS — a
+          # silently dead backup is exactly the failure mode the SPOF
+          # assessment flagged.
+          expr: |
+            time() - kube_cronjob_status_last_successful_time{namespace="netbird-backup", cronjob="netbird-vps-backup"} > 26 * 3600
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "NetBird VPS backup has not succeeded in 26h+"
+            description: "The netbird-vps-backup CronJob's last successful run is over 26 hours old. Check the job logs in the netbird-backup namespace (ssh reachability of netbird.renner.uno, the netbird-backup-ssh ExternalSecret, PVC space)."
+
+        - alert: NetbirdVpsBackupNeverRan
+          # The CronJob exists but has never succeeded — most likely the
+          # one-time OpenBao provisioning step
+          # (bao kv put homelab/infrastructure/netbird-backup ssh-key=@…)
+          # hasn't been done, so the ExternalSecret cannot sync and the
+          # job pod fails on the missing secret mount.
+          expr: |
+            kube_cronjob_created{namespace="netbird-backup", cronjob="netbird-vps-backup"}
+            unless
+            kube_cronjob_status_last_successful_time{namespace="netbird-backup", cronjob="netbird-vps-backup"}
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            summary: "NetBird VPS backup has never succeeded"
+            description: "The netbird-vps-backup CronJob exists but has no successful run. If this is the initial rollout: the private key must be provisioned once by hand — bao kv put homelab/infrastructure/netbird-backup ssh-key=@<keyfile> — before the ExternalSecret can sync."

kubernetes/infrastructure/kustomization.yaml

@@ -16,6 +16,7 @@ resources:
   - ceph-csi-drivers/
   - cloudnative-pg/
   - netbird-ingress/
+  - netbird-backup/
   - coredns-config/
   - k8s-gateway/
   - external-dns-unifi/

kubernetes/infrastructure/namespaces/namespaces.yaml

@@ -3,6 +3,13 @@ kind: Namespace
 metadata:
   name: apps
 ---
+# Pull-based off-VPS backup of the NetBird controller state
+# (kubernetes/infrastructure/netbird-backup).
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: netbird-backup
+---
 apiVersion: v1
 kind: Namespace
 metadata:

kubernetes/infrastructure/netbird-backup/cronjob.yaml

@@ -0,0 +1,97 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: netbird-vps-backup
+  namespace: netbird-backup
+  labels:
+    app.kubernetes.io/name: netbird-backup
+spec:
+  # Nightly at 02:45, well ahead of the 07:45 Velero Schedule
+  # (post-crds/velero-schedules.yaml) that ships this PVC offsite.
+  schedule: "45 2 * * *"
+  timeZone: Europe/Berlin
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 2
+  failedJobsHistoryLimit: 5
+  startingDeadlineSeconds: 3600
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 600
+      ttlSecondsAfterFinished: 259200 # 3 d
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: netbird-backup
+        spec:
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            runAsGroup: 65534
+            fsGroup: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: pull
+              # ci-ansible ships an ssh client and standard coreutils and
+              # is pulled from the local registry; a dedicated slim image
+              # is not worth maintaining for one nightly ssh pull.
+              image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+              command:
+                - sh
+                - -ec
+                - |
+                  # The key's authorized_keys entry on the VPS is
+                  # restrict,command=… — the server runs the dump script
+                  # regardless of what we request, streaming a tar.gz.
+                  # ssh demands 0600 on the key; secret mounts are group-
+                  # readable, so copy first. The host key is PINNED (no
+                  # TOFU): captured 2026-07-26 via ssh-keyscan and
+                  # cross-checked against the workstation's known_hosts.
+                  # If the VPS is ever reprovisioned with a new host key,
+                  # update this line — the job failing loudly on a key
+                  # mismatch is the desired behavior.
+                  install -m 600 /ssh/ssh-key /tmp/key
+                  printf '%s\n' 'netbird.renner.uno ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKCe8LRXbyn6Y+5V4ia4ErnBEcTFzHI1fAnsTvT77HJU' > /tmp/known_hosts
+                  # Leftover .part files from aborted earlier runs are
+                  # outside the retention glob — clear them up front.
+                  rm -f /backup/*.part
+                  ts=$(date +%Y-%m-%d)
+                  ssh -i /tmp/key \
+                      -o StrictHostKeyChecking=yes \
+                      -o UserKnownHostsFile=/tmp/known_hosts \
+                      -o ConnectTimeout=15 \
+                      root@netbird.renner.uno > "/backup/netbird-$ts.tar.gz.part"
+                  gzip -t "/backup/netbird-$ts.tar.gz.part"
+                  mv "/backup/netbird-$ts.tar.gz.part" "/backup/netbird-$ts.tar.gz"
+                  # Retention: newest 14 dumps (Velero keeps 30 d offsite).
+                  ls -1t /backup/netbird-*.tar.gz | tail -n +15 | xargs -r rm --
+                  ls -lh /backup/
+              env:
+                - name: HOME
+                  value: /tmp
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              resources:
+                requests:
+                  cpu: 25m
+                  memory: 64Mi
+                limits:
+                  memory: 256Mi
+              volumeMounts:
+                - name: ssh-key
+                  mountPath: /ssh
+                  readOnly: true
+                - name: backup
+                  mountPath: /backup
+          volumes:
+            - name: ssh-key
+              secret:
+                secretName: netbird-backup-ssh
+                defaultMode: 0440
+            - name: backup
+              persistentVolumeClaim:
+                claimName: netbird-backup

kubernetes/infrastructure/netbird-backup/externalsecret.yaml

@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: netbird-backup-ssh
+  namespace: netbird-backup
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: netbird-backup-ssh
+  # OpenBao path homelab/infrastructure/netbird-backup, key `ssh-key`:
+  # the private half of the dedicated backup keypair whose public half
+  # is pinned (forced-command, restrict) in the VPS's authorized_keys by
+  # the netbird-server role. ONE-TIME provisioning by hand (same pattern
+  # as the rotator bootstrap credentials):
+  #   bao kv put homelab/infrastructure/netbird-backup ssh-key=@<keyfile>
+  dataFrom:
+    - extract:
+        key: infrastructure/netbird-backup

kubernetes/infrastructure/netbird-backup/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Pull-based backup of the NetBird controller VPS's management store.
+# The VPS side (dump script + restricted SSH key) is deployed by the
+# netbird-server Ansible role; the Velero Schedule that ships the PVC
+# offsite lives in post-crds/velero-schedules.yaml; the staleness alert
+# in apps/monitoring-rules/prometheusrule-netbird-backup.yaml.
+resources:
+  - externalsecret.yaml
+  - pvc.yaml
+  - cronjob.yaml

kubernetes/infrastructure/netbird-backup/pvc.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: netbird-backup
+  namespace: netbird-backup
+spec:
+  # 14 daily dumps of a small SQLite store plus config — 5Gi is
+  # generous headroom.
+  storageClassName: ceph-block
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml

@@ -26,6 +26,7 @@ spec:
         - influxdb
         - ingress
         - monitoring
+        - netbird-backup # netbird-backup-ssh: the VPS-pull backup key
         - paperless
         - velero
   provider:

kubernetes/infrastructure/post-crds/velero-schedules.yaml

@@ -257,3 +257,29 @@ spec:
     storageLocation: default
     ttl: 720h0m0s
     snapshotMoveData: true
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: netbird-backup-daily
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: homelab-backup
+spec:
+  # Daily at 07:45 local (next free stagger slot after apps-daily at
+  # 07:15). Ships the netbird-backup PVC — nightly pulls of the NetBird
+  # controller VPS's management store (SQLite dump + config, pulled at
+  # 02:45 by the netbird-vps-backup CronJob) — offsite. This is the
+  # only copy of the mesh state that survives losing the Hetzner VPS.
+  # Added 2026-07-26 (SPOF assessment: controller store had zero backup
+  # coverage). No CNPG in this namespace.
+  schedule: "45 7 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    includedNamespaces:
+      - netbird-backup
+    includedResources:
+      - persistentvolumeclaims
+    storageLocation: default
+    ttl: 720h0m0s
+    snapshotMoveData: true