diff --git a/ansible/roles/netbird-server/files/netbird-backup-dump b/ansible/roles/netbird-server/files/netbird-backup-dump new file mode 100644 index 00000000..a4c4ee74 --- /dev/null +++ b/ansible/roles/netbird-server/files/netbird-backup-dump @@ -0,0 +1,18 @@ +#!/bin/sh +# Forced-command target for the netbird-backup SSH key (see the +# "Authorize the cluster backup key" task): takes a CONSISTENT snapshot +# of the management SQLite store via the online-backup API (safe against +# a live writer, unlike copying store.db) and streams a gzipped tar to +# stdout. The key's authorized_keys entry is restricted to exactly this +# script — the in-cluster puller cannot run anything else. +# +# Deliberately store.db ONLY: /opt/netbird/config is fully regenerable +# from the netbird-server role's templates + vault, and including it +# would put the plaintext TURN secret (turnserver.conf) into every +# nightly tarball. Restore = re-run the role (rebuilds config), then +# untar this dump into /opt/netbird/management/. +set -eu +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'" +tar -C "$tmp" -czf - store.db diff --git a/ansible/roles/netbird-server/tasks/main.yml b/ansible/roles/netbird-server/tasks/main.yml index 4e0277e6..3e8e699a 100644 --- a/ansible/roles/netbird-server/tasks/main.yml +++ b/ansible/roles/netbird-server/tasks/main.yml @@ -77,6 +77,30 @@ register: docker_compose_result changed_when: "'Started' in docker_compose_result.stdout or 'Creating' in docker_compose_result.stdout" +# Off-VPS backup of the management store (SPOF assessment 2026-07-25: +# this SQLite DB — peers, setup keys, groups, routes — had zero backup +# coverage; losing the VPS meant re-enrolling the entire mesh). The VPS +# cannot reach any PBS (both PBS names resolve to the public proxy, +# ports closed), so the CLUSTER pulls: a CronJob +# (kubernetes/infrastructure/netbird-backup) ssh's in with a dedicated +# key that is locked to the dump script below, and lands the tarball on +# a Ceph PVC that Velero ships offsite. The dump is store.db ONLY — +# /opt/netbird/config is regenerable from this role's templates + vault +# and would otherwise leak the TURN secret into every tarball. Restore: +# re-run this role (rebuilds config), then untar the dump into +# /opt/netbird/management/. +- name: Install the backup dump script + ansible.builtin.copy: + src: netbird-backup-dump + dest: /usr/local/bin/netbird-backup-dump + mode: "0755" + +- name: Authorize the cluster backup key (forced command, no shell) + ansible.posix.authorized_key: + user: root + key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeW/L9bHMTSa5/Qb0ClppCf4Z0hL39JwzI6tBmKd2FG netbird-backup@homelab-cluster" + key_options: 'restrict,command="/usr/local/bin/netbird-backup-dump"' + - name: Wait for management container to be ready ansible.builtin.pause: seconds: 10 diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index ffd75801..5a04463e 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -7,6 +7,7 @@ resources: - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml - prometheusrule-netbird-availability.yaml + - prometheusrule-netbird-backup.yaml - prometheusrule-netbird-ingress.yaml - prometheusrule-velero.yaml - prometheusrule-woodpecker-rotation.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-netbird-backup.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-backup.yaml new file mode 100644 index 00000000..d12d3d93 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-netbird-backup.yaml @@ -0,0 +1,44 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: netbird-backup + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus selects on this label. + release: kube-prometheus-stack +spec: + groups: + - name: netbird-backup.rules + rules: + - alert: NetbirdVpsBackupStale + # The nightly pull of the controller VPS's management store + # hasn't succeeded in over 26 h (one missed cycle + slack). + # This dump is the only copy of the mesh state (peers, setup + # keys, routes) that survives losing the Hetzner VPS — a + # silently dead backup is exactly the failure mode the SPOF + # assessment flagged. + expr: | + time() - kube_cronjob_status_last_successful_time{namespace="netbird-backup", cronjob="netbird-vps-backup"} > 26 * 3600 + for: 30m + labels: + severity: critical + annotations: + summary: "NetBird VPS backup has not succeeded in 26h+" + description: "The netbird-vps-backup CronJob's last successful run is over 26 hours old. Check the job logs in the netbird-backup namespace (ssh reachability of netbird.renner.uno, the netbird-backup-ssh ExternalSecret, PVC space)." + + - alert: NetbirdVpsBackupNeverRan + # The CronJob exists but has never succeeded — most likely the + # one-time OpenBao provisioning step + # (bao kv put homelab/infrastructure/netbird-backup ssh-key=@…) + # hasn't been done, so the ExternalSecret cannot sync and the + # job pod fails on the missing secret mount. + expr: | + kube_cronjob_created{namespace="netbird-backup", cronjob="netbird-vps-backup"} + unless + kube_cronjob_status_last_successful_time{namespace="netbird-backup", cronjob="netbird-vps-backup"} + for: 6h + labels: + severity: warning + annotations: + summary: "NetBird VPS backup has never succeeded" + description: "The netbird-vps-backup CronJob exists but has no successful run. If this is the initial rollout: the private key must be provisioned once by hand — bao kv put homelab/infrastructure/netbird-backup ssh-key=@ — before the ExternalSecret can sync." diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index 2b0645c5..3b34a28a 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -16,6 +16,7 @@ resources: - ceph-csi-drivers/ - cloudnative-pg/ - netbird-ingress/ + - netbird-backup/ - coredns-config/ - k8s-gateway/ - external-dns-unifi/ diff --git a/kubernetes/infrastructure/namespaces/namespaces.yaml b/kubernetes/infrastructure/namespaces/namespaces.yaml index 30369283..d804d5b6 100644 --- a/kubernetes/infrastructure/namespaces/namespaces.yaml +++ b/kubernetes/infrastructure/namespaces/namespaces.yaml @@ -3,6 +3,13 @@ kind: Namespace metadata: name: apps --- +# Pull-based off-VPS backup of the NetBird controller state +# (kubernetes/infrastructure/netbird-backup). +apiVersion: v1 +kind: Namespace +metadata: + name: netbird-backup +--- apiVersion: v1 kind: Namespace metadata: diff --git a/kubernetes/infrastructure/netbird-backup/cronjob.yaml b/kubernetes/infrastructure/netbird-backup/cronjob.yaml new file mode 100644 index 00000000..c97a7212 --- /dev/null +++ b/kubernetes/infrastructure/netbird-backup/cronjob.yaml @@ -0,0 +1,97 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: netbird-vps-backup + namespace: netbird-backup + labels: + app.kubernetes.io/name: netbird-backup +spec: + # Nightly at 02:45, well ahead of the 07:45 Velero Schedule + # (post-crds/velero-schedules.yaml) that ships this PVC offsite. + schedule: "45 2 * * *" + timeZone: Europe/Berlin + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 5 + startingDeadlineSeconds: 3600 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 600 + ttlSecondsAfterFinished: 259200 # 3 d + template: + metadata: + labels: + app.kubernetes.io/name: netbird-backup + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: pull + # ci-ansible ships an ssh client and standard coreutils and + # is pulled from the local registry; a dedicated slim image + # is not worth maintaining for one nightly ssh pull. + image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29 + command: + - sh + - -ec + - | + # The key's authorized_keys entry on the VPS is + # restrict,command=… — the server runs the dump script + # regardless of what we request, streaming a tar.gz. + # ssh demands 0600 on the key; secret mounts are group- + # readable, so copy first. The host key is PINNED (no + # TOFU): captured 2026-07-26 via ssh-keyscan and + # cross-checked against the workstation's known_hosts. + # If the VPS is ever reprovisioned with a new host key, + # update this line — the job failing loudly on a key + # mismatch is the desired behavior. + install -m 600 /ssh/ssh-key /tmp/key + printf '%s\n' 'netbird.renner.uno ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKCe8LRXbyn6Y+5V4ia4ErnBEcTFzHI1fAnsTvT77HJU' > /tmp/known_hosts + # Leftover .part files from aborted earlier runs are + # outside the retention glob — clear them up front. + rm -f /backup/*.part + ts=$(date +%Y-%m-%d) + ssh -i /tmp/key \ + -o StrictHostKeyChecking=yes \ + -o UserKnownHostsFile=/tmp/known_hosts \ + -o ConnectTimeout=15 \ + root@netbird.renner.uno > "/backup/netbird-$ts.tar.gz.part" + gzip -t "/backup/netbird-$ts.tar.gz.part" + mv "/backup/netbird-$ts.tar.gz.part" "/backup/netbird-$ts.tar.gz" + # Retention: newest 14 dumps (Velero keeps 30 d offsite). + ls -1t /backup/netbird-*.tar.gz | tail -n +15 | xargs -r rm -- + ls -lh /backup/ + env: + - name: HOME + value: /tmp + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + memory: 256Mi + volumeMounts: + - name: ssh-key + mountPath: /ssh + readOnly: true + - name: backup + mountPath: /backup + volumes: + - name: ssh-key + secret: + secretName: netbird-backup-ssh + defaultMode: 0440 + - name: backup + persistentVolumeClaim: + claimName: netbird-backup diff --git a/kubernetes/infrastructure/netbird-backup/externalsecret.yaml b/kubernetes/infrastructure/netbird-backup/externalsecret.yaml new file mode 100644 index 00000000..e49c03fd --- /dev/null +++ b/kubernetes/infrastructure/netbird-backup/externalsecret.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbird-backup-ssh + namespace: netbird-backup +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + name: netbird-backup-ssh + # OpenBao path homelab/infrastructure/netbird-backup, key `ssh-key`: + # the private half of the dedicated backup keypair whose public half + # is pinned (forced-command, restrict) in the VPS's authorized_keys by + # the netbird-server role. ONE-TIME provisioning by hand (same pattern + # as the rotator bootstrap credentials): + # bao kv put homelab/infrastructure/netbird-backup ssh-key=@ + dataFrom: + - extract: + key: infrastructure/netbird-backup diff --git a/kubernetes/infrastructure/netbird-backup/kustomization.yaml b/kubernetes/infrastructure/netbird-backup/kustomization.yaml new file mode 100644 index 00000000..0be931db --- /dev/null +++ b/kubernetes/infrastructure/netbird-backup/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Pull-based backup of the NetBird controller VPS's management store. +# The VPS side (dump script + restricted SSH key) is deployed by the +# netbird-server Ansible role; the Velero Schedule that ships the PVC +# offsite lives in post-crds/velero-schedules.yaml; the staleness alert +# in apps/monitoring-rules/prometheusrule-netbird-backup.yaml. +resources: + - externalsecret.yaml + - pvc.yaml + - cronjob.yaml diff --git a/kubernetes/infrastructure/netbird-backup/pvc.yaml b/kubernetes/infrastructure/netbird-backup/pvc.yaml new file mode 100644 index 00000000..d8b2f600 --- /dev/null +++ b/kubernetes/infrastructure/netbird-backup/pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: netbird-backup + namespace: netbird-backup +spec: + # 14 daily dumps of a small SQLite store plus config — 5Gi is + # generous headroom. + storageClassName: ceph-block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml b/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml index 1a0243d4..b850a7b3 100644 --- a/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml +++ b/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml @@ -26,6 +26,7 @@ spec: - influxdb - ingress - monitoring + - netbird-backup # netbird-backup-ssh: the VPS-pull backup key - paperless - velero provider: diff --git a/kubernetes/infrastructure/post-crds/velero-schedules.yaml b/kubernetes/infrastructure/post-crds/velero-schedules.yaml index 3aff79f7..890fff41 100644 --- a/kubernetes/infrastructure/post-crds/velero-schedules.yaml +++ b/kubernetes/infrastructure/post-crds/velero-schedules.yaml @@ -257,3 +257,29 @@ spec: storageLocation: default ttl: 720h0m0s snapshotMoveData: true +--- +apiVersion: velero.io/v1 +kind: Schedule +metadata: + name: netbird-backup-daily + namespace: velero + labels: + app.kubernetes.io/part-of: homelab-backup +spec: + # Daily at 07:45 local (next free stagger slot after apps-daily at + # 07:15). Ships the netbird-backup PVC — nightly pulls of the NetBird + # controller VPS's management store (SQLite dump + config, pulled at + # 02:45 by the netbird-vps-backup CronJob) — offsite. This is the + # only copy of the mesh state that survives losing the Hetzner VPS. + # Added 2026-07-26 (SPOF assessment: controller store had zero backup + # coverage). No CNPG in this namespace. + schedule: "45 7 * * *" + useOwnerReferencesInBackup: false + template: + includedNamespaces: + - netbird-backup + includedResources: + - persistentvolumeclaims + storageLocation: default + ttl: 720h0m0s + snapshotMoveData: true