From 5a4f7efd5ca6de27545800129571773b5bbfc95c Mon Sep 17 00:00:00 2001 From: atlas-bot Date: Mon, 28 Sep 2026 16:37:43 +0000 Subject: [PATCH] feat(ha-restore-test): throwaway HA backup restore test outside production (REN-339) Recovers ha-pg from Barman into ha-pg-restore (no WAL archiver) and restores the home-assistant-daily Velero backup into namespace ha-restore-test. Two Jobs log the runbook C.1/C.2 checks. The /config probe is deny-all and never runs Home Assistant. Nothing in namespace home-assistant changes. Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/ha-restore-test/cluster.yaml | 43 ++++++++++ .../ha-restore-test/config-check-job.yaml | 77 +++++++++++++++++ .../hetzner-creds-externalsecret.yaml | 18 ++++ .../apps/ha-restore-test/kustomization.yaml | 17 ++++ .../apps/ha-restore-test/namespace.yaml | 8 ++ .../apps/ha-restore-test/network-policy.yaml | 85 +++++++++++++++++++ .../apps/ha-restore-test/objectstore.yaml | 30 +++++++ .../apps/ha-restore-test/pg-check-job.yaml | 65 ++++++++++++++ .../apps/ha-restore-test/velero-restore.yaml | 29 +++++++ .../clusters/homelab/ha-restore-test.yaml | 24 ++++++ .../post-crds/openbao-clustersecretstore.yaml | 1 + 11 files changed, 397 insertions(+) create mode 100644 kubernetes/apps/ha-restore-test/cluster.yaml create mode 100644 kubernetes/apps/ha-restore-test/config-check-job.yaml create mode 100644 kubernetes/apps/ha-restore-test/hetzner-creds-externalsecret.yaml create mode 100644 kubernetes/apps/ha-restore-test/kustomization.yaml create mode 100644 kubernetes/apps/ha-restore-test/namespace.yaml create mode 100644 kubernetes/apps/ha-restore-test/network-policy.yaml create mode 100644 kubernetes/apps/ha-restore-test/objectstore.yaml create mode 100644 kubernetes/apps/ha-restore-test/pg-check-job.yaml create mode 100644 kubernetes/apps/ha-restore-test/velero-restore.yaml create mode 100644 kubernetes/clusters/homelab/ha-restore-test.yaml diff --git a/kubernetes/apps/ha-restore-test/cluster.yaml b/kubernetes/apps/ha-restore-test/cluster.yaml new file mode 100644 index 00000000..41ee4268 --- /dev/null +++ b/kubernetes/apps/ha-restore-test/cluster.yaml @@ -0,0 +1,43 @@ +# Runbook C.1: recover the recorder DB from ha-pg's Barman backups to the +# latest point (last base backup + archived WAL). +# +# DELIBERATELY NO `plugins:` block. With no WAL archiver this cluster never +# writes to s3://renner-uno-k8s-cnpg/ha-pg; see objectstore.yaml. +# +# One instance, no walStorage, ceph-block-ephemeral (reclaim Delete) so +# nothing outlives the namespace. 32Gi matches ha-pg's PGDATA: the restored +# data plus the WAL replayed since the last base backup. +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ha-pg-restore + namespace: ha-restore-test +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + enableSuperuserAccess: true + storage: + storageClass: ceph-block-ephemeral + size: 32Gi + bootstrap: + recovery: + source: ha-pg + database: homeassistant + owner: homeassistant + externalClusters: + - name: ha-pg + plugin: + name: barman-cloud.cloudnative-pg.io + parameters: + barmanObjectName: ha-pg-store-ro + serverName: ha-pg + postgresql: + parameters: + shared_buffers: "512MB" + max_connections: "50" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 1536Mi diff --git a/kubernetes/apps/ha-restore-test/config-check-job.yaml b/kubernetes/apps/ha-restore-test/config-check-job.yaml new file mode 100644 index 00000000..e82a55ca --- /dev/null +++ b/kubernetes/apps/ha-restore-test/config-check-job.yaml @@ -0,0 +1,77 @@ +# Runbook C.2 checks on the restored /config. NEVER run Home Assistant on this +# data: it would connect to MQTT, Zigbee2MQTT, Shelly and ESPHome and act on +# real devices. This pod only reads files, mounted read-only, and +# config-probe-deny-all in network-policy.yaml blocks all its traffic. +# +# The pod stays Pending until Velero has created the PVC. It prints file +# metadata and counts only: no secret values, secrets.yaml key names only. +apiVersion: batch/v1 +kind: Job +metadata: + name: ha-config-check + namespace: ha-restore-test +spec: + backoffLimit: 0 + activeDeadlineSeconds: 14400 + template: + metadata: + labels: + app.kubernetes.io/name: ha-config-probe + spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + # HA writes /config as root; the probe needs to read 0600 files. + # No capabilities, read-only volume and root filesystem. + runAsUser: 0 + runAsGroup: 0 + seccompProfile: { type: RuntimeDefault } + containers: + - name: check + image: alpine/git:2.54.0 + command: + - sh + - -c + - | + set -u + cd /config + echo "checked_at $(date -u +%FT%TZ)" + du -sh /config 2>/dev/null + for f in .storage/core.config_entries .storage/core.entity_registry secrets.yaml configuration.yaml; do + if [ -s "$f" ]; then + echo "OK $f bytes=$(wc -c <"$f") mtime=$(date -u -r "$f" +%FT%TZ)" + else + echo "FAIL $f missing or empty" + fi + done + echo "config_entries: $(grep -c '"entry_id"' .storage/core.config_entries)" + echo "entity_registry entity_ids: $(grep -o '"entity_id":' .storage/core.entity_registry | wc -l)" + echo "newest .storage files (restore point):" + for f in .storage/*; do echo "$(date -u -r "$f" +%FT%TZ) $f"; done | sort -r | head -5 + echo "secrets.yaml key names:" + grep -E '^[A-Za-z0-9_]+:' secrets.yaml | cut -d: -f1 | tr '\n' ' '; echo + echo "git:" + git -c safe.directory=/config log -1 --format='%H %cI %s' + git -c safe.directory=/config rev-parse --abbrev-ref HEAD + echo "git status (uncommitted/untracked vs HEAD):" + git -c safe.directory=/config status --porcelain | head -100 + echo "yaml files:" + find . -path ./.git -prune -o \( -name '*.yaml' -o -name '*.yml' \) -print | sort | head -300 + echo "done $(date -u +%FT%TZ)" + env: + - { name: HOME, value: /tmp } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: config, mountPath: /config, readOnly: true } + - { name: tmp, mountPath: /tmp } + volumes: + - name: config + persistentVolumeClaim: + claimName: home-assistant-config + readOnly: true + - name: tmp + emptyDir: {} diff --git a/kubernetes/apps/ha-restore-test/hetzner-creds-externalsecret.yaml b/kubernetes/apps/ha-restore-test/hetzner-creds-externalsecret.yaml new file mode 100644 index 00000000..7c77466e --- /dev/null +++ b/kubernetes/apps/ha-restore-test/hetzner-creds-externalsecret.yaml @@ -0,0 +1,18 @@ +# Same shared Barman credential ha-pg uses (infrastructure/hetzner-s3). Needs +# ha-restore-test on the openbao ClusterSecretStore allowlist +# (infrastructure/post-crds/openbao-clustersecretstore.yaml). +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: hetzner-s3-creds + namespace: ha-restore-test +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + name: hetzner-s3-creds + dataFrom: + - extract: + key: infrastructure/hetzner-s3 diff --git a/kubernetes/apps/ha-restore-test/kustomization.yaml b/kubernetes/apps/ha-restore-test/kustomization.yaml new file mode 100644 index 00000000..02f5d8ad --- /dev/null +++ b/kubernetes/apps/ha-restore-test/kustomization.yaml @@ -0,0 +1,17 @@ +# THROWAWAY: HA backup restore test outside production (REN-339, runbook +# REN-249 C.1 + C.2). Nothing here touches namespace home-assistant. Remove the +# whole directory and clusters/homelab/ha-restore-test.yaml once the results +# are on the card; Flux prune then deletes the namespace and everything in it. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - network-policy.yaml + # C.1 recorder DB: recover ha-pg from Barman into a new Cluster. + - hetzner-creds-externalsecret.yaml + - objectstore.yaml + - cluster.yaml + - pg-check-job.yaml + # C.2 /config PVC: Velero restore + read-only probe. + - velero-restore.yaml + - config-check-job.yaml diff --git a/kubernetes/apps/ha-restore-test/namespace.yaml b/kubernetes/apps/ha-restore-test/namespace.yaml new file mode 100644 index 00000000..44341ceb --- /dev/null +++ b/kubernetes/apps/ha-restore-test/namespace.yaml @@ -0,0 +1,8 @@ +# The `-restore-test` suffix is load-bearing: the monthly Velero restore-test +# sweep (infrastructure/velero/restore-test.sh) purges Released PVs whose +# claimRef matches `.*-restore-test/.*`. The Velero-restored PVCs land on +# ceph-block (Retain), so their PVs outlive this namespace until that sweep. +apiVersion: v1 +kind: Namespace +metadata: + name: ha-restore-test diff --git a/kubernetes/apps/ha-restore-test/network-policy.yaml b/kubernetes/apps/ha-restore-test/network-policy.yaml new file mode 100644 index 00000000..a8e5b8d8 --- /dev/null +++ b/kubernetes/apps/ha-restore-test/network-policy.yaml @@ -0,0 +1,85 @@ +--- +# The restored /config holds MQTT, Zigbee2MQTT, Shelly and ESPHome credentials. +# The probe pod must never reach anything. Deny rules win over every allow +# rule Cilium unions for the endpoint, so this holds even if some other policy +# selects the probe. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: config-probe-deny-all + namespace: ha-restore-test +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: ha-config-probe + ingressDeny: + - fromEntities: + - all + egressDeny: + - toEntities: + - all +--- +# Recovered cluster: the CNPG operator (database ns) polls the instance +# manager on :8000; the pg-check Job reads on :5432. Nothing else gets in. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: ha-pg-restore-ingress + namespace: ha-restore-test +spec: + endpointSelector: + matchLabels: + cnpg.io/cluster: ha-pg-restore + ingress: + - fromEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": database + app.kubernetes.io/name: cloudnative-pg + toPorts: + - ports: + - port: "8000" + protocol: TCP + - fromEndpoints: + - matchLabels: + app.kubernetes.io/name: ha-pg-check + toPorts: + - ports: + - port: "5432" + protocol: TCP + # PMTUD, same reason as ha-pg-allow-cnpg-operator in home-assistant-ns.yaml. + - fromEntities: + - cluster + icmps: + - fields: + - { type: 3, family: IPv4 } + - { type: 1, family: IPv6 } + - { type: 2, family: IPv6 } +--- +# The pg-check Job talks to the recovered cluster and DNS, nothing else. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: ha-pg-check-egress + namespace: ha-restore-test +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: ha-pg-check + egress: + - toEndpoints: + - matchLabels: + cnpg.io/cluster: ha-pg-restore + toPorts: + - ports: + - port: "5432" + protocol: TCP + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP diff --git a/kubernetes/apps/ha-restore-test/objectstore.yaml b/kubernetes/apps/ha-restore-test/objectstore.yaml new file mode 100644 index 00000000..16926b1c --- /dev/null +++ b/kubernetes/apps/ha-restore-test/objectstore.yaml @@ -0,0 +1,30 @@ +# READ-ONLY view of ha-pg's Barman prefix, used only as the recovery source. +# +# Two clusters archiving WAL to one destinationPath corrupt both (see +# ../home-assistant/ha-pg-objectstore.yaml). That cannot happen here: the +# recovered Cluster in cluster.yaml has NO `plugins` entry, so nothing archives +# WAL or takes base backups through this store. No retentionPolicy either, so +# nothing here can prune ha-pg's backups. +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ha-pg-store-ro + namespace: ha-restore-test +spec: + configuration: + destinationPath: s3://renner-uno-k8s-cnpg/ha-pg + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/apps/ha-restore-test/pg-check-job.yaml b/kubernetes/apps/ha-restore-test/pg-check-job.yaml new file mode 100644 index 00000000..ce7c1132 --- /dev/null +++ b/kubernetes/apps/ha-restore-test/pg-check-job.yaml @@ -0,0 +1,65 @@ +# Runbook C.1 checks. Waits for the recovered cluster, then prints the +# results to its log (read with `view`; no exec needed). +apiVersion: batch/v1 +kind: Job +metadata: + name: ha-pg-check + namespace: ha-restore-test +spec: + backoffLimit: 2 + activeDeadlineSeconds: 14400 + template: + metadata: + labels: + app.kubernetes.io/name: ha-pg-check + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 26 + runAsGroup: 26 + seccompProfile: { type: RuntimeDefault } + containers: + - name: check + image: ghcr.io/cloudnative-pg/postgresql:16 + env: + - name: PGHOST + value: ha-pg-restore-rw + - name: PGDATABASE + value: homeassistant + - name: PGUSER + value: postgres + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: ha-pg-restore-superuser + key: password + command: + - bash + - -c + - | + set -u + echo "start $(date -u +%FT%TZ)" + until pg_isready -q; do sleep 15; done + until psql -Atqc 'select 1' >/dev/null 2>&1; do sleep 15; done + echo "ready $(date -u +%FT%TZ)" + psql -X -v ON_ERROR_STOP=1 <<'SQL' + \pset footer off + select now() as checked_at, pg_is_in_recovery() as in_recovery; + select to_timestamp(max(last_updated_ts)) as states_max_last_updated from states; + select count(*) as statistics_rows from statistics; + select count(*) as statistics_short_term_rows from statistics_short_term; + select schema_version, changed from schema_changes order by change_id desc limit 1; + select pg_size_pretty(pg_database_size('homeassistant')) as db_size; + select timeline_id, redo_lsn from pg_control_checkpoint(); + SQL + echo "done $(date -u +%FT%TZ)" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: tmp, mountPath: /tmp } + volumes: + - name: tmp + emptyDir: {} diff --git a/kubernetes/apps/ha-restore-test/velero-restore.yaml b/kubernetes/apps/ha-restore-test/velero-restore.yaml new file mode 100644 index 00000000..9465f1aa --- /dev/null +++ b/kubernetes/apps/ha-restore-test/velero-restore.yaml @@ -0,0 +1,29 @@ +# Runbook C.2: restore the latest home-assistant-daily backup into +# ha-restore-test. PVCs only (Flux owns every other object). +# +# Velero cannot filter by object name. The label narrows the restore to the +# four PVCs the home-assistant Kustomization owns (home-assistant-config, +# esphome-data, shelly-manager-data, zigbee2mqtt-data); only +# home-assistant-config is mounted. Nothing mounts the other three. +# +# namespaceMapping means nothing is written to namespace home-assistant. +# The `-rtest-` in the name lets the monthly restore-test sweep reclaim the +# data-mover staging PVs in velero/ (see restore-test.sh purge_released_pvs). +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: home-assistant-daily-rtest-ren339 + namespace: velero +spec: + scheduleName: home-assistant-daily + includedNamespaces: + - home-assistant + namespaceMapping: + home-assistant: ha-restore-test + includedResources: + - persistentvolumeclaims + - persistentvolumes + labelSelector: + matchLabels: + kustomize.toolkit.fluxcd.io/name: home-assistant + restorePVs: true diff --git a/kubernetes/clusters/homelab/ha-restore-test.yaml b/kubernetes/clusters/homelab/ha-restore-test.yaml new file mode 100644 index 00000000..fd8d89e7 --- /dev/null +++ b/kubernetes/clusters/homelab/ha-restore-test.yaml @@ -0,0 +1,24 @@ +# THROWAWAY (REN-339): HA backup restore test outside production. Delete this +# file and apps/ha-restore-test/ when the results are on the card; prune: true +# then removes the namespace, the recovered cluster and the Velero Restore. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ha-restore-test + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/apps/ha-restore-test + prune: true + # Restores take a while; the Jobs report the results in their logs. + wait: false + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman diff --git a/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml b/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml index 588ce1f3..898d1028 100644 --- a/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml +++ b/kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml @@ -22,6 +22,7 @@ spec: - crowdsec - evcc - flux-system # flux-system-git-auth: syncs the rotated flux gitea PAT into Flux's git secret + - ha-restore-test # THROWAWAY (REN-339): Barman creds for the ha-pg restore test; remove with apps/ha-restore-test/ - home-assistant - immich - influxdb -- 2.54.0