kubernetes/apps/ha-restore-test/cluster.yaml

@@ -0,0 +1,43 @@
+# Runbook C.1: recover the recorder DB from ha-pg's Barman backups to the
+# latest point (last base backup + archived WAL).
+#
+# DELIBERATELY NO `plugins:` block. With no WAL archiver this cluster never
+# writes to s3://renner-uno-k8s-cnpg/ha-pg; see objectstore.yaml.
+#
+# One instance, no walStorage, ceph-block-ephemeral (reclaim Delete) so
+# nothing outlives the namespace. 32Gi matches ha-pg's PGDATA: the restored
+# data plus the WAL replayed since the last base backup.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: ha-pg-restore
+  namespace: ha-restore-test
+spec:
+  instances: 1
+  imageName: ghcr.io/cloudnative-pg/postgresql:16
+  enableSuperuserAccess: true
+  storage:
+    storageClass: ceph-block-ephemeral
+    size: 32Gi
+  bootstrap:
+    recovery:
+      source: ha-pg
+      database: homeassistant
+      owner: homeassistant
+  externalClusters:
+    - name: ha-pg
+      plugin:
+        name: barman-cloud.cloudnative-pg.io
+        parameters:
+          barmanObjectName: ha-pg-store-ro
+          serverName: ha-pg
+  postgresql:
+    parameters:
+      shared_buffers: "512MB"
+      max_connections: "50"
+  resources:
+    requests:
+      cpu: 200m
+      memory: 512Mi
+    limits:
+      memory: 1536Mi

kubernetes/apps/ha-restore-test/config-check-job.yaml

@@ -0,0 +1,77 @@
+# Runbook C.2 checks on the restored /config. NEVER run Home Assistant on this
+# data: it would connect to MQTT, Zigbee2MQTT, Shelly and ESPHome and act on
+# real devices. This pod only reads files, mounted read-only, and
+# config-probe-deny-all in network-policy.yaml blocks all its traffic.
+#
+# The pod stays Pending until Velero has created the PVC. It prints file
+# metadata and counts only: no secret values, secrets.yaml key names only.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ha-config-check
+  namespace: ha-restore-test
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 14400
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ha-config-probe
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        # HA writes /config as root; the probe needs to read 0600 files.
+        # No capabilities, read-only volume and root filesystem.
+        runAsUser: 0
+        runAsGroup: 0
+        seccompProfile: { type: RuntimeDefault }
+      containers:
+        - name: check
+          image: alpine/git:2.54.0
+          command:
+            - sh
+            - -c
+            - |
+              set -u
+              cd /config
+              echo "checked_at $(date -u +%FT%TZ)"
+              du -sh /config 2>/dev/null
+              for f in .storage/core.config_entries .storage/core.entity_registry secrets.yaml configuration.yaml; do
+                if [ -s "$f" ]; then
+                  echo "OK   $f bytes=$(wc -c <"$f") mtime=$(date -u -r "$f" +%FT%TZ)"
+                else
+                  echo "FAIL $f missing or empty"
+                fi
+              done
+              echo "config_entries: $(grep -c '"entry_id"' .storage/core.config_entries)"
+              echo "entity_registry entity_ids: $(grep -o '"entity_id":' .storage/core.entity_registry | wc -l)"
+              echo "newest .storage files (restore point):"
+              for f in .storage/*; do echo "$(date -u -r "$f" +%FT%TZ) $f"; done | sort -r | head -5
+              echo "secrets.yaml key names:"
+              grep -E '^[A-Za-z0-9_]+:' secrets.yaml | cut -d: -f1 | tr '\n' ' '; echo
+              echo "git:"
+              git -c safe.directory=/config log -1 --format='%H %cI %s'
+              git -c safe.directory=/config rev-parse --abbrev-ref HEAD
+              echo "git status (uncommitted/untracked vs HEAD):"
+              git -c safe.directory=/config status --porcelain | head -100
+              echo "yaml files:"
+              find . -path ./.git -prune -o \( -name '*.yaml' -o -name '*.yml' \) -print | sort | head -300
+              echo "done $(date -u +%FT%TZ)"
+          env:
+            - { name: HOME, value: /tmp }
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: { drop: ["ALL"] }
+          volumeMounts:
+            - { name: config, mountPath: /config, readOnly: true }
+            - { name: tmp, mountPath: /tmp }
+      volumes:
+        - name: config
+          persistentVolumeClaim:
+            claimName: home-assistant-config
+            readOnly: true
+        - name: tmp
+          emptyDir: {}

kubernetes/apps/ha-restore-test/hetzner-creds-externalsecret.yaml

@@ -0,0 +1,18 @@
+# Same shared Barman credential ha-pg uses (infrastructure/hetzner-s3). Needs
+# ha-restore-test on the openbao ClusterSecretStore allowlist
+# (infrastructure/post-crds/openbao-clustersecretstore.yaml).
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hetzner-s3-creds
+  namespace: ha-restore-test
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hetzner-s3-creds
+  dataFrom:
+    - extract:
+        key: infrastructure/hetzner-s3

kubernetes/apps/ha-restore-test/kustomization.yaml

@@ -0,0 +1,17 @@
+# THROWAWAY: HA backup restore test outside production (REN-339, runbook
+# REN-249 C.1 + C.2). Nothing here touches namespace home-assistant. Remove the
+# whole directory and clusters/homelab/ha-restore-test.yaml once the results
+# are on the card; Flux prune then deletes the namespace and everything in it.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - network-policy.yaml
+  # C.1 recorder DB: recover ha-pg from Barman into a new Cluster.
+  - hetzner-creds-externalsecret.yaml
+  - objectstore.yaml
+  - cluster.yaml
+  - pg-check-job.yaml
+  # C.2 /config PVC: Velero restore + read-only probe.
+  - velero-restore.yaml
+  - config-check-job.yaml

kubernetes/apps/ha-restore-test/namespace.yaml

@@ -0,0 +1,8 @@
+# The `-restore-test` suffix is load-bearing: the monthly Velero restore-test
+# sweep (infrastructure/velero/restore-test.sh) purges Released PVs whose
+# claimRef matches `.*-restore-test/.*`. The Velero-restored PVCs land on
+# ceph-block (Retain), so their PVs outlive this namespace until that sweep.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ha-restore-test

kubernetes/apps/ha-restore-test/network-policy.yaml

@@ -0,0 +1,85 @@
+---
+# The restored /config holds MQTT, Zigbee2MQTT, Shelly and ESPHome credentials.
+# The probe pod must never reach anything. Deny rules win over every allow
+# rule Cilium unions for the endpoint, so this holds even if some other policy
+# selects the probe.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: config-probe-deny-all
+  namespace: ha-restore-test
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: ha-config-probe
+  ingressDeny:
+    - fromEntities:
+        - all
+  egressDeny:
+    - toEntities:
+        - all
+---
+# Recovered cluster: the CNPG operator (database ns) polls the instance
+# manager on :8000; the pg-check Job reads on :5432. Nothing else gets in.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ha-pg-restore-ingress
+  namespace: ha-restore-test
+spec:
+  endpointSelector:
+    matchLabels:
+      cnpg.io/cluster: ha-pg-restore
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": database
+            app.kubernetes.io/name: cloudnative-pg
+      toPorts:
+        - ports:
+            - port: "8000"
+              protocol: TCP
+    - fromEndpoints:
+        - matchLabels:
+            app.kubernetes.io/name: ha-pg-check
+      toPorts:
+        - ports:
+            - port: "5432"
+              protocol: TCP
+    # PMTUD, same reason as ha-pg-allow-cnpg-operator in home-assistant-ns.yaml.
+    - fromEntities:
+        - cluster
+      icmps:
+        - fields:
+            - { type: 3,   family: IPv4 }
+            - { type: 1,   family: IPv6 }
+            - { type: 2,   family: IPv6 }
+---
+# The pg-check Job talks to the recovered cluster and DNS, nothing else.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ha-pg-check-egress
+  namespace: ha-restore-test
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: ha-pg-check
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            cnpg.io/cluster: ha-pg-restore
+      toPorts:
+        - ports:
+            - port: "5432"
+              protocol: TCP
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+            - port: "53"
+              protocol: TCP

kubernetes/apps/ha-restore-test/objectstore.yaml

@@ -0,0 +1,30 @@
+# READ-ONLY view of ha-pg's Barman prefix, used only as the recovery source.
+#
+# Two clusters archiving WAL to one destinationPath corrupt both (see
+# ../home-assistant/ha-pg-objectstore.yaml). That cannot happen here: the
+# recovered Cluster in cluster.yaml has NO `plugins` entry, so nothing archives
+# WAL or takes base backups through this store. No retentionPolicy either, so
+# nothing here can prune ha-pg's backups.
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: ha-pg-store-ro
+  namespace: ha-restore-test
+spec:
+  configuration:
+    destinationPath: s3://renner-uno-k8s-cnpg/ha-pg
+    endpointURL: https://fsn1.your-objectstorage.com
+    s3Credentials:
+      accessKeyId:
+        name: hetzner-s3-creds
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: hetzner-s3-creds
+        key: ACCESS_SECRET_KEY
+      region:
+        name: hetzner-s3-creds
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip

kubernetes/apps/ha-restore-test/pg-check-job.yaml

@@ -0,0 +1,65 @@
+# Runbook C.1 checks. Waits for the recovered cluster, then prints the
+# results to its log (read with `view`; no exec needed).
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ha-pg-check
+  namespace: ha-restore-test
+spec:
+  backoffLimit: 2
+  activeDeadlineSeconds: 14400
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ha-pg-check
+    spec:
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 26
+        runAsGroup: 26
+        seccompProfile: { type: RuntimeDefault }
+      containers:
+        - name: check
+          image: ghcr.io/cloudnative-pg/postgresql:16
+          env:
+            - name: PGHOST
+              value: ha-pg-restore-rw
+            - name: PGDATABASE
+              value: homeassistant
+            - name: PGUSER
+              value: postgres
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: ha-pg-restore-superuser
+                  key: password
+          command:
+            - bash
+            - -c
+            - |
+              set -u
+              echo "start $(date -u +%FT%TZ)"
+              until pg_isready -q; do sleep 15; done
+              until psql -Atqc 'select 1' >/dev/null 2>&1; do sleep 15; done
+              echo "ready $(date -u +%FT%TZ)"
+              psql -X -v ON_ERROR_STOP=1 <<'SQL'
+              \pset footer off
+              select now() as checked_at, pg_is_in_recovery() as in_recovery;
+              select to_timestamp(max(last_updated_ts)) as states_max_last_updated from states;
+              select count(*) as statistics_rows from statistics;
+              select count(*) as statistics_short_term_rows from statistics_short_term;
+              select schema_version, changed from schema_changes order by change_id desc limit 1;
+              select pg_size_pretty(pg_database_size('homeassistant')) as db_size;
+              select timeline_id, redo_lsn from pg_control_checkpoint();
+              SQL
+              echo "done $(date -u +%FT%TZ)"
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: { drop: ["ALL"] }
+          volumeMounts:
+            - { name: tmp, mountPath: /tmp }
+      volumes:
+        - name: tmp
+          emptyDir: {}

kubernetes/apps/ha-restore-test/velero-restore.yaml

@@ -0,0 +1,29 @@
+# Runbook C.2: restore the latest home-assistant-daily backup into
+# ha-restore-test. PVCs only (Flux owns every other object).
+#
+# Velero cannot filter by object name. The label narrows the restore to the
+# four PVCs the home-assistant Kustomization owns (home-assistant-config,
+# esphome-data, shelly-manager-data, zigbee2mqtt-data); only
+# home-assistant-config is mounted. Nothing mounts the other three.
+#
+# namespaceMapping means nothing is written to namespace home-assistant.
+# The `-rtest-` in the name lets the monthly restore-test sweep reclaim the
+# data-mover staging PVs in velero/ (see restore-test.sh purge_released_pvs).
+apiVersion: velero.io/v1
+kind: Restore
+metadata:
+  name: home-assistant-daily-rtest-ren339
+  namespace: velero
+spec:
+  scheduleName: home-assistant-daily
+  includedNamespaces:
+    - home-assistant
+  namespaceMapping:
+    home-assistant: ha-restore-test
+  includedResources:
+    - persistentvolumeclaims
+    - persistentvolumes
+  labelSelector:
+    matchLabels:
+      kustomize.toolkit.fluxcd.io/name: home-assistant
+  restorePVs: true

kubernetes/clusters/homelab/ha-restore-test.yaml

@@ -0,0 +1,24 @@
+# THROWAWAY (REN-339): HA backup restore test outside production. Delete this
+# file and apps/ha-restore-test/ when the results are on the card; prune: true
+# then removes the namespace, the recovered cluster and the Velero Restore.
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: ha-restore-test
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 1m
+  timeout: 10m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/apps/ha-restore-test
+  prune: true
+  # Restores take a while; the Jobs report the results in their logs.
+  wait: false
+  dependsOn:
+    - name: infrastructure
+    - name: infrastructure-crds
+    - name: infrastructure-ceph-cluster
+    - name: infrastructure-cnpg-barman

kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml

@@ -22,6 +22,7 @@ spec:
         - crowdsec
         - evcc
         - flux-system # flux-system-git-auth: syncs the rotated flux gitea PAT into Flux's git secret
+        - ha-restore-test # THROWAWAY (REN-339): Barman creds for the ha-pg restore test; remove with apps/ha-restore-test/
         - home-assistant
         - immich
         - influxdb