From c27825279ec1c7c530af0a63b5626566303c116c Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 15:42:30 +0200 Subject: [PATCH] backup: cut authentik, immich, velero over to Hetzner Stage 2 of the offsite backup migration, applying the pattern validated end-to-end on paperless (WAL archive + base backup + scratch restore). CNPG (authentik ak-pg, immich im-pg): - repoint ObjectStore -> s3://renner-uno-k8s-cnpg/ @ fsn1 - creds barman-cloud-creds -> hetzner-s3-creds (wired into kustomizations) - drop encryption: AES256 (Hetzner has no barman-usable SSE; see #646) Velero: - BSL -> renner-uno-k8s-velero @ fsn1, virtual-hosted (s3ForcePathStyle false), region fsn1, checksumAlgorithm "" (required on Hetzner) - credentials -> cloud-credentials-hetzner - Kopia encrypts client-side, so velero keeps at-rest encryption. Garage creds/secrets left in place for one-revert rollback. After this deploys + validates, the Garage Prometheus scrape/alerts can be dropped and nas-1 freed for nightly shutdown. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../infrastructure/velero/helmrelease.yaml | 13 ++++++------ .../infrastructure/velero/kustomization.yaml | 1 + kubernetes/seed/authentik/kustomization.yaml | 1 + kubernetes/seed/authentik/objectstore.yaml | 20 +++++++++---------- kubernetes/seed/immich/kustomization.yaml | 1 + kubernetes/seed/immich/objectstore.yaml | 17 +++++++++------- 6 files changed, 30 insertions(+), 23 deletions(-) diff --git a/kubernetes/infrastructure/velero/helmrelease.yaml b/kubernetes/infrastructure/velero/helmrelease.yaml index f08f3274..6c43aafe 100644 --- a/kubernetes/infrastructure/velero/helmrelease.yaml +++ b/kubernetes/infrastructure/velero/helmrelease.yaml @@ -40,7 +40,7 @@ spec: name: plugins credentials: useSecret: true - existingSecret: cloud-credentials + existingSecret: cloud-credentials-hetzner # Enable CSI feature so snapshots go through VolumeSnapshotClass # (defined in rook-ceph-cluster/volumesnapshotclass.yaml) and get # moved to Garage via the built-in data mover. @@ -62,14 +62,15 @@ spec: backupStorageLocation: - name: default provider: aws - bucket: k8s-velero + bucket: renner-uno-k8s-velero default: true config: - region: garage - s3Url: https://s3.nas-1.renner.uno - s3ForcePathStyle: "true" + region: fsn1 + s3Url: https://fsn1.your-objectstorage.com + # Hetzner requires virtual-hosted-style addressing. + s3ForcePathStyle: "false" # Documented workaround for aws-sdk-go-v2 checksum mismatch - # on self-hosted S3 (Ceph, Backblaze). Harmless on Garage. + # on non-AWS S3 (Ceph/Garage/Hetzner). Required on Hetzner. checksumAlgorithm: "" volumeSnapshotLocation: [] # Native (non-CSI) VSL snapshots disabled — CSI is the path. diff --git a/kubernetes/infrastructure/velero/kustomization.yaml b/kubernetes/infrastructure/velero/kustomization.yaml index 5006805e..aa6fe896 100644 --- a/kubernetes/infrastructure/velero/kustomization.yaml +++ b/kubernetes/infrastructure/velero/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - secret.sops.yaml + - hetzner-creds.sops.yaml - snapshot-readonly-storageclass.yaml - node-agent-config.yaml - helmrelease.yaml diff --git a/kubernetes/seed/authentik/kustomization.yaml b/kubernetes/seed/authentik/kustomization.yaml index 7d776873..74dc76f6 100644 --- a/kubernetes/seed/authentik/kustomization.yaml +++ b/kubernetes/seed/authentik/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - barman-creds.sops.yaml + - hetzner-creds.sops.yaml - objectstore.yaml - cluster.yaml - scheduled-backup.yaml diff --git a/kubernetes/seed/authentik/objectstore.yaml b/kubernetes/seed/authentik/objectstore.yaml index 4e4150c9..6bec3d80 100644 --- a/kubernetes/seed/authentik/objectstore.yaml +++ b/kubernetes/seed/authentik/objectstore.yaml @@ -6,25 +6,25 @@ metadata: namespace: authentik spec: configuration: - destinationPath: s3://k8s-cnpg/ak-pg - endpointURL: https://s3.nas-1.renner.uno + # Offsite Hetzner Object Storage (fsn1) — replaces Garage on nas-1. + # No SSE: Hetzner supports only SSE-C, unusable by barman-cloud + # (plugin #646), so at-rest encryption is unavailable; transit is TLS. + destinationPath: s3://renner-uno-k8s-cnpg/ak-pg + endpointURL: https://fsn1.your-objectstorage.com s3Credentials: accessKeyId: - name: barman-cloud-creds + name: hetzner-s3-creds key: ACCESS_KEY_ID secretAccessKey: - name: barman-cloud-creds + name: hetzner-s3-creds key: ACCESS_SECRET_KEY - # Garage requires an explicit region — without it boto3's - # HeadBucket call (used by barman-cloud-check-wal-archive during - # CNPG bootstrap.recovery) returns 400 Bad Request. + # Region is required (REGION=fsn1) — without it boto3's HeadBucket + # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. region: - name: barman-cloud-creds + name: hetzner-s3-creds key: REGION wal: compression: gzip - encryption: AES256 data: compression: gzip - encryption: AES256 retentionPolicy: "30d" diff --git a/kubernetes/seed/immich/kustomization.yaml b/kubernetes/seed/immich/kustomization.yaml index 84b53a32..4dc33efc 100644 --- a/kubernetes/seed/immich/kustomization.yaml +++ b/kubernetes/seed/immich/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - barman-creds.sops.yaml + - hetzner-creds.sops.yaml - objectstore.yaml - cluster.yaml - scheduled-backup.yaml diff --git a/kubernetes/seed/immich/objectstore.yaml b/kubernetes/seed/immich/objectstore.yaml index e04f1376..51827ddb 100644 --- a/kubernetes/seed/immich/objectstore.yaml +++ b/kubernetes/seed/immich/objectstore.yaml @@ -6,22 +6,25 @@ metadata: namespace: immich spec: configuration: - destinationPath: s3://k8s-cnpg/im-pg - endpointURL: https://s3.nas-1.renner.uno + # Offsite Hetzner Object Storage (fsn1) — replaces Garage on nas-1. + # No SSE: Hetzner supports only SSE-C, unusable by barman-cloud + # (plugin #646), so at-rest encryption is unavailable; transit is TLS. + destinationPath: s3://renner-uno-k8s-cnpg/im-pg + endpointURL: https://fsn1.your-objectstorage.com s3Credentials: accessKeyId: - name: barman-cloud-creds + name: hetzner-s3-creds key: ACCESS_KEY_ID secretAccessKey: - name: barman-cloud-creds + name: hetzner-s3-creds key: ACCESS_SECRET_KEY + # Region is required (REGION=fsn1) — without it boto3's HeadBucket + # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. region: - name: barman-cloud-creds + name: hetzner-s3-creds key: REGION wal: compression: gzip - encryption: AES256 data: compression: gzip - encryption: AES256 retentionPolicy: "30d" -- 2.54.0