From b27c4c8afe26eec70677d13d46b74c7bb8ec86ea Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 13:10:43 +0200 Subject: [PATCH 1/6] opentofu: provision Hetzner Object Storage backup buckets Add the offsite S3 backup target that decouples cluster backups from nas-1's local Garage, so nas-1 can be powered off nightly to save power (and as a genuine DR improvement: backups + bootstrap secrets no longer share a box with, and survive a rebuild of, the cluster). Hetzner has no API to generate S3 credentials (Console-only, confirmed empirically against the live hcloud API), so provisioning is two-part: the keypair is created by hand in the Console, then the aminueza/minio provider creates the buckets (k8s-cnpg, k8s-velero, k8s-secrets, k8s-migration) over the S3 API. - object_storage.tf: minio provider (fsn1) + for_each bucket resources - variables_hetzner.tf: endpoint/region/keypair vars + enable flag - providers.tf: aminueza/minio ~> 3.33 - deploy-infrastructure.yml: wire TF_VAR_hetzner_s3_{access,secret}_key from Woodpecker secrets into plan + apply Bucket names mirror the current Garage layout so the cluster-side change is only an endpoint/region/addressing swap (handled in follow-up commits, staged paperless-first). Co-Authored-By: Claude Opus 4.8 (1M context) --- .woodpecker/deploy-infrastructure.yml | 11 +++++ opentofu/.terraform.lock.hcl | 60 +++++++++++++++++++-------- opentofu/object_storage.tf | 52 +++++++++++++++++++++++ opentofu/providers.tf | 4 ++ opentofu/variables_hetzner.tf | 31 ++++++++++++++ 5 files changed, 140 insertions(+), 18 deletions(-) create mode 100644 opentofu/object_storage.tf create mode 100644 opentofu/variables_hetzner.tf diff --git a/.woodpecker/deploy-infrastructure.yml b/.woodpecker/deploy-infrastructure.yml index 3a6a6910..aee2e973 100644 --- a/.woodpecker/deploy-infrastructure.yml +++ b/.woodpecker/deploy-infrastructure.yml @@ -42,6 +42,13 @@ steps: from_secret: unifi_api_key TF_VAR_unifi_api_url: from_secret: unifi_api_url + # Hetzner Object Storage S3 keypair (offsite backup buckets). + # Only consumed once var.enable_hetzner_buckets = true; harmless + # (unset) until the Console-generated secrets are added. + TF_VAR_hetzner_s3_access_key: + from_secret: hetzner_s3_access_key + TF_VAR_hetzner_s3_secret_key: + from_secret: hetzner_s3_secret_key commands: - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" - | @@ -93,6 +100,10 @@ steps: from_secret: unifi_api_key TF_VAR_unifi_api_url: from_secret: unifi_api_url + TF_VAR_hetzner_s3_access_key: + from_secret: hetzner_s3_access_key + TF_VAR_hetzner_s3_secret_key: + from_secret: hetzner_s3_secret_key SSH_PRIVATE_KEY: from_secret: ssh_private_key PROXMOX_TOKEN_ID: diff --git a/opentofu/.terraform.lock.hcl b/opentofu/.terraform.lock.hcl index 552e2123..9820d4ea 100644 --- a/opentofu/.terraform.lock.hcl +++ b/opentofu/.terraform.lock.hcl @@ -1,6 +1,27 @@ # This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. +provider "registry.opentofu.org/aminueza/minio" { + version = "3.38.1" + constraints = "~> 3.33" + hashes = [ + "h1:RPiEWHf8qQFmd8ahWPy53WDMHuOWhVATVThltPp32BI=", + "zh:0467780e774ae205f8c9e761fd505d135234c230951c0151ee9ffd69df6604d5", + "zh:300882809f0e13fbba3525570038cbd4a46c2a8c63a30507d9819fcdab0b8eff", + "zh:384d324b5bc5792b413553af6a0461600bd114c511391ca6d9291fdc599d0ef5", + "zh:49a23532f4f32822d60782f717ce7a0725521a95802b74950b3e7f28467be131", + "zh:56ad7321a51af1e287ad733746c580c27817b513c8525f4377551c77bb601969", + "zh:7923fe9e3ef27406de277634da38e297f70cce4eebd30af4d85351da5f75814a", + "zh:96323556c643c196490f34817ffe6b389a567b93bec837ba53f2b66c05edf601", + "zh:967e16527644dbc4260becf89894814db10c87d0914961e79ad4383c7bc9e5c7", + "zh:ae4dc3072b1b0e68bdcb29f1ea067aa3f32ccddf9201f07422ff6b7333ca8b45", + "zh:cfbb1f9cd75cd4deddd0aebdc16ed2ac66a4dd14878ac1b1af43f075401ea7d6", + "zh:d56c2e782501faea059f2da7dd2ef27c9eeae721a1d3d4486a67a0ef1fce3530", + "zh:edd1cec091e873bd718d7a4a70c84c094a6c26a17bcfc4acf3719dd9816bee09", + "zh:fc231f1f0e7c5c8756c9a5582abce3230d642786f09f0a72fc5326502e6b7110", + ] +} + provider "registry.opentofu.org/bpg/proxmox" { version = "0.96.0" constraints = "~> 0.95" @@ -24,24 +45,6 @@ provider "registry.opentofu.org/bpg/proxmox" { ] } -provider "registry.opentofu.org/hashicorp/aws" { - version = "5.100.0" - constraints = "~> 5.0" - hashes = [ - "h1:zef23ac/YWw9O2FepFWRs+my9iWWUkniL4dT4LnCKjU=", - "zh:1a41f3ee26720fee7a9a0a361890632a1701b5dc1cf5355dc651ddbe115682ff", - "zh:30457f36690c19307921885cc5e72b9dbeba369445815903acd5c39ac0e41e7a", - "zh:42c22674d5f23f6309eaf3ac3a4f1f8b66b566c1efe1dcb0dd2fb30c17ce1f78", - "zh:4cc271c795ff8ce6479ec2d11a8ba65a0a9ed6331def6693f4b9dccb6e662838", - "zh:60932aa376bb8c87cd1971240063d9d38ba6a55502c867fdbb9f5361dc93d003", - "zh:864e42784bde77b18393ebfcc0104cea9123da5f4392e8a059789e296952eefa", - "zh:9750423138bb01ecaa5cec1a6691664f7783d301fb1628d3b64a231b6b564e0e", - "zh:e5d30c4dec271ef9d6fe09f48237ec6cfea1036848f835b4e47f274b48bda5a7", - "zh:e62bd314ae97b43d782e0841b13e68a3f8ec85cc762004f973ce5ce7b6cdbfd0", - "zh:ea851a3c072528a4445ac6236ba2ce58ffc99ec466019b0bd0e4adde63a248e4", - ] -} - provider "registry.opentofu.org/hashicorp/null" { version = "3.2.4" hashes = [ @@ -59,3 +62,24 @@ provider "registry.opentofu.org/hashicorp/null" { ] } +provider "registry.opentofu.org/ubiquiti-community/unifi" { + version = "0.53.0" + constraints = "~> 0.41" + hashes = [ + "h1:+Pwff8jbNGhEJR8nBc04Xk765MR8mtCa0rsBgZNknQA=", + "zh:088a95939d4f87f5ed0ba7c6ea0dffd695bd372b4814c5eb6cb9b40bb884f9b6", + "zh:3dd892c44f49ef67c5301730e6b593f766f50cdad8a7d91eb7bbc74d0c015774", + "zh:4de2a45312383a67af55896942c199f1112c483a336be03c833a6840dc797e4b", + "zh:4df88c90e7676ab3494693ec3de8611b6c33d629f36f40b892e077cc76c37448", + "zh:52930b904ce21395ba1eb6a2c3a6760f1e9a45f411f8b531a72893abf81e6780", + "zh:5f3e323eda5d12f46aa6280ab70b9cdaf17a8b3347b9b13c6675490767088739", + "zh:657d33639f96570c41c9a5b2dac19705bb8b3d002fc47ce33bb533096f8ec02a", + "zh:6df5f3cf23b54a2cad2053469c307e860b3b1abd05c1a4af88fff859715788d0", + "zh:7595095286283bda0afb40434e4904b416cad67440833355299f12f354263d80", + "zh:7e37ae7f6b31caec4b0af7ca3f0e075e06ffdf9ef9807e48008f1f6503cf7f50", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:ad565fd355e690b9625f84f222e91316aa00d1d1f159c31d68d9c63c163767dc", + "zh:da2b3822e903bcc9be21775f9795e0a68b3753103d867756ab2539d586fdf710", + "zh:dad9011bbb983912c931845c194bd3bb35e0d535c41c7daf395076c0aa23ce05", + ] +} diff --git a/opentofu/object_storage.tf b/opentofu/object_storage.tf new file mode 100644 index 00000000..240b230b --- /dev/null +++ b/opentofu/object_storage.tf @@ -0,0 +1,52 @@ +# Hetzner Object Storage — offsite S3 backup target. +# +# Replaces the local Garage S3 on nas-1 (s3.nas-1.renner.uno) as the +# backup target for CNPG Barman, Velero, cluster-access secrets, and +# migrate-* tarballs. Moving these offsite decouples every cluster +# backup from nas-1 (so it can be powered off nightly to save power) and +# is a genuine DR improvement: backups + bootstrap secrets no longer +# live on the same box, and survive a full cluster/site rebuild. +# +# Two-part provisioning, because Hetzner has no API to generate S3 +# credentials (Console-only — confirmed empirically against the live +# hcloud API: every object-storage endpoint 404s): +# 1. MANUAL, one-time: Console -> Object Storage -> Credentials -> +# Generate. Yields an Access Key + Secret Key. +# 2. AUTOMATED (this file): the aminueza/minio provider creates the +# buckets over the S3 API using that keypair. +# +# Activation sequence (keeps the running infra pipeline green until ready): +# a. Generate the keypair in the Console. +# b. Add Woodpecker secrets: hetzner_s3_access_key, hetzner_s3_secret_key. +# c. Flip var.enable_hetzner_buckets default to true (one-line commit). +# d. Run deploy-infrastructure with TOFU_APPLY=true. +# While enable_hetzner_buckets = false the for_each is empty, so the +# minio provider is never configured and empty creds are harmless. + +provider "minio" { + minio_server = var.hetzner_s3_endpoint # host only, no scheme + minio_region = var.hetzner_s3_region + minio_user = var.hetzner_s3_access_key + minio_password = var.hetzner_s3_secret_key + minio_ssl = true +} + +locals { + # Bucket names mirror the current Garage layout so the cluster-side + # change is only an endpoint/region/addressing swap. All DNS-safe + # (hyphens, no underscores) as Hetzner virtual-hosted-style requires. + hetzner_buckets = [ + "k8s-cnpg", # CloudNativePG Barman base backups + WAL (ak-pg, pprl-pg, im-pg) + "k8s-velero", # Velero PVC backups (Kopia data mover) + "k8s-secrets", # SOPS talosconfig/kubeconfig for cluster bootstrap + "k8s-migration" # one-shot PVC tarballs from migrate-* pipelines + ] +} + +resource "minio_s3_bucket" "k8s" { + for_each = var.enable_hetzner_buckets ? toset(local.hetzner_buckets) : toset([]) + + bucket = each.value + acl = "private" + object_locking = false +} diff --git a/opentofu/providers.tf b/opentofu/providers.tf index 4a91f78c..44a63436 100644 --- a/opentofu/providers.tf +++ b/opentofu/providers.tf @@ -19,6 +19,10 @@ terraform { source = "ubiquiti-community/unifi" version = "~> 0.41" } + minio = { + source = "aminueza/minio" + version = "~> 3.33" + } } } diff --git a/opentofu/variables_hetzner.tf b/opentofu/variables_hetzner.tf new file mode 100644 index 00000000..29d1142c --- /dev/null +++ b/opentofu/variables_hetzner.tf @@ -0,0 +1,31 @@ +variable "hetzner_s3_endpoint" { + type = string + description = "Hetzner Object Storage S3 endpoint host (NO scheme). Location-bound, e.g. fsn1.your-objectstorage.com" + default = "fsn1.your-objectstorage.com" +} + +variable "hetzner_s3_region" { + type = string + description = "Hetzner Object Storage region string (location id), e.g. fsn1" + default = "fsn1" +} + +variable "hetzner_s3_access_key" { + type = string + description = "Hetzner Object Storage S3 access key. Console -> Object Storage -> Credentials -> Generate. Inject via TF_VAR_hetzner_s3_access_key (Woodpecker secret hetzner_s3_access_key)." + sensitive = true + default = "" +} + +variable "hetzner_s3_secret_key" { + type = string + description = "Hetzner Object Storage S3 secret key (shown once at creation). Inject via TF_VAR_hetzner_s3_secret_key (Woodpecker secret hetzner_s3_secret_key)." + sensitive = true + default = "" +} + +variable "enable_hetzner_buckets" { + type = bool + description = "Gate for Hetzner bucket creation. Set false to keep the minio provider inert (e.g. before the S3 keypair / Woodpecker secrets exist). True provisions the buckets." + default = true +} -- 2.54.0 From d3f94d359be3acac983fa4d6c3f0f28aca7aa858 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 13:34:34 +0200 Subject: [PATCH 2/6] opentofu: pin unifi provider to 0.49.x (unblock plan) deploy-infrastructure's `tofu plan` was failing for everyone (not just this branch): CI runs `tofu init -upgrade`, and the loose `~> 0.41` constraint resolved to unifi 0.50.0+, which changed the dns_record `ttl` schema from an integer to a Go duration string. Every plan then died on the existing integer TTLs: Error: Invalid Time Duration String Value unifi_dns_record.renner_uno (ttl = 300) unifi_dns_record.authentik_db (ttl = 60) Bisected the provider: 0.49.0 is the last version that accepts integer TTLs; 0.50.0 introduced the break. Pin to ~> 0.49.0 so the existing config keeps validating. (A future bump past 0.49 needs those TTLs rewritten to duration strings first.) Also regenerates the lock: adds aminueza/minio, drops the stale unused hashicorp/aws entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- opentofu/.terraform.lock.hcl | 92 +++++++++++++++++++----------------- opentofu/providers.tf | 9 +++- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/opentofu/.terraform.lock.hcl b/opentofu/.terraform.lock.hcl index 9820d4ea..6167a5e1 100644 --- a/opentofu/.terraform.lock.hcl +++ b/opentofu/.terraform.lock.hcl @@ -23,63 +23,67 @@ provider "registry.opentofu.org/aminueza/minio" { } provider "registry.opentofu.org/bpg/proxmox" { - version = "0.96.0" + version = "0.111.0" constraints = "~> 0.95" hashes = [ - "h1:BXlbaB7LBlyQcrR1Tpzbq39oxDs10kTpcONK+gppXnM=", - "zh:0840a64ddffdbca48e816f4f2cb338208d80f4d982ed53e6f0482c890ea1b1d6", - "zh:1f07c0aa02b24d81fae010e83c2a375d84c6910f4f861a37764c2c717ae5adf7", - "zh:2d8680abbeab700af7a0f87df505919fcdb314fb777637ddad7ecec626965468", - "zh:3e32f68c236c4b2ccf522ed875514eb579419a9798b783880a4293c375795eef", - "zh:454d031293eaf86d9c97eb748dd9c59f2d454dd2c8ee93e83d2d4125bbb7508f", - "zh:515dbfaeeec99edbd1095314fea10ed16d01c37c9047720e39fb7dd9bb0e1fd2", - "zh:55d4449fe8c50ee2ea57b5c02c2baef51b6be3df21ce0469abd0c952cf3e2ddd", - "zh:86c82e416e95a35fb41ad1eef27acdfbb18fd6e2c53a21c97b6b189f4cd78312", - "zh:88cee9f0e45e0ebed32adb8dc8a6778dbd8f058ad33a890a4cb528f8358da2b4", - "zh:9f0511e933100ad2f0a1dae159f7547057816c35ee3a18a1e7d43077866073e6", - "zh:b76ee0064bf97550fa2d9c492e668621bf214f68072d5c9d3a3adaa95fdc4630", - "zh:bba13b469e15e373c2e9181420ea7d4f8f1996b7b249c28151063588c61a14e6", - "zh:dd164d24926f52b00a9b7e53f83837c5c20a960fcb6925653c360cd83d85d7a5", - "zh:e31d207ea2eaa68a55dd9b13773ddec9b79e86c69fd16a429d916130d93f4ffb", + "h1:RHGueoNj9kiaBMD5mbWFudB4sSvMuqlZJ8UfMfmgnss=", + "zh:07afbe99e262f36b6285df259fa5b371bdb0ca62515a6da486ee004562ddcc1d", + "zh:0b3f17c240a488c8768dc8415c8b80b285495c801a300a784bae677a49eff8bc", + "zh:1fae807c8aec1b51685197414881ad2f62815e1258e3a02686dd938db0ff73eb", + "zh:324766437eb30912c304d5cb3a7d08386e9aef00673bdb0cc09e0b2e37a72782", + "zh:59280417f562df0b7a2ef9b18c4b760237b3e2b10e1a4a9a0d7082f8dcb8bdf6", + "zh:62d702525c44eafce794c9fdda2914bd2fb747cf1ce225d94f28592e2a365a05", + "zh:792d19ac8448c4ecec245fc5b7c9980123971e5f55812d5ba653eb377e373b71", + "zh:8a1e9531bd4ae0d47c433de5cbef552d337b2777c443c182b11f19a77fedf439", + "zh:9787b9d153527882875150fcb970122a4bec24af430a1bc96626d0be6d4056ee", + "zh:ba4db2859844bbac4aa55af6277c35f125fbd15e7acbdb41cb0c2fa593b7a441", + "zh:cac48236b323ab32a0c3f7cfcec63658eec28c5c7113ea6ca4903ccbe6de564b", + "zh:d3ca52ad4aa8e270e5e65158da3499cd5c87d0080e9b7553ac64fde07666b76c", + "zh:e139f53d785d7f91934057a9c255bb12399ea8930058673fe12fbe42eb141ecd", "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", ] } provider "registry.opentofu.org/hashicorp/null" { - version = "3.2.4" + version = "3.3.0" hashes = [ - "h1:jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9MhT0=", - "zh:1769783386610bed8bb1e861a119fe25058be41895e3996d9216dd6bb8a7aee3", - "zh:32c62a9387ad0b861b5262b41c5e9ed6e940eda729c2a0e58100e6629af27ddb", - "zh:339bf8c2f9733fce068eb6d5612701144c752425cebeafab36563a16be460fb2", - "zh:36731f23343aee12a7e078067a98644c0126714c4fe9ac930eecb0f2361788c4", - "zh:3d106c7e32a929e2843f732625a582e562ff09120021e510a51a6f5d01175b8d", - "zh:74bcb3567708171ad83b234b92c9d63ab441ef882b770b0210c2b14fdbe3b1b6", - "zh:90b55bdbffa35df9204282251059e62c178b0ac7035958b93a647839643c0072", - "zh:ae24c0e5adc692b8f94cb23a000f91a316070fdc19418578dcf2134ff57cf447", - "zh:b5c10d4ad860c4c21273203d1de6d2f0286845edf1c64319fa2362df526b5f58", - "zh:e05bbd88e82e1d6234988c85db62fd66f11502645838fff594a2ec25352ecd80", + "h1:0r7+t8CqzjfBgHgEiJGBCw+McEUdRXliMdF+Hk29d8o=", + "zh:083dcc0bec53f8abfa3f2aa2ce9d732a9675338fd60ae7d61162e25db7cb08bf", + "zh:19f7456b5a2ad16595860974714bfdb25b87bc16356ea9d5c7453892aaa27864", + "zh:222c0ed1fed4e4c677ebe626104dbfdba66763e264de0d9c27c58ce60104ee69", + "zh:271711d6caa7dd5a4e9b79fe8c679fab61a840bcf80040a0f5ebb425d1b27d97", + "zh:5adcf35f30baaea13f80c2a2c774deb9369892719493049687e23476c9dff40f", + "zh:5bcfd19df16e73d7f0ad75bd09e2b3b86cf6700d09822d585d68304b71de1d97", + "zh:604edecf263e38674decb35bb4e0e048fdc951f26fa103c33065ff9728f0313b", + "zh:782acbfb4fa4807e273e588fe45b4aaea9dd0fd1136f76ec3200f6f4db3af8d6", + "zh:84411a596d528fe67294e5c1cfd0c2036b08802497bcc4215ce518924f3c9a4a", + "zh:85e79eecf3f5348975cffec3016b0eba3baf605646102d4348796ccd2df2e5f6", + "zh:95669535ca17aeefef307ebfd59ce6930953173baae5637e8cbbf0297ec7ad58", + "zh:d04d9b177747bfd66b4a45b5d911a2a7822aa8451f5e35621971fb7a4206b530", + "zh:e6d9c924475283e90833450a14a732f4deb6d9bb131db8f86ab856e894270836", + "zh:ebcab0c8a1334c86ed7cfa53f571a17ad6d27e9901f27a8854ea622a74b54bb6", + "zh:ef9c757bb2c83d2103811a3d86b6ec5be06b0ffc337b84db1582d023bce7cdcd", ] } provider "registry.opentofu.org/ubiquiti-community/unifi" { - version = "0.53.0" - constraints = "~> 0.41" + version = "0.49.0" + constraints = "~> 0.49.0" hashes = [ - "h1:+Pwff8jbNGhEJR8nBc04Xk765MR8mtCa0rsBgZNknQA=", - "zh:088a95939d4f87f5ed0ba7c6ea0dffd695bd372b4814c5eb6cb9b40bb884f9b6", - "zh:3dd892c44f49ef67c5301730e6b593f766f50cdad8a7d91eb7bbc74d0c015774", - "zh:4de2a45312383a67af55896942c199f1112c483a336be03c833a6840dc797e4b", - "zh:4df88c90e7676ab3494693ec3de8611b6c33d629f36f40b892e077cc76c37448", - "zh:52930b904ce21395ba1eb6a2c3a6760f1e9a45f411f8b531a72893abf81e6780", - "zh:5f3e323eda5d12f46aa6280ab70b9cdaf17a8b3347b9b13c6675490767088739", - "zh:657d33639f96570c41c9a5b2dac19705bb8b3d002fc47ce33bb533096f8ec02a", - "zh:6df5f3cf23b54a2cad2053469c307e860b3b1abd05c1a4af88fff859715788d0", - "zh:7595095286283bda0afb40434e4904b416cad67440833355299f12f354263d80", - "zh:7e37ae7f6b31caec4b0af7ca3f0e075e06ffdf9ef9807e48008f1f6503cf7f50", + "h1:8JBZbUlJaGtC6HnmHlp4dziyLkV+vI+FTbGz8M9Osoo=", + "zh:1769d5fdbd10d2fcb932c94694c607b94da2f550fd76ea723add1521a11b0157", + "zh:1ad979910253a99b245be7b99fb9aa5261b9fdbe8eda55992fb1397e22942eee", + "zh:22a8d7675da2a73322416ce260e6aee1e75a5fd89222672391908fb5d4398d3f", + "zh:268874d973a44bb613ea20599954a87e6ffe7e011714cb9634c04b6d6e9973e9", + "zh:421bec329d6340a847e87d1ba95d541628c54608a8026750d59fb68446674c9e", + "zh:48cc2e738148814cb84241c74143c29c5a37de9ce7ad2899d0c18c64092ccc21", + "zh:66a52ebb13ebc89cdad340c90b50e90d09caad765b81dd0c68c7cd2d0c828a19", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:ad565fd355e690b9625f84f222e91316aa00d1d1f159c31d68d9c63c163767dc", - "zh:da2b3822e903bcc9be21775f9795e0a68b3753103d867756ab2539d586fdf710", - "zh:dad9011bbb983912c931845c194bd3bb35e0d535c41c7daf395076c0aa23ce05", + "zh:89d71896685af39546ae95916403758269c5a60b5a12de041eab6dd969ba36fb", + "zh:d16169fa5e088bfc0fdc993b2e114f0e1204646fd979f5efcc64a79ada29c802", + "zh:d4485bfc5dd1c2cf2ca58c1840a24975b231fd2226b5790f980be7173e9f5c6e", + "zh:d4d976ce264342895516980389255e794826eaa2bc31f805c750c587b40a6445", + "zh:e97c2712fcbd492f78e4152fbf7c61d78f428f73087b47cac08145d88542169d", + "zh:fa9b9092dd122417917ecbd0bcbd51aaa3637499b24a31365a74f310c4dd8725", ] } diff --git a/opentofu/providers.tf b/opentofu/providers.tf index 44a63436..102085b0 100644 --- a/opentofu/providers.tf +++ b/opentofu/providers.tf @@ -16,8 +16,13 @@ terraform { version = "~> 0.95" } unifi = { - source = "ubiquiti-community/unifi" - version = "~> 0.41" + source = "ubiquiti-community/unifi" + # Pinned to 0.49.x — the last line that accepts an integer + # dns_record `ttl`. 0.50.0 switched `ttl` to a Go duration string, + # which breaks every plan against the existing integer TTLs + # (ttl = 300, 60). Bumping past 0.49 requires rewriting those TTLs + # to duration strings ("300s", "60s") first. + version = "~> 0.49.0" } minio = { source = "aminueza/minio" -- 2.54.0 From cd5a59f8710303abac40e8973d44a4a04a0645a9 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 13:38:19 +0200 Subject: [PATCH 3/6] opentofu: prefix Hetzner bucket names (global-namespace collision) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First apply created k8s-cnpg/k8s-secrets/k8s-migration but k8s-velero failed: "The requested bucket name is not available." Hetzner Object Storage uses a global bucket namespace (virtual-hosted style → buckets are endpoint subdomains), so generic k8s-* names collide with other customers. Prefix all buckets with the domain (renner-uno-k8s-*) to stay unique and consistent. The three already-created plain buckets are empty and get recreated under the new names by the targeted apply. Co-Authored-By: Claude Opus 4.8 (1M context) --- opentofu/object_storage.tf | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/opentofu/object_storage.tf b/opentofu/object_storage.tf index 240b230b..4a154487 100644 --- a/opentofu/object_storage.tf +++ b/opentofu/object_storage.tf @@ -32,14 +32,16 @@ provider "minio" { } locals { - # Bucket names mirror the current Garage layout so the cluster-side - # change is only an endpoint/region/addressing swap. All DNS-safe - # (hyphens, no underscores) as Hetzner virtual-hosted-style requires. + # Hetzner Object Storage uses a GLOBAL bucket namespace (virtual-hosted + # style: each bucket is a subdomain of the endpoint), so generic names + # like "k8s-velero" collide with other customers. Prefix with the + # domain to stay unique. All DNS-safe (lowercase, hyphens, no + # underscores) per S3 + virtual-hosted-style rules. hetzner_buckets = [ - "k8s-cnpg", # CloudNativePG Barman base backups + WAL (ak-pg, pprl-pg, im-pg) - "k8s-velero", # Velero PVC backups (Kopia data mover) - "k8s-secrets", # SOPS talosconfig/kubeconfig for cluster bootstrap - "k8s-migration" # one-shot PVC tarballs from migrate-* pipelines + "renner-uno-k8s-cnpg", # CloudNativePG Barman base backups + WAL (ak-pg, pprl-pg, im-pg) + "renner-uno-k8s-velero", # Velero PVC backups (Kopia data mover) + "renner-uno-k8s-secrets", # SOPS talosconfig/kubeconfig for cluster bootstrap + "renner-uno-k8s-migration" # one-shot PVC tarballs from migrate-* pipelines ] } -- 2.54.0 From 625f068014724346857415086ecbe219b713ed3c Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 13:42:22 +0200 Subject: [PATCH 4/6] opentofu: drop unused migration bucket The migrate-* PVC-tarball workflow is retired, so renner-uno-k8s-migration is no longer needed. Remove it; the (empty) bucket is destroyed by the targeted apply. Remaining backup buckets: cnpg, velero, secrets. Co-Authored-By: Claude Opus 4.8 (1M context) --- opentofu/object_storage.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/opentofu/object_storage.tf b/opentofu/object_storage.tf index 4a154487..e6b8b6f4 100644 --- a/opentofu/object_storage.tf +++ b/opentofu/object_storage.tf @@ -38,10 +38,9 @@ locals { # domain to stay unique. All DNS-safe (lowercase, hyphens, no # underscores) per S3 + virtual-hosted-style rules. hetzner_buckets = [ - "renner-uno-k8s-cnpg", # CloudNativePG Barman base backups + WAL (ak-pg, pprl-pg, im-pg) - "renner-uno-k8s-velero", # Velero PVC backups (Kopia data mover) - "renner-uno-k8s-secrets", # SOPS talosconfig/kubeconfig for cluster bootstrap - "renner-uno-k8s-migration" # one-shot PVC tarballs from migrate-* pipelines + "renner-uno-k8s-cnpg", # CloudNativePG Barman base backups + WAL (ak-pg, pprl-pg, im-pg) + "renner-uno-k8s-velero", # Velero PVC backups (Kopia data mover) + "renner-uno-k8s-secrets", # SOPS talosconfig/kubeconfig for cluster bootstrap ] } -- 2.54.0 From 62a00e029013a8c7df76cb7c752e6b39111b77a4 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 14:03:33 +0200 Subject: [PATCH 5/6] kubernetes: add SOPS-encrypted Hetzner S3 credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New secrets holding the Hetzner Object Storage keypair, ready for the per-app backup cutover: - seed/{paperless,authentik,immich}/hetzner-creds.sops.yaml → Secret/hetzner-s3-creds (ACCESS_KEY_ID, ACCESS_SECRET_KEY, REGION=fsn1) - infrastructure/velero/hetzner-creds.sops.yaml → Secret/cloud-credentials-hetzner (cloud INI blob) These are not referenced by any kustomization yet, so they're inert — the existing Garage creds keep driving backups until each ObjectStore / BSL is repointed (staged, paperless-first). Also gitignore *.plaintext.yaml so the pre-encryption template can never be committed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 3 +++ .../velero/hetzner-creds.sops.yaml | 23 +++++++++++++++++ .../seed/authentik/hetzner-creds.sops.yaml | 25 +++++++++++++++++++ .../seed/immich/hetzner-creds.sops.yaml | 25 +++++++++++++++++++ .../seed/paperless/hetzner-creds.sops.yaml | 25 +++++++++++++++++++ 5 files changed, 101 insertions(+) create mode 100644 kubernetes/infrastructure/velero/hetzner-creds.sops.yaml create mode 100644 kubernetes/seed/authentik/hetzner-creds.sops.yaml create mode 100644 kubernetes/seed/immich/hetzner-creds.sops.yaml create mode 100644 kubernetes/seed/paperless/hetzner-creds.sops.yaml diff --git a/.gitignore b/.gitignore index 8eb478f7..09409932 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ kubernetes/access-staging/ # Trivy scan outputs (generated in CI workspace, never commit) trivy-iac.json trivy-image-*.json + +# Plaintext secret templates awaiting SOPS encryption (never commit) +*.plaintext.yaml diff --git a/kubernetes/infrastructure/velero/hetzner-creds.sops.yaml b/kubernetes/infrastructure/velero/hetzner-creds.sops.yaml new file mode 100644 index 00000000..74197cf5 --- /dev/null +++ b/kubernetes/infrastructure/velero/hetzner-creds.sops.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cloud-credentials-hetzner + namespace: velero +type: Opaque +stringData: + cloud: ENC[AES256_GCM,data:ecR9UB3OUEmRrURplx9Ht2Vfee+2zhq9vREMlodL2PXNqSDmKQZngi92WqOPtRnXX9nnym/YiAur2dzZnZcKXJwvbpKVoZC6ObRTYurp5QFsDfK32lEsk0+8lAvujdwBObtNlo+r/pul2xtSDUnF3g==,iv:UhrGwmtbgEhsGC/Y+4LwDOCNuC4iENB0/JUtsoKWiTg=,tag:cXmlxTWE/FGWVoj9MMN7MQ==,type:str] +sops: + age: + - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzZWVKQmJUTFZKeHVycStW + ZzFwaGNmR3EyTVo5a2xFc2hTZ21vdWNNOWg0CllvS2pwZW42bHI3UkRsYXpaUmNS + blBuSW9aNGJ6S00xQVJHU1VpMU5XYkkKLS0tIGx5K0dueTJOSmNLcFdkcU01ZWJ0 + bHUwSm5ud3d1RlBVdjQ3V3ptZ1RBb28KuQypeI/t0usiVwFUVx1wxT4d94m9+UIk + poffQTm3uY/W4eMBkT233iI3eipIw4b7NAvokM0UWM3uTodTu1loVw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-06-25T12:02:18Z" + mac: ENC[AES256_GCM,data:+gX06F0a3CCuoYp+3nmgWugqJ58slKa+lw6igGIm4e+Bf0Y7w5cAHp2BG9CPg146kfrdxETS3h8prsQJH56PkFPxLWwg6c1/cviyXTKAWXyGHZZyojh52AIvCEpUYulTJzpMdAhKVHOQiEqB0vuFMPjapE3/FayBXDula7C23hI=,iv:GmvsyEBqVzpAp+4Zvf6EjbovYXUHKW3CMtodAJ62+uc=,tag:vFWYFmDwEyXohs/AQFUfIQ==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.10.2 diff --git a/kubernetes/seed/authentik/hetzner-creds.sops.yaml b/kubernetes/seed/authentik/hetzner-creds.sops.yaml new file mode 100644 index 00000000..9b20b748 --- /dev/null +++ b/kubernetes/seed/authentik/hetzner-creds.sops.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: hetzner-s3-creds + namespace: authentik +type: Opaque +stringData: + ACCESS_KEY_ID: ENC[AES256_GCM,data:wCRNe1MXQ0KpyHyZZMZBDkBWoyA=,iv:8k0fnSH7Vy/jktFy1Ku3YhtRB9T+bTdtC/+LHzoESa4=,tag:639q1bpg0hdefJIX6ROSAw==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:5hlkvKEl1kzPq20H4x8B5rbiJlRGMjAOy+NfNXbfIq2DOp67Wry/Bw==,iv:taHkfSUH7PIuO/TcwJWEHfnlS1Dp7dj7LXuUrci2PVA=,tag:7WlnmAFJcndngoDt+ycNSw==,type:str] + REGION: ENC[AES256_GCM,data:mrUW8A==,iv:QXCLCMqjbniXF14BTzPnOKKVoFxW0NaiUwWtSHrtpUI=,tag:HSgcHBm1ILwr8shaQN3Qxg==,type:str] +sops: + age: + - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpRnRsVWsraVVKc0ZucVJE + TUh2eThncHFESXhSOG0zUmcvVmRqREV2cWtvCnBDM0tuZFhKS1dWcTBJdlErYkF1 + S2hsZFBuR3IraXVPaFlpb2IxN2gyeDQKLS0tIFVDUGI3S1lXNUdiMkZqdFdGT1A1 + NDFjbUdBZ1lscHFHdnBYQnBTVnB1cXcKT/WA4B3qyHqD9IOxVVav2oXLbOJ6xOLs + /3lQjr2YN1nwgmvcApznZZ2LRm8rFK8Yr2QyVMpD6TLKUkxtJfSh6Q== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-06-25T12:02:18Z" + mac: ENC[AES256_GCM,data:BdC8HclDUK4z63XnV7YwJkxr3k/Czsvh24mt4a26XYCZKZrBNE9SOtv8gCeI07V6BurItTBldVpfGgTwEL3F2zYUlV9n1XLv+poocgDTAea5svkAgEllkfDPu/YnC6v3TdBh8vwVVKJmXQKuUtXUQDTm1yA13fZgmbQGfpZu9Co=,iv:WQmLCLm12o1ohwGXUQJRc8QN+y9Uv7ydP2v1gBQ1sIk=,tag:GWx4EBR5eXFNCFAiWXf/Cw==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.10.2 diff --git a/kubernetes/seed/immich/hetzner-creds.sops.yaml b/kubernetes/seed/immich/hetzner-creds.sops.yaml new file mode 100644 index 00000000..2f404f0b --- /dev/null +++ b/kubernetes/seed/immich/hetzner-creds.sops.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: hetzner-s3-creds + namespace: immich +type: Opaque +stringData: + ACCESS_KEY_ID: ENC[AES256_GCM,data:KkRzqE3uUn9zWFunRqJ8gDaPCp8=,iv:lKw7g+AXhleqH4h8bLfBFnTYJG++UCW1SNn3/RcC/rE=,tag:637zgYEV1WwOfdMHZdgjnQ==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:DNjVOwOBra2mtUhPwRr1QKPN9uOmYgB9iR/okg46tP9U6po3JZFxZA==,iv:oo0sQjTgZ1Q3EwTIFdNIR8ZtVfXNy46hS/vTQdGecv0=,tag:wJcS9pKKrlQSojevGgPZ+g==,type:str] + REGION: ENC[AES256_GCM,data:djfCcw==,iv:AqJSsqkE66TB5qzNrqOALXRfPbOCx4rWV2VidvQ84A8=,tag:xy7YsZ3Y0OEVVH7aRe6gzw==,type:str] +sops: + age: + - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvcDk0L04xb0pFNUJPdGpX + eVpXRE1hdk5ic0ZmRkpoVjBjMjBrZzJPdFQ0Ck9Denk4dmJPSS9UeXFFdktKUUha + aTNrcXBBNGtoZTAzOHZKQkdVWElvY1kKLS0tIFFNZENtL1R3czRuSXBkSEJJc1h2 + a1RVVFpFNFlCVXpxUGMydzhTMXNxNWMKqeskO2ESVk6gjNeVhSaeO890O1qApmFZ + OEAfF8ZepKXA/fNOWKfjHHopmyCebuZ74Z/wnWOmhTzODL0k57b78g== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-06-25T12:02:18Z" + mac: ENC[AES256_GCM,data:QnemJCVJCMV7N14rLSLzE5WJeS+Bf5yQ240hdtZu9DtV/mrHdN3TdBah/KbIZzUUhfxZMpmmMpO490RxvhFEyNRu/ALvwKjeLzFV/dL0+tZ2KTS2Ish55DJ49upPuuZbwxCKN0xNgVT9L6ahhiUjEBB1Mp4lH1ttL1TwD2a5Sg0=,iv:j2V7uAaNWu7B/3UD/fk+gTAgpyK2OafwOGCM8/IIwtk=,tag:xS50bP8l7XkYXAxpPr3LDg==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.10.2 diff --git a/kubernetes/seed/paperless/hetzner-creds.sops.yaml b/kubernetes/seed/paperless/hetzner-creds.sops.yaml new file mode 100644 index 00000000..891083be --- /dev/null +++ b/kubernetes/seed/paperless/hetzner-creds.sops.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: hetzner-s3-creds + namespace: paperless +type: Opaque +stringData: + ACCESS_KEY_ID: ENC[AES256_GCM,data:ZL8AjmSaksieE7aeuR6Io1NAJoE=,iv:p5oiqgYmNwSJ+zqtKTZ7Fm9Q4iAYsrNiXenic/O76+U=,tag:M5ES84+mEzJkWBlD+lYiYg==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:lGsUEAVgneDd46oaRlkNGlraO+aBpjbrR4+NFcXzL2QKVVey4w584Q==,iv:m5Ncf706DC+n5G0q9bUExHkD4aKU3btxzK6rzKhWo1A=,tag:0sQeXSEoyd0X8mqgu/iKAA==,type:str] + REGION: ENC[AES256_GCM,data:vD7PPA==,iv:VahVc1lR8+FfOW15+oh3F+zsNuOgHWZX0UYqi4nxw0I=,tag:glMxb5nulOzWwenNR4CfRQ==,type:str] +sops: + age: + - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDZkY0MUZTQkZ4N2tjTE1t + c094d1ZqYlZKNXQvQ3VXeEJTTkFPNG9vU0NJCkJ1bzJXdkVlYjdoU3o3SXhPeE1M + M3hSMEsyLzlWS0VQMlNwWHJDanhEV1UKLS0tIEhueWo0bzNQZ2FzTmNTbnlJZEFJ + aFRoVkp0WDI1S04yNjg1Ty8wUGNYbXcKIuSgLzrWpLvXMfLy1hGS0kASRpDOjp0S + Sd4jQajsS5Ets2P2JWGM58dPWlnoLJPaSvGudxTM09/AtGKU/hoLUA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-06-25T12:02:18Z" + mac: ENC[AES256_GCM,data:7yJxUC3AarJvh2/7xiOf+ShHw76uZrRTBy4dPnRZCsThilLK33iFFNanNPq63RxblrrRWz5KOJLNTummwKC2LsyL9gNXrnYk4B0yWaXn+lTrXOv1upbxuaxY8th8i/9sGbF9hqlFDoTwswfFZyIvyuRVp3tPVrPHysCQppJQ/Xw=,iv:OL1RM7T9fcyka7lFQc4e//qcz10GPyUSmVll1ojt+N0=,tag:rDcN1XWpy20GsavxT7YnYQ==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.10.2 -- 2.54.0 From 46db5ba409ef651c32ea52feaf18cc931524d7cb Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 14:07:22 +0200 Subject: [PATCH 6/6] paperless: cut CNPG backups over to Hetzner Object Storage Stage 1 of the offsite backup migration (paperless-first). Repoint the pprl-pg ObjectStore from local Garage (s3.nas-1.renner.uno) to Hetzner fsn1, and switch its credentials to the new hetzner-s3-creds secret. - destinationPath: s3://k8s-cnpg/pprl-pg -> s3://renner-uno-k8s-cnpg/pprl-pg - endpointURL: s3.nas-1.renner.uno -> fsn1.your-objectstorage.com - s3Credentials: barman-cloud-creds -> hetzner-s3-creds The ObjectStore CRD has no path-style knob; barman/boto3 default (auto) addressing works against Hetzner's Ceph RGW. Garage creds/secret are left in place for a one-revert rollback. Validate (WAL archive + base backup + restore) before broadening to authentik/immich/velero. Co-Authored-By: Claude Opus 4.8 (1M context) --- kubernetes/seed/paperless/kustomization.yaml | 1 + kubernetes/seed/paperless/objectstore.yaml | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/kubernetes/seed/paperless/kustomization.yaml b/kubernetes/seed/paperless/kustomization.yaml index 84b53a32..4dc33efc 100644 --- a/kubernetes/seed/paperless/kustomization.yaml +++ b/kubernetes/seed/paperless/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - barman-creds.sops.yaml + - hetzner-creds.sops.yaml - objectstore.yaml - cluster.yaml - scheduled-backup.yaml diff --git a/kubernetes/seed/paperless/objectstore.yaml b/kubernetes/seed/paperless/objectstore.yaml index b41b732e..f8921c72 100644 --- a/kubernetes/seed/paperless/objectstore.yaml +++ b/kubernetes/seed/paperless/objectstore.yaml @@ -6,17 +6,20 @@ metadata: namespace: paperless spec: configuration: - destinationPath: s3://k8s-cnpg/pprl-pg - endpointURL: https://s3.nas-1.renner.uno + # Offsite Hetzner Object Storage (fsn1) — replaces local Garage on + # nas-1 so paperless backups survive nas-1 being powered off. barman + # uses boto3 default (auto) addressing; Hetzner (Ceph RGW) accepts it. + destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg + endpointURL: https://fsn1.your-objectstorage.com s3Credentials: accessKeyId: - name: barman-cloud-creds + name: hetzner-s3-creds key: ACCESS_KEY_ID secretAccessKey: - name: barman-cloud-creds + name: hetzner-s3-creds key: ACCESS_SECRET_KEY region: - name: barman-cloud-creds + name: hetzner-s3-creds key: REGION wal: compression: gzip -- 2.54.0