homelab/opentofu/object_storage.tf

# Hetzner Object Storage — offsite S3 backup target.
#
# Replaces the local Garage S3 on nas-1 (s3.nas-1.renner.uno) as the
# backup target for CNPG Barman, Velero, cluster-access secrets, and
# migrate-* tarballs. Moving these offsite decouples every cluster
# backup from nas-1 (so it can be powered off nightly to save power) and
# is a genuine DR improvement: backups + bootstrap secrets no longer
# live on the same box, and survive a full cluster/site rebuild.
#
# Two-part provisioning, because Hetzner has no API to generate S3
# credentials (Console-only — confirmed empirically against the live
# hcloud API: every object-storage endpoint 404s):
#   1. MANUAL, one-time: Console -> Object Storage -> Credentials ->
#      Generate. Yields an Access Key + Secret Key.
#   2. AUTOMATED (this file): the aminueza/minio provider creates the
#      buckets over the S3 API using that keypair.
#
# Activation sequence (keeps the running infra pipeline green until ready):
#   a. Generate the keypair in the Console.
#   b. Add Woodpecker secrets: hetzner_s3_access_key, hetzner_s3_secret_key.
#   c. Flip var.enable_hetzner_buckets default to true (one-line commit).
#   d. Run deploy-infrastructure with TOFU_APPLY=true.
# While enable_hetzner_buckets = false the for_each is empty, so the
# minio provider is never configured and empty creds are harmless.

provider "minio" {
  minio_server   = var.hetzner_s3_endpoint # host only, no scheme
  minio_region   = var.hetzner_s3_region
  minio_user     = var.hetzner_s3_access_key
  minio_password = var.hetzner_s3_secret_key
  minio_ssl      = true
}

locals {
  # Hetzner Object Storage uses a GLOBAL bucket namespace (virtual-hosted
  # style: each bucket is a subdomain of the endpoint), so generic names
  # like "k8s-velero" collide with other customers. Prefix with the
  # domain to stay unique. All DNS-safe (lowercase, hyphens, no
  # underscores) per S3 + virtual-hosted-style rules.
  hetzner_buckets = [
    "renner-uno-k8s-cnpg",    # CloudNativePG Barman base backups + WAL (ak-pg, pprl-pg, im-pg)
    "renner-uno-k8s-velero",  # Velero PVC backups (Kopia data mover)
    "renner-uno-k8s-secrets", # SOPS talosconfig/kubeconfig for cluster bootstrap
    "renner-uno-tfstate",     # OpenTofu remote state (migrated off the nas-1 Garage S3)
    "renner-uno-k8s-mariadb", # mariadb-operator logical dumps (wordpress-mariadb)
  ]
}

resource "minio_s3_bucket" "k8s" {
  for_each = var.enable_hetzner_buckets ? toset(local.hetzner_buckets) : toset([])

  bucket = each.value
  acl    = "private"

  # object_locking can only be set at bucket-creation time (S3/MinIO
  # semantics) — the provider silently ignores changes to this field on an
  # existing bucket (no ForceNew, no Update handling: see
  # aminueza/minio resource_minio_s3_bucket.go, Create-only consumption of
  # ObjectLocking). These four buckets are already live (created with
  # object_locking = false), so flipping this to true here would be a
  # silent no-op, NOT a fix. Enabling object-lock requires destroying and
  # recreating each bucket (full data migration) — tracked separately,
  # see gitea issue #191 discussion. Do not set this to true without also
  # planning the recreation.
  object_locking = false
}

# Versioning: safe to enable on an already-existing bucket (unlike
# object_locking, this is a real, live S3/MinIO configuration knob, not a
# create-time-only attribute). This is the recoverability fix for issue
# #191: a leaked/misused S3 credential can no longer permanently destroy
# Velero backups, CNPG WAL archives, cluster secrets, or tfstate with a
# plain DeleteObject — the prior version survives until explicitly purged
# by version ID.
resource "minio_s3_bucket_versioning" "k8s" {
  for_each = var.enable_hetzner_buckets ? toset(local.hetzner_buckets) : toset([])

  bucket = minio_s3_bucket.k8s[each.key].bucket

  versioning_configuration {
    status = "Enabled"
  }
}

locals {
  # Buckets that receive high-churn, app-managed backups (Barman WAL/base
  # backups, Kopia PVC snapshots via Velero) — versioning alone would let
  # noncurrent versions accumulate forever as those tools prune and
  # re-upload. Cap noncurrent-version retention to match the retention
  # window the app layer already enforces, so old S3 versions don't outlive
  # the backups that superseded them by more than that window:
  #   - CNPG ObjectStore retentionPolicy: "30d" (kubernetes/apps/*/objectstore.yaml)
  #   - Velero Schedule ttl: 720h0m0s = 30d (kubernetes/infrastructure/post-crds/velero-schedules.yaml)
  # renner-uno-k8s-secrets and renner-uno-tfstate hold small, low-churn
  # objects (kubeconfig/talosconfig, tfstate) with no existing retention
  # precedent in this repo — left uncapped (keep every version) since
  # storage cost there is negligible.
  hetzner_buckets_noncurrent_expiry_days = {
    "renner-uno-k8s-cnpg"    = 30
    "renner-uno-k8s-velero"  = 30
    "renner-uno-k8s-mariadb" = 30 # Backup maxRetention: 720h (kubernetes/apps/wordpress/mariadb.yaml)
  }
}

resource "minio_s3_bucket_lifecycle" "k8s_noncurrent_expiry" {
  for_each = var.enable_hetzner_buckets ? local.hetzner_buckets_noncurrent_expiry_days : {}

  bucket = minio_s3_bucket.k8s[each.key].bucket

  rule {
    id     = "expire-noncurrent-versions"
    status = "Enabled"

    noncurrent_version_expiration {
      noncurrent_days = each.value
    }
  }

  depends_on = [minio_s3_bucket_versioning.k8s]
}