From 0ce60164e1c9ec5b064ff5bed581043c2a483112 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Sat, 27 Jun 2026 12:17:33 +0200 Subject: [PATCH] ci: decommission migration-only rebuild-k8s; point access tooling at Hetzner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rebuild-k8s.yml was the one-shot PVE->K8s migration/rebuild pipeline; it is no longer needed and still pointed at the retired Garage S3 on the now-off nas-1. Remove it (DR restore is via Velero + flux bootstrap). The live cluster-access path is kept and migrated off Garage onto Hetzner Object Storage (renner-uno-k8s-secrets, fsn1.your-objectstorage.com, region fsn1, hetzner_s3_* keypair): - scripts/ci/fetch-kubeconfig.sh: Garage -> Hetzner; env vars renamed GARAGE_SECRETS_* -> S3_ACCESS_KEY_ID/S3_SECRET_ACCESS_KEY. - .woodpecker/e2e.yml, e2e-test.yml: pass hetzner_s3_access_key/secret_key (the live cron e2e pipeline needs this — it was broken against dead nas-1). - scripts/import-cluster-access.sh: Garage -> Hetzner endpoint/bucket/region. NOTE: the workstation aws-cli "secrets" profile must now hold the Hetzner Object Storage keypair. Also: delete the orphaned test-garage-smoke play + pipeline, and fix the two dangling rebuild-k8s.yml references (zfs-tuning play comment; update-audit prompt Talos-pin file list). Assumes cluster-access objects (kubeconfig.sops/talosconfig.sops) were migrated into renner-uno-k8s-secrets; with rebuild-k8s gone, nothing auto-refreshes them, which is fine for a stable cluster. Co-Authored-By: Claude Opus 4.8 (1M context) --- .woodpecker/e2e-test.yml | 8 +- .woodpecker/e2e.yml | 14 +- .woodpecker/rebuild-k8s.yml | 279 ------------------ .woodpecker/test-garage-smoke.yml | 27 -- .../plays/configure-pve-k8s-zfs-tuning.yml | 6 +- ansible/plays/test-garage-smoke.yml | 81 ----- ci/agents/update-audit/prompt.md | 4 +- scripts/ci/fetch-kubeconfig.sh | 22 +- scripts/import-cluster-access.sh | 20 +- 9 files changed, 37 insertions(+), 424 deletions(-) delete mode 100644 .woodpecker/rebuild-k8s.yml delete mode 100644 .woodpecker/test-garage-smoke.yml delete mode 100644 ansible/plays/test-garage-smoke.yml diff --git a/.woodpecker/e2e-test.yml b/.woodpecker/e2e-test.yml index 78f8ff64..76f56183 100644 --- a/.woodpecker/e2e-test.yml +++ b/.woodpecker/e2e-test.yml @@ -8,10 +8,10 @@ steps: environment: SOPS_AGE_KEY: from_secret: sops_age_key - GARAGE_SECRETS_KEY_ID: - from_secret: garage_secrets_key_id - GARAGE_SECRETS_SECRET_KEY: - from_secret: garage_secrets_secret_key + S3_ACCESS_KEY_ID: + from_secret: hetzner_s3_access_key + S3_SECRET_ACCESS_KEY: + from_secret: hetzner_s3_secret_key commands: - bash scripts/ci/fetch-kubeconfig.sh - bash kubernetes/e2e-test.sh diff --git a/.woodpecker/e2e.yml b/.woodpecker/e2e.yml index d89dcaf1..3b71fa16 100644 --- a/.woodpecker/e2e.yml +++ b/.woodpecker/e2e.yml @@ -1,7 +1,5 @@ -# Standalone e2e test pipeline. Same script as the rebuild-k8s `e2e-test` -# stage, runnable independently so regressions (cert renewal, OSD flap, -# observability gap, OIDC config drift, RGW outage) are caught between -# rebuilds. +# Standalone e2e test pipeline — catches regressions (cert renewal, OSD flap, +# observability gap, OIDC config drift, RGW outage) on a schedule. # # Triggers: # - cron : configure schedule via Woodpecker UI / API (Cron tab on this @@ -19,10 +17,10 @@ steps: environment: SOPS_AGE_KEY: from_secret: sops_age_key - GARAGE_SECRETS_KEY_ID: - from_secret: garage_secrets_key_id - GARAGE_SECRETS_SECRET_KEY: - from_secret: garage_secrets_secret_key + S3_ACCESS_KEY_ID: + from_secret: hetzner_s3_access_key + S3_SECRET_ACCESS_KEY: + from_secret: hetzner_s3_secret_key commands: - bash scripts/ci/fetch-kubeconfig.sh - bash kubernetes/e2e-test.sh diff --git a/.woodpecker/rebuild-k8s.yml b/.woodpecker/rebuild-k8s.yml deleted file mode 100644 index 0e38a7db..00000000 --- a/.woodpecker/rebuild-k8s.yml +++ /dev/null @@ -1,279 +0,0 @@ -when: - - event: manual - evaluate: 'PIPELINE == "rebuild-k8s"' - -steps: - - name: recreate-vms - image: gitea.renner.uno/gitea_admin/ci-tofu:2026-04-29 - directory: opentofu - environment: - TF_IN_AUTOMATION: "true" - TF_VAR_proxmox_api_url: - from_secret: proxmox_api_url - TF_VAR_proxmox_user: - from_secret: proxmox_user - TF_VAR_proxmox_token_id: - from_secret: proxmox_token_id - TF_VAR_proxmox_token_secret: - from_secret: proxmox_token_secret - AWS_ACCESS_KEY_ID: - from_secret: tf_garage_access - AWS_SECRET_ACCESS_KEY: - from_secret: tf_garage_secret - AWS_REGION: "us-east-1" - TF_VAR_aws_provider_access_key: - from_secret: tf_rfs_access - TF_VAR_aws_provider_secret_key: - from_secret: tf_rfs_secret - TF_VAR_unifi_api_key: - from_secret: unifi_api_key - TF_VAR_unifi_api_url: - from_secret: unifi_api_url - SSH_PRIVATE_KEY: - from_secret: ssh_private_key - commands: - - mkdir -p ~/.ssh - - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config - - export TF_VAR_ssh_private_key="$(cat ~/.ssh/id_ed25519)" - # Ensure Talos image exists on all Proxmox nodes - - | - TALOS_VERSION="v1.12.6" - TALOS_SCHEMATIC="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" - TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw" - TALOS_FILENAME="talos-$TALOS_VERSION-nocloud-amd64.raw" - API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret" - for node in jupiter saturn neptune; do - EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \ - -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true) - if [ "$EXISTS" -gt 0 ]; then - echo "Talos image already exists on $node" - continue - fi - echo "Downloading Talos image to $node..." - curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/download-url" \ - -H "Authorization: PVEAPIToken=$API_TOKEN" \ - --data-urlencode "url=$TALOS_URL" \ - --data-urlencode "content=import" \ - --data-urlencode "filename=$TALOS_FILENAME" \ - --max-time 600 - for i in $(seq 1 60); do - sleep 10 - EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \ - -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true) - if [ "$EXISTS" -gt 0 ]; then echo "Download complete on $node"; break; fi - done - done - - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" - # Replace all K8s VMs (3 controlplane + 1 edge worker), retry individual failures up to 3 times. - # `-target` scopes the plan to the K8s modules only — without it, drift on unrelated VMs/LXCs - # (e.g. from manual qm move-disk) gets pulled into the apply. - - | - TARGETS="-target=module.k8s_1 -target=module.k8s_2 -target=module.k8s_3 -target=module.k8s_edge" - REPLACE_ALL="-replace=module.k8s_1.proxmox_virtual_environment_vm.this -replace=module.k8s_2.proxmox_virtual_environment_vm.this -replace=module.k8s_3.proxmox_virtual_environment_vm.this -replace=module.k8s_edge.proxmox_virtual_environment_vm.this" - if tofu apply -auto-approve $TARGETS $REPLACE_ALL; then - echo "All VMs replaced successfully" - else - echo "Some VMs failed, retrying individually..." - failed="" - for mod in k8s_1 k8s_2 k8s_3 k8s_edge; do - success=0 - for attempt in 1 2 3; do - echo "Checking module.$mod (attempt $attempt)..." - if tofu plan -detailed-exitcode -target="module.$mod" >/dev/null 2>&1; then - echo " module.$mod is up to date" - success=1 - break - fi - echo " Retrying module.$mod..." - if tofu apply -auto-approve -target="module.$mod" -replace="module.$mod.proxmox_virtual_environment_vm.this"; then - success=1 - break - fi - sleep 10 - done - [ "$success" -eq 1 ] || failed="$failed $mod" - done - if [ -n "$failed" ]; then - echo "FAILED to recreate modules:$failed" - exit 1 - fi - fi - - echo "VM recreation complete" - - - name: wipe-backups - # Cold-start aid: empty the CNPG (k8s-cnpg) and Velero (k8s-velero) - # backup buckets in Garage. Runs only when WIPE_BACKUPS=true is passed - # (e.g. `woodpecker pipeline create ... --var WIPE_BACKUPS=true`). - # - # Position: after recreate-vms so the old cluster's VMs (and therefore - # its barman-cloud + Velero pods) are destroyed before we wipe — no - # more writers, no race where the old cluster lays down fresh objects - # between our `aws s3 rm` and the new cluster's first seed. - # - # Without this step the seed Jobs in the new cluster gate on the - # contents of the corresponding s3://k8s-cnpg//base/ path: - # if a stale base from a prior run is present, the gate exits 0 and - # CNPG bootstrap.recovery tries to recover from data that doesn't - # match — recovery Job exhausts backoffLimit and the cluster sticks - # in "Setting up primary". - when: - - evaluate: 'WIPE_BACKUPS == "true"' - # Bumped to 2026-05-01 for the aws-cli addition baked into ci-k8s. - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01 - environment: - GARAGE_ENDPOINT: https://s3.nas-1.renner.uno - CNPG_KEY_ID: - from_secret: garage_cnpg_key_id - CNPG_SECRET_KEY: - from_secret: garage_cnpg_secret_key - VELERO_KEY_ID: - from_secret: garage_velero_key_id - VELERO_SECRET_KEY: - from_secret: garage_velero_secret_key - commands: - - | - set -euo pipefail - log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - - wipe_bucket() { - local bucket="$1" key_id="$2" secret_key="$3" - log "Wiping s3://$bucket ..." - AWS_ACCESS_KEY_ID="$key_id" AWS_SECRET_ACCESS_KEY="$secret_key" \ - aws --endpoint-url "$GARAGE_ENDPOINT" s3 rm --recursive "s3://$bucket/" >/dev/null - local rem - rem=$(AWS_ACCESS_KEY_ID="$key_id" AWS_SECRET_ACCESS_KEY="$secret_key" \ - aws --endpoint-url "$GARAGE_ENDPOINT" s3 ls --recursive --summarize "s3://$bucket/" \ - | awk '/Total Objects:/ {print $3}') - log " remaining objects in $bucket: ${rem:-?}" - [ "${rem:-0}" = "0" ] || { log " bucket NOT empty after wipe — failing"; return 1; } - } - - wipe_bucket k8s-cnpg "$CNPG_KEY_ID" "$CNPG_SECRET_KEY" - wipe_bucket k8s-velero "$VELERO_KEY_ID" "$VELERO_SECRET_KEY" - log "wipe-backups complete." - - - name: tune-cp-zvols - # Set sync=disabled on the freshly-created K8s CP VM zvols before Talos - # boots, so etcd's WAL fsync isn't amplified through the local-zfs pool. - # See ansible/plays/configure-pve-k8s-zfs-tuning.yml for the durability - # tradeoff. - image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29 - environment: - ANSIBLE_FORCE_COLOR: "true" - ANSIBLE_HOST_KEY_CHECKING: "false" - SSH_PRIVATE_KEY: - from_secret: ssh_private_key - VAULT_PASS: - from_secret: vault_pass - commands: - - mkdir -p ~/.ssh - - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config - - cd ansible - - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass - - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass - - ansible-playbook -i inventory/pve_hosts.yml plays/configure-pve-k8s-zfs-tuning.yml -v - - - name: talos-bootstrap - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29 - commands: - # Wait for VMs to boot into maintenance mode - - echo "Waiting 60s for VMs to boot..." - - sleep 60 - - | - for node in k8s-1 k8s-2 k8s-3 k8s-edge; do - echo -n " DNS $node.srv.renner.uno: " - getent hosts "$node.srv.renner.uno" || echo "UNRESOLVED" - done - - cd kubernetes - - bash bootstrap.sh talos - - - name: wipe-osd-disks - # Rook OSDs live on /dev/nvme0n1 (PCI-passthrough on each PVE node). The disks - # survive VM destroy/recreate, but the mon DB on Talos EPHEMERAL does not — so - # after a rebuild the new Ceph cluster has a fresh FSID and refuses to adopt the - # old BlueStore-signed disks ("skipping osd.N: belonging to a different ceph - # cluster"). Wipe them here, before flux-bootstrap, so OSDs come up clean. - # Edge worker (k8s-edge) has no OSD and is skipped. - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29 - commands: - - export TALOSCONFIG="$(pwd)/kubernetes/access-staging/talosconfig" - - | - set -e - for node in k8s-1 k8s-2 k8s-3; do - echo "Wiping nvme0n1 on $node..." - talosctl wipe disk nvme0n1 \ - -n "$node.srv.renner.uno" \ - -e "k8s-1.srv.renner.uno" \ - --method FAST - done - - - name: export-access - # Encrypt the staging talos/kubeconfig and upload to Garage - # (s3://k8s-secrets/cluster-access/). Replaces the previous "git commit - # && git push" mechanism, which raced with concurrent commits on master - # and forced workstation users to `git pull --rebase` after each rebuild. - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01 - environment: - SOPS_AGE_KEY: - from_secret: sops_age_key - GARAGE_SECRETS_KEY_ID: - from_secret: garage_secrets_key_id - GARAGE_SECRETS_SECRET_KEY: - from_secret: garage_secrets_secret_key - commands: - - mkdir -p ~/.config/sops/age - - echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt - - | - set -e - AGE_RECIPIENT="age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3" - if [[ ! -f kubernetes/access-staging/talosconfig || ! -f kubernetes/access-staging/kubeconfig ]]; then - echo "Access files not found; talos-bootstrap may have failed" - exit 1 - fi - TMPDIR=$(mktemp -d) - trap 'rm -rf "$TMPDIR"' EXIT - sops -e --age "$AGE_RECIPIENT" --input-type binary --output-type binary kubernetes/access-staging/talosconfig > "$TMPDIR/talosconfig.sops" - sops -e --age "$AGE_RECIPIENT" --input-type binary --output-type binary kubernetes/access-staging/kubeconfig > "$TMPDIR/kubeconfig.sops" - export AWS_ACCESS_KEY_ID="$GARAGE_SECRETS_KEY_ID" - export AWS_SECRET_ACCESS_KEY="$GARAGE_SECRETS_SECRET_KEY" - export AWS_DEFAULT_REGION=garage - aws --endpoint-url https://s3.nas-1.renner.uno \ - s3 cp "$TMPDIR/talosconfig.sops" s3://k8s-secrets/cluster-access/talosconfig.sops - aws --endpoint-url https://s3.nas-1.renner.uno \ - s3 cp "$TMPDIR/kubeconfig.sops" s3://k8s-secrets/cluster-access/kubeconfig.sops - - - name: flux-bootstrap - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29 - environment: - GITEA_TOKEN: - from_secret: gitea_token - SOPS_AGE_KEY: - from_secret: sops_age_key - commands: - - mkdir -p ~/.config/sops/age - - echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt - - cd kubernetes - - bash bootstrap.sh flux - - - name: wait-apps - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29 - commands: - - cd kubernetes - - bash bootstrap.sh wait-apps - - - name: e2e-test - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01 - environment: - SOPS_AGE_KEY: - from_secret: sops_age_key - GARAGE_SECRETS_KEY_ID: - from_secret: garage_secrets_key_id - GARAGE_SECRETS_SECRET_KEY: - from_secret: garage_secrets_secret_key - commands: - - bash scripts/ci/fetch-kubeconfig.sh - - bash kubernetes/e2e-test.sh diff --git a/.woodpecker/test-garage-smoke.yml b/.woodpecker/test-garage-smoke.yml deleted file mode 100644 index de0444c9..00000000 --- a/.woodpecker/test-garage-smoke.yml +++ /dev/null @@ -1,27 +0,0 @@ -when: - - event: manual - evaluate: 'PIPELINE == "test-garage-smoke"' - -steps: - - name: run-playbook - image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29 - environment: - ANSIBLE_FORCE_COLOR: "true" - ANSIBLE_HOST_KEY_CHECKING: "false" - SSH_PRIVATE_KEY: - from_secret: ssh_private_key - VAULT_PASS: - from_secret: vault_pass - PROXMOX_TOKEN_ID: - from_secret: proxmox_token_id - PROXMOX_TOKEN_SECRET: - from_secret: proxmox_token_secret - commands: - - mkdir -p ~/.ssh - - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - 'printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config' - - cd ansible - - echo "$$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass - - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass - - ansible-playbook -i inventory/ plays/test-garage-smoke.yml -v diff --git a/ansible/plays/configure-pve-k8s-zfs-tuning.yml b/ansible/plays/configure-pve-k8s-zfs-tuning.yml index 40c106f7..e2083016 100644 --- a/ansible/plays/configure-pve-k8s-zfs-tuning.yml +++ b/ansible/plays/configure-pve-k8s-zfs-tuning.yml @@ -18,9 +18,9 @@ # # Idempotent: skips zvols that already have sync=disabled. # -# Wired into .woodpecker/rebuild-k8s.yml between `recreate-vms` and -# `talos-bootstrap` so freshly-created zvols get the property before Talos -# starts writing. +# Run manually against freshly-created CP zvols so they get the property +# before Talos starts writing (previously wired into the now-removed +# rebuild-k8s pipeline between `recreate-vms` and `talos-bootstrap`). - name: Set sync=disabled on K8s CP VM zvols hosts: pve_cluster diff --git a/ansible/plays/test-garage-smoke.yml b/ansible/plays/test-garage-smoke.yml deleted file mode 100644 index 67d98b85..00000000 --- a/ansible/plays/test-garage-smoke.yml +++ /dev/null @@ -1,81 +0,0 @@ ---- -# Smoke test for the Garage S3 endpoint on nas-1. Reads the velero-key -# creds on-host, then delegates an aws-cli roundtrip (list/put/list/get/ -# compare/delete) to the CI runner so DNS + reverse-proxy + TLS are -# actually exercised externally. -- name: Garage smoke test - hosts: synology_nas - gather_facts: false - - vars: - garage_data_dir: "/volume1/docker/garage" - garage_endpoint: "https://s3.nas-1.renner.uno" - garage_region: "garage" - garage_test_bucket: "k8s-velero" - garage_test_object: "smoke-test.txt" - - tasks: - - name: Read velero-key creds from nas-1 - ansible.builtin.shell: | - cat {{ garage_data_dir }}/config/keys/velero-key.creds - become: true - register: velero_creds - changed_when: false - no_log: true - - - name: Parse KEY_ID + SECRET_KEY - ansible.builtin.set_fact: - velero_key_id: "{{ velero_creds.stdout_lines | select('match', '^KEY_ID=') | map('regex_replace', '^KEY_ID=', '') | first }}" - velero_secret: "{{ velero_creds.stdout_lines | select('match', '^SECRET_KEY=') | map('regex_replace', '^SECRET_KEY=', '') | first }}" - no_log: true - - - name: Network reachability — DNS + TLS + unauth HEAD (no secrets) - delegate_to: localhost - ansible.builtin.shell: | - echo "=== DNS ===" - getent hosts s3.nas-1.renner.uno || nslookup s3.nas-1.renner.uno || true - echo "=== HTTPS HEAD (expect 400/403, proves TLS + proxy) ===" - curl -sS -o /dev/null -D - --max-time 10 {{ garage_endpoint }}/ || true - echo "=== cert subject ===" - echo | openssl s_client -servername s3.nas-1.renner.uno -connect s3.nas-1.renner.uno:443 2>/dev/null | openssl x509 -noout -subject -issuer -dates || true - register: net_probe - changed_when: false - - - name: Show network probe - ansible.builtin.debug: - var: net_probe.stdout_lines - - - name: S3 roundtrip (list/put/list/get+diff/delete) - delegate_to: localhost - environment: - AWS_ACCESS_KEY_ID: "{{ velero_key_id }}" - AWS_SECRET_ACCESS_KEY: "{{ velero_secret }}" - AWS_DEFAULT_REGION: "{{ garage_region }}" - ansible.builtin.shell: | - set -e - echo "=== list (pre) ===" - aws --endpoint-url {{ garage_endpoint }} s3 ls s3://{{ garage_test_bucket }} 2>&1 || echo "(empty bucket is OK)" - echo "=== put ===" - echo "smoke-test $(date -u +%FT%TZ)" > /tmp/garage-smoke.txt - aws --endpoint-url {{ garage_endpoint }} s3 cp /tmp/garage-smoke.txt s3://{{ garage_test_bucket }}/{{ garage_test_object }} 2>&1 - echo "=== list (post) ===" - aws --endpoint-url {{ garage_endpoint }} s3 ls s3://{{ garage_test_bucket }}/{{ garage_test_object }} 2>&1 - echo "=== get + compare ===" - aws --endpoint-url {{ garage_endpoint }} s3 cp s3://{{ garage_test_bucket }}/{{ garage_test_object }} /tmp/garage-smoke-back.txt 2>&1 - diff /tmp/garage-smoke.txt /tmp/garage-smoke-back.txt && echo "roundtrip match" - echo "=== delete ===" - aws --endpoint-url {{ garage_endpoint }} s3 rm s3://{{ garage_test_bucket }}/{{ garage_test_object }} 2>&1 - echo "=== done ===" - register: smoke - changed_when: false - failed_when: false - no_log: true - - - name: Show S3 roundtrip output (command rendering + env redacted) - ansible.builtin.debug: - msg: "{{ smoke.stdout_lines | default([]) + (smoke.stderr_lines | default([])) }}" - - - name: Fail if roundtrip did not succeed - ansible.builtin.fail: - msg: "S3 roundtrip failed — see previous task output for details" - when: smoke.rc is defined and smoke.rc != 0 diff --git a/ci/agents/update-audit/prompt.md b/ci/agents/update-audit/prompt.md index 089191a8..dfb8afc6 100644 --- a/ci/agents/update-audit/prompt.md +++ b/ci/agents/update-audit/prompt.md @@ -24,8 +24,8 @@ The current working directory is the project root (where `ansible/`, `opentofu/` For each, find the **current pin in the repo** (Read/Glob/Grep), then look up the **latest upstream release** (WebFetch/WebSearch), and report how far behind it is. -1. **Talos Linux** — current pin appears in `opentofu/talos_image.tf`, - `.woodpecker/rebuild-k8s.yml`, and `kubernetes/talos/controlplane.yaml`. Latest: +1. **Talos Linux** — current pin appears in `opentofu/talos_image.tf` and + `kubernetes/talos/controlplane.yaml`. Latest: `https://api.github.com/repos/siderolabs/talos/releases/latest`. 2. **Kubernetes** — pinned via the kubelet/installer in `kubernetes/talos/`. Note that the Kubernetes version is coupled to the Talos release; report the latest diff --git a/scripts/ci/fetch-kubeconfig.sh b/scripts/ci/fetch-kubeconfig.sh index 6a39ca63..e47bf1c4 100755 --- a/scripts/ci/fetch-kubeconfig.sh +++ b/scripts/ci/fetch-kubeconfig.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash -# Fetch kubeconfig.sops from Garage and decrypt to ~/.kube/config. -# Used by every Woodpecker step that needs cluster access. +# Fetch kubeconfig.sops from Hetzner Object Storage and decrypt to ~/.kube/config. +# Used by every Woodpecker step that needs cluster access (the e2e pipelines). # # Required env (typically from_secret in the calling pipeline): -# GARAGE_SECRETS_KEY_ID, GARAGE_SECRETS_SECRET_KEY — secrets-key creds for k8s-secrets -# SOPS_AGE_KEY — age private key (Flux/CI keyring) +# S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY — Hetzner Object Storage keypair +# SOPS_AGE_KEY — age private key (Flux/CI keyring) set -euo pipefail -: "${GARAGE_SECRETS_KEY_ID:?GARAGE_SECRETS_KEY_ID not set}" -: "${GARAGE_SECRETS_SECRET_KEY:?GARAGE_SECRETS_SECRET_KEY not set}" +: "${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID not set}" +: "${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY not set}" : "${SOPS_AGE_KEY:?SOPS_AGE_KEY not set}" mkdir -p ~/.config/sops/age ~/.kube @@ -17,11 +17,11 @@ echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt TMP=$(mktemp) trap 'rm -f "$TMP"' EXIT -AWS_ACCESS_KEY_ID="$GARAGE_SECRETS_KEY_ID" \ -AWS_SECRET_ACCESS_KEY="$GARAGE_SECRETS_SECRET_KEY" \ -AWS_DEFAULT_REGION=garage \ - aws --endpoint-url https://s3.nas-1.renner.uno \ - s3 cp s3://k8s-secrets/cluster-access/kubeconfig.sops "$TMP" +AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY_ID" \ +AWS_SECRET_ACCESS_KEY="$S3_SECRET_ACCESS_KEY" \ +AWS_DEFAULT_REGION=fsn1 \ + aws --endpoint-url https://fsn1.your-objectstorage.com \ + s3 cp s3://renner-uno-k8s-secrets/cluster-access/kubeconfig.sops "$TMP" sops -d --input-type binary --output-type binary "$TMP" > ~/.kube/config chmod 600 ~/.kube/config diff --git a/scripts/import-cluster-access.sh b/scripts/import-cluster-access.sh index 2139c02d..1efa7e40 100755 --- a/scripts/import-cluster-access.sh +++ b/scripts/import-cluster-access.sh @@ -1,15 +1,17 @@ #!/usr/bin/env bash set -euo pipefail -# Import SOPS-encrypted talosconfig and kubeconfig from Garage (s3://k8s-secrets/cluster-access/) -# into local ~/.talos/config and ~/.kube/config. +# Import SOPS-encrypted talosconfig and kubeconfig from Hetzner Object Storage +# (s3://renner-uno-k8s-secrets/cluster-access/) into local ~/.talos/config and +# ~/.kube/config. # -# Requires aws-cli + sops on the workstation, and Garage `secrets-key` credentials reachable -# either via env (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY) or via an aws-cli profile passed in -# AWS_PROFILE. +# Requires aws-cli + sops on the workstation, and Hetzner Object Storage +# credentials reachable either via env (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY) +# or via an aws-cli profile passed in AWS_PROFILE (the "secrets" profile must +# hold the Hetzner Object Storage keypair). -ENDPOINT="https://s3.nas-1.renner.uno" -BUCKET="k8s-secrets" +ENDPOINT="https://fsn1.your-objectstorage.com" +BUCKET="renner-uno-k8s-secrets" PREFIX="cluster-access" export AWS_PROFILE="secrets" @@ -21,9 +23,9 @@ TMPDIR=$(mktemp -d) trap 'rm -rf "$TMPDIR"' EXIT echo "Fetching access files from $ENDPOINT/$BUCKET/$PREFIX/ ..." -AWS_DEFAULT_REGION=garage aws --endpoint-url "$ENDPOINT" \ +AWS_DEFAULT_REGION=fsn1 aws --endpoint-url "$ENDPOINT" \ s3 cp "s3://$BUCKET/$PREFIX/talosconfig.sops" "$TMPDIR/talosconfig.sops" -AWS_DEFAULT_REGION=garage aws --endpoint-url "$ENDPOINT" \ +AWS_DEFAULT_REGION=fsn1 aws --endpoint-url "$ENDPOINT" \ s3 cp "s3://$BUCKET/$PREFIX/kubeconfig.sops" "$TMPDIR/kubeconfig.sops" mkdir -p ~/.talos ~/.kube