Merge pull request 'ci: decommission migration-only rebuild-k8s; point access tooling at Hetzner' (#73) from decouple-garage-decomm-rebuild into master
Reviewed-on: gitea_admin/homelab#73
This commit was merged in pull request #73.
This commit is contained in:
@@ -8,10 +8,10 @@ steps:
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
GARAGE_SECRETS_KEY_ID:
|
||||
from_secret: garage_secrets_key_id
|
||||
GARAGE_SECRETS_SECRET_KEY:
|
||||
from_secret: garage_secrets_secret_key
|
||||
S3_ACCESS_KEY_ID:
|
||||
from_secret: hetzner_s3_access_key
|
||||
S3_SECRET_ACCESS_KEY:
|
||||
from_secret: hetzner_s3_secret_key
|
||||
commands:
|
||||
- bash scripts/ci/fetch-kubeconfig.sh
|
||||
- bash kubernetes/e2e-test.sh
|
||||
|
||||
+6
-8
@@ -1,7 +1,5 @@
|
||||
# Standalone e2e test pipeline. Same script as the rebuild-k8s `e2e-test`
|
||||
# stage, runnable independently so regressions (cert renewal, OSD flap,
|
||||
# observability gap, OIDC config drift, RGW outage) are caught between
|
||||
# rebuilds.
|
||||
# Standalone e2e test pipeline — catches regressions (cert renewal, OSD flap,
|
||||
# observability gap, OIDC config drift, RGW outage) on a schedule.
|
||||
#
|
||||
# Triggers:
|
||||
# - cron : configure schedule via Woodpecker UI / API (Cron tab on this
|
||||
@@ -19,10 +17,10 @@ steps:
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
GARAGE_SECRETS_KEY_ID:
|
||||
from_secret: garage_secrets_key_id
|
||||
GARAGE_SECRETS_SECRET_KEY:
|
||||
from_secret: garage_secrets_secret_key
|
||||
S3_ACCESS_KEY_ID:
|
||||
from_secret: hetzner_s3_access_key
|
||||
S3_SECRET_ACCESS_KEY:
|
||||
from_secret: hetzner_s3_secret_key
|
||||
commands:
|
||||
- bash scripts/ci/fetch-kubeconfig.sh
|
||||
- bash kubernetes/e2e-test.sh
|
||||
|
||||
@@ -1,279 +0,0 @@
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'PIPELINE == "rebuild-k8s"'
|
||||
|
||||
steps:
|
||||
- name: recreate-vms
|
||||
image: gitea.renner.uno/gitea_admin/ci-tofu:2026-04-29
|
||||
directory: opentofu
|
||||
environment:
|
||||
TF_IN_AUTOMATION: "true"
|
||||
TF_VAR_proxmox_api_url:
|
||||
from_secret: proxmox_api_url
|
||||
TF_VAR_proxmox_user:
|
||||
from_secret: proxmox_user
|
||||
TF_VAR_proxmox_token_id:
|
||||
from_secret: proxmox_token_id
|
||||
TF_VAR_proxmox_token_secret:
|
||||
from_secret: proxmox_token_secret
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: tf_garage_access
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: tf_garage_secret
|
||||
AWS_REGION: "us-east-1"
|
||||
TF_VAR_aws_provider_access_key:
|
||||
from_secret: tf_rfs_access
|
||||
TF_VAR_aws_provider_secret_key:
|
||||
from_secret: tf_rfs_secret
|
||||
TF_VAR_unifi_api_key:
|
||||
from_secret: unifi_api_key
|
||||
TF_VAR_unifi_api_url:
|
||||
from_secret: unifi_api_url
|
||||
SSH_PRIVATE_KEY:
|
||||
from_secret: ssh_private_key
|
||||
commands:
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
|
||||
- chmod 600 ~/.ssh/id_ed25519
|
||||
- printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
|
||||
- export TF_VAR_ssh_private_key="$(cat ~/.ssh/id_ed25519)"
|
||||
# Ensure Talos image exists on all Proxmox nodes
|
||||
- |
|
||||
TALOS_VERSION="v1.12.6"
|
||||
TALOS_SCHEMATIC="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
|
||||
TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
|
||||
TALOS_FILENAME="talos-$TALOS_VERSION-nocloud-amd64.raw"
|
||||
API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
|
||||
for node in jupiter saturn neptune; do
|
||||
EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
|
||||
-H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
|
||||
if [ "$EXISTS" -gt 0 ]; then
|
||||
echo "Talos image already exists on $node"
|
||||
continue
|
||||
fi
|
||||
echo "Downloading Talos image to $node..."
|
||||
curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/download-url" \
|
||||
-H "Authorization: PVEAPIToken=$API_TOKEN" \
|
||||
--data-urlencode "url=$TALOS_URL" \
|
||||
--data-urlencode "content=import" \
|
||||
--data-urlencode "filename=$TALOS_FILENAME" \
|
||||
--max-time 600
|
||||
for i in $(seq 1 60); do
|
||||
sleep 10
|
||||
EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
|
||||
-H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
|
||||
if [ "$EXISTS" -gt 0 ]; then echo "Download complete on $node"; break; fi
|
||||
done
|
||||
done
|
||||
- tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"
|
||||
# Replace all K8s VMs (3 controlplane + 1 edge worker), retry individual failures up to 3 times.
|
||||
# `-target` scopes the plan to the K8s modules only — without it, drift on unrelated VMs/LXCs
|
||||
# (e.g. from manual qm move-disk) gets pulled into the apply.
|
||||
- |
|
||||
TARGETS="-target=module.k8s_1 -target=module.k8s_2 -target=module.k8s_3 -target=module.k8s_edge"
|
||||
REPLACE_ALL="-replace=module.k8s_1.proxmox_virtual_environment_vm.this -replace=module.k8s_2.proxmox_virtual_environment_vm.this -replace=module.k8s_3.proxmox_virtual_environment_vm.this -replace=module.k8s_edge.proxmox_virtual_environment_vm.this"
|
||||
if tofu apply -auto-approve $TARGETS $REPLACE_ALL; then
|
||||
echo "All VMs replaced successfully"
|
||||
else
|
||||
echo "Some VMs failed, retrying individually..."
|
||||
failed=""
|
||||
for mod in k8s_1 k8s_2 k8s_3 k8s_edge; do
|
||||
success=0
|
||||
for attempt in 1 2 3; do
|
||||
echo "Checking module.$mod (attempt $attempt)..."
|
||||
if tofu plan -detailed-exitcode -target="module.$mod" >/dev/null 2>&1; then
|
||||
echo " module.$mod is up to date"
|
||||
success=1
|
||||
break
|
||||
fi
|
||||
echo " Retrying module.$mod..."
|
||||
if tofu apply -auto-approve -target="module.$mod" -replace="module.$mod.proxmox_virtual_environment_vm.this"; then
|
||||
success=1
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
[ "$success" -eq 1 ] || failed="$failed $mod"
|
||||
done
|
||||
if [ -n "$failed" ]; then
|
||||
echo "FAILED to recreate modules:$failed"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
- echo "VM recreation complete"
|
||||
|
||||
- name: wipe-backups
|
||||
# Cold-start aid: empty the CNPG (k8s-cnpg) and Velero (k8s-velero)
|
||||
# backup buckets in Garage. Runs only when WIPE_BACKUPS=true is passed
|
||||
# (e.g. `woodpecker pipeline create ... --var WIPE_BACKUPS=true`).
|
||||
#
|
||||
# Position: after recreate-vms so the old cluster's VMs (and therefore
|
||||
# its barman-cloud + Velero pods) are destroyed before we wipe — no
|
||||
# more writers, no race where the old cluster lays down fresh objects
|
||||
# between our `aws s3 rm` and the new cluster's first seed.
|
||||
#
|
||||
# Without this step the seed Jobs in the new cluster gate on the
|
||||
# contents of the corresponding s3://k8s-cnpg/<cluster>/base/ path:
|
||||
# if a stale base from a prior run is present, the gate exits 0 and
|
||||
# CNPG bootstrap.recovery tries to recover from data that doesn't
|
||||
# match — recovery Job exhausts backoffLimit and the cluster sticks
|
||||
# in "Setting up primary".
|
||||
when:
|
||||
- evaluate: 'WIPE_BACKUPS == "true"'
|
||||
# Bumped to 2026-05-01 for the aws-cli addition baked into ci-k8s.
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01
|
||||
environment:
|
||||
GARAGE_ENDPOINT: https://s3.nas-1.renner.uno
|
||||
CNPG_KEY_ID:
|
||||
from_secret: garage_cnpg_key_id
|
||||
CNPG_SECRET_KEY:
|
||||
from_secret: garage_cnpg_secret_key
|
||||
VELERO_KEY_ID:
|
||||
from_secret: garage_velero_key_id
|
||||
VELERO_SECRET_KEY:
|
||||
from_secret: garage_velero_secret_key
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
wipe_bucket() {
|
||||
local bucket="$1" key_id="$2" secret_key="$3"
|
||||
log "Wiping s3://$bucket ..."
|
||||
AWS_ACCESS_KEY_ID="$key_id" AWS_SECRET_ACCESS_KEY="$secret_key" \
|
||||
aws --endpoint-url "$GARAGE_ENDPOINT" s3 rm --recursive "s3://$bucket/" >/dev/null
|
||||
local rem
|
||||
rem=$(AWS_ACCESS_KEY_ID="$key_id" AWS_SECRET_ACCESS_KEY="$secret_key" \
|
||||
aws --endpoint-url "$GARAGE_ENDPOINT" s3 ls --recursive --summarize "s3://$bucket/" \
|
||||
| awk '/Total Objects:/ {print $3}')
|
||||
log " remaining objects in $bucket: ${rem:-?}"
|
||||
[ "${rem:-0}" = "0" ] || { log " bucket NOT empty after wipe — failing"; return 1; }
|
||||
}
|
||||
|
||||
wipe_bucket k8s-cnpg "$CNPG_KEY_ID" "$CNPG_SECRET_KEY"
|
||||
wipe_bucket k8s-velero "$VELERO_KEY_ID" "$VELERO_SECRET_KEY"
|
||||
log "wipe-backups complete."
|
||||
|
||||
- name: tune-cp-zvols
|
||||
# Set sync=disabled on the freshly-created K8s CP VM zvols before Talos
|
||||
# boots, so etcd's WAL fsync isn't amplified through the local-zfs pool.
|
||||
# See ansible/plays/configure-pve-k8s-zfs-tuning.yml for the durability
|
||||
# tradeoff.
|
||||
image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
|
||||
environment:
|
||||
ANSIBLE_FORCE_COLOR: "true"
|
||||
ANSIBLE_HOST_KEY_CHECKING: "false"
|
||||
SSH_PRIVATE_KEY:
|
||||
from_secret: ssh_private_key
|
||||
VAULT_PASS:
|
||||
from_secret: vault_pass
|
||||
commands:
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
|
||||
- chmod 600 ~/.ssh/id_ed25519
|
||||
- printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
|
||||
- cd ansible
|
||||
- echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
|
||||
- export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
|
||||
- ansible-playbook -i inventory/pve_hosts.yml plays/configure-pve-k8s-zfs-tuning.yml -v
|
||||
|
||||
- name: talos-bootstrap
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29
|
||||
commands:
|
||||
# Wait for VMs to boot into maintenance mode
|
||||
- echo "Waiting 60s for VMs to boot..."
|
||||
- sleep 60
|
||||
- |
|
||||
for node in k8s-1 k8s-2 k8s-3 k8s-edge; do
|
||||
echo -n " DNS $node.srv.renner.uno: "
|
||||
getent hosts "$node.srv.renner.uno" || echo "UNRESOLVED"
|
||||
done
|
||||
- cd kubernetes
|
||||
- bash bootstrap.sh talos
|
||||
|
||||
- name: wipe-osd-disks
|
||||
# Rook OSDs live on /dev/nvme0n1 (PCI-passthrough on each PVE node). The disks
|
||||
# survive VM destroy/recreate, but the mon DB on Talos EPHEMERAL does not — so
|
||||
# after a rebuild the new Ceph cluster has a fresh FSID and refuses to adopt the
|
||||
# old BlueStore-signed disks ("skipping osd.N: belonging to a different ceph
|
||||
# cluster"). Wipe them here, before flux-bootstrap, so OSDs come up clean.
|
||||
# Edge worker (k8s-edge) has no OSD and is skipped.
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29
|
||||
commands:
|
||||
- export TALOSCONFIG="$(pwd)/kubernetes/access-staging/talosconfig"
|
||||
- |
|
||||
set -e
|
||||
for node in k8s-1 k8s-2 k8s-3; do
|
||||
echo "Wiping nvme0n1 on $node..."
|
||||
talosctl wipe disk nvme0n1 \
|
||||
-n "$node.srv.renner.uno" \
|
||||
-e "k8s-1.srv.renner.uno" \
|
||||
--method FAST
|
||||
done
|
||||
|
||||
- name: export-access
|
||||
# Encrypt the staging talos/kubeconfig and upload to Garage
|
||||
# (s3://k8s-secrets/cluster-access/). Replaces the previous "git commit
|
||||
# && git push" mechanism, which raced with concurrent commits on master
|
||||
# and forced workstation users to `git pull --rebase` after each rebuild.
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
GARAGE_SECRETS_KEY_ID:
|
||||
from_secret: garage_secrets_key_id
|
||||
GARAGE_SECRETS_SECRET_KEY:
|
||||
from_secret: garage_secrets_secret_key
|
||||
commands:
|
||||
- mkdir -p ~/.config/sops/age
|
||||
- echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt
|
||||
- |
|
||||
set -e
|
||||
AGE_RECIPIENT="age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3"
|
||||
if [[ ! -f kubernetes/access-staging/talosconfig || ! -f kubernetes/access-staging/kubeconfig ]]; then
|
||||
echo "Access files not found; talos-bootstrap may have failed"
|
||||
exit 1
|
||||
fi
|
||||
TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
sops -e --age "$AGE_RECIPIENT" --input-type binary --output-type binary kubernetes/access-staging/talosconfig > "$TMPDIR/talosconfig.sops"
|
||||
sops -e --age "$AGE_RECIPIENT" --input-type binary --output-type binary kubernetes/access-staging/kubeconfig > "$TMPDIR/kubeconfig.sops"
|
||||
export AWS_ACCESS_KEY_ID="$GARAGE_SECRETS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$GARAGE_SECRETS_SECRET_KEY"
|
||||
export AWS_DEFAULT_REGION=garage
|
||||
aws --endpoint-url https://s3.nas-1.renner.uno \
|
||||
s3 cp "$TMPDIR/talosconfig.sops" s3://k8s-secrets/cluster-access/talosconfig.sops
|
||||
aws --endpoint-url https://s3.nas-1.renner.uno \
|
||||
s3 cp "$TMPDIR/kubeconfig.sops" s3://k8s-secrets/cluster-access/kubeconfig.sops
|
||||
|
||||
- name: flux-bootstrap
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29
|
||||
environment:
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
commands:
|
||||
- mkdir -p ~/.config/sops/age
|
||||
- echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt
|
||||
- cd kubernetes
|
||||
- bash bootstrap.sh flux
|
||||
|
||||
- name: wait-apps
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29
|
||||
commands:
|
||||
- cd kubernetes
|
||||
- bash bootstrap.sh wait-apps
|
||||
|
||||
- name: e2e-test
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
GARAGE_SECRETS_KEY_ID:
|
||||
from_secret: garage_secrets_key_id
|
||||
GARAGE_SECRETS_SECRET_KEY:
|
||||
from_secret: garage_secrets_secret_key
|
||||
commands:
|
||||
- bash scripts/ci/fetch-kubeconfig.sh
|
||||
- bash kubernetes/e2e-test.sh
|
||||
@@ -1,27 +0,0 @@
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'PIPELINE == "test-garage-smoke"'
|
||||
|
||||
steps:
|
||||
- name: run-playbook
|
||||
image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
|
||||
environment:
|
||||
ANSIBLE_FORCE_COLOR: "true"
|
||||
ANSIBLE_HOST_KEY_CHECKING: "false"
|
||||
SSH_PRIVATE_KEY:
|
||||
from_secret: ssh_private_key
|
||||
VAULT_PASS:
|
||||
from_secret: vault_pass
|
||||
PROXMOX_TOKEN_ID:
|
||||
from_secret: proxmox_token_id
|
||||
PROXMOX_TOKEN_SECRET:
|
||||
from_secret: proxmox_token_secret
|
||||
commands:
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
|
||||
- chmod 600 ~/.ssh/id_ed25519
|
||||
- 'printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config'
|
||||
- cd ansible
|
||||
- echo "$$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
|
||||
- export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
|
||||
- ansible-playbook -i inventory/ plays/test-garage-smoke.yml -v
|
||||
@@ -18,9 +18,9 @@
|
||||
#
|
||||
# Idempotent: skips zvols that already have sync=disabled.
|
||||
#
|
||||
# Wired into .woodpecker/rebuild-k8s.yml between `recreate-vms` and
|
||||
# `talos-bootstrap` so freshly-created zvols get the property before Talos
|
||||
# starts writing.
|
||||
# Run manually against freshly-created CP zvols so they get the property
|
||||
# before Talos starts writing (previously wired into the now-removed
|
||||
# rebuild-k8s pipeline between `recreate-vms` and `talos-bootstrap`).
|
||||
|
||||
- name: Set sync=disabled on K8s CP VM zvols
|
||||
hosts: pve_cluster
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
---
|
||||
# Smoke test for the Garage S3 endpoint on nas-1. Reads the velero-key
|
||||
# creds on-host, then delegates an aws-cli roundtrip (list/put/list/get/
|
||||
# compare/delete) to the CI runner so DNS + reverse-proxy + TLS are
|
||||
# actually exercised externally.
|
||||
- name: Garage smoke test
|
||||
hosts: synology_nas
|
||||
gather_facts: false
|
||||
|
||||
vars:
|
||||
garage_data_dir: "/volume1/docker/garage"
|
||||
garage_endpoint: "https://s3.nas-1.renner.uno"
|
||||
garage_region: "garage"
|
||||
garage_test_bucket: "k8s-velero"
|
||||
garage_test_object: "smoke-test.txt"
|
||||
|
||||
tasks:
|
||||
- name: Read velero-key creds from nas-1
|
||||
ansible.builtin.shell: |
|
||||
cat {{ garage_data_dir }}/config/keys/velero-key.creds
|
||||
become: true
|
||||
register: velero_creds
|
||||
changed_when: false
|
||||
no_log: true
|
||||
|
||||
- name: Parse KEY_ID + SECRET_KEY
|
||||
ansible.builtin.set_fact:
|
||||
velero_key_id: "{{ velero_creds.stdout_lines | select('match', '^KEY_ID=') | map('regex_replace', '^KEY_ID=', '') | first }}"
|
||||
velero_secret: "{{ velero_creds.stdout_lines | select('match', '^SECRET_KEY=') | map('regex_replace', '^SECRET_KEY=', '') | first }}"
|
||||
no_log: true
|
||||
|
||||
- name: Network reachability — DNS + TLS + unauth HEAD (no secrets)
|
||||
delegate_to: localhost
|
||||
ansible.builtin.shell: |
|
||||
echo "=== DNS ==="
|
||||
getent hosts s3.nas-1.renner.uno || nslookup s3.nas-1.renner.uno || true
|
||||
echo "=== HTTPS HEAD (expect 400/403, proves TLS + proxy) ==="
|
||||
curl -sS -o /dev/null -D - --max-time 10 {{ garage_endpoint }}/ || true
|
||||
echo "=== cert subject ==="
|
||||
echo | openssl s_client -servername s3.nas-1.renner.uno -connect s3.nas-1.renner.uno:443 2>/dev/null | openssl x509 -noout -subject -issuer -dates || true
|
||||
register: net_probe
|
||||
changed_when: false
|
||||
|
||||
- name: Show network probe
|
||||
ansible.builtin.debug:
|
||||
var: net_probe.stdout_lines
|
||||
|
||||
- name: S3 roundtrip (list/put/list/get+diff/delete)
|
||||
delegate_to: localhost
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID: "{{ velero_key_id }}"
|
||||
AWS_SECRET_ACCESS_KEY: "{{ velero_secret }}"
|
||||
AWS_DEFAULT_REGION: "{{ garage_region }}"
|
||||
ansible.builtin.shell: |
|
||||
set -e
|
||||
echo "=== list (pre) ==="
|
||||
aws --endpoint-url {{ garage_endpoint }} s3 ls s3://{{ garage_test_bucket }} 2>&1 || echo "(empty bucket is OK)"
|
||||
echo "=== put ==="
|
||||
echo "smoke-test $(date -u +%FT%TZ)" > /tmp/garage-smoke.txt
|
||||
aws --endpoint-url {{ garage_endpoint }} s3 cp /tmp/garage-smoke.txt s3://{{ garage_test_bucket }}/{{ garage_test_object }} 2>&1
|
||||
echo "=== list (post) ==="
|
||||
aws --endpoint-url {{ garage_endpoint }} s3 ls s3://{{ garage_test_bucket }}/{{ garage_test_object }} 2>&1
|
||||
echo "=== get + compare ==="
|
||||
aws --endpoint-url {{ garage_endpoint }} s3 cp s3://{{ garage_test_bucket }}/{{ garage_test_object }} /tmp/garage-smoke-back.txt 2>&1
|
||||
diff /tmp/garage-smoke.txt /tmp/garage-smoke-back.txt && echo "roundtrip match"
|
||||
echo "=== delete ==="
|
||||
aws --endpoint-url {{ garage_endpoint }} s3 rm s3://{{ garage_test_bucket }}/{{ garage_test_object }} 2>&1
|
||||
echo "=== done ==="
|
||||
register: smoke
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
|
||||
- name: Show S3 roundtrip output (command rendering + env redacted)
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ smoke.stdout_lines | default([]) + (smoke.stderr_lines | default([])) }}"
|
||||
|
||||
- name: Fail if roundtrip did not succeed
|
||||
ansible.builtin.fail:
|
||||
msg: "S3 roundtrip failed — see previous task output for details"
|
||||
when: smoke.rc is defined and smoke.rc != 0
|
||||
@@ -24,8 +24,8 @@ The current working directory is the project root (where `ansible/`, `opentofu/`
|
||||
For each, find the **current pin in the repo** (Read/Glob/Grep), then look up the
|
||||
**latest upstream release** (WebFetch/WebSearch), and report how far behind it is.
|
||||
|
||||
1. **Talos Linux** — current pin appears in `opentofu/talos_image.tf`,
|
||||
`.woodpecker/rebuild-k8s.yml`, and `kubernetes/talos/controlplane.yaml`. Latest:
|
||||
1. **Talos Linux** — current pin appears in `opentofu/talos_image.tf` and
|
||||
`kubernetes/talos/controlplane.yaml`. Latest:
|
||||
`https://api.github.com/repos/siderolabs/talos/releases/latest`.
|
||||
2. **Kubernetes** — pinned via the kubelet/installer in `kubernetes/talos/`. Note
|
||||
that the Kubernetes version is coupled to the Talos release; report the latest
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fetch kubeconfig.sops from Garage and decrypt to ~/.kube/config.
|
||||
# Used by every Woodpecker step that needs cluster access.
|
||||
# Fetch kubeconfig.sops from Hetzner Object Storage and decrypt to ~/.kube/config.
|
||||
# Used by every Woodpecker step that needs cluster access (the e2e pipelines).
|
||||
#
|
||||
# Required env (typically from_secret in the calling pipeline):
|
||||
# GARAGE_SECRETS_KEY_ID, GARAGE_SECRETS_SECRET_KEY — secrets-key creds for k8s-secrets
|
||||
# SOPS_AGE_KEY — age private key (Flux/CI keyring)
|
||||
# S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY — Hetzner Object Storage keypair
|
||||
# SOPS_AGE_KEY — age private key (Flux/CI keyring)
|
||||
set -euo pipefail
|
||||
|
||||
: "${GARAGE_SECRETS_KEY_ID:?GARAGE_SECRETS_KEY_ID not set}"
|
||||
: "${GARAGE_SECRETS_SECRET_KEY:?GARAGE_SECRETS_SECRET_KEY not set}"
|
||||
: "${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID not set}"
|
||||
: "${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY not set}"
|
||||
: "${SOPS_AGE_KEY:?SOPS_AGE_KEY not set}"
|
||||
|
||||
mkdir -p ~/.config/sops/age ~/.kube
|
||||
@@ -17,11 +17,11 @@ echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt
|
||||
TMP=$(mktemp)
|
||||
trap 'rm -f "$TMP"' EXIT
|
||||
|
||||
AWS_ACCESS_KEY_ID="$GARAGE_SECRETS_KEY_ID" \
|
||||
AWS_SECRET_ACCESS_KEY="$GARAGE_SECRETS_SECRET_KEY" \
|
||||
AWS_DEFAULT_REGION=garage \
|
||||
aws --endpoint-url https://s3.nas-1.renner.uno \
|
||||
s3 cp s3://k8s-secrets/cluster-access/kubeconfig.sops "$TMP"
|
||||
AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY_ID" \
|
||||
AWS_SECRET_ACCESS_KEY="$S3_SECRET_ACCESS_KEY" \
|
||||
AWS_DEFAULT_REGION=fsn1 \
|
||||
aws --endpoint-url https://fsn1.your-objectstorage.com \
|
||||
s3 cp s3://renner-uno-k8s-secrets/cluster-access/kubeconfig.sops "$TMP"
|
||||
|
||||
sops -d --input-type binary --output-type binary "$TMP" > ~/.kube/config
|
||||
chmod 600 ~/.kube/config
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Import SOPS-encrypted talosconfig and kubeconfig from Garage (s3://k8s-secrets/cluster-access/)
|
||||
# into local ~/.talos/config and ~/.kube/config.
|
||||
# Import SOPS-encrypted talosconfig and kubeconfig from Hetzner Object Storage
|
||||
# (s3://renner-uno-k8s-secrets/cluster-access/) into local ~/.talos/config and
|
||||
# ~/.kube/config.
|
||||
#
|
||||
# Requires aws-cli + sops on the workstation, and Garage `secrets-key` credentials reachable
|
||||
# either via env (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY) or via an aws-cli profile passed in
|
||||
# AWS_PROFILE.
|
||||
# Requires aws-cli + sops on the workstation, and Hetzner Object Storage
|
||||
# credentials reachable either via env (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY)
|
||||
# or via an aws-cli profile passed in AWS_PROFILE (the "secrets" profile must
|
||||
# hold the Hetzner Object Storage keypair).
|
||||
|
||||
ENDPOINT="https://s3.nas-1.renner.uno"
|
||||
BUCKET="k8s-secrets"
|
||||
ENDPOINT="https://fsn1.your-objectstorage.com"
|
||||
BUCKET="renner-uno-k8s-secrets"
|
||||
PREFIX="cluster-access"
|
||||
export AWS_PROFILE="secrets"
|
||||
|
||||
@@ -21,9 +23,9 @@ TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
echo "Fetching access files from $ENDPOINT/$BUCKET/$PREFIX/ ..."
|
||||
AWS_DEFAULT_REGION=garage aws --endpoint-url "$ENDPOINT" \
|
||||
AWS_DEFAULT_REGION=fsn1 aws --endpoint-url "$ENDPOINT" \
|
||||
s3 cp "s3://$BUCKET/$PREFIX/talosconfig.sops" "$TMPDIR/talosconfig.sops"
|
||||
AWS_DEFAULT_REGION=garage aws --endpoint-url "$ENDPOINT" \
|
||||
AWS_DEFAULT_REGION=fsn1 aws --endpoint-url "$ENDPOINT" \
|
||||
s3 cp "s3://$BUCKET/$PREFIX/kubeconfig.sops" "$TMPDIR/kubeconfig.sops"
|
||||
|
||||
mkdir -p ~/.talos ~/.kube
|
||||
|
||||
Reference in New Issue
Block a user