homelab/kubernetes/bootstrap.sh

#!/usr/bin/env bash
set -euo pipefail

# K8s Cluster Bootstrap — split into phases so CI can export access as early as possible.
#
# Usage:
#   ./bootstrap.sh              # run all phases locally
#   ./bootstrap.sh talos        # Talos apply + K8s bootstrap + kubeconfig (writes access-staging/)
#   ./bootstrap.sh flux         # reads access-staging/, installs flux, waits for infrastructure
#   ./bootstrap.sh wait-apps    # waits for apps/authentik/paperless to reconcile
#   ./bootstrap.sh vmtest       # throwaway QEMU VMs, then the talos phase
#                               # against them unchanged. Needs root for the
#                               # provisioner and leaves nothing behind.
#   ./bootstrap.sh hcloudtest   # the same rehearsal on Hetzner Cloud, which is
#                               # the only place the registry fallback can be
#                               # tested: gitea.renner.uno has no external
#                               # ingress, so a node there genuinely cannot
#                               # reach it. BILLS BY THE HOUR — the machines
#                               # stay up until hcloudclean. Access files land in
#                               # ~/.local/state/homelab-hcloudtest/.
#   ./bootstrap.sh hcloudclean  # deletes everything hcloudtest created, by
#                               # label, and the restore test's deploy keys.
#
# Restore test on the hcloudtest cluster (production tree, read-only source,
# kubernetes/restore-test/flux-root.yaml):
#   KUBECONFIG=~/.local/state/homelab-hcloudtest/kubeconfig \
#   TALOSCONFIG=~/.local/state/homelab-hcloudtest/talosconfig \
#   FLUX_SOURCE=restore-test ./bootstrap.sh flux
#
# Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls
# from is itself Flux-managed, so a cold rebuild cannot pull from it until the
# cluster already exists. Set FLUX_SOURCE=github to bootstrap Flux from the
# external GitHub mirror instead — the only way out of that loop:
#   FLUX_SOURCE=github GITHUB_TOKEN=<pat> ./bootstrap.sh flux
# The mirror is kept current server-side by Gitea's native push-mirror. Full
# cold-rebuild procedure: kubernetes/RUNBOOK-cold-rebuild.md.
#
# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt

TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# Cilium's single source of truth. Talos ships `cni: none` (talos/patches/
# cluster.yaml) and nothing else, so the CNI has to be installed here, between
# `talosctl bootstrap` and Flux — Flux's own controllers are pods and cannot
# reconcile the network they need in order to start. Everything the install
# needs is read out of this file at run time so there is no second copy of a
# version or a value to keep in step. See PROPOSAL-cilium-single-owner.md.
CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml"
K8SGW_HR="$REPO_ROOT/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml"
ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging"
NODES=(k8s-1 k8s-2 k8s-3)
# No dedicated workers since the k8s-edge VM was retired (2026-07-02) —
# hardware-bound services live on the bare-metal edge-1 Arch host
# (ansible/docs/edge-host.md).
WORKERS=()
ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
DOMAIN="srv.renner.uno"
DNS_SERVER="10.2.40.1"  # UDM (old PVE DNS VIP 10.2.40.53 decommissioned)
# Cluster API endpoint — Talos VIP (10.2.40.60) elected via etcd, fronted by
# UniFi DNS A record k8s.renner.uno. Defined in talos/patches/k8s-{1,2,3}.yaml.
K8S_API_HOST="k8s.renner.uno"
# Which site patch the nodes get: talos/patches/site-$SITE.yaml. The three
# variables above and this one are everything the vmtest phase has to change in
# order to run this same code against throwaway VMs — if that list ever grows,
# the test is drifting away from the thing it claims to exercise.
SITE="${SITE:-homelab}"
# Where this script writes the cluster access files. These are the paths
# talosctl and kubectl read from anyway, so unset they resolve to exactly the
# old behaviour. Set, the whole run stays out of $HOME — which is what lets the
# vmtest phase drive a throwaway cluster without replacing the operator's
# production access with a VM's.
TALOSCONFIG_PATH="${TALOSCONFIG:-$HOME/.talos/config}"
KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}"
case "$KUBECONFIG_PATH" in
    *:*) echo "KUBECONFIG holds multiple paths; set it to a single file" >&2; exit 1 ;;
esac
# Addresses to apply to, in the order of NODES. Empty means resolve the node
# FQDNs, which is right for real hardware on a network that has DNS for it.
NODE_ADDRS=()
GITEA_HOST="gitea.renner.uno"
GITEA_OWNER="gitea_admin"
GITEA_REPO="homelab"
# Flux git source: "gitea" (default, in-cluster) or "github" (break-glass mirror
# for a cold rebuild — see the usage note above and RUNBOOK-cold-rebuild.md).
FLUX_SOURCE="${FLUX_SOURCE:-gitea}"
# GitHub break-glass mirror (github.com/konairius/Homelab), kept current by
# Gitea's native push-mirror. Overridable in case the mirror moves.
GITHUB_OWNER="${GITHUB_OWNER:-konairius}"
GITHUB_REPO="${GITHUB_REPO:-Homelab}"
# Owner of the private ghcr.io namespace the rebuild-critical images are
# mirrored into, and the username half of the node registry credential. Matches
# GHCR_USER in .woodpecker/build-ci-images.yml, which pushes them.
GHCR_PULL_USER="${GHCR_PULL_USER:-konairius}"
FLUX_PATH="kubernetes/clusters/homelab"
# Branch Flux reconciles from, and that `flux bootstrap` commits its own sync
# manifests into. master for a real rebuild.
#
# Overridable so the phase can be rehearsed without writing to master — point it
# at a throwaway branch, which has to exist already: `flux bootstrap` clones the
# branch and fails with `couldn't find remote ref` rather than creating it.
#
#     git push origin origin/master:refs/heads/rehearsal-1
#     FLUX_BRANCH=rehearsal-1 ./bootstrap.sh flux
#
# Nothing else about the phase changes, which is the point. Delete the branch
# afterwards.
FLUX_BRANCH="${FLUX_BRANCH:-master}"
SOPS_KEY="$HOME/.config/sops/age/keys.txt"

PHASE="${1:-all}"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

is_tty() { [[ -t 1 ]]; }

step() {
    local _ignored=$1
    local label="$2"
    if is_tty; then
        printf "${BLUE}==>${NC} %s\n" "$label"
    else
        printf "==> %s\n" "$label"
    fi
}

# One EXIT trap for the whole script. Handlers register here rather than calling
# `trap ... EXIT` themselves, because a second registration replaces the first
# instead of chaining — which is how a secrets-cleanup handler silently disabled
# the VM teardown.
SECRETS_TMPDIR=""
on_exit() {
    [[ -n "$SECRETS_TMPDIR" ]] && rm -rf "$SECRETS_TMPDIR"
    declare -F cleanup_vmtest >/dev/null && cleanup_vmtest
    # No teardown here: provisioning is its own phase and the machines it leaves
    # behind are its product. `hcloudclean` is what deletes them.
    # Only when this run actually booked machines. The function is always
    # defined, so testing for it fired on the qemu phase too.
    if [[ "${HCLOUD_PROVISIONED:-0}" == "1" ]]; then
        echo -e "${YELLOW}Hetzner machines are UP and billed until deleted.${NC}"
        echo "  ./kubernetes/bootstrap.sh hcloudclean   # when finished"
    fi
    return 0
}
trap on_exit EXIT

fail() { echo -e "\n${RED}FAILED:${NC} $1"; exit 1; }
ok()   { echo -e "${GREEN}OK${NC}"; }

# Reads the Ready condition of a Flux Kustomization, without going through the
# table `flux get` prints.
#
# That table is NAME REVISION SUSPENDED READY MESSAGE, and every wait below used
# to pick READY as the fourth field. REVISION is empty until the first
# successful reconcile — precisely the state these waits exist to catch — so in
# the failure case the columns shift left and the fourth field lands on the
# message. Matching it against "False" then never fires, and a persistently
# failing Kustomization reads as merely slow. That cost a full two-hour run:
# infrastructure had been failing its server-side dry-run from the first minute,
# and the counter meant to catch it never moved.
#
# Echoes "<status>|<message>". An empty status means the object does not exist
# yet; kubectl's own error is carried through as the message rather than
# discarded, so "not created yet" stays distinguishable from "lookup broke".
ks_ready_condition() {
    local name="$1" out jp
    jp='{.status.conditions[?(@.type=="Ready")].status}'
    jp+='|{.status.conditions[?(@.type=="Ready")].message}'
    if out=$(kubectl -n flux-system get kustomization "$name" -o jsonpath="$jp" 2>&1); then
        printf '%s' "$out"
    else
        printf '|%s' "$out"
    fi
}

resolve_node_ip() {
    local fqdn=$1
    dig +short "$fqdn" A @"$DNS_SERVER" | head -1
}

phase_runs() {
    [[ "$PHASE" == "all" || "$PHASE" == "$1" ]]
}
# The restore test targets the cluster bootstrap.sh hcloudtest builds.
if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wait-apps" ]]; then
    fail "FLUX_SOURCE=restore-test runs the flux and wait-apps phases only"
fi
OPENBAO_NS="openbao"

# Where the secrets come from. On a workstation, the operator's pass store. In
# the in-cluster Job there is no pass and no GPG key — and there should not be:
# the pass store opens everything, and it stays on the operator's machine. Set
# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
# operator streams in when the run asks for them:
#
#     pass show homelab/openbao/seal-key \
#       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
#
# They exist in the pod's filesystem for the length of the run and are never
# written to a Secret in this cluster, so they do not land in its etcd or in
# any backup of it. The seal key alone becomes a Secret in the cluster being
# built, which is what lets that OpenBao unseal itself.
OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}"

# Every secret the run needs, named once, in one place. The list is here and
# nowhere else on purpose: when it was spread over the call sites, each was read
# at the moment it was needed, so a missing one surfaced twenty minutes into a
# run — and a second copy of the same path in a runbook or a wrapper is a copy
# that goes stale.
#
# The git token for `flux bootstrap`, chosen by the source it will authenticate
# against — which is the whole point and is what the previous expression got
# wrong. It read ${GITHUB_TOKEN:-${GITEA_TOKEN:-}}, taking whichever variable
# happened to be set: on a workstation with GITEA_TOKEN exported and no
# GITHUB_TOKEN, a break-glass run handed a Gitea token to the GitHub API and
# died on `401 Bad credentials` after the cluster was already built.
#
# `gh auth token` as the GitHub fallback because that is the credential the
# operator already has on this machine, and it is what the vmtest secret feeder
# has always used. Emptiness is not checked here — require_secrets rejects an
# empty result and names what is missing.
#
# gh is called plainly, with its stderr left alone. Swallowing it would turn a
# failed login into an empty string, and require_secrets would then report
# "came back empty" for a credential that exists but could not be read — a zero
# that cannot be told apart from a fault. Its non-zero exit propagates, which is
# what makes require_secrets print the reason instead.
git_token_for_source() {
    if [[ "$FLUX_SOURCE" == "github" || "$FLUX_SOURCE" == "restore-test" ]]; then
        if [[ -n "${GITHUB_TOKEN:-}" ]]; then
            printf '%s' "$GITHUB_TOKEN"
        else
            gh auth token
        fi
    else
        printf '%s' "${GITEA_TOKEN:-}"
    fi
}

# Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
OPENBAO_REQUIRED_SECRETS=(
    "sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
    # Which token depends on FLUX_SOURCE, and that is not a detail: in a real
    # cold rebuild the in-cluster Gitea cannot serve the cluster being rebuilt,
    # so the break-glass path pulls from the GitHub mirror. A run that demands a
    # Gitea token is a run that cannot rehearse the disaster it exists for.
    # The variable names are the ones the usage header and RUNBOOK-cold-rebuild
    # tell the operator to export. An invented third name was worse than a typo:
    # `${VAR:?}` inside eval terminates the shell outright, so the documented
    # recovery command exited with no message and no cleanup at all.
    #
    # Emptiness is not tested here on purpose — require_secrets rejects an empty
    # file and names what is missing.
    "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
    "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
    "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
    # The static seal key. The snapshots are encrypted under it, so it has to
    # be the one the backed-up cluster ran with — a fresh key restores nothing.
    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
    # Read by the node registry auth (see registry_auth_patch): the images an app
    # needs while the cluster is still being built are mirrored to a private
    # ghcr.io namespace, and a node has to be able to pull them before anything
    # in the cluster exists to hold a credential. Demanded up front like the
    # rest, because a rebuild that discovers it missing has already wiped the
    # disks.
    "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
)
# Asked for by name only, never by a bare require_secrets: a rebuild of the
# real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
# Hetzner project homelab-backup-verify, which holds nothing but throwaway
# machines.
HCLOUD_REQUIRED_SECRETS=(
    "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|bao kv get -mount=homelab -field=token infrastructure/hcloud-backup-verify"
)

# Collects all of them up front, into one directory, so everything after this
# point reads files and takes the same code path whether the run is on a laptop
# during a real recovery or in the in-cluster Job.
#
# The only difference between those two is where the material comes from, and it
# is confined to this function: on a laptop the commands in the list produce it
# from pass; in the Job the operator streams it in. Everything downstream cannot
# tell the difference — which is the point, because the recovery path must not be
# a path that only gets exercised during a recovery.
# require_secrets [name...]
#
# With no arguments, demands every entry in OPENBAO_REQUIRED_SECRETS. With
# names, demands only those — which is what lets the talos phase stand alone.
#
# That phase used to need nothing but the machines, and `bootstrap.sh talos` is
# documented as the part that can run unattended. Node registry auth changed
# that: the machine config now carries a pull credential, so the phase needs
# exactly one secret. Demanding all six there would drag in the OpenBao seal
# key, which deliberately never reaches CI — and demanding none, as before,
# left the phase to discover the missing file halfway through `talosctl gen`.
require_secrets() {
    local entry name what how
    local -a wanted=()
    if [[ $# -gt 0 ]]; then
        for entry in "${OPENBAO_REQUIRED_SECRETS[@]}" "${HCLOUD_REQUIRED_SECRETS[@]}"; do
            for name in "$@"; do
                [[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry")
            done
        done
        [[ ${#wanted[@]} -eq $# ]] \
            || fail "require_secrets asked for [$*] but the list defines only: ${wanted[*]%%|*}"
    else
        wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
    fi

    # Which branch: fetch from pass, or wait for someone to stream files in.
    #
    # The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
    # FUNCTION sets it. Once a subset call had run — hcloudtest asks for
    # ghcr-pull and hcloud-token before anything else — every later call found
    # the variable populated and took the wait branch, printing streaming
    # instructions for a Job that does not exist on a workstation where pass was
    # available all along. The run sat there while three nodes stayed in
    # maintenance.
    #
    # SECRETS_SELF_FETCH records which world we are in, and it is decided once:
    # an empty OPENBAO_SECRETS_DIR on entry means pass, an inherited one means
    # the Job. Later calls follow that decision instead of re-deriving it from
    # state this function has since changed.
    if [[ -z "$OPENBAO_SECRETS_DIR" || "${SECRETS_SELF_FETCH:-0}" == "1" ]]; then
        if [[ -z "$OPENBAO_SECRETS_DIR" ]]; then
            OPENBAO_SECRETS_DIR=$(mktemp -d)
            chmod 700 "$OPENBAO_SECRETS_DIR"
        fi
        SECRETS_SELF_FETCH=1
        # Registered by name, not with a second `trap ... EXIT`: a second one
        # REPLACES the first, and the first is the vmtest teardown. Setting it
        # here left four VMs, their bridges and the sudo keepalive running after
        # a workstation rehearsal — silently, and only on the laptop path, since
        # in-cluster this branch is not taken.
        SECRETS_TMPDIR="$OPENBAO_SECRETS_DIR"
        echo -n "Collecting secrets... "
        for entry in "${wanted[@]}"; do
            name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
            # Already collected by an earlier call — a subset call fetches only
            # what it asked for, so the full call that follows must fill the
            # gaps rather than re-run every command. `pass show` on a locked
            # store prompts, and prompting twice for the same key is how an
            # unattended run turns into a hung one.
            [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name" 2>"$OPENBAO_SECRETS_DIR/.err"; then
                echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
                fail "could not obtain the $what with: $how"
            fi
            [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
                || fail "the $what came back empty from: $how"
        done
        ok
        return 0
    fi

    mkdir -p "$OPENBAO_SECRETS_DIR"
    local entry name what how missing
    for i in $(seq 1 180); do
        missing=()
        for entry in "${wanted[@]}"; do
            name=${entry%%|*}
            [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] || missing+=("$entry")
        done
        [[ ${#missing[@]} -eq 0 ]] && { echo "All secrets present."; return 0; }
        if [[ $i -eq 1 ]]; then
            echo -e "${YELLOW}Waiting for ${#missing[@]} secret(s). Stream each in with:${NC}" >&2
            for entry in "${missing[@]}"; do
                name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
                echo "  # $what" >&2
                echo "  $how | kubectl -n $OPENBAO_NS_HINT exec -i \$POD -- sh -c 'cat > $OPENBAO_SECRETS_DIR/$name'" >&2
            done
        fi
        sleep 10
        [[ $i -eq 180 ]] && fail "still missing: ${missing[*]%%|*}"
    done
}
OPENBAO_NS_HINT=vmtest

# Reads one secret by its name in the list above. In the Job that is the file the
# operator streamed in; on a workstation it runs the command the list names,
# which for these is `pass show <path>`.
#
# The list is the only place the paths appear. They used to sit here as
# constants as well, which is how the same value ends up defined twice and one
# copy goes stale.
read_secret() {
    local name=$1
    [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
        || fail "$name is not in $OPENBAO_SECRETS_DIR — require_secrets runs first and should have caught this"
    cat "$OPENBAO_SECRETS_DIR/$name"
}

# Hetzner Object Storage, where the offsite backups live. Read with the restore
# key (s3-access, s3-secret).
S3_ENDPOINT="fsn1.your-objectstorage.com"
S3_REGION="fsn1"

# One request, signed by curl with the restore key. The body goes to stdout;
# anything but a 2xx fails, with S3's error document on stderr.
s3_request() {
    local method=$1 path=$2 body code
    body=$(mktemp)
    if ! code=$(curl -sS -o "$body" -w '%{http_code}' -X "$method" \
            --aws-sigv4 "aws:amz:$S3_REGION:s3" \
            --user "$(read_secret s3-access):$(read_secret s3-secret)" \
            "https://$S3_ENDPOINT/$path"); then
        rm -f "$body"
        return 1
    fi
    if [[ "$code" != 2?? ]]; then
        echo "$method $path: HTTP $code" >&2
        cat "$body" >&2
        echo >&2
        rm -f "$body"
        return 1
    fi
    cat "$body"
    rm -f "$body"
}

# The "directories" directly under PREFIX in BUCKET, one per line, without
# PREFIX and the trailing slash.
s3_dirs() {
    local bucket=$1 prefix=$2 xml
    xml=$(s3_request GET "$bucket?list-type=2&delimiter=%2F&prefix=${prefix//\//%2F}") || return 1
    [[ "$xml" == *"<IsTruncated>false</IsTruncated>"* ]] \
        || { echo "the listing of $bucket/$prefix is truncated or not a listing" >&2; return 1; }
    # awk, not grep: no match is not an error.
    awk -v p="$prefix" '{
        while (match($0, /<CommonPrefixes><Prefix>[^<]*<\/Prefix>/)) {
            d = substr($0, RSTART + 24, RLENGTH - 33)
            $0 = substr($0, RSTART + RLENGTH)
            sub(/\/$/, "", d)
            print substr(d, length(p) + 1)
        }
    }' <<<"$xml"
}

# The bucket every CNPG cluster recovers from (cnpg-object-stores/). The
# restore test archives to CNPG_TEST_BUCKET instead: a test incarnation with a
# base backup in the production bucket would be the newest there, and the next
# real recovery would read the test's data.
CNPG_BUCKET="renner-uno-k8s-cnpg"
CNPG_TEST_BUCKET="renner-uno-k8s-cnpg-rtest"
CNPG_STORES_DIR="$REPO_ROOT/kubernetes/cnpg-object-stores"

# cnpg-paths, the ConfigMap cnpg-object-stores takes its paths from. Written
# once per cluster incarnation: an existing one belongs to clusters already
# archiving under its paths and is kept.
#
# For each source ObjectStore in cnpg-object-stores/, cluster <x>:
#   CNPG_<X>_SOURCE   the candidate prefix whose <x>/base/ holds the newest
#                     completed base backup. Candidates: <x>, <x>-g<14 digits> and the
#                     store's cnpg.renner.uno/source-prefixes.
#   CNPG_<X>_ARCHIVE  <x>-g<now> in ARCHIVE_BUCKET: new, so empty, so the
#                     plugin's empty-archive check passes rather than being
#                     skipped.
write_cnpg_paths() {
    local archive_bucket=$1
    local gen dirs name extra var p listing newest best best_at id info
    local -a literals=()
    echo "Writing the CNPG backup paths (cnpg-paths)..."
    command -v yq >/dev/null || fail "yq not found (reads $CNPG_STORES_DIR)"
    if kubectl -n flux-system get configmap cnpg-paths -o name >/dev/null 2>"$OPENBAO_SECRETS_DIR/.err"; then
        echo "  already present, kept"
        return 0
    fi
    grep -q NotFound "$OPENBAO_SECRETS_DIR/.err" \
        || { cat "$OPENBAO_SECRETS_DIR/.err" >&2; fail "could not read the cnpg-paths ConfigMap"; }

    gen=g$(date -u +%Y%m%d%H%M%S)
    dirs=$(s3_dirs "$CNPG_BUCKET" "") || fail "could not list $CNPG_BUCKET"
    while IFS=$'\t' read -r name extra; do
        best=''
        best_at=''
        for p in $dirs; do
            [[ "$p" == "$name" || "$p" =~ ^${name}-g[0-9]{14}$ || " $extra " == *" $p "* ]] || continue
            listing=$(s3_dirs "$CNPG_BUCKET" "$p/$name/base/") \
                || fail "could not list $CNPG_BUCKET/$p/$name/base/"
            # Newest completed backup: barman uploads backup.info with
            # status=STARTED when a backup begins and rewrites it with DONE or
            # FAILED when it ends.
            newest=''
            for id in $(sort -r <<<"$listing"); do
                [[ "$id" > "$best_at" ]] || break
                info=$(s3_request GET "$CNPG_BUCKET/$p/$name/base/$id/backup.info") \
                    || fail "could not read $CNPG_BUCKET/$p/$name/base/$id/backup.info"
                if grep -qx 'status=DONE' <<<"$info"; then
                    newest=$id
                    break
                fi
                echo "  $name: skipping $p/$name/base/$id ($(grep '^status=' <<<"$info" || echo 'no status'))"
            done
            if [[ -n "$newest" ]]; then
                best=$p
                best_at=$newest
            fi
        done
        [[ -n "$best" ]] \
            || fail "no completed base backup of $name in $CNPG_BUCKET (looked under $name, $name-g<timestamp>${extra:+, $extra})"
        var=CNPG_$(tr 'a-z-' 'A-Z_' <<<"$name")
        literals+=("--from-literal=${var}_SOURCE=s3://$CNPG_BUCKET/$best"
                   "--from-literal=${var}_ARCHIVE=s3://$archive_bucket/$name-$gen")
        echo "  $name: recovers from $CNPG_BUCKET/$best (base backup $best_at), archives to $archive_bucket/$name-$gen"
    done < <(yq -r 'select(.kind == "ObjectStore" and .metadata.annotations["cnpg.renner.uno/server-name"] != null)
                | [.metadata.annotations["cnpg.renner.uno/server-name"],
                   (.metadata.annotations["cnpg.renner.uno/source-prefixes"] // "")] | @tsv' \
                "$CNPG_STORES_DIR"/*.yaml)
    [[ ${#literals[@]} -gt 0 ]] \
        || fail "no source ObjectStore (cnpg.renner.uno/server-name) in $CNPG_STORES_DIR"
    kubectl -n flux-system create configmap cnpg-paths "${literals[@]}" >/dev/null \
        || fail "could not create the cnpg-paths ConfigMap"
    ok
}

# The restore test's archive bucket. Created when missing; on a rerun it
# already exists, which S3 reports as an error.
create_cnpg_test_bucket() {
    local out
    out=$(s3_request PUT "$CNPG_TEST_BUCKET" 2>&1) && return 0
    [[ "$out" == *BucketAlreadyOwnedByYou* ]] && return 0
    echo "$out" >&2
    fail "could not create $CNPG_TEST_BUCKET"
}

# Empties and deletes the restore test's archive bucket. Each pass deletes the
# first page of keys and lists again, so no continuation token is needed.
delete_cnpg_test_bucket() {
    local xml key
    while :; do
        if ! xml=$(s3_request GET "$CNPG_TEST_BUCKET?list-type=2" 2>&1); then
            [[ "$xml" == *NoSuchBucket* ]] && return 0
            echo "$xml" >&2
            echo "  WARNING: $CNPG_TEST_BUCKET not emptied — check the console" >&2
            return 0
        fi
        while read -r key; do
            [[ -n "$key" ]] || continue
            s3_request DELETE "$CNPG_TEST_BUCKET/$key" >/dev/null \
                || { echo "  WARNING: $CNPG_TEST_BUCKET/$key not deleted — check the console" >&2; return 0; }
        done < <(awk '{
            while (match($0, /<Key>[^<]*<\/Key>/)) {
                print substr($0, RSTART + 5, RLENGTH - 11)
                $0 = substr($0, RSTART + RLENGTH)
            }
        }' <<<"$xml")
        [[ "$xml" == *"<IsTruncated>true</IsTruncated>"* ]] || break
    done
    if s3_request DELETE "$CNPG_TEST_BUCKET" >/dev/null; then
        echo "  deleted bucket $CNPG_TEST_BUCKET"
    else
        echo "  WARNING: bucket $CNPG_TEST_BUCKET not deleted — check the console" >&2
    fi
}

# Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase
# against it unchanged — same shape as the vmtest phase below, and for the same
# reason: a rehearsal that runs different code proves less than one that does
# not.
#
# WHY THIS EXISTS, given vmtest already rehearses a rebuild locally.
#
# The one thing vmtest cannot test is the registry fallback. The nodes now carry
# machine.registries.mirrors with gitea.renner.uno first and ghcr.io second, and
# the whole point of the second endpoint is the window where the first does not
# exist. At home it always exists — the production cluster is running and
# answers on the LAN — so containerd never reaches the fallback and it stays
# unproven. From Hetzner, Gitea does not answer at all: it has no external
# ingress.
#
# WHAT IT COSTS, because this one bills by the hour.
#
# Teardown is by LABEL, not by remembered ID, and runs from the single EXIT
# trap. A run that dies between `server create` and its own bookkeeping is
# therefore still cleanable — by the next run, or by `bootstrap.sh hcloudclean`,
# which does nothing but the sweep. Anything this phase creates carries
# HCLOUD_LABEL so that sweep can find it without being told.
#
# WHAT IT IS.
#
# The production config on new hardware: cluster.yaml, site-hcloud.yaml (what
# the Hetzner site is), containerd-layer-retention.yaml and hcloud-N.yaml (each
# machine). Nothing stands in for the home network. Flux is a separate run,
# FLUX_SOURCE=restore-test bootstrap.sh flux, because it needs the OpenBao seal
# key. Ceph replicates over the pod network there: Hetzner networks are layer 3
# (restore-test/flux-root.yaml).
#
# Containment: an outbound Hetzner firewall on every node admits udp and icmp
# anywhere and tcp anywhere except the addresses of proxy.renner.uno, the public
# edge in front of production.
# Talos version and Image Factory schematic, read out of the installer image in
# cluster.yaml. Both throwaway-cluster phases need them and neither may state
# them: a second copy is a copy that goes stale, and the day it did the test
# would quietly stop booting the image a rebuild installs.
talos_image_coords() {
    command -v yq >/dev/null || fail "yq not found"
    local image
    image=$(yq -r '.machine.install.image' "$TALOS_DIR/patches/cluster.yaml")
    VM_SCHEMATIC=${image#*metal-installer/}
    VM_SCHEMATIC=${VM_SCHEMATIC%%:*}
    VM_VERSION=${image##*:}
    [[ ${#VM_SCHEMATIC} -eq 64 && "$VM_VERSION" == v* ]] \
        || fail "could not read a schematic and version out of '$image'"
}

HCLOUD_LABEL="managed-by=bootstrap-hcloudtest"

# The disks a machine has are stated in its Talos patch and read from there, so
# the provisioner carries no second copy. Emits one size in whole GB per raw
# volume the patch declares off the system disk, in order.
hcloud_volumes_for_node() {
    local patch="$TALOS_DIR/patches/$1.yaml" size gib
    [[ -f "$patch" ]] || fail "$patch not found — the machine config is the list"
    while read -r size; do
        [[ -n "$size" && "$size" != "null" ]] || continue
        # Talos states GiB; the API bills GB. Round up, so the volume can never
        # come in under the minSize the machine config asked for.
        gib=${size%GiB}
        [[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as <n>GiB"
        echo $(( (gib * 1074 + 999) / 1000 ))
    done < <(yq -r 'select(.kind == "RawVolumeConfig" and .provisioning.diskSelector.match != "system_disk") | .provisioning.minSize' "$patch")
}

# Deletes everything this phase created, and NOTHING else.
#
# The project (homelab-backup-verify) also runs the backup-verify CronJob's
# machines. `hcloud server list -l ""` lists EVERY server, so an unset
# HCLOUD_LABEL would turn this function into a project wipe: the label is
# checked before it is used, and every candidate must also carry the name
# prefix this phase assigns.
HCLOUD_NAME_PREFIX="hcloud"
# Outlives the run: the test cluster's talosconfig and kubeconfig.
HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest"
# The public edge in front of production. The test cluster must not reach it.
HCLOUD_PROXY_HOST="proxy.renner.uno"
# Hetzner's limit on effective rules per firewall and direction.
HCLOUD_FW_RULE_LIMIT=500
# Hetzner's limit on prefixes in one rule (not documented; 101 is rejected).
HCLOUD_FW_RULE_PREFIXES=100

# The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when
# missing: a throwaway server in rescue mode writes the Image Factory hcloud
# image to its disk, which is then snapshotted. Matched by version and
# schematic. Called directly, not in $(...), so that fail stops the run.
hcloud_talos_snapshot() {
    local labels="os=talos,talos-version=$VM_VERSION,talos-schematic=${VM_SCHEMATIC:0:16}"
    local builder="$HCLOUD_NAME_PREFIX-imgbuild" key_dir addr ids listing
    listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \
        || fail "could not list the Talos snapshots"
    mapfile -t ids <<<"$listing"
    if [[ -n "${ids[0]:-}" ]]; then
        HCLOUD_IMAGE=${ids[0]}
        return 0
    fi
    echo "  no snapshot for $VM_VERSION/${VM_SCHEMATIC:0:16}, building one"
    key_dir=$(mktemp -d)
    ssh-keygen -q -t ed25519 -N '' -C "$builder" -f "$key_dir/key"
    if hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=name | grep -qx "$builder"; then
        hcloud ssh-key delete "$builder" >/dev/null || fail "could not replace the stale image builder key"
    fi
    hcloud ssh-key create --name "$builder" --label "$HCLOUD_LABEL" \
        --public-key-from-file "$key_dir/key.pub" >/dev/null || fail "could not upload the image builder key"
    hcloud server create --name "$builder" --type cx23 --image debian-12 \
        --location "${HCLOUD_LOCATION:-fsn1}" --ssh-key "$builder" --label "$HCLOUD_LABEL" >/dev/null \
        || fail "could not create the image builder"
    # From here a failure leaves a billed server: remove it before failing.
    builder_fail() {
        hcloud server delete "$builder" >/dev/null || echo "  WARNING: $builder not deleted" >&2
        hcloud ssh-key delete "$builder" >/dev/null || echo "  WARNING: ssh key $builder not deleted" >&2
        rm -rf "$key_dir"
        fail "$1"
    }
    hcloud server enable-rescue "$builder" --ssh-key "$builder" >/dev/null || builder_fail "could not enable rescue on $builder"
    hcloud server reset "$builder" >/dev/null || builder_fail "could not reset $builder into rescue"
    addr=$(hcloud server ip "$builder") || builder_fail "could not read the address of $builder"
    for i in $(seq 1 30); do
        ssh -i "$key_dir/key" -o BatchMode=yes -o ConnectTimeout=10 \
            -o UserKnownHostsFile="$key_dir/known_hosts" -o StrictHostKeyChecking=accept-new \
            "root@$addr" 'grep -q rescue /etc/hostname || hostname | grep -q rescue' && break
        sleep 10
        [[ $i -eq 30 ]] && builder_fail "$builder never came up in rescue"
    done
    ssh -i "$key_dir/key" -o BatchMode=yes -o UserKnownHostsFile="$key_dir/known_hosts" "root@$addr" \
        "set -eo pipefail; curl -fsSL https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz | xz -d | dd of=/dev/sda bs=4M conv=fsync" \
        || builder_fail "writing the Talos image on $builder failed"
    hcloud server poweroff "$builder" >/dev/null || builder_fail "could not power off $builder"
    hcloud server create-image "$builder" --type snapshot \
        --description "talos $VM_VERSION ${VM_SCHEMATIC:0:16}" --label "$labels" \
        || builder_fail "could not snapshot $builder"
    listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \
        || builder_fail "could not list the Talos snapshots"
    mapfile -t ids <<<"$listing"
    [[ -n "${ids[0]:-}" ]] || builder_fail "the snapshot of $builder is not listed under $labels"
    HCLOUD_IMAGE=${ids[0]}
    hcloud server delete "$builder" >/dev/null || echo "  WARNING: $builder not deleted" >&2
    hcloud ssh-key delete "$builder" >/dev/null || echo "  WARNING: ssh key $builder not deleted" >&2
    rm -rf "$key_dir"
}

# One address as a string of bits: 32 for IPv4, 128 for IPv6. Returns 1, with
# the reason on stderr, for anything else.
ip_to_bits() {
    local addr=$1 bits="" part v i head tail
    local -a parts=() hparts=() tparts=()
    if [[ "$addr" == *:* ]]; then
        if [[ "$addr" == *::* ]]; then
            head=${addr%%::*}; tail=${addr#*::}
            [[ -n "$head" ]] && IFS=: read -r -a hparts <<<"$head"
            [[ -n "$tail" ]] && IFS=: read -r -a tparts <<<"$tail"
            parts=("${hparts[@]}")
            for (( i = ${#hparts[@]} + ${#tparts[@]}; i < 8; i++ )); do parts+=(0); done
            parts+=("${tparts[@]}")
        else
            IFS=: read -r -a parts <<<"$addr"
        fi
        [[ ${#parts[@]} -eq 8 ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; }
        for part in "${parts[@]}"; do
            [[ "$part" =~ ^[0-9a-fA-F]{1,4}$ ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; }
            v=$(( 16#$part ))
            for (( i = 15; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done
        done
    else
        IFS=. read -r -a parts <<<"$addr"
        [[ ${#parts[@]} -eq 4 ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
        for part in "${parts[@]}"; do
            [[ "$part" =~ ^[0-9]{1,3}$ ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
            v=$(( 10#$part ))
            (( v <= 255 )) || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
            for (( i = 7; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done
        done
    fi
    printf '%s' "$bits"
}

# A prefix given as bits, in CIDR notation. $2 is the address length (32|128).
bits_to_cidr() {
    local p=$1 width=$2 full out="" i
    full=$p
    while (( ${#full} < width )); do full+="0"; done
    if (( width == 32 )); then
        for (( i = 0; i < 32; i += 8 )); do out+="${out:+.}$(( 2#${full:i:8} ))"; done
    else
        for (( i = 0; i < 128; i += 16 )); do out+="${out:+:}$(printf '%x' $(( 2#${full:i:16} )))"; done
    fi
    printf '%s/%s\n' "$out" "${#p}"
}

# cidr_complement <address>...: the prefixes that, with the given addresses
# of one family, cover 0.0.0.0/0 or ::/0 exactly once. Returns 1, with the
# reason on stderr, on a malformed or mixed-family list.
cidr_complement() {
    local width=32 a bits
    local -a excl=()
    [[ "$1" == *:* ]] && width=128
    for a in "$@"; do
        [[ ( "$a" == *:* && $width -eq 128 ) || ( "$a" != *:* && $width -eq 32 ) ]] \
            || { echo "cidr_complement: '$a' is not of the same family as '$1'" >&2; return 1; }
        bits=$(ip_to_bits "$a") || return 1
        excl+=("$bits")
    done
    _cidr_walk() {
        local p=$1 x hit=0
        for x in "${excl[@]}"; do
            [[ "$x" == "$p"* ]] && { hit=1; break; }
        done
        if (( hit == 0 )); then
            bits_to_cidr "$p" "$width"
            return 0
        fi
        (( ${#p} == width )) && return 0
        _cidr_walk "${p}0"
        _cidr_walk "${p}1"
    }
    _cidr_walk ""
}

# Addresses of HCLOUD_PROXY_HOST for one record type (A|AAAA), over DoH: the
# operator's LAN resolver intercepts plain DNS. Returns 1, with the reason on
# stderr, when the lookup fails.
hcloud_proxy_addrs() {
    local type=$1 num answer
    case "$type" in
        A) num=1 ;;
        AAAA) num=28 ;;
        *) echo "hcloud_proxy_addrs: unknown type $type" >&2; return 1 ;;
    esac
    answer=$(curl -sS --max-time 10 -H 'accept: application/dns-json' \
        "https://cloudflare-dns.com/dns-query?name=$HCLOUD_PROXY_HOST&type=$type") \
        || { echo "could not resolve $HCLOUD_PROXY_HOST $type over DoH" >&2; return 1; }
    jq -e '.Status == 0' >/dev/null <<<"$answer" \
        || { echo "DoH lookup of $HCLOUD_PROXY_HOST $type failed: $answer" >&2; return 1; }
    jq -r --argjson t "$num" '.Answer[]? | select(.type == $t) | .data' <<<"$answer" | sort -u
}

# Outbound firewall for the nodes: udp and icmp anywhere, tcp anywhere except
# HCLOUD_PROXY_HOST. Attached before an earlier run's egress firewall is
# removed, so no node is ever without one.
hcloud_apply_egress() {
    local fw="hcloudtest-node-egress-$$" v4 v6 c4 c6 n count fw_id fw_name srv_id
    local -a a4=() a6=() tcp_dst=()
    v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)"
    v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)"
    [[ -n "$v4" ]] && mapfile -t a4 <<<"$v4"
    [[ -n "$v6" ]] && mapfile -t a6 <<<"$v6"
    (( ${#a4[@]} + ${#a6[@]} > 0 )) || fail "$HCLOUD_PROXY_HOST resolved to no address — refusing to build the egress firewall"
    echo "  blocking tcp to $HCLOUD_PROXY_HOST: ${a4[*]} ${a6[*]}"
    if (( ${#a4[@]} > 0 )); then
        c4=$(cidr_complement "${a4[@]}") || fail "could not compute the IPv4 complement of ${a4[*]}"
    else
        c4=0.0.0.0/0
    fi
    if (( ${#a6[@]} > 0 )); then
        c6=$(cidr_complement "${a6[@]}") || fail "could not compute the IPv6 complement of ${a6[*]}"
    else
        c6=::/0
    fi
    mapfile -t tcp_dst <<<"$c4"$'\n'"$c6"
    # Effective rules are counted per destination prefix; 0.0.0.0/0 and ::/0
    # carry udp and icmp.
    count=$(printf '%s\n' 0.0.0.0/0 ::/0 "${tcp_dst[@]}" | sort -u | grep -c .)
    (( count < HCLOUD_FW_RULE_LIMIT )) \
        || fail "the egress firewall needs $count effective rules, Hetzner allows $HCLOUD_FW_RULE_LIMIT per firewall"
    hcloud firewall create --name "$fw" --label "$HCLOUD_LABEL" >/dev/null || fail "could not create $fw"
    hcloud firewall add-rule "$fw" --direction out --protocol udp --port any \
        --destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw"
    hcloud firewall add-rule "$fw" --direction out --protocol icmp \
        --destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw"
    for (( n = 0; n < ${#tcp_dst[@]}; n += HCLOUD_FW_RULE_PREFIXES )); do
        hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \
            --destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]:n:HCLOUD_FW_RULE_PREFIXES}")" \
            --description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw"
    done
    for n in "${HCLOUD_NODES[@]}"; do
        hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \
            || fail "could not attach $fw to $n"
    done
    while read -r fw_id fw_name; do
        [[ -n "$fw_id" && "$fw_name" == hcloudtest-node-egress-* && "$fw_name" != "$fw" ]] || continue
        while read -r srv_id; do
            [[ -n "$srv_id" ]] || continue
            hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \
                || fail "could not detach $fw_name from server $srv_id"
        done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id')
        hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name"
        echo "  removed $fw_name"
    done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
    echo "  $fw: $count effective rules"
}

# Runs restore-test/egress-check.yaml in the cluster KUBECONFIG names and fails
# unless every expectation holds.
hcloud_egress_check() {
    local v4 v6 addrs phase=""
    step 0 "Checking egress containment from a pod..."
    v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)"
    v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)"
    addrs=$(printf '%s\n' "$v4" "$v6" | awk 'NF' | paste -sd' ' -)
    [[ -n "$addrs" ]] || fail "$HCLOUD_PROXY_HOST resolved to no address"
    kubectl delete pod egress-check --ignore-not-found --wait=true >/dev/null \
        || fail "could not remove the previous egress-check pod"
    sed "s|PROXY_ADDRS_PLACEHOLDER|$addrs|" "$REPO_ROOT/kubernetes/restore-test/egress-check.yaml" \
        | kubectl apply -f - >/dev/null || fail "could not start the egress check"
    for i in $(seq 1 48); do
        phase=$(kubectl get pod egress-check -o jsonpath='{.status.phase}') || phase=""
        [[ "$phase" == Succeeded || "$phase" == Failed ]] && break
        sleep 5
    done
    if [[ "$phase" != Succeeded && "$phase" != Failed ]]; then
        kubectl describe pod egress-check | sed 's/^/  /' || true
        fail "the egress check never finished (phase '${phase:-none}') — do not start Flux"
    fi
    kubectl logs egress-check | sed 's/^/  /' || true
    [[ "$phase" == "Succeeded" ]] || fail "egress containment does not hold — do not start Flux"
    kubectl delete pod egress-check --wait=false >/dev/null
    ok
}

cleanup_hcloudtest() {
    command -v hcloud >/dev/null || return 0
    [[ -n "${HCLOUD_TOKEN:-}" ]] || return 0
    [[ -n "${HCLOUD_LABEL:-}" ]] \
        || { echo "  REFUSING to sweep: HCLOUD_LABEL is empty, which would match everything" >&2; return 0; }

    local id name
    # Servers first: a firewall still attached to one cannot be deleted.
    while read -r id name; do
        [[ -n "$id" ]] || continue
        case "$name" in
            "$HCLOUD_NAME_PREFIX"-*) ;;
            *) echo "  REFUSING to delete $name ($id): carries the label but not the name prefix" >&2; continue ;;
        esac
        echo "  deleting server $name"
        hcloud server delete "$id" >/dev/null \
            || echo "  WARNING: server $name not deleted — check the console" >&2
    done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)

    for id in $(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
        echo "  deleting firewall $id"
        hcloud firewall delete "$id" >/dev/null \
            || echo "  WARNING: firewall $id not deleted — check the console" >&2
    done
    for id in $(hcloud volume list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
        echo "  deleting volume $id"
        hcloud volume delete "$id" >/dev/null \
            || echo "  WARNING: volume $id not deleted — check the console" >&2
    done
    for id in $(hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
        echo "  deleting ssh key $id"
        hcloud ssh-key delete "$id" >/dev/null \
            || echo "  WARNING: ssh key $id not deleted — check the console" >&2
    done
    # Last: a network with a server still attached cannot be deleted, and a
    # leftover one takes its name with it, so the next run cannot recreate it.
    for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
        echo "  deleting network $id"
        hcloud network delete "$id" >/dev/null \
            || echo "  WARNING: network $id not deleted — check the console" >&2
    done
}

# The restore test's source, as restore-test/flux-root.yaml names it: sets
# restore_url and restore_repo (<owner>/<repo>).
restore_test_repo() {
    command -v yq >/dev/null || fail "yq not found (reads restore-test/flux-root.yaml)"
    restore_url=$(yq -r 'select(.kind == "GitRepository" and .metadata.name == "restore-test") | .spec.url' \
        "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml")
    [[ "$restore_url" =~ ^ssh://git@github\.com/([^/]+)/([^/]+)\.git$ ]] \
        || fail "restore-test/flux-root.yaml names '$restore_url', not ssh://git@github.com/<owner>/<repo>.git"
    restore_repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}"
}

# Sweep only. For the case where a previous run was killed hard enough that its
# own trap never ran, and for a cron job to call before it starts.
if [[ "$PHASE" == "hcloudclean" ]]; then
    echo -e "${YELLOW}=== Phase: Hetzner cleanup ===${NC}"
    command -v hcloud >/dev/null || fail "hcloud not found"
    command -v bao >/dev/null    || fail "bao not found (the Hetzner token is read from OpenBao)"
    require_secrets hcloud-token
    HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN
    cleanup_hcloudtest
    require_secrets s3-access s3-secret
    delete_cnpg_test_bucket
    # The restore test's read-only deploy keys on the GitHub mirror.
    if command -v gh >/dev/null; then
        restore_test_repo
        for key_id in $(gh api "repos/$restore_repo/keys" \
                --jq '.[] | select(.title | startswith("restore-test ")) | .id'); do
            echo "  deleting deploy key $key_id"
            gh api -X DELETE "repos/$restore_repo/keys/$key_id" \
                || echo "  WARNING: deploy key $key_id not deleted — check the repo settings" >&2
        done
    else
        echo "  WARNING: gh not found — restore-test deploy keys on the GitHub mirror not checked" >&2
    fi
    ok
    exit 0
fi

if [[ "$PHASE" == "hcloudtest" ]]; then
    echo -e "${YELLOW}=== Phase: Hetzner rebuild test ===${NC}"
    command -v hcloud >/dev/null   || fail "hcloud not found — https://github.com/hetznercloud/cli"
    command -v talosctl >/dev/null || fail "talosctl not found"
    command -v yq >/dev/null       || fail "yq not found"
    command -v jq >/dev/null       || fail "jq not found (needed to read the SAN pool and server attachments)"
    command -v curl >/dev/null     || fail "curl not found (needed to learn this host's public address)"
    command -v bao >/dev/null      || fail "bao not found (the Hetzner token is read from OpenBao)"
    command -v kubectl >/dev/null  || fail "kubectl not found (runs the egress check)"

    # Three control planes, all carrying workloads — the production shape. One
    # machine is a node, not a cluster: etcd has no quorum to lose and nothing
    # schedules elsewhere, so a rebuild that only ever ran on a single box
    # proves nothing about the one that has three.
    HCLOUD_NODES=(hcloud-1 hcloud-2 hcloud-3)
    for n in "${HCLOUD_NODES[@]}"; do
        [[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
    done
    [[ -f "$TALOS_DIR/patches/site-hcloud.yaml" ]] || fail "site-hcloud.yaml not found"

    require_secrets hcloud-token ghcr-pull
    HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN

    # No sweep before creating: machines from an earlier run are the input to
    # this one, adopted below. The names are fixed, so duplicates cannot
    # accumulate. `hcloudclean` is what deletes.

    step 0 "Looking up the Talos snapshot..."
    talos_image_coords
    HCLOUD_IMAGE=""
    hcloud_talos_snapshot
    [[ -n "$HCLOUD_IMAGE" ]] || fail "no Talos snapshot"
    echo "  snapshot $HCLOUD_IMAGE"

    # The Talos API is unauthenticated in maintenance mode, which is how
    # apply-config reaches a node that has no config yet. On a public address
    # that has to be closed to everyone but this host for the minutes it takes.
    step 0 "Creating a firewall for this host only..."
    # Both families, and each with its own prefix length. The first version
    # asked ifconfig.me once and appended /32: this host answered over IPv6, and
    # Hetzner rejected `2003:d6:…:86d2/32` because /32 of that address is not a
    # network boundary. A single-address rule is /32 for v4 and /128 for v6.
    #
    # Both are added rather than whichever happened to answer, because the rule
    # has to admit the address talosctl actually leaves from, and that is chosen
    # per connection by the resolver — not by whatever this lookup returned.
    # No `local` here: this phase runs in the script body, not in a function,
    # and `local` is a runtime error there — one `bash -n` does not catch.
    # One family failing is ordinary — a host without IPv6 is not broken — so
    # neither attempt aborts the run on its own. Their stderr is kept rather
    # than discarded, and reported if BOTH fail: at that point the reason is the
    # diagnosis, and "could not determine this host's address" without it sends
    # you looking at the wrong thing.
    fw_sources=()
    fw_err=$(mktemp)
    v4=$(curl -4 -fsS --max-time 10 https://ifconfig.me 2>>"$fw_err") && [[ -n "$v4" ]] && fw_sources+=("$v4/32")
    v6=$(curl -6 -fsS --max-time 10 https://ifconfig.me 2>>"$fw_err") && [[ -n "$v6" ]] && fw_sources+=("$v6/128")
    if [[ ${#fw_sources[@]} -eq 0 ]]; then
        cat "$fw_err" >&2
        rm -f "$fw_err"
        fail "could not determine this host's public address over either IPv4 or IPv6"
    fi
    rm -f "$fw_err"
    fw_src=$(IFS=,; echo "${fw_sources[*]}")

    hcloud firewall create --name "hcloudtest-$$" --label "$HCLOUD_LABEL" >/dev/null \
        || fail "could not create the firewall"
    hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 50000 \
        --source-ips "$fw_src" --description "Talos API, this host only" >/dev/null \
        || fail "could not add the Talos API rule"
    hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 6443 \
        --source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \
        || fail "could not add the Kubernetes API rule"
    echo "  open to $fw_src only"
    # Servers held from an earlier run get this firewall first; only then are
    # the earlier runs' inbound firewalls detached and deleted, so no server is
    # ever without one. Egress firewalls are replaced by hcloud_apply_egress.
    while read -r srv; do
        [[ -n "$srv" ]] || continue
        hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$srv" >/dev/null \
            || fail "could not attach this run's firewall to $srv"
    done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name)
    while read -r fw_id fw_name; do
        [[ -n "$fw_id" && "$fw_name" != "hcloudtest-$$" && "$fw_name" != hcloudtest-node-egress-* ]] || continue
        while read -r srv_id; do
            [[ -n "$srv_id" ]] || continue
            hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \
                || fail "could not detach $fw_name from server $srv_id"
        done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id')
        hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name"
        echo "  removed $fw_name"
    done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
    ok

    # Default type cpx52 (12 vCPU, 24 GB, 480 GB local disk): the hcloud-N
    # patches put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system
    # disk. cax* are ARM and do not match the amd64 schematic.
    #
    # Two segments only, unlike the homelab's split in
    # NETWORK-traffic-classes.md: here the node address carries management,
    # cluster-internal and external alike, and Ceph replication has its own.
    #
    # The order is load-bearing: it decides which kernel interface each segment
    # becomes, and the machine config names those interfaces.
    HCLOUD_SEGMENTS=(cluster storage)
    # Segment ranges are read from what the cluster declares, never invented
    # here — the disks' rule applied to networks:
    #   cluster  the subnet the site patch pins node addresses to.
    #   storage  the SAN pool the ceph-san attachment allocates from, widened
    #            to the containing /24 because the fabric wants a subnet.
    # The fabric reserves the first address of a network range as its gateway.
    # For the storage net that is the SAN pool's first address — a collision
    # that matters only if cross-node SAN traffic on this site is ever
    # attempted, and is left unresolved until it is.
    seg_range() {
        case "$1" in
            cluster)
                yq -r 'select(.machine) | .machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;;
            storage)
                local pool
                pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range')
                [[ "$pool" == *.0/2[4-9] ]] \
                    || fail "cannot derive a /24 from the SAN pool '$pool'"
                printf '%s/24' "${pool%.*}.0" ;;
        esac
    }
    # Read from the location rather than stated a second time.
    HCLOUD_ZONE=$(hcloud location describe "${HCLOUD_LOCATION:-fsn1}" -o format='{{.NetworkZone}}') \
        || fail "could not read the network zone of ${HCLOUD_LOCATION:-fsn1}"
    [[ -n "$HCLOUD_ZONE" ]] \
        || fail "${HCLOUD_LOCATION:-fsn1} reported no network zone"

    step 0 "Creating ${#HCLOUD_SEGMENTS[@]} private network(s) in $HCLOUD_ZONE..."
    for seg in "${HCLOUD_SEGMENTS[@]}"; do
        net="hcloudtest-$seg"
        want_range=$(seg_range "$seg")
        [[ -n "$want_range" && "$want_range" != null* ]] \
            || fail "could not derive the $seg range from the cluster config"
        # Reused only if the range matches. A stale one fails later, at the
        # attachment, naming the wrong thing.
        if have_range=$(hcloud network describe "$net" -o format='{{.IPRange}}'); then
            if [[ "$have_range" == "$want_range" ]]; then
                echo "  $net (already there)"
                continue
            fi
            echo "  $net carries $have_range, this run wants $want_range — replacing"
            hcloud network delete "$net" >/dev/null \
                || fail "could not replace $net — detach its servers first, or run hcloudclean"
        fi
        hcloud network create --name "$net" --ip-range "$want_range" \
            --label "$HCLOUD_LABEL" >/dev/null \
            || fail "could not create network $net"
        hcloud network add-subnet "$net" --network-zone "$HCLOUD_ZONE" \
            --type cloud --ip-range "$want_range" >/dev/null \
            || fail "could not add a subnet to $net"
        echo "  $net  $want_range"
    done
    ok

    # From here on machines exist and are billed; on_exit says so.
    HCLOUD_PROVISIONED=1
    step 0 "Creating ${#HCLOUD_NODES[@]} server(s)..."
    NODE_ADDRS=()
    # Servers this test already holds. Capacity is the binding constraint, so a
    # machine in hand is worth keeping across runs.
    #
    # Listed by label: a same-named server without it falls through to the
    # create path and fails on "name already used", rather than being adopted
    # and wiped.
    existing_servers=$(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name)
    for n in "${HCLOUD_NODES[@]}"; do
        if grep -qx "$n" <<<"$existing_servers"; then
            # Rebuilt, not reused as-is: a configured Talos rejects a fresh
            # apply-config, and `talosctl reset` needs the previous run's PKI,
            # which is gone. Rebuilding from the snapshot returns the node to
            # maintenance mode, reachable without any old secret.
            echo "  $n: reusing the server we already hold"
            hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \
                || fail "could not rebuild $n back into maintenance mode"
            # This run's firewall was attached above, before the old ones went.
        else
            # Retried, because "error during placement (resource_unavailable)"
            # is the provider being momentarily out of that type in that
            # location, not a wrong request. It happened on the first cx43 run
            # and the same type in the same location created fine minutes later.
            # For a test meant to run on a schedule that distinction matters:
            # without the retry every capacity blip is a red run someone has to
            # come and look at.
            create_err=""
            for attempt in $(seq 1 30); do
                create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \
                    --type "${HCLOUD_TYPE:-cpx52}" --location "${HCLOUD_LOCATION:-fsn1}" \
                    --firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break
                grep -qi 'resource_unavailable\|placement' <<<"$create_err" \
                    || fail "could not create $n: ${create_err//$'\n'/ | }"
                [[ $attempt -eq 1 ]] && echo "  $n: no capacity yet, retrying"
                sleep 20
                [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cpx52} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes"
            done
        fi
        addr=$(hcloud server ip "$n") || fail "could not read the address of $n"
        [[ -n "$addr" ]] || fail "$n has no public address"

        # Attached in HCLOUD_SEGMENTS order; the interface names follow from it.
        # An adopted server keeps its attachments through a rebuild, and
        # attaching twice is an error, so an existing one is skipped.
        for seg in "${HCLOUD_SEGMENTS[@]}"; do
            net="hcloudtest-$seg"
            net_id=$(hcloud network describe "$net" -o json | jq -r '.id')
            if hcloud server describe "$n" -o json \
                | jq -e --argjson id "$net_id" 'any(.private_net[]; .network == $id)' >/dev/null; then
                continue
            fi
            hcloud server attach-to-network "$n" --network "$net" >/dev/null \
                || fail "could not attach $n to $net"
        done

        # The OSD device. Every node carries one, which is the production shape
        # — k8s-1/2/3 all hold storage — and is only possible here because
        # Hetzner attaches volumes to any server, control plane included. The
        # qemu provisioner cannot, which is why that site has to split into one
        # control plane and three workers.
        #
        #
        # Neither formatted nor mounted. Rook needs the raw device, and an
        # --automount volume arrives with a filesystem on it that makes Ceph
        # skip the disk as already in use.
        #
        # Recreated rather than reused, even on an adopted server: a volume from
        # the previous run carries a bluestore signature, and Rook adopts a
        # device that already looks like an OSD. Volumes are not the scarce
        # resource here; servers are.
        if hcloud volume describe "$n-osd" >/dev/null; then
            echo "  $n: discarding the previous OSD volume"
            # An unattached leftover is ordinary after an interrupted run, so
            # this failure is tolerated while the delete below is not.
            hcloud volume detach "$n-osd" >/dev/null || true
            hcloud volume delete "$n-osd" >/dev/null \
                || fail "could not delete the previous OSD volume for $n"
        fi
        yq -e 'select(.kind == "RawVolumeConfig" and .name == "osd-data")' "$TALOS_DIR/patches/$n.yaml" >/dev/null \
            || fail "$n declares no osd-data RawVolumeConfig — a node with no OSD device is not this cluster's shape"
        # A raw volume on the system disk needs no Hetzner volume.
        vol_size=$(hcloud_volumes_for_node "$n")
        if [[ -n "$vol_size" ]]; then
            hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \
                --label "$HCLOUD_LABEL" >/dev/null \
                || fail "could not create the OSD volume for $n"
            echo "  $n at $addr, with a ${vol_size}GB OSD volume"
        else
            echo "  $n at $addr, OSD on the system disk"
        fi
        NODE_ADDRS+=("$addr")
    done
    ok

    # The nodes have to reach each other, and until now the firewall admitted
    # only this host. One node did not care; three do — etcd peers on 2380, the
    # apiserver, kubelet on 10250 and Cilium's VXLAN all run between them, and
    # with the rules above alone etcd simply never forms a quorum.
    #
    # Added after creation because the addresses do not exist before it. Rules
    # apply to an attached firewall immediately, so the ordering costs nothing.
    #
    # All ports between the nodes rather than an enumerated list: this is a
    # throwaway cluster whose members are each other's only peers, and guessing
    # the full port set of Talos plus Cilium is how a rebuild fails on the one
    # port nobody wrote down. The outside still sees only 50000 and 6443, and
    # only from this host.
    step 0 "Opening the cluster to itself..."
    node_src=$(IFS=,; printf '%s' "${NODE_ADDRS[*]/%//32}")
    for proto in tcp udp; do
        hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol "$proto" \
            --port any --source-ips "$node_src" \
            --description "intra-cluster $proto" >/dev/null \
            || fail "could not open $proto between the nodes"
    done
    echo "  $node_src"
    ok

    # Before any node is configured.
    step 0 "Closing tcp egress to $HCLOUD_PROXY_HOST (Hetzner firewall)..."
    hcloud_apply_egress
    ok

    # Handover: the variables the vmtest phase sets, and HCLOUDTEST.
    SITE=hcloud
    NODES=("${HCLOUD_NODES[@]}")
    WORKERS=()
    ALL_NODES=("${NODES[@]}")
    K8S_API_HOST="${NODE_ADDRS[0]}"
    HCLOUDTEST=1

    # Isolation from the production cluster. Both halves are required: the
    # _PATH variables are derived before any phase runs, and the talos phase
    # writes to those.
    mkdir -p "$HCLOUD_STATE_DIR"
    chmod 700 "$HCLOUD_STATE_DIR"
    export TALOSCONFIG="$HCLOUD_STATE_DIR/talosconfig"
    export KUBECONFIG="$HCLOUD_STATE_DIR/kubeconfig"
    TALOSCONFIG_PATH="$TALOSCONFIG"
    KUBECONFIG_PATH="$KUBECONFIG"

    PHASE=talos
fi

# Provision throwaway QEMU VMs, then run the Talos phase below against
# them unchanged. Deliberately not a copy of that phase: it sets the site
# variables and falls through, because a rehearsal that runs different code
# proves less than one that does not. Everything it changes is listed here — if
# that list grows, the test is drifting away from what it claims to exercise.
#
# What it cannot cover, and what therefore still has to be true on hardware:
# Ceph (the provisioner gives extra disks to workers only), the mgmt and v30iot
# VLANs and the Layer2 VIP (one flat NIC per VM), and Flux (its own phase).
#
# Needs root for the QEMU provisioner (CNI and KVM), and only for that step.
if [[ "$PHASE" == "vmtest" ]]; then
    echo -e "${YELLOW}=== Phase: VM rebuild test ===${NC}"
    command -v talosctl >/dev/null || fail "talosctl not found"
    command -v yq >/dev/null       || fail "yq not found"
    command -v jq >/dev/null       || fail "jq not found (needed to read OpenBao's seal status)"
    # The node patches are the list; the VM count follows from them rather than
    # being stated twice. vm-1 is the control plane, the rest are the workers
    # that carry an OSD device — see the comment on NODES below for why.
    VM_CP_NODES=(vm-1)
    VM_WORKER_NODES=(vm-2 vm-3 vm-4)
    VM_CPS=${#VM_CP_NODES[@]}
    VM_WORKER_COUNT=${#VM_WORKER_NODES[@]}
    for n in "${VM_CP_NODES[@]}" "${VM_WORKER_NODES[@]}"; do
        [[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
    done
    [[ -f "$TALOS_DIR/patches/site-qemu.yaml" ]] || fail "site-qemu.yaml not found"

    # Talos version and Image Factory schematic, from the installer image in
    # cluster.yaml — see talos_image_coords above for why neither phase states
    # them itself.
    talos_image_coords

    # sun_path caps a unix socket path at 108 bytes and the provisioner puts a
    # qemu monitor socket under the state directory, so this has to stay short.
    # A path under the repo or a long mktemp dir silently fails to boot.
    # Overridable because in-cluster the VM disks belong on the mounted scratch
    # volume, not on the container's own writable layer. Keep whatever is set
    # SHORT for the reason above.
    VM_STATE="${VM_STATE:-/tmp/bootstrap-vmtest}"
    [[ ${#VM_STATE} -le 50 ]] \
        || fail "VM_STATE '$VM_STATE' is ${#VM_STATE} bytes; keep it under 50 so the qemu monitor socket fits in sun_path's 108"
    # cleanup_vmtest runs `sudo rm -rf "$VM_STATE"`, so refuse anything that is
    # not a nested absolute path.
    [[ $VM_STATE == /*/?* ]] \
        || fail "VM_STATE '$VM_STATE' must be an absolute path at least two levels deep"
    VM_CLUSTER=bootstrap-vmtest
    VM_DIR=$(mktemp -d)
    TALOSCTL_BIN=$(command -v talosctl)

    # Hard isolation from the production cluster: talosctl and kubectl both
    # honour these, and the phase below writes to them rather than to $HOME.
    export TALOSCONFIG="$VM_DIR/talosconfig"
    export KUBECONFIG="$VM_DIR/kubeconfig"
    TALOSCONFIG_PATH="$TALOSCONFIG"
    KUBECONFIG_PATH="$KUBECONFIG"

    # The provisioner needs root for KVM and for the bridges and taps it builds.
    # On a workstation that means sudo; in the Job that runs this in-cluster the
    # process is already root and sudo is not installed, so the elevation is a
    # variable rather than a literal.
    SUDO_KEEPALIVE=""
    if [[ $EUID -eq 0 ]]; then
        SUDO=""
    else
        SUDO="sudo -E"
        # A full run outlasts sudo's timestamp, and the teardown at the end then
        # prompts for a password with nobody watching — the VMs survive and the
        # next run collides with them. Take it once here and keep the timestamp
        # warm for as long as the phase lasts.
        sudo -v || fail "the QEMU provisioner needs root (CNI and KVM)"
        ( while sudo -n true; do sleep 50; done ) &
        SUDO_KEEPALIVE=$!
    fi

    cleanup_vmtest() {
        [[ -n "$SUDO_KEEPALIVE" ]] && kill "$SUDO_KEEPALIVE" >/dev/null || true
        # Diagnosing anything inside the VMs is impossible once they are gone,
        # and the teardown fires the moment a run ends — including when it ends
        # badly, which is exactly when the VMs are worth looking at. VMTEST_KEEP
        # holds them, and the pod, until someone deletes the Job.
        if [[ "${VMTEST_KEEP:-0}" == "1" ]]; then
            echo -e "${YELLOW}VMTEST_KEEP=1 — leaving the VMs up.${NC}"
            echo "  talosctl --talosconfig $TALOSCONFIG -n 10.5.0.2 ..."
            echo "  delete the Job when finished; nothing else reclaims this."
            sleep infinity
        fi
        echo -n "Destroying throwaway VMs... "
        $SUDO "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
        # destroy removes the cluster's own directory but leaves the state root.
        $SUDO rm -rf "$VM_STATE"
        rm -rf "$VM_DIR"
        echo "done"
    }
    # No `trap ... EXIT` here — on_exit calls cleanup_vmtest once it is defined.

    # A run killed hard never reaches its EXIT trap, so the previous cluster can
    # still be holding the provisioner's CIDR. Clear it first rather than
    # failing on a collision that looks like a networking problem.
    if [[ -d "$VM_STATE/$VM_CLUSTER" ]]; then
        echo -n "Removing a cluster left behind by an earlier run... "
        $SUDO "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
        $SUDO rm -rf "$VM_STATE"
        echo "done"
    fi

    step 0 "Creating $VM_CPS control plane and $VM_WORKER_COUNT workers in maintenance mode..."
    # maintenance means the provisioner applies no configuration of its own, so
    # every byte of machine config below comes from talos/patches.
    #
    # The control plane gets 6GiB. At 2560MiB it ran with ~250MiB available and
    # its page cache squeezed to 386MiB, so every etcd fsync hit a system in
    # permanent reclaim: linearized reads took 5-20s against an expected 100ms,
    # the apiserver became unreachable, and the run stalled with no error to
    # read. That failure says nothing about whether the repo can rebuild a
    # cluster, so it is bought off with memory rather than tuned around.
    #
    # The workers get 8GiB for the same reason. At 4GiB a storage node sat at
    # 99% of its memory requests once it carried an OSD, a mon and a mgr, and
    # what could not be scheduled any more was kube-multus-ds and whereabouts —
    # the CNI itself. Without Multus the OSDs came up with no ceph-san
    # interface and crash-looped on "unable to find any IPv4 address in
    # networks 10.99.0.0/29", which reads like an addressing bug and is a
    # memory one. Sizing the VMs to the workload they actually run is cheaper
    # than shrinking the Ceph requests, which would test something other than
    # what production runs.
    #
    # Workers get 4 CPUs, one constraint later: with memory solved they sat at
    # 69% of CPU requests, and what could no longer schedule was the
    # rook-ceph-operator itself. Nothing was left driving the cluster — two
    # OSDs crash-looped unattended and the PVCs waited on a provisioner whose
    # operator was Pending.
    #
    # The disks are sized the same way. At 10GiB the system disk yielded an
    # 8.36GB /var, which the infrastructure image set filled to 82%: kubelet
    # tainted all four nodes node.kubernetes.io/disk-pressure, no pod could be
    # scheduled any more, and Ceph's own monitor sat Pending while the symptom
    # surfaced three layers away as a Cilium HelmRelease stuck InProgress. The
    # second disk is the Ceph OSD; at 8GiB, three of them replicated 3x left
    # 8GiB usable, less than OpenBao's two PVCs alone ask for. Both are raw
    # sparse files, so the larger geometry costs only what is actually written.
    #
    # The control plane gets 16GiB. 6 was not enough and neither was 8: at both
    # it reached 99% of allocatable memory once the infrastructure layer was up,
    # and a control plane with nothing left over does not stay a control plane.
    # The kubelet admits the node-critical CNI DaemonSets by evicting whatever
    # ranks below them, and kube-apiserver — a static pod at
    # system-cluster-critical — is what ranks below them:
    #
    #   message: Pod was preempted by Kubelet to accommodate a critical pod.
    #   reason:  Preempting
    #
    # The priority ordering is not the defect and must not be tuned away. It is
    # deliberate: a node without its CNI cannot run anything, so the CNI wins.
    # What the eviction reports is the shortfall underneath it, and the only
    # honest fix is to remove the shortfall. Production does this by having
    # ~95GiB per node; the throwaway cluster does it here.
    #
    # This is not a spare-capacity argument for the workers. vm-1 is the only
    # control plane and still takes workload, so it carries the static pods, the
    # CNI, and its share of the infrastructure layer at once — which is the
    # asymmetry with production, where every node is a control plane and none of
    # them is small.
    $SUDO "$TALOSCTL_BIN" cluster create qemu \
        --name "$VM_CLUSTER" \
        --state "$VM_STATE" \
        --presets iso,maintenance \
        --controlplanes "$VM_CPS" \
        --workers "$VM_WORKER_COUNT" \
        --talos-version "$VM_VERSION" \
        --schematic-id "$VM_SCHEMATIC" \
        --memory-controlplanes 16GiB \
        --memory-workers 8GiB \
        --cpus-controlplanes 2 \
        --cpus-workers 4 \
        --disks virtio:40GiB,virtio:24GiB \
        --talosconfig-destination "$TALOSCONFIG" \
        || fail "could not create the throwaway VMs"
    $SUDO chown -R "$(id -u):$(id -g)" "$VM_STATE" "$TALOSCONFIG"
    ok

    # The provisioner hands out its CIDR sequentially from .2; .1 is the
    # gateway and the load balancer it puts in front of the API. If that ever
    # changes, the apply loop below reports which address stopped answering
    # rather than failing silently.
    # One control plane and three workers, not three control planes: the
    # provisioner gives the second disk only to workers ("disks after the first
    # one are added only to worker machines"), and an OSD needs it. Three of
    # them because the CephCluster runs three mons with allowMultiplePerNode
    # false over pools that are replica 3 across failureDomain host.
    NODES=("${VM_CP_NODES[@]}")
    WORKERS=("${VM_WORKER_NODES[@]}")
    ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
    NODE_ADDRS=()
    for i in $(seq 0 $(( ${#ALL_NODES[@]} - 1 ))); do
        NODE_ADDRS+=("10.5.0.$(( i + 2 ))")
    done
    SITE=qemu
    K8S_API_HOST=10.5.0.1
    VMTEST=1

    # Run the real phases, not a reimplementation of them.
    #
    # Default is the Talos phase alone: that needs nothing but the machines, so
    # it is the part that can run unattended. VMTEST_FULL=1 continues into flux
    # and wait-apps — the whole bootstrap against VMs — which additionally needs
    # GITEA_TOKEN and a FLUX_BRANCH to write to, because `flux bootstrap`
    # commits its sync manifests into the branch it reconciles.
    if [[ "${VMTEST_FULL:-0}" == "1" ]]; then
        [[ "$FLUX_BRANCH" != "master" ]] \
            || fail "VMTEST_FULL would have flux bootstrap commit into master — point FLUX_BRANCH at a throwaway branch that already exists"
        echo -e "${YELLOW}Continuing into the flux phase against branch ${FLUX_BRANCH}.${NC}"
        PHASE=all
    else
        PHASE=talos
    fi
fi

# Before anything destructive, and after the vmtest block has decided which
# phases will run. A rebuild that will need these has no business wiping disks
# first, only to discover an hour later that pass is locked or a token is unset —
# by then the old cluster is gone and the new one cannot be finished.
#
# In the Job this is where the file-waiting branch blocks, which is the right
# moment too: the operator has just started the run and is watching it.
if phase_runs flux; then
    require_secrets
fi

if phase_runs talos; then
    echo -e "${YELLOW}=== Phase: Talos bootstrap ===${NC}"
    command -v talosctl >/dev/null || fail "talosctl not found"
    command -v kubectl >/dev/null  || fail "kubectl not found"
    command -v helm >/dev/null   || fail "helm not found (needed to install the CNI)"
    command -v yq >/dev/null     || fail "yq not found (needed to read the Cilium HelmRelease)"
    [[ -f "$TALOS_DIR/patches/cluster.yaml"         ]] || fail "cluster.yaml patch not found"
    [[ -f "$TALOS_DIR/patches/containerd-layer-retention.yaml" ]] || fail "containerd-layer-retention.yaml patch not found"
    [[ -f "$CILIUM_HR"                              ]] || fail "cilium helmrelease not found at $CILIUM_HR"

    step 0 "Generating Talos configs..."
    TALOS_GEN_DIR=$(mktemp -d)
    talosctl gen config homelab "https://${K8S_API_HOST}:6443" \
        --output "$TALOS_GEN_DIR" \
        --force
    mkdir -p "$(dirname "$TALOSCONFIG_PATH")"
    cp "$TALOS_GEN_DIR/talosconfig" "$TALOSCONFIG_PATH"

    # Lets a node pull the images it needs before the registry that normally
    # serves them exists.
    #
    # Several apps need an image while the cluster is still being built —
    # paperless's fetch-bridge-cert init container, zigbee2mqtt's seed-data init
    # container, the trmnl renderer, and the CNI plugins every pod depends on.
    # They are referenced at gitea.renner.uno, which is correct for normal
    # operation and useless during a rebuild, because Gitea runs inside the
    # cluster being rebuilt.
    #
    # `mirrors` gives containerd an ordered list to try. Gitea is named first, so
    # nothing changes once the cluster is up: pulls stay local and never leave
    # the network. ghcr.io is second and only answers when the first cannot,
    # which is exactly the rebuild window. The manifests therefore never name an
    # external registry — the fallback is a property of the node, not of the
    # deployment.
    #
    # Containerd keeps the repository path when it falls through, which is why
    # the images are published under konairius/ on BOTH registries
    # (.woodpecker/build-ci-images.yml). Verified equal: the same build pushes
    # one digest to both paths.
    #
    # The ghcr namespace is private, so it needs a credential — and the usual
    # place for one, an imagePullSecret, fails here twice over: it would have to
    # exist in five namespaces before any of them exists, and it would come from
    # OpenBao, which needs Ceph, which needs these very images.
    # machine.registries.config is containerd's own auth, in place before the
    # first pod is scheduled and applying to every pull on the node regardless
    # of namespace. Field names follow .docker/config.json
    # (pkg/machinery/config/types/v1alpha1, RegistryAuthConfig).
    #
    # The patch is written into the in-memory secrets dir, never into the repo
    # or the generated config directory, and dies with the Job like the material
    # it carries.
    registry_auth_patch() {
        local out="$OPENBAO_SECRETS_DIR/registry-auth.yaml" token
        token=$(read_secret ghcr-pull) || fail "could not read the GHCR pull token"
        ( umask 077; cat > "$out" <<EOF
machine:
  registries:
    mirrors:
      gitea.renner.uno:
        endpoints:
          - https://gitea.renner.uno
          - https://ghcr.io
    config:
      ghcr.io:
        auth:
          username: "${GHCR_PULL_USER}"
          password: "${token}"
EOF
        ) || fail "could not write the registry auth patch"
        printf '%s' "$out"
    }
    # OUTSIDE the command substitution below, and that placement is the fix for
    # a bug this had: require_secrets prints "All secrets present." on stdout,
    # and calling it inside registry_auth_patch made that line part of the
    # captured path. talosctl then reported
    #   open All secrets present. | /tmp/…/registry-auth.yaml: no such file
    # and retried 60 times while the node sat in maintenance. A function whose
    # value is its stdout must be the only thing writing there.
    #
    # This phase can run on its own (`bootstrap.sh talos`), in which case
    # nothing has demanded the secrets yet — the full require_secrets is gated
    # on the flux phase. Ask for the one this needs, and only that one: pulling
    # in the whole list would make an unattended talos run depend on the OpenBao
    # seal key, which never reaches CI.
    require_secrets ghcr-pull
    REGISTRY_PATCH=$(registry_auth_patch)
    [[ -s "$REGISTRY_PATCH" ]] \
        || fail "the registry auth patch was not written to a readable file: '$REGISTRY_PATCH'"

    step 0 "Applying Talos configs to nodes..."
    apply_node_config() {
        local node=$1 base_config=$2 fixed_addr=${3:-}
        local fqdn="${node}.${DOMAIN}"
        echo "  ${fixed_addr:-$fqdn}:"
        for i in $(seq 1 60); do
            # A fixed address skips DNS. Real hardware has a name to resolve;
            # a throwaway VM has an address and nothing else.
            if [[ -n "$fixed_addr" ]]; then
                node_ip="$fixed_addr"
            else
                node_ip=$(resolve_node_ip "$fqdn")
            fi
            if [[ -z "$node_ip" ]]; then
                echo "    attempt $i/60: DNS returned no IP"
                sleep 10
                continue
            fi
            if err=$(talosctl apply-config --insecure -n "$node_ip" \
                --file "$base_config" \
                --config-patch "@$TALOS_DIR/patches/cluster.yaml" \
                --config-patch "@$TALOS_DIR/patches/site-${SITE}.yaml" \
                --config-patch "@$TALOS_DIR/patches/containerd-layer-retention.yaml" \
                --config-patch "@$TALOS_DIR/patches/${node}.yaml" \
                --config-patch "@$REGISTRY_PATCH" 2>&1); then
                echo -e "    applied via $node_ip ${GREEN}(OK)${NC}"
                return 0
            fi
            echo "    attempt $i/60: $node_ip - ${err//$'\n'/ | }"
            sleep 10
        done
        fail "Timeout waiting for ${fixed_addr:-$fqdn}"
    }

    # One loop over ALL_NODES rather than one per role, so NODE_ADDRS can be a
    # single list in the same order. Control planes come first in ALL_NODES,
    # which is what decides the base config each node gets.
    for idx in "${!ALL_NODES[@]}"; do
        if [[ $idx -lt ${#NODES[@]} ]]; then
            base="$TALOS_GEN_DIR/controlplane.yaml"
        else
            base="$TALOS_GEN_DIR/worker.yaml"
        fi
        apply_node_config "${ALL_NODES[$idx]}" "$base" "${NODE_ADDRS[$idx]:-}"
    done

    echo -n "Configuring talosctl... "
    # `gen config` leaves talosconfig endpoints empty; they are the Talos API
    # addresses and unrelated to cluster.controlPlane.endpoint.
    if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
        talosctl config endpoint "${NODE_ADDRS[@]}"
        talosctl config node "${NODE_ADDRS[0]}"
    else
        talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
        talosctl config node "${NODES[0]}.${DOMAIN}"
    fi
    ok

    step 0 "Waiting for every node to come back configured..."
    # Asks each machine directly, by address, and expects it to answer with the
    # hostname its patch assigns. That is what has to be true before etcd is
    # bootstrapped: the config took, the node rebooted into it, and the machine
    # at that address is the one we think it is.
    #
    # This used to wait on `talosctl get members`, which was wrong twice over.
    # It queried through the configured endpoint list rather than each node, so
    # apid refused it with `PermissionDenied: no request forwarding` — the same
    # thing that broke the bootstrap call. And membership comes from the
    # discovery service, because the Kubernetes registry is disabled: a cold
    # rebuild would have been waiting on discovery.talos.dev being reachable,
    # before the cluster it is trying to rebuild even exists.
    #
    # Each node installs to disk and reboots first, so the window is generous.
    for idx in "${!ALL_NODES[@]}"; do
        want="${ALL_NODES[$idx]}"
        addr="${NODE_ADDRS[$idx]:-${want}.${DOMAIN}}"
        echo -n "  $want ($addr)... "
        node_err=""
        for i in $(seq 1 60); do
            got=$(talosctl -n "$addr" -e "$addr" get hostname \
                    -o jsonpath='{.spec.hostname}' 2>"$TALOS_GEN_DIR/hostname.err") || true
            node_err=$(cat "$TALOS_GEN_DIR/hostname.err")
            [[ "$got" == "$want" ]] && break
            sleep 10
            if [[ $i -eq 60 ]]; then
                echo
                [[ -n "$node_err" ]] && echo "  last talosctl error: $node_err" >&2
                fail "$addr never came back as $want (last answer: '${got:-none}')"
            fi
        done
        ok
    done

    echo -n "Bootstrapping Kubernetes... "
    # -n AND -e on one control plane, deliberately. etcd bootstrap is the one
    # call apid refuses to forward: reached through any other node it fails with
    # `PermissionDenied: no request forwarding`. Without them talosctl picks an
    # endpoint from the configured list — with more than one control plane, or a
    # list that includes workers, that is a coin flip.
    if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
        bootstrap_target="${NODE_ADDRS[0]}"
    else
        bootstrap_target="${NODES[0]}.${DOMAIN}"
    fi
    # Re-running against an already-bootstrapped cluster is not an error — that
    # is why this does not simply fail on a non-zero exit. Anything that is not
    # "already exists" is shown rather than swallowed, because a silent failure
    # here surfaces much later as an API that never answers.
    #
    # And it RETRIES, because "bootstrap is not available yet" is a moment, not
    # a fault. The wait above only establishes that the node answers with its
    # hostname; etcd accepting the bootstrap call comes later, and how much
    # later depends on the machine. On qemu the gap is short enough to miss. On
    # a Hetzner cloud server it is not: the single attempt failed with
    # FailedPrecondition, the run carried on regardless, and then sat out its
    # full timeout waiting for an apiserver that could never start because etcd
    # had never been bootstrapped.
    bootstrap_err=""
    for i in $(seq 1 60); do
        if bootstrap_err=$(talosctl bootstrap -n "$bootstrap_target" -e "$bootstrap_target" 2>&1); then
            break
        fi
        if grep -qi 'already\|AlreadyExists' <<<"$bootstrap_err"; then
            bootstrap_err=""
            break
        fi
        # Not yet ready is the expected answer for the first attempts; anything
        # else is reported once and then retried anyway, since the alternative
        # is failing on a condition that may still clear.
        if ! grep -qi 'not available yet\|connection refused\|FailedPrecondition' <<<"$bootstrap_err"; then
            [[ $i -eq 1 ]] && echo -e "\n${YELLOW}bootstrap reported:${NC} ${bootstrap_err//$'\n'/ | }"
        fi
        sleep 10
        [[ $i -eq 60 ]] && { echo; echo "${bootstrap_err//$'\n'/ | }" >&2; fail "etcd never accepted the bootstrap call"; }
    done
    ok

    echo -n "Getting kubeconfig... "
    kubeconfig_err=""
    for i in $(seq 1 30); do
        mkdir -p "$(dirname "$KUBECONFIG_PATH")"
        if kubeconfig_err=$(talosctl kubeconfig "$KUBECONFIG_PATH" --force 2>&1); then
            sed -i "s|server:.*|server: https://${K8S_API_HOST}:6443|" "$KUBECONFIG_PATH"
            break
        fi
        sleep 5
        # Held back while retrying — the first attempts fail by design — but
        # printed if it never works, because that message is the diagnosis.
        [[ $i -eq 30 ]] && { echo; echo "$kubeconfig_err" >&2; fail "Couldn't get kubeconfig"; }
    done
    ok

    echo -n "Waiting for the Kubernetes API to answer... "
    # Having a kubeconfig is not the same as having an apiserver. `talosctl
    # kubeconfig` succeeds as soon as Talos can hand out credentials, which is
    # before kube-apiserver is serving on the endpoint the file points at — so
    # the very next command could fail with an EOF against a port nothing is
    # listening on yet. On hardware the gap usually closes on its own; on a
    # scratch cluster it does not, which is how this was found.
    # `get --raw /version` needs the server; `kubectl version` would answer from
    # the client alone and pass against nothing at all.
    api_err=""
    for i in $(seq 1 60); do
        api_err=$(kubectl get --raw /version 2>&1) && break
        sleep 5
        if [[ $i -eq 60 ]]; then
            echo "$api_err" >&2
            fail "Kubernetes API at https://${K8S_API_HOST}:6443 never answered"
        fi
    done
    ok

    step 0 "Installing Cilium (CNI)..."
    # Must run before the node-Ready wait below: with `cni: none` the kubelet
    # reports NotReady until a CNI is present, so that loop would time out.
    #
    # Chart repo, version constraint and values all come out of the HelmRelease
    # so this cannot drift from what Flux will reconcile minutes later. helm
    # accepts the semver range verbatim and resolves it the same way Flux does.
    #
    # serviceMonitor is the one override, and it is a bootstrap constraint
    # rather than a value: its CRD arrives with the monitoring stack much
    # later, and the object would fail to apply. Nothing else is overridden,
    # which is what lets helm-controller adopt this release instead of
    # replacing it.
    CILIUM_VALUES=$(mktemp)
    yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES"
    grep -q 'policyMapMax' "$CILIUM_VALUES" \
        || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR"
    # The site's part, from the cilium-site ConfigMap in the site patch: the
    # HelmRelease reads it through valuesFrom, under its own values.
    CILIUM_SITE_VALUES=$(mktemp)
    yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "cilium-site") | .contents' \
        "$TALOS_DIR/patches/site-${SITE}.yaml" \
        | yq -r '.data["values.yaml"]' > "$CILIUM_SITE_VALUES"
    grep -q 'routingMode' "$CILIUM_SITE_VALUES" \
        || fail "no cilium-site values (routingMode) in talos/patches/site-${SITE}.yaml"

    # Cilium's cni.confPath is a subdirectory of /etc/cni/net.d that Multus
    # creates (#832). /etc/cni is read-only to the kubelet on Talos, so the
    # agent's DirectoryOrCreate mount cannot make it, and on an empty cluster
    # Multus is not there yet. A hostNetwork pod per node creates it; there is
    # no pod network before Cilium.
    cni_conf_dir=$(yq -r '.cni.confPath' "$CILIUM_VALUES")
    if [[ "$cni_conf_dir" == /etc/cni/net.d/* ]]; then
        step 0 "Creating $cni_conf_dir on every node..."
        # kube-controller-manager creates kube-system/default shortly after the API answers.
        for i in $(seq 1 30); do
            kubectl -n kube-system get serviceaccount default >/dev/null && break
            sleep 5
            [[ $i -eq 30 ]] && fail "kube-system/default ServiceAccount never appeared"
        done
        # A pod bound to a node that has not registered yet is garbage collected.
        for node in "${ALL_NODES[@]}"; do
            for i in $(seq 1 60); do
                kubectl get node "$node" >/dev/null && break
                sleep 5
                [[ $i -eq 60 ]] && fail "$node never registered with the API server"
            done
        done
        for node in "${ALL_NODES[@]}"; do
            kubectl -n kube-system delete pod "cni-confdir-$node" --ignore-not-found --wait=true >/dev/null \
                || fail "could not remove an earlier cni-confdir pod on $node"
            kubectl apply -f - >/dev/null <<EOF || fail "could not create the cni-confdir pod on $node"
apiVersion: v1
kind: Pod
metadata:
  name: cni-confdir-$node
  namespace: kube-system
spec:
  nodeName: $node
  hostNetwork: true
  restartPolicy: Never
  tolerations:
    - operator: Exists
  containers:
    - name: mkdir
      image: docker.io/library/busybox:1.37
      command: ["mkdir", "-p", "/host/${cni_conf_dir#/etc/cni/net.d/}"]
      volumeMounts:
        - name: netd
          mountPath: /host
  volumes:
    - name: netd
      hostPath:
        path: /etc/cni/net.d
        type: Directory
EOF
        done
        for node in "${ALL_NODES[@]}"; do
            kubectl -n kube-system wait --for=jsonpath='{.status.phase}'=Succeeded \
                "pod/cni-confdir-$node" --timeout=5m >/dev/null \
                || fail "creating $cni_conf_dir on $node did not succeed (kubectl -n kube-system describe pod cni-confdir-$node)"
            kubectl -n kube-system delete pod "cni-confdir-$node" >/dev/null
        done
        ok
    fi

    helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null
    helm repo update cilium >/dev/null
    helm upgrade --install cilium cilium/cilium \
        --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \
        --namespace kube-system \
        -f "$CILIUM_SITE_VALUES" \
        -f "$CILIUM_VALUES" \
        --set prometheus.serviceMonitor.enabled=false \
        --wait --timeout 10m
    rm -f "$CILIUM_VALUES" "$CILIUM_SITE_VALUES"
    ok

    # Multus is the only writer of /etc/cni/net.d (#832); without it the kubelet
    # reports "cni plugin not initialized" and nothing schedules, Flux included.
    # Applied from the manifests Flux owns, which adopts them.
    step 0 "Installing Multus (the CNI entry point)..."
    kubectl apply -k "$REPO_ROOT/kubernetes/infrastructure/multus" >/dev/null \
        || fail "could not apply kubernetes/infrastructure/multus"
    kubectl -n kube-system rollout status daemonset/kube-multus-ds-subdir --timeout=10m >/dev/null \
        || fail "Multus never became ready (kubectl -n kube-system get pods -l app=multus)"
    ok

    step 0 "Waiting for K8s nodes..."
    expected=${#ALL_NODES[@]}
    # Before the loop, because 0 == 0 would satisfy its exit condition on the
    # first pass and report a complete Talos phase for a cluster with no nodes
    # in it. `expected` comes from ALL_NODES, which a provisioning phase hands
    # over — an empty one is a bug in that phase, not a cluster that is ready.
    [[ "$expected" -gt 0 ]] \
        || fail "ALL_NODES is empty, so this wait would pass against any cluster, including one with no nodes"
    nodes_err=""
    for i in $(seq 1 60); do
        # The STATUS column, matched whole. `grep -c Ready` counted NotReady
        # too — the substring is in it — so this waited for nodes to register
        # rather than to become ready, and passed while every one of them was
        # still NotReady.
        nodes_out=$(kubectl get nodes --no-headers 2>"$TALOS_GEN_DIR/nodes.err") || true
        nodes_err=$(cat "$TALOS_GEN_DIR/nodes.err")
        ready=$(awk '$2 == "Ready"' <<<"$nodes_out" | grep -c . || true)
        if [[ "$ready" -eq "$expected" ]]; then
            # Say what was accepted. Silence on success made this step
            # unauditable: a run that printed "Waiting for K8s nodes..." and
            # then "Talos phase complete" was indistinguishable from one that
            # matched 0 against 0, and the log gave no way to tell which had
            # happened. A count that is also the pass condition is the one
            # number worth printing.
            echo "$ready/$expected Ready"
            break
        fi
        sleep 5
        if [[ $i -eq 60 ]]; then
            echo
            [[ -n "$nodes_err" ]] && echo "last kubectl error: $nodes_err" >&2
            echo "${nodes_out:-  (no nodes listed)}" >&2
            fail "Not all nodes ready (got $ready/$expected)"
        fi
    done

    # The nodes' resolver asks k8s-gateway first (ResolverConfig in the site
    # patch), and CoreDNS forwards external names to the node. Until k8s-gateway
    # answers, every lookup waits out the node's 2 s read timeout and CoreDNS
    # gives up first: pods resolve nothing, Flux included. So it is installed
    # here, from the values Flux will reconcile (the HelmRelease, under the
    # site's k8s-gateway-site values, as valuesFrom merges them), and Flux
    # adopts the release.
    step 0 "Installing k8s-gateway (the nodes' first resolver)..."
    k8sgw_values=$(mktemp)
    k8sgw_site_values=$(mktemp)
    yq 'select(.kind == "HelmRelease") | .spec.values' "$K8SGW_HR" > "$k8sgw_values"
    yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "k8s-gateway-site") | .contents' \
        "$TALOS_DIR/patches/site-${SITE}.yaml" \
        | yq -r '.data["values.yaml"]' > "$k8sgw_site_values"
    grep -q 'dnsPolicy' "$k8sgw_site_values" \
        || fail "no k8s-gateway-site values (dnsPolicy) in talos/patches/site-${SITE}.yaml"
    helm upgrade --install k8s-gateway \
        "$(yq -r 'select(.kind == "OCIRepository") | .spec.url' "$K8SGW_HR")" \
        --namespace kube-system \
        -f "$k8sgw_site_values" \
        -f "$k8sgw_values" \
        --wait --timeout 5m >/dev/null \
        || fail "k8s-gateway did not come up (kubectl -n kube-system get pods -l app.kubernetes.io/name=k8s-gateway)"
    rm -f "$k8sgw_values" "$k8sgw_site_values"
    ok

    # Stash configs for subsequent phases / export-access step. Skipped under
    # vmtest and hcloudtest: those are a throwaway cluster's credentials, and a
    # later production `bootstrap.sh flux` copies access-staging/ over the
    # operator's kubeconfig and talosconfig.
    if [[ "${HCLOUDTEST:-0}" == "1" ]]; then
        hcloud_egress_check
        echo -e "${GREEN}Talos phase complete.${NC} Access files in $HCLOUD_STATE_DIR/ (not staged)"
    elif [[ "${VMTEST:-0}" == "1" ]]; then
        # Nodes being Ready says nothing about storage. The CephCluster picks
        # OSD nodes by a label out of the machine config and their device by the
        # partition label RawVolumeConfig creates, so assert both here — running
        # it by hand afterwards races the teardown, and a test that does not
        # check its own premise is worth little.
        step 0 "Checking the OSD prerequisites Ceph selects on..."
        for idx in $(seq "${#NODES[@]}" $(( ${#ALL_NODES[@]} - 1 ))); do
            vm_node="${ALL_NODES[$idx]}"
            vm_addr="${NODE_ADDRS[$idx]}"
            vols=$(talosctl -n "$vm_addr" -e "$vm_addr" get discoveredvolumes 2>&1) \
                || fail "could not read volumes from $vm_node ($vm_addr): ${vols//$'\n'/ | }"
            awk '$NF == "r-osd-data" {found=1} END {exit !found}' <<<"$vols" \
                || fail "$vm_node has no r-osd-data partition — RawVolumeConfig did not take"
            kubectl get node "$vm_node" \
                -o jsonpath='{.metadata.labels.storage\.renner\.uno/ceph-osd}' \
                | grep -q true \
                || fail "$vm_node is missing the storage.renner.uno/ceph-osd label"
            echo -e "  $vm_node: partition and label ${GREEN}(OK)${NC}"
        done

        echo -e "${GREEN}VM rebuild test passed.${NC} ${#ALL_NODES[@]} nodes Ready on the repo's own config."
    else
        mkdir -p "$ACCESS_STAGING"
        cp "$TALOSCONFIG_PATH" "$ACCESS_STAGING/talosconfig"
        cp "$KUBECONFIG_PATH"  "$ACCESS_STAGING/kubeconfig"
        echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
    fi
fi

# Helper: load staged access files into home dir (for later CI steps)
# Bring OpenBao up far enough that External Secrets can read from it.
#
# This was eight commands typed by hand, and it sits in the middle of the cold
# rebuild: infrastructure-crds waits on the ClusterSecretStore, which needs an
# unsealed OpenBao with Kubernetes auth configured, and every app Kustomization
# waits on infrastructure-crds. A rebuild could not reach a reconciled state
# without someone at a keyboard.
#
# The seal key stays where it was — the operator's `pass` store — and is read
# here at run time. It is never written to the repo and never reaches CI, which
# is the reason this was left manual in the first place. It reaches the cluster
# as the openbao-seal-key Secret, created by the flux phase before Flux starts
# OpenBao, and from then on every replica unseals itself.
#
# Everything below is idempotent: it is safe to re-run against an OpenBao that
# is already unsealed and configured, which is what makes it usable as part of
# `bootstrap.sh all` rather than a one-shot.

bao_exec() {
    local pod=$1; shift
    kubectl -n "$OPENBAO_NS" exec -i "$pod" -- "$@"
}

# Where the OpenBao raft snapshots already are: inside the Velero backup, in the
# kopia repository the data mover writes to. Nothing new is produced for the
# rebuild — the nightly snapshot CronJob and the openbao-daily Velero Schedule
# have been doing this all along.
#
# Reading them needs three things, all of which live OUTSIDE the cluster, which
# is the point: bucket credentials from pass (their own key, not Velero's),
# kopia, and the repository password. The last is not a secret — Velero
# hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
# the offsite backup's confidentiality rests on bucket access alone.
#
# This is what breaks the circle. Velero's own credentials come from an
# ExternalSecret, which needs the ClusterSecretStore, which needs an unsealed
# OpenBao — so Velero cannot be the thing that restores OpenBao.
OPENBAO_KOPIA_BUCKET="renner-uno-k8s-velero"
OPENBAO_KOPIA_PREFIX="kopia/openbao/"
OPENBAO_S3_ENDPOINT="$S3_ENDPOINT"
OPENBAO_S3_REGION="$S3_REGION"
# Velero's hardcoded backup repository password (pkg/repository/keys/keys.go).
OPENBAO_KOPIA_PASSWORD="static-passw0rd"

restore_openbao() {
    local pod=$1
    step 0 "Restoring OpenBao from the offsite backup..."
    command -v kopia >/dev/null || fail "kopia not found — needed to read the Velero backup repository without Velero"

    local work kopia_err
    work=$(mktemp -d)
    export KOPIA_CONFIG_PATH="$work/repository.config"
    export KOPIA_CACHE_DIRECTORY="$work/cache"
    export KOPIA_LOG_DIR="$work/logs"
    export KOPIA_CHECK_FOR_UPDATES=false
    export KOPIA_PASSWORD="$OPENBAO_KOPIA_PASSWORD"

    # Assigned first, not interpolated into the argument list. `fail` inside
    # $( ) exits only the subshell, and a failed substitution in an argument does
    # not stop the outer command — so a missing key produced an EMPTY
    # --access-key and an authentication error instead of the real reason.
    local s3_access s3_secret
    s3_access=$(read_secret s3-access) || fail "no s3-access secret"
    s3_secret=$(read_secret s3-secret) || fail "no s3-secret secret"

    # Nothing here is expected to fail, so nothing here discards its error. This
    # is the path that only runs during a genuine disaster; it is the last place
    # to be guessing why something did not work.
    echo -n "  connecting to the backup repository (read-only)... "
    if ! kopia_err=$(kopia repository connect s3 \
            --bucket "$OPENBAO_KOPIA_BUCKET" \
            --prefix "$OPENBAO_KOPIA_PREFIX" \
            --endpoint "$OPENBAO_S3_ENDPOINT" \
            --region "$OPENBAO_S3_REGION" \
            --access-key "$s3_access" \
            --secret-access-key "$s3_secret" \
            --readonly 2>&1); then
        echo; echo "$kopia_err" >&2
        fail "could not open the backup repository at $OPENBAO_S3_ENDPOINT/$OPENBAO_KOPIA_BUCKET/$OPENBAO_KOPIA_PREFIX"
    fi
    ok

    # Newest kopia snapshot, then the newest .snap inside it. The PVC holds a
    # week of them, so the last line is the most recent.
    local ksnap snapfile listing
    listing=$(kopia snapshot list --all 2>&1) \
        || { echo "$listing" >&2; fail "could not list snapshots in the backup repository"; }
    ksnap=$(tail -1 <<<"$listing" | awk '{print $4}')
    [[ -n "$ksnap" ]] \
        || { echo "$listing" >&2; fail "could not read a snapshot id out of the listing above"; }
    snapfile=$(kopia ls "$ksnap" 2>&1 | grep -o 'openbao-raft-[0-9-]*\.snap' | sort | tail -1) || true
    [[ -n "$snapfile" ]] || fail "no openbao-raft-*.snap in kopia snapshot $ksnap"
    echo "  newest snapshot: $snapfile"

    if ! kopia_err=$(kopia restore "$ksnap/$snapfile" "$work/restore.snap" 2>&1); then
        echo "$kopia_err" >&2
        fail "could not extract $snapfile from the backup"
    fi
    [[ -s "$work/restore.snap" ]] || fail "$snapfile came out empty"
    echo "  extracted $(stat -c%s "$work/restore.snap") bytes"

    # A snapshot can only be loaded into a running OpenBao, so the fresh one is
    # initialised first. Under the static seal, init hands out a recovery key
    # rather than an unseal key and the server unseals itself; the only thing
    # used from its output is the root token, for the next few seconds, and
    # then it stops existing: restoring replaces the keyring with the
    # snapshot's, after which the root token and recovery key from pass apply.
    #
    # No -force. Without it OpenBao checks that the snapshot was written under
    # the seal key this server runs with, and refuses a mismatch by name
    # ("possibly the snapshot is using a different autoseal key"). That check
    # is the one guard against restoring under the wrong seal-key Secret;
    # -force would skip it and leave a server that can never unseal.
    #
    # The init output is not echoed on a parse failure: it carries the root
    # token and the recovery key of a server that is about to be overwritten,
    # but a log line is a log line.
    echo -n "  initialising the empty server (throwaway recovery key)... "
    local init_json root
    init_json=$(bao_exec "$pod" bao operator init -recovery-shares=1 -recovery-threshold=1 -format=json 2>&1) \
        || { echo; echo "$init_json" >&2; fail "could not initialise OpenBao"; }
    root=$(jq -r '.root_token // empty' <<<"$init_json")
    [[ -n "$root" ]] || fail "init output carried no root token"
    ok

    # Sealed after init means the static seal is not working — the Secret is
    # missing or malformed — and the restore below cannot run against it.
    echo -n "  waiting for it to unseal itself"
    for i in $(seq 1 30); do
        if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
            | jq -e '.sealed == false' >/dev/null; then
            echo; break
        fi
        printf '.'
        sleep 2
        if [[ $i -eq 30 ]]; then
            echo
            cat "$OPENBAO_SECRETS_DIR/.baoerr" >&2
            fail "$pod did not unseal itself after init — check the openbao-seal-key Secret and the pod log"
        fi
    done

    echo -n "  loading the snapshot... "
    kubectl -n "$OPENBAO_NS" cp "$work/restore.snap" "$pod:/tmp/restore.snap" >/dev/null \
        || fail "could not copy the snapshot into $pod"
    local out
    out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore /tmp/restore.snap 2>&1) \
        || { echo; echo "$out" >&2; fail "snapshot restore rejected"; }
    ok

    rm -rf "$work"
    # The server seals itself against the keyring that came with the snapshot
    # and unseals again from the static seal — the same key the snapshot was
    # written under. bring_up_openbao waits for that on every replica.
    echo "  restored — the server re-seals and unseals itself from the seal key"
}

bring_up_openbao() {
    step 0 "Bringing OpenBao up..."
    command -v jq >/dev/null   || fail "jq not found (needed to read OpenBao's seal status)"

    # OpenBao is deployed by the infrastructure Kustomization. What this step
    # depends on is therefore that Flux has applied it — not that some number of
    # minutes have gone by. A guessed duration is not a condition: the first
    # version failed after ten minutes on a slow cluster, and raising the number
    # only moves the guess.
    #
    # So the loop ends on one of two facts, both read from the cluster: the
    # StatefulSet exists, or Flux reports the Kustomization has failed. The
    # elapsed-time bound below is a backstop against hanging forever, not the
    # mechanism — it is deliberately far longer than any real reconcile.
    local pods=() ks_state ks_bad=0
    echo -n "  waiting for Flux to apply the openbao StatefulSet"
    for i in $(seq 1 720); do
        if kubectl -n "$OPENBAO_NS" get statefulset openbao >/dev/null 2>&1; then
            echo
            break
        fi
        ks_state=$(ks_ready_condition infrastructure)
        # Only a persistent failure counts. The first version matched any message
        # containing "error" or "failed", which fires on `failed to download
        # artifact from source-controller` — a routine transient in the minutes
        # after bootstrap that Flux retries on its own, and one that aborted the
        # rebuild. A wait built on cluster facts is no better than a guessed
        # timeout if it reacts to a fact that is not the one it means.
        if [[ "${ks_state%%|*}" == "False" ]] && grep -qiE 'failed|error' <<<"${ks_state#*|}"; then
            (( ks_bad++ )) || true
        else
            ks_bad=0
        fi
        if (( ks_bad >= 30 )); then
            echo
            echo "  infrastructure Ready=${ks_state%%|*} — ${ks_state#*|}" >&2
            fail "the infrastructure Kustomization has been failing for five minutes — OpenBao will not appear"
        fi
        (( i % 18 == 0 )) && { echo; echo "  infrastructure Ready=${ks_state%%|*} — ${ks_state#*|}"; echo -n "  waiting"; }
        printf '.'
        sleep 10
        [[ $i -eq 720 ]] && { echo; fail "openbao StatefulSet never appeared, and Flux never reported a failure — something is wedged"; }
    done

    # The StatefulSet exists; its pods follow.
    for i in $(seq 1 60); do
        mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
            -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
        [[ ${#pods[@]} -gt 0 ]] && break
        sleep 10
        [[ $i -eq 60 ]] && fail "the openbao StatefulSet exists but produced no pods"
    done
    echo "  found ${#pods[@]} pod(s): ${pods[*]}"

    # A pod object existing is not a pod that can answer. Waiting only for the
    # name meant `bao status` ran against a container still pulling or in Init,
    # its error was thrown away, and an empty answer parsed as "not initialised"
    # — which routes into the restore branch, the one branch that must never be
    # entered by mistake. Wait for the container to be running first.
    echo -n "  waiting for ${pods[0]} to answer"
    local status="" status_err=""
    # Bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to
    # provision its volume.
    for i in $(seq 1 180); do
        if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then
            echo; break
        fi
        status_err=$(cat "$OPENBAO_SECRETS_DIR/.baoerr")
        # `bao status` exits 2 when sealed and still prints valid JSON — that is
        # an answer, not a failure.
        #
        # Ask whether the field is PRESENT, not whether it is true. `jq -e
        # '.initialized'` tests the value, so a brand-new OpenBao answering
        # {"initialized": false, "sealed": true} — a complete and correct
        # answer, and the only one a cold rebuild can give — read as no answer
        # at all. The wait then ran its full 90 rounds against a healthy pod and
        # failed with "never answered bao status", one step after the cluster
        # had finally come up on its own.
        if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi
        printf '.'
        sleep 10
        if [[ $i -eq 180 ]]; then
            echo
            echo "  last error: $status_err" >&2
            fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised"
        fi
    done

    # `bao status` reports Initialized either way, so it distinguishes "needs a
    # key" from "needs initialising". Those must not be confused: initialising an
    # OpenBao that already holds data is how the data gets lost.
    # Read presence separately from value. jq's `//` treats false as absent, so
    # `.initialized // empty` on the one answer a rebuild always gives —
    # {"initialized": false} — yields an empty string, and the guard below then
    # reports that OpenBao's state could not be read at all. Same mistake as the
    # `jq -e` in the wait above, one line apart: false is a value, not a gap.
    local initialized
    initialized=$(jq -r 'if has("initialized") then (.initialized | tostring) else "" end' <<<"$status")
    [[ -n "$initialized" ]] \
        || { echo "$status" >&2; fail "could not read initialised state from bao status"; }

    if [[ "$initialized" != "true" ]]; then
        # An uninitialised OpenBao is restored from the offsite snapshot, never
        # initialised fresh: initialising over an existing keyring is how the
        # data is lost, and a genuinely new OpenBao is a decision rather than a
        # step (MIGRATION-sops-openbao.md).
        restore_openbao "${pods[0]}"
    fi

    # Wait for every replica the StatefulSet wants, not the ones that happen to
    # exist right now. On a cold start only openbao-0 exists when this runs:
    # podManagementPolicy is OrderedReady, so openbao-1 is created once
    # openbao-0 passes its readiness probe and openbao-2 after that. Each
    # unseals itself from the static seal; nothing here types a key.
    #
    # Waiting for the whole chain rather than the first pod matters because the
    # openbao Service sets publishNotReadyAddresses — the chart does this so
    # replicas can reach each other to join raft before any of them is Ready —
    # so a replica that is still starting is a live backend on the service IP.
    # Clients balanced onto it get `dial tcp <svc>:8200: connect: no route to
    # host`, and the failure surfaces nowhere near here: external-secrets'
    # ClusterSecretStore flaps, every ExternalSecret reports "could not get
    # secret data from provider", and infrastructure-crds — which waits on those
    # — never goes Ready, taking all twenty app Kustomizations with it.
    local want
    want=$(kubectl -n "$OPENBAO_NS" get statefulset openbao -o jsonpath='{.spec.replicas}') \
        || fail "could not read the openbao StatefulSet's replica count"
    [[ "$want" =~ ^[0-9]+$ ]] || fail "openbao StatefulSet reported a non-numeric replica count: '$want'"
    echo "  waiting for $want replica(s) to unseal themselves"

    local unsealed=0 round pod
    local -A seen=()
    for round in $(seq 1 180); do
        mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
            -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
        unsealed=0
        for pod in "${pods[@]}"; do
            if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
                | jq -e '.sealed == false' >/dev/null; then
                [[ -n "${seen[$pod]:-}" ]] || { echo -e "  $pod: ${GREEN}unsealed${NC}"; seen[$pod]=1; }
                unsealed=$(( unsealed + 1 ))
            fi
        done
        (( unsealed >= want )) && break
        sleep 10
        if [[ $round -eq 180 ]]; then
            # A replica that is running and still sealed did not get a working
            # seal. Say which, with its own answer, rather than "timed out".
            for pod in "${pods[@]}"; do
                [[ -n "${seen[$pod]:-}" ]] && continue
                echo "  $pod:" >&2
                bao_exec "$pod" bao status 2>&1 | sed 's/^/    /' >&2 || true
            done
            fail "only $unsealed of $want OpenBao replicas unsealed after 30 minutes — a sealed one means the static seal failed (openbao-seal-key Secret, pod log)"
        fi
    done
    ok
}

load_staged_access() {
    [[ -f "$ACCESS_STAGING/talosconfig" && -f "$ACCESS_STAGING/kubeconfig" ]] \
        || fail "access-staging/ missing talosconfig or kubeconfig (run talos phase first)"
    # To the paths talosctl and kubectl will actually read, not to $HOME. An
    # operator with KUBECONFIG pointing somewhere else — an ordinary setup — got
    # the staged config written where nothing would look at it, and every step
    # after this ran against whatever cluster the old KUBECONFIG named. Wrong
    # target, no error.
    mkdir -p "$(dirname "$TALOSCONFIG_PATH")" "$(dirname "$KUBECONFIG_PATH")"
    cp "$ACCESS_STAGING/talosconfig" "$TALOSCONFIG_PATH"
    cp "$ACCESS_STAGING/kubeconfig"  "$KUBECONFIG_PATH"
}

if phase_runs flux; then
    echo -e "${YELLOW}=== Phase: FluxCD bootstrap ===${NC}"
    command -v kubectl >/dev/null || fail "kubectl not found"
    command -v flux >/dev/null    || fail "flux not found"
    # In the in-cluster Job neither the SOPS key nor the Gitea token is present,
    # and neither should be baked in. Both are streamed into the secrets
    # directory when the run asks for them, the same way the seal key is.
    # Unconditional: the same collection step on a laptop and in the Job, so the
    # recovery path is the one that gets exercised.
    require_secrets
    SOPS_KEY="$OPENBAO_SECRETS_DIR/sops-age.key"
    [[ -f "$SOPS_KEY" ]] || fail "SOPS age key not found at $SOPS_KEY"
    # One collected token, handed to whichever variable the chosen source reads.
    # `flux bootstrap gitea` and `flux bootstrap github` each take theirs from
    # the environment.
    case "$FLUX_SOURCE" in
        gitea)  GITEA_TOKEN=$(read_secret git-token);  export GITEA_TOKEN ;;
        github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;;
        restore-test)
            GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN
            command -v gh >/dev/null || fail "gh not found (registers the read-only deploy key)"
            command -v jq >/dev/null || fail "jq not found (reads the DoH answer for the egress check)"
            # The throwaway cluster carries production secrets, and this phase
            # copies access files over KUBECONFIG_PATH and TALOSCONFIG_PATH:
            # neither may be the operator's production file.
            [[ "$KUBECONFIG_PATH" != "$HOME/.kube/config" && "$TALOSCONFIG_PATH" != "$HOME/.talos/config" ]] \
                || fail "FLUX_SOURCE=restore-test needs KUBECONFIG and TALOSCONFIG pointed at the test cluster ($HCLOUD_STATE_DIR)"
            ;;
        *)      fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github|restore-test)" ;;
    esac
    # The restore test reads its access from KUBECONFIG/TALOSCONFIG, which
    # hcloudtest writes; access-staging/ may hold an older run's files.
    [[ "$PHASE" == "flux" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access
    if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
        # Whatever the paths say, the cluster itself must be the test cluster,
        # before anything below writes a secret to it.
        node_names=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') \
            || fail "cannot list the nodes of the target cluster"
        [[ -n "$node_names" ]] || fail "the target cluster has no nodes"
        for n in $node_names; do
            [[ "$n" == "$HCLOUD_NAME_PREFIX"-* ]] \
                || fail "node '$n' is not a $HCLOUD_NAME_PREFIX-* test node — refusing the restore test against this cluster"
        done
        hcloud_egress_check
    fi

    echo -n "Creating SOPS age secret... "
    kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f -
    kubectl create secret generic sops-age -n flux-system \
        --from-file=age.agekey="$SOPS_KEY" \
        --dry-run=client -o yaml | kubectl apply -f -
    ok

    # The static seal key, before Flux brings OpenBao up: a pod whose env
    # references a Secret that does not exist is never started, and OpenBao
    # reads the key at start. Left alone when the cluster already holds the
    # same value. A DIFFERENT value is refused rather than applied: the data
    # is encrypted under whichever key OpenBao started with, and swapping the
    # Secret would leave every replica sealed for good at its next restart.
    # That is a migration, not a bootstrap step.
    #
    # --from-literal, not --from-file: `pass show` ends its output with a
    # newline, the substitution strips it, and OpenBao rejects a key that is
    # not exactly 32 bytes of base64. The key sits in this process's argument
    # list for the length of the call, as it did in the pod's before.
    echo -n "Creating OpenBao seal key secret... "
    seal_key=$(read_secret seal-key) || fail "could not read the seal key"
    kubectl create namespace "$OPENBAO_NS" --dry-run=client -o yaml | kubectl apply -f -
    if existing=$(kubectl -n "$OPENBAO_NS" get secret openbao-seal-key \
            -o go-template='{{index .data "key" | base64decode}}' 2>"$OPENBAO_SECRETS_DIR/.err"); then
        [[ "$existing" == "$seal_key" ]] \
            || fail "openbao-seal-key in $OPENBAO_NS holds a different key than pass — not replacing it (OpenBao's data is written under the key it started with)"
        echo "already present"
    elif grep -q NotFound "$OPENBAO_SECRETS_DIR/.err"; then
        kubectl -n "$OPENBAO_NS" create secret generic openbao-seal-key \
            --from-literal=key="$seal_key" >/dev/null \
            || fail "could not create the openbao-seal-key Secret"
        ok
    else
        echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
        fail "could not read the openbao-seal-key Secret"
    fi
    unset seal_key existing

    # Before Flux: cnpg-object-stores fails without it rather than render empty
    # paths, and the paths must exist before any Cluster bootstraps.
    if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
        create_cnpg_test_bucket
        write_cnpg_paths "$CNPG_TEST_BUCKET"
    else
        write_cnpg_paths "$CNPG_BUCKET"
    fi

    if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
        # The production tree through kubernetes/restore-test/flux-root.yaml,
        # whose patches are the test's whole deviation from production. Nothing
        # is committed anywhere: the source is the GitHub mirror, read through a
        # deploy key GitHub itself holds to read-only, so the test cluster
        # cannot push (image automation) even with production's manifests.
        step 0 "Bootstrapping FluxCD for the restore test (restore-test/flux-root.yaml)..."
        flux install --components-extra=image-reflector-controller,image-automation-controller \
            || fail "flux install failed"
        restore_test_repo
        deploy_key="$OPENBAO_SECRETS_DIR/restore-test-git"
        rm -f "$deploy_key" "$deploy_key.pub"
        ssh-keygen -q -t ed25519 -N '' -C "restore-test" -f "$deploy_key"
        gh api -X POST "repos/$restore_repo/keys" \
            -f title="restore-test $(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            -f key="$(cat "$deploy_key.pub")" -F read_only=true >/dev/null \
            || fail "could not register the read-only deploy key on $restore_repo"
        flux create secret git restore-test-git \
            --url="$restore_url" \
            --private-key-file="$deploy_key" >/dev/null \
            || fail "could not create the restore-test-git Secret"
        # On FLUX_BRANCH, so a branch can be tried here before it is merged.
        # shellcheck disable=SC2016 # $b is a jq variable
        yq -y --arg b "$FLUX_BRANCH" \
            'if .kind == "GitRepository" and .metadata.name == "restore-test" then .spec.ref.branch = $b else . end' \
            "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml" | kubectl apply -f - >/dev/null \
            || fail "could not apply restore-test/flux-root.yaml"
        echo "  source: $restore_url, branch $FLUX_BRANCH"
        ok
    elif [[ "$FLUX_SOURCE" == "github" ]]; then
        step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..."
        echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}"
        echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}"
        # GITHUB_TOKEN is read from the environment by `flux bootstrap github`.
        #
        # --secret-name, and this is what makes the break-glass path survive its
        # own success. The default is `flux-system`, which is also the target of
        # the flux-system-git-auth ExternalSecret in post-crds — that one carries
        # the GITEA credentials, which is correct for the steady state where the
        # source is in-cluster Gitea.
        #
        # Sharing the name means the moment infrastructure-crds reconciles and
        # ESO resolves, the GitHub token is overwritten with Gitea credentials
        # while the GitRepository still points at github.com. Flux then stops
        # being able to read its own source:
        #
        #   GitOperationFailed: unable to list remote for
        #   'https://github.com/.../Homelab.git': authentication required:
        #   Invalid username or token
        #
        # The rebuild destroys its git source at the exact moment it starts
        # working, which is why it stayed hidden: no earlier run ever got the
        # ExternalSecrets to resolve at all.
        #
        # A separate name keeps the two credentials apart. `flux-system` still
        # receives the Gitea credentials and is simply unused until the source is
        # switched back to Gitea — the manual step the warning above names.
        flux bootstrap github \
            --owner="$GITHUB_OWNER" \
            --repository="$GITHUB_REPO" \
            --branch="$FLUX_BRANCH" \
            --path="$FLUX_PATH" \
            --secret-name=flux-system-breakglass \
            --components-extra=image-reflector-controller,image-automation-controller \
            --personal \
            --token-auth --timeout 15m 2>&1
    else
        step 0 "Bootstrapping FluxCD from in-cluster Gitea..."
        flux bootstrap gitea \
            --hostname="$GITEA_HOST" \
            --owner="$GITEA_OWNER" \
            --repository="$GITEA_REPO" \
            --branch="$FLUX_BRANCH" \
            --path="$FLUX_PATH" \
            --components-extra=image-reflector-controller,image-automation-controller \
            --personal \
            --token-auth --timeout 15m 2>&1
    fi

    bring_up_openbao

    step 0 "Waiting for infrastructure..."
    for i in $(seq 1 60); do
        # The Ready condition, not the printed table: `grep -c True` used to
        # match SUSPENDED=True as well, so a Kustomization Flux is not
        # reconciling at all read as ready and let the run proceed.
        ready=$(ks_ready_condition infrastructure)
        if [[ "${ready%%|*}" == "True" ]]; then
            break
        fi
        sleep 10
        if [[ $i -eq 60 ]]; then
            echo -e "${YELLOW}Warning: infrastructure not ready after 10m, continuing...${NC}"
            flux get kustomizations 2>&1 | sed 's/^/    /' | head -12
        fi
    done

    step 0 "Waiting for Ceph cluster..."
    for i in $(seq 1 60); do
        health=$(kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)
        if [[ "$health" == "Ready" ]]; then
            break
        fi
        sleep 10
        if [[ $i -eq 60 ]]; then
            # Advisory, so it continues — but a warning that says only "not ready"
            # sends the reader back to the cluster for information this step
            # already had.
            echo -e "${YELLOW}Warning: Ceph not ready after 10m (phase: ${health:-none}), continuing...${NC}"
            kubectl -n rook-ceph get cephcluster -o wide 2>&1 | sed 's/^/    /'
            kubectl -n rook-ceph get pods --no-headers 2>&1 \
                | awk '$3 != "Running" && $3 != "Completed"' | head -10 | sed 's/^/    /'
        fi
    done

    step 0 "Waiting for Ceph Object Store..."
    for i in $(seq 1 60); do
        phase=$(kubectl get cephobjectstore -n rook-ceph migration -o jsonpath='{.status.phase}' 2>/dev/null || true)
        if [[ "$phase" == "Ready" ]]; then
            break
        fi
        sleep 10
        if [[ $i -eq 60 ]]; then
            echo -e "${YELLOW}Warning: Object Store not ready after 10m (phase: ${phase:-none}), continuing...${NC}"
            kubectl -n rook-ceph get cephobjectstore 2>&1 | sed 's/^/    /'
        fi
    done
fi

if phase_runs wait-apps; then
    echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}"
    command -v kubectl >/dev/null || fail "kubectl not found"
    command -v flux >/dev/null    || fail "flux not found"
    # The restore test reads its access from KUBECONFIG/TALOSCONFIG.
    [[ "$PHASE" == "wait-apps" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access

    # Wait for a Flux Kustomization to become Ready=True. Prints a status line
    # whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a
    # silent-but-progressing reconcile is still visible in CI output. Returns
    # 0 on Ready, 1 on timeout (caller decides whether to fail or continue).
    wait_kustomization() {
        local name="$1" label="$2" max_iters="$3"
        local interval=10 elapsed=0 prev_status=""
        local timeout_min=$(( max_iters * interval / 60 ))
        step 0 "Waiting for ${label} (timeout ${timeout_min}m)..."
        for _ in $(seq 1 "$max_iters"); do
            local cond ready msg status
            cond=$(ks_ready_condition "$name")
            ready="${cond%%|*}"
            msg="${cond#*|}"
            if [[ "$ready" == "True" ]]; then
                printf "  [%4ds] Ready ✓\n" "$elapsed"
                return 0
            fi
            status="${ready:-?}: ${msg:-(no status yet)}"
            # Print on first iteration, on state change, or as a 60-s heartbeat.
            # 240 chars is enough to see the full Flux stalled-resource message.
            if [[ "$prev_status" != "$status" ]] || (( elapsed > 0 && elapsed % 60 == 0 )); then
                printf "  [%4ds] %s\n" "$elapsed" "${status:0:240}"
                prev_status="$status"
            fi
            sleep "$interval"
            elapsed=$(( elapsed + interval ))
        done
        printf "  [%4ds] TIMEOUT after %sm — full status follows\n" "$elapsed" "$timeout_min"
        # Dump the full untruncated status + Events for the offending resources
        # so the post-mortem doesn't need a second cluster trip.
        flux get kustomization "$name" 2>&1 | sed 's/^/    /' || true
        kubectl get events --sort-by=.lastTimestamp -A 2>/dev/null \
            | grep -i -E "warning|error" | tail -20 | sed 's/^/    /' || true
        echo -e "${YELLOW}Warning: ${label} not Ready, continuing anyway${NC}"
        # 0, not 1. Called bare under `set -e`, a non-zero return ends the script
        # immediately — so this said "continuing anyway" and then stopped, and
        # every wait after it never ran. The message was the truth; the return
        # value contradicted it.
        return 0
    }

    # Each app KS now owns its CNPG database (+ any data PVCs) directly — the
    # per-app `seed-<app>` KSes that used to bootstrap them were folded into the
    # app KSes when the seed layer was retired. Each wait below gates on the app
    # KS, whose readiness includes its CNPG cluster coming up (bootstrap.recovery)
    # and, through the restore KS it depends on, its volumes restored by Velero.
    # Timeouts stay generous to cover first-time cluster bootstrap on fresh Ceph.
    # apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up
    # is genuinely 5–10 min on a fresh OSD-wiped cluster).
    wait_kustomization gitea               "Gitea (incl. data-gitea-0 restore)"                  360
    wait_kustomization apps                "apps (after restore-gitea)"                          360
    wait_kustomization influxdb            "InfluxDB (HR install + token Jobs)"                  600
    wait_kustomization woodpecker          "Woodpecker CI (incl. wp-pg CNPG backend)"            600
    wait_kustomization authentik           "Authentik (incl. ak-pg CNPG backend)"                600
    wait_kustomization monitoring          "kube-prometheus-stack (Loki + Prom + AM + Grafana)"  180
    wait_kustomization music-assistant     "Music Assistant (incl. data PVC)"                    300
    wait_kustomization paperless           "Paperless (incl. pprl-pg CNPG + data PVCs)"          600
    wait_kustomization home-assistant      "Home Assistant (HAOS backup → native restore)"       180
    wait_kustomization audiobookshelf      "Audiobookshelf (incl. library/metadata PVCs)"        900
    wait_kustomization navidrome           "Navidrome (incl. data + music library PVCs)"         600
    wait_kustomization immich              "Immich (incl. im-pg CNPG + library PVC)"             1800

    # Wait for cert-manager to finish issuing every Certificate. Kustomization
    # Ready only means the Ingress + Certificate manifests are applied; cert-
    # manager then runs an async ACME DNS-01 challenge per Certificate (~30s–2m
    # via Cloudflare). e2e's TLS check fails on the ingress-nginx default fake
    # cert if any Certificate is still Issuing when the test runs.
    wait_certificates() {
        local max_iters=90 interval=10 elapsed=0 prev_status=""
        local timeout_min=$(( max_iters * interval / 60 ))
        step 0 "Waiting for all cert-manager Certificates Ready (timeout ${timeout_min}m)..."
        for _ in $(seq 1 "$max_iters"); do
            local total ready not_ready_names status
            total=$(kubectl get certificate -A --no-headers 2>/dev/null | wc -l | tr -d ' ')
            ready=$(kubectl get certificate -A --no-headers 2>/dev/null | awk '$3=="True"' | wc -l | tr -d ' ')
            if [[ "$total" -gt 0 && "$total" == "$ready" ]]; then
                printf "  [%4ds] All %s Certificates Ready ✓\n" "$elapsed" "$total"
                return 0
            fi
            not_ready_names=$(kubectl get certificate -A --no-headers 2>/dev/null \
                | awk '$3!="True" {print $1"/"$2}' | head -5 | tr '\n' ' ')
            status="${ready:-0}/${total:-?} Ready, waiting: ${not_ready_names:-(none)}"
            if [[ "$prev_status" != "$status" ]] || (( elapsed > 0 && elapsed % 60 == 0 )); then
                printf "  [%4ds] %s\n" "$elapsed" "${status:0:140}"
                prev_status="$status"
            fi
            sleep "$interval"
            elapsed=$(( elapsed + interval ))
        done
        printf "  [%4ds] TIMEOUT after %sm\n" "$elapsed" "$timeout_min"
        echo -e "${YELLOW}Warning: not all Certificates Ready, continuing anyway${NC}"
        kubectl get certificate -A --no-headers | awk '$3!="True"' | head -10
        # 0 for the same reason as wait_kustomization: under `set -e` a non-zero
        # return here ends the run despite the message saying otherwise.
        return 0
    }
    wait_certificates

    echo ""
    echo -e "${YELLOW}=== Verification ===${NC}"
    # Whole-field matches. `grep -c Ready` counts NotReady too — the substring is
    # in it — so this block reported every node healthy on a cluster where none
    # was. `grep -c True` on HelmReleases matched the SUSPENDED column and any
    # message containing the word.
    #
    # awk also avoids grep's exit 1 on zero matches, which under pipefail ended
    # the run here with no output at all.
    echo -n "Nodes: "
    kubectl get nodes --no-headers | awk '$2 == "Ready"' | wc -l | tr -d '\n'
    echo "/${#ALL_NODES[@]}"

    # Ready count against a denominator: "12 ready" reads as success whether the
    # cluster has 12 HelmReleases or 40. Both numbers come from the objects, not
    # from `flux get`'s table, whose columns shift when REVISION is empty.
    echo -n "HelmReleases: "
    hr_total=$(kubectl get helmrelease --all-namespaces --no-headers | wc -l)
    hr_ready=$(kubectl get helmrelease --all-namespaces -o jsonpath=\
'{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \
        | grep -c '^True$') || true
    echo "${hr_ready:-0}/${hr_total} ready"

    echo -n "Ceph: "
    kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.ceph.health}' 2>/dev/null || echo "pending"
    echo ""

    echo -n "Ingress (local): "
    curl -sk --connect-timeout 5 --resolve status.renner.uno:443:10.2.99.86 https://status.renner.uno -o /dev/null -w "%{http_code}" 2>/dev/null || echo "N/A"
    echo ""
fi