claude-botandClaude Opus 5.5
a5f2bc39e4
Review of #1481 (merged with it open): - The Job printed the operator's streaming command with the pod's HOME expanded (/work/.config/...), unusable on the workstation. The defaults stay a literal ~ now; eval expands it on the workstation. - 'Collecting secrets' captured stderr, where age-plugin-yubikey asks for the PIN and touch: a silent hang. stderr stays on the terminal. - Runbook/migration doc still named pass as the DR store in places; tooling list gains age + age-plugin-yubikey; the file list gains hetzner-cloud-api-secret.age. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2674 lines
136 KiB
Bash
Executable File
2674 lines
136 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# K8s Cluster Bootstrap — split into phases so CI can export access as early as possible.
|
||
#
|
||
# Usage:
|
||
# ./bootstrap.sh # run all phases locally
|
||
# ./bootstrap.sh talos # Talos apply + K8s bootstrap + kubeconfig (writes access-staging/)
|
||
# ./bootstrap.sh flux # reads access-staging/, installs flux, waits for infrastructure
|
||
# ./bootstrap.sh wait-apps # waits for apps/authentik/paperless to reconcile
|
||
# ./bootstrap.sh vmtest # throwaway QEMU VMs, then the talos phase
|
||
# # against them unchanged. Needs root for the
|
||
# # provisioner and leaves nothing behind.
|
||
# ./bootstrap.sh hcloudtest # the same rehearsal on Hetzner Cloud, which is
|
||
# # the only place the registry fallback can be
|
||
# # tested: gitea.renner.uno has no external
|
||
# # ingress, so a node there genuinely cannot
|
||
# # reach it. BILLS BY THE HOUR — the machines
|
||
# # stay up until hcloudclean. Access files land in
|
||
# # ~/.local/state/homelab-hcloudtest/.
|
||
# ./bootstrap.sh hcloudclean # deletes everything hcloudtest created, by
|
||
# # label, and the restore test's deploy keys.
|
||
#
|
||
# Restore test on the hcloudtest cluster (production tree, read-only source,
|
||
# kubernetes/restore-test/flux-root.yaml):
|
||
# KUBECONFIG=~/.local/state/homelab-hcloudtest/kubeconfig \
|
||
# TALOSCONFIG=~/.local/state/homelab-hcloudtest/talosconfig \
|
||
# FLUX_SOURCE=restore-test ./bootstrap.sh flux
|
||
#
|
||
# Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls
|
||
# from is itself Flux-managed, so a cold rebuild cannot pull from it until the
|
||
# cluster already exists. Set FLUX_SOURCE=github to bootstrap Flux from the
|
||
# external GitHub mirror instead — the only way out of that loop:
|
||
# FLUX_SOURCE=github GITHUB_TOKEN=<pat> ./bootstrap.sh flux
|
||
# The mirror is kept current server-side by Gitea's native push-mirror. Full
|
||
# cold-rebuild procedure: kubernetes/RUNBOOK-cold-rebuild.md.
|
||
#
|
||
# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt,
|
||
# age + age-plugin-yubikey (the DR secrets in ~/.config/homelab-secrets)
|
||
|
||
TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
|
||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||
# Cilium's single source of truth. Talos ships `cni: none` (talos/patches/
|
||
# cluster.yaml) and nothing else, so the CNI has to be installed here, between
|
||
# `talosctl bootstrap` and Flux — Flux's own controllers are pods and cannot
|
||
# reconcile the network they need in order to start. Everything the install
|
||
# needs is read out of this file at run time so there is no second copy of a
|
||
# version or a value to keep in step. See PROPOSAL-cilium-single-owner.md.
|
||
CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml"
|
||
K8SGW_HR="$REPO_ROOT/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml"
|
||
ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging"
|
||
NODES=(k8s-1 k8s-2 k8s-3)
|
||
# No dedicated workers since the k8s-edge VM was retired (2026-07-02) —
|
||
# hardware-bound services live on the bare-metal edge-1 Arch host
|
||
# (ansible/docs/edge-host.md).
|
||
WORKERS=()
|
||
ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
|
||
DOMAIN="srv.renner.uno"
|
||
DNS_SERVER="10.2.40.1" # UDM (old PVE DNS VIP 10.2.40.53 decommissioned)
|
||
# Cluster API endpoint — Talos VIP (10.2.40.60) elected via etcd, fronted by
|
||
# UniFi DNS A record k8s.renner.uno. Defined in talos/patches/k8s-{1,2,3}.yaml.
|
||
K8S_API_HOST="k8s.renner.uno"
|
||
# Which site patch the nodes get: talos/patches/site-$SITE.yaml. The three
|
||
# variables above and this one are everything the vmtest phase has to change in
|
||
# order to run this same code against throwaway VMs — if that list ever grows,
|
||
# the test is drifting away from the thing it claims to exercise.
|
||
SITE="${SITE:-homelab}"
|
||
# Where this script writes the cluster access files. These are the paths
|
||
# talosctl and kubectl read from anyway, so unset they resolve to exactly the
|
||
# old behaviour. Set, the whole run stays out of $HOME — which is what lets the
|
||
# vmtest phase drive a throwaway cluster without replacing the operator's
|
||
# production access with a VM's.
|
||
TALOSCONFIG_PATH="${TALOSCONFIG:-$HOME/.talos/config}"
|
||
KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}"
|
||
case "$KUBECONFIG_PATH" in
|
||
*:*) echo "KUBECONFIG holds multiple paths; set it to a single file" >&2; exit 1 ;;
|
||
esac
|
||
# Addresses to apply to, in the order of NODES. Empty means resolve the node
|
||
# FQDNs, which is right for real hardware on a network that has DNS for it.
|
||
NODE_ADDRS=()
|
||
GITEA_HOST="gitea.renner.uno"
|
||
GITEA_OWNER="gitea_admin"
|
||
GITEA_REPO="homelab"
|
||
# Flux git source: "gitea" (default, in-cluster) or "github" (break-glass mirror
|
||
# for a cold rebuild — see the usage note above and RUNBOOK-cold-rebuild.md).
|
||
FLUX_SOURCE="${FLUX_SOURCE:-gitea}"
|
||
# GitHub break-glass mirror (github.com/konairius/Homelab), kept current by
|
||
# Gitea's native push-mirror. Overridable in case the mirror moves.
|
||
GITHUB_OWNER="${GITHUB_OWNER:-konairius}"
|
||
GITHUB_REPO="${GITHUB_REPO:-Homelab}"
|
||
# Owner of the private ghcr.io namespace the rebuild-critical images are
|
||
# mirrored into, and the username half of the node registry credential. Matches
|
||
# GHCR_USER in .woodpecker/build-ci-images.yml, which pushes them.
|
||
GHCR_PULL_USER="${GHCR_PULL_USER:-konairius}"
|
||
FLUX_PATH="kubernetes/clusters/homelab"
|
||
# Branch Flux reconciles from, and that `flux bootstrap` commits its own sync
|
||
# manifests into. main for a real rebuild.
|
||
#
|
||
# Overridable so the phase can be rehearsed without writing to main — point it
|
||
# at a throwaway branch, which has to exist already: `flux bootstrap` clones the
|
||
# branch and fails with `couldn't find remote ref` rather than creating it.
|
||
#
|
||
# git push origin origin/main:refs/heads/rehearsal-1
|
||
# FLUX_BRANCH=rehearsal-1 ./bootstrap.sh flux
|
||
#
|
||
# Nothing else about the phase changes, which is the point. Delete the branch
|
||
# afterwards.
|
||
FLUX_BRANCH="${FLUX_BRANCH:-main}"
|
||
SOPS_KEY="$HOME/.config/sops/age/keys.txt"
|
||
|
||
PHASE="${1:-all}"
|
||
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m'
|
||
|
||
is_tty() { [[ -t 1 ]]; }
|
||
|
||
step() {
|
||
local _ignored=$1
|
||
local label="$2"
|
||
if is_tty; then
|
||
printf "${BLUE}==>${NC} %s\n" "$label"
|
||
else
|
||
printf "==> %s\n" "$label"
|
||
fi
|
||
}
|
||
|
||
# One EXIT trap for the whole script. Handlers register here rather than calling
|
||
# `trap ... EXIT` themselves, because a second registration replaces the first
|
||
# instead of chaining — which is how a secrets-cleanup handler silently disabled
|
||
# the VM teardown.
|
||
SECRETS_TMPDIR=""
|
||
on_exit() {
|
||
[[ -n "$SECRETS_TMPDIR" ]] && rm -rf "$SECRETS_TMPDIR"
|
||
declare -F cleanup_vmtest >/dev/null && cleanup_vmtest
|
||
# No teardown here: provisioning is its own phase and the machines it leaves
|
||
# behind are its product. `hcloudclean` is what deletes them.
|
||
# Only when this run actually booked machines. The function is always
|
||
# defined, so testing for it fired on the qemu phase too.
|
||
if [[ "${HCLOUD_PROVISIONED:-0}" == "1" ]]; then
|
||
echo -e "${YELLOW}Hetzner machines are UP and billed until deleted.${NC}"
|
||
echo " ./kubernetes/bootstrap.sh hcloudclean # when finished"
|
||
fi
|
||
return 0
|
||
}
|
||
trap on_exit EXIT
|
||
|
||
fail() { echo -e "\n${RED}FAILED:${NC} $1"; exit 1; }
|
||
ok() { echo -e "${GREEN}OK${NC}"; }
|
||
|
||
# Reads the Ready condition of a Flux Kustomization, without going through the
|
||
# table `flux get` prints.
|
||
#
|
||
# That table is NAME REVISION SUSPENDED READY MESSAGE, and every wait below used
|
||
# to pick READY as the fourth field. REVISION is empty until the first
|
||
# successful reconcile — precisely the state these waits exist to catch — so in
|
||
# the failure case the columns shift left and the fourth field lands on the
|
||
# message. Matching it against "False" then never fires, and a persistently
|
||
# failing Kustomization reads as merely slow. That cost a full two-hour run:
|
||
# infrastructure had been failing its server-side dry-run from the first minute,
|
||
# and the counter meant to catch it never moved.
|
||
#
|
||
# Echoes "<status>|<message>". An empty status means the object does not exist
|
||
# yet; kubectl's own error is carried through as the message rather than
|
||
# discarded, so "not created yet" stays distinguishable from "lookup broke".
|
||
ks_ready_condition() {
|
||
local name="$1" out jp
|
||
jp='{.status.conditions[?(@.type=="Ready")].status}'
|
||
jp+='|{.status.conditions[?(@.type=="Ready")].message}'
|
||
if out=$(kubectl -n flux-system get kustomization "$name" -o jsonpath="$jp" 2>&1); then
|
||
printf '%s' "$out"
|
||
else
|
||
printf '|%s' "$out"
|
||
fi
|
||
}
|
||
|
||
resolve_node_ip() {
|
||
local fqdn=$1
|
||
dig +short "$fqdn" A @"$DNS_SERVER" | head -1
|
||
}
|
||
|
||
phase_runs() {
|
||
[[ "$PHASE" == "all" || "$PHASE" == "$1" ]]
|
||
}
|
||
# The restore test targets the cluster bootstrap.sh hcloudtest builds.
|
||
if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wait-apps" ]]; then
|
||
fail "FLUX_SOURCE=restore-test runs the flux and wait-apps phases only"
|
||
fi
|
||
OPENBAO_NS="openbao"
|
||
|
||
# Where the secrets come from. On a workstation, age files in
|
||
# HOMELAB_SECRETS_DIR, encrypted to the operator's YubiKeys and an offline paper
|
||
# key (a copy sits on nas-2). HOMELAB_AGE_IDENTITY picks what decrypts them: the
|
||
# YubiKey identity stubs by default (touch per file), the paper key's file when
|
||
# both sticks are gone. In the in-cluster Job there is neither — and there
|
||
# should not be: those keys open everything, and they stay with the operator.
|
||
# Set OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
|
||
# operator streams in when the run asks for them:
|
||
#
|
||
# age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age \
|
||
# | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
|
||
#
|
||
# They exist in the pod's filesystem for the length of the run and are never
|
||
# written to a Secret in this cluster, so they do not land in its etcd or in
|
||
# any backup of it. The seal key alone becomes a Secret in the cluster being
|
||
# built, which is what lets that OpenBao unseal itself.
|
||
OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}"
|
||
|
||
# Every secret the run needs, named once, in one place. The list is here and
|
||
# nowhere else on purpose: when it was spread over the call sites, each was read
|
||
# at the moment it was needed, so a missing one surfaced twenty minutes into a
|
||
# run — and a second copy of the same path in a runbook or a wrapper is a copy
|
||
# that goes stale.
|
||
#
|
||
# The git token for `flux bootstrap`, chosen by the source it will authenticate
|
||
# against — which is the whole point and is what the previous expression got
|
||
# wrong. It read ${GITHUB_TOKEN:-${GITEA_TOKEN:-}}, taking whichever variable
|
||
# happened to be set: on a workstation with GITEA_TOKEN exported and no
|
||
# GITHUB_TOKEN, a break-glass run handed a Gitea token to the GitHub API and
|
||
# died on `401 Bad credentials` after the cluster was already built.
|
||
#
|
||
# `gh auth token` as the GitHub fallback because that is the credential the
|
||
# operator already has on this machine, and it is what the vmtest secret feeder
|
||
# has always used. Emptiness is not checked here — require_secrets rejects an
|
||
# empty result and names what is missing.
|
||
#
|
||
# gh is called plainly, with its stderr left alone. Swallowing it would turn a
|
||
# failed login into an empty string, and require_secrets would then report
|
||
# "came back empty" for a credential that exists but could not be read — a zero
|
||
# that cannot be told apart from a fault. Its non-zero exit propagates, which is
|
||
# what makes require_secrets print the reason instead.
|
||
git_token_for_source() {
|
||
if [[ "$FLUX_SOURCE" == "github" || "$FLUX_SOURCE" == "restore-test" ]]; then
|
||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||
printf '%s' "$GITHUB_TOKEN"
|
||
else
|
||
gh auth token
|
||
fi
|
||
else
|
||
printf '%s' "${GITEA_TOKEN:-}"
|
||
fi
|
||
}
|
||
|
||
# Literal ~ on purpose: the commands are eval'd on the workstation, which
|
||
# expands it there, and the in-cluster Job prints them for the operator to run
|
||
# on that workstation, where the Job's own HOME (/work) would be wrong.
|
||
# shellcheck disable=SC2088
|
||
[[ -n "${HOMELAB_SECRETS_DIR:-}" ]] || HOMELAB_SECRETS_DIR='~/.config/homelab-secrets'
|
||
# shellcheck disable=SC2088
|
||
[[ -n "${HOMELAB_AGE_IDENTITY:-}" ]] || HOMELAB_AGE_IDENTITY='~/.config/age/yubikeys.txt'
|
||
AGE_SECRET="age -d -i $HOMELAB_AGE_IDENTITY $HOMELAB_SECRETS_DIR"
|
||
|
||
# Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
|
||
OPENBAO_REQUIRED_SECRETS=(
|
||
"sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
|
||
# Which token depends on FLUX_SOURCE, and that is not a detail: in a real
|
||
# cold rebuild the in-cluster Gitea cannot serve the cluster being rebuilt,
|
||
# so the break-glass path pulls from the GitHub mirror. A run that demands a
|
||
# Gitea token is a run that cannot rehearse the disaster it exists for.
|
||
# The variable names are the ones the usage header and RUNBOOK-cold-rebuild
|
||
# tell the operator to export. An invented third name was worse than a typo:
|
||
# `${VAR:?}` inside eval terminates the shell outright, so the documented
|
||
# recovery command exited with no message and no cleanup at all.
|
||
#
|
||
# Emptiness is not tested here on purpose — require_secrets rejects an empty
|
||
# file and names what is missing.
|
||
"git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
|
||
"s3-access|Hetzner restore access key|$AGE_SECRET/s3-access.age"
|
||
"s3-secret|Hetzner restore secret key|$AGE_SECRET/s3-secret.age"
|
||
# The static seal key. The snapshots are encrypted under it, so it has to
|
||
# be the one the backed-up cluster ran with — a fresh key restores nothing.
|
||
"seal-key|OpenBao seal key|$AGE_SECRET/seal-key.age"
|
||
# Read by the node registry auth (see registry_auth_patch): the images an app
|
||
# needs while the cluster is still being built are mirrored to a private
|
||
# ghcr.io namespace, and a node has to be able to pull them before anything
|
||
# in the cluster exists to hold a credential. Demanded up front like the
|
||
# rest, because a rebuild that discovers it missing has already wiped the
|
||
# disks.
|
||
"ghcr-pull|GHCR pull token for the image mirror|$AGE_SECRET/ghcr-pull.age"
|
||
)
|
||
# Asked for by name only, never by a bare require_secrets: a rebuild of the
|
||
# real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
|
||
# Hetzner project homelab-backup-verify, which holds nothing but throwaway
|
||
# machines.
|
||
HCLOUD_REQUIRED_SECRETS=(
|
||
"hcloud-token|Hetzner Cloud API token for the throwaway test cluster|bao kv get -mount=homelab -field=token infrastructure/hcloud-backup-verify"
|
||
)
|
||
|
||
# Collects all of them up front, into one directory, so everything after this
|
||
# point reads files and takes the same code path whether the run is on a laptop
|
||
# during a real recovery or in the in-cluster Job.
|
||
#
|
||
# The only difference between those two is where the material comes from, and it
|
||
# is confined to this function: on a laptop the commands in the list decrypt
|
||
# it from HOMELAB_SECRETS_DIR; in the Job the operator streams it in. Everything downstream cannot
|
||
# tell the difference — which is the point, because the recovery path must not be
|
||
# a path that only gets exercised during a recovery.
|
||
# require_secrets [name...]
|
||
#
|
||
# With no arguments, demands every entry in OPENBAO_REQUIRED_SECRETS. With
|
||
# names, demands only those — which is what lets the talos phase stand alone.
|
||
#
|
||
# That phase used to need nothing but the machines, and `bootstrap.sh talos` is
|
||
# documented as the part that can run unattended. Node registry auth changed
|
||
# that: the machine config now carries a pull credential, so the phase needs
|
||
# exactly one secret. Demanding all six there would drag in the OpenBao seal
|
||
# key, which deliberately never reaches CI — and demanding none, as before,
|
||
# left the phase to discover the missing file halfway through `talosctl gen`.
|
||
require_secrets() {
|
||
local entry name what how
|
||
local -a wanted=()
|
||
if [[ $# -gt 0 ]]; then
|
||
for entry in "${OPENBAO_REQUIRED_SECRETS[@]}" "${HCLOUD_REQUIRED_SECRETS[@]}"; do
|
||
for name in "$@"; do
|
||
[[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry")
|
||
done
|
||
done
|
||
[[ ${#wanted[@]} -eq $# ]] \
|
||
|| fail "require_secrets asked for [$*] but the list defines only: ${wanted[*]%%|*}"
|
||
else
|
||
wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
|
||
fi
|
||
|
||
# Which branch: decrypt locally, or wait for someone to stream files in.
|
||
#
|
||
# The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
|
||
# FUNCTION sets it. Once a subset call had run — hcloudtest asks for
|
||
# ghcr-pull and hcloud-token before anything else — every later call found
|
||
# the variable populated and took the wait branch, printing streaming
|
||
# instructions for a Job that does not exist on a workstation where pass was
|
||
# available all along. The run sat there while three nodes stayed in
|
||
# maintenance.
|
||
#
|
||
# SECRETS_SELF_FETCH records which world we are in, and it is decided once:
|
||
# an empty OPENBAO_SECRETS_DIR on entry means local age decryption, an inherited one means
|
||
# the Job. Later calls follow that decision instead of re-deriving it from
|
||
# state this function has since changed.
|
||
if [[ -z "$OPENBAO_SECRETS_DIR" || "${SECRETS_SELF_FETCH:-0}" == "1" ]]; then
|
||
if [[ -z "$OPENBAO_SECRETS_DIR" ]]; then
|
||
OPENBAO_SECRETS_DIR=$(mktemp -d)
|
||
chmod 700 "$OPENBAO_SECRETS_DIR"
|
||
fi
|
||
SECRETS_SELF_FETCH=1
|
||
# Registered by name, not with a second `trap ... EXIT`: a second one
|
||
# REPLACES the first, and the first is the vmtest teardown. Setting it
|
||
# here left four VMs, their bridges and the sudo keepalive running after
|
||
# a workstation rehearsal — silently, and only on the laptop path, since
|
||
# in-cluster this branch is not taken.
|
||
SECRETS_TMPDIR="$OPENBAO_SECRETS_DIR"
|
||
echo -n "Collecting secrets... "
|
||
for entry in "${wanted[@]}"; do
|
||
name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
|
||
# Already collected by an earlier call — a subset call fetches only
|
||
# what it asked for, so the full call that follows must fill the
|
||
# gaps rather than re-run every command. Each decryption waits for
|
||
# a YubiKey touch, and asking twice for the same key is how an
|
||
# unattended run turns into a hung one.
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
|
||
# stderr stays on the terminal: age-plugin-yubikey asks for the
|
||
# PIN and the touch there, and a captured prompt is a silent hang.
|
||
echo
|
||
if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name"; then
|
||
fail "could not obtain the $what with: $how"
|
||
fi
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
|
||
|| fail "the $what came back empty from: $how"
|
||
done
|
||
ok
|
||
return 0
|
||
fi
|
||
|
||
mkdir -p "$OPENBAO_SECRETS_DIR"
|
||
local entry name what how missing
|
||
for i in $(seq 1 180); do
|
||
missing=()
|
||
for entry in "${wanted[@]}"; do
|
||
name=${entry%%|*}
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] || missing+=("$entry")
|
||
done
|
||
[[ ${#missing[@]} -eq 0 ]] && { echo "All secrets present."; return 0; }
|
||
if [[ $i -eq 1 ]]; then
|
||
echo -e "${YELLOW}Waiting for ${#missing[@]} secret(s). Stream each in with:${NC}" >&2
|
||
for entry in "${missing[@]}"; do
|
||
name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
|
||
echo " # $what" >&2
|
||
echo " $how | kubectl -n $OPENBAO_NS_HINT exec -i \$POD -- sh -c 'cat > $OPENBAO_SECRETS_DIR/$name'" >&2
|
||
done
|
||
fi
|
||
sleep 10
|
||
[[ $i -eq 180 ]] && fail "still missing: ${missing[*]%%|*}"
|
||
done
|
||
}
|
||
OPENBAO_NS_HINT=vmtest
|
||
|
||
# Reads one secret by its name in the list above. In the Job that is the file the
|
||
# operator streamed in; on a workstation it runs the command the list names,
|
||
# which for these is `age -d` of a file in HOMELAB_SECRETS_DIR.
|
||
#
|
||
# The list is the only place the paths appear. They used to sit here as
|
||
# constants as well, which is how the same value ends up defined twice and one
|
||
# copy goes stale.
|
||
read_secret() {
|
||
local name=$1
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
|
||
|| fail "$name is not in $OPENBAO_SECRETS_DIR — require_secrets runs first and should have caught this"
|
||
cat "$OPENBAO_SECRETS_DIR/$name"
|
||
}
|
||
|
||
# Hetzner Object Storage, where the offsite backups live. Read with the restore
|
||
# key (s3-access, s3-secret).
|
||
S3_ENDPOINT="fsn1.your-objectstorage.com"
|
||
S3_REGION="fsn1"
|
||
|
||
# One request, signed by curl with the restore key. The body goes to stdout;
|
||
# anything but a 2xx fails, with S3's error document on stderr.
|
||
s3_request() {
|
||
local method=$1 path=$2 body code
|
||
body=$(mktemp)
|
||
if ! code=$(curl -sS -o "$body" -w '%{http_code}' -X "$method" \
|
||
--aws-sigv4 "aws:amz:$S3_REGION:s3" \
|
||
--user "$(read_secret s3-access):$(read_secret s3-secret)" \
|
||
"https://$S3_ENDPOINT/$path"); then
|
||
rm -f "$body"
|
||
return 1
|
||
fi
|
||
if [[ "$code" != 2?? ]]; then
|
||
echo "$method $path: HTTP $code" >&2
|
||
cat "$body" >&2
|
||
echo >&2
|
||
rm -f "$body"
|
||
return 1
|
||
fi
|
||
cat "$body"
|
||
rm -f "$body"
|
||
}
|
||
|
||
# The "directories" directly under PREFIX in BUCKET, one per line, without
|
||
# PREFIX and the trailing slash.
|
||
s3_dirs() {
|
||
local bucket=$1 prefix=$2 xml
|
||
xml=$(s3_request GET "$bucket?list-type=2&delimiter=%2F&prefix=${prefix//\//%2F}") || return 1
|
||
[[ "$xml" == *"<IsTruncated>false</IsTruncated>"* ]] \
|
||
|| { echo "the listing of $bucket/$prefix is truncated or not a listing" >&2; return 1; }
|
||
# awk, not grep: no match is not an error.
|
||
awk -v p="$prefix" '{
|
||
while (match($0, /<CommonPrefixes><Prefix>[^<]*<\/Prefix>/)) {
|
||
d = substr($0, RSTART + 24, RLENGTH - 33)
|
||
$0 = substr($0, RSTART + RLENGTH)
|
||
sub(/\/$/, "", d)
|
||
print substr(d, length(p) + 1)
|
||
}
|
||
}' <<<"$xml"
|
||
}
|
||
|
||
# The bucket every CNPG cluster recovers from (cnpg-object-stores/). The
|
||
# restore test archives to CNPG_TEST_BUCKET instead: a test incarnation with a
|
||
# base backup in the production bucket would be the newest there, and the next
|
||
# real recovery would read the test's data.
|
||
CNPG_BUCKET="renner-uno-k8s-cnpg"
|
||
CNPG_TEST_BUCKET="renner-uno-k8s-cnpg-rtest"
|
||
CNPG_STORES_DIR="$REPO_ROOT/kubernetes/cnpg-object-stores"
|
||
|
||
# cnpg-paths, the ConfigMap cnpg-object-stores takes its paths from. Written
|
||
# once per cluster incarnation: an existing one belongs to clusters already
|
||
# archiving under its paths and is kept.
|
||
#
|
||
# For each source ObjectStore in cnpg-object-stores/, cluster <x>:
|
||
# CNPG_<X>_SOURCE the candidate prefix whose <x>/base/ holds the newest
|
||
# completed base backup. Candidates: <x>, <x>-g<14 digits> and the
|
||
# store's cnpg.renner.uno/source-prefixes.
|
||
# CNPG_<X>_ARCHIVE <x>-g<now> in ARCHIVE_BUCKET: new, so empty, so the
|
||
# plugin's empty-archive check passes rather than being
|
||
# skipped.
|
||
write_cnpg_paths() {
|
||
local archive_bucket=$1
|
||
local gen dirs name extra var p listing newest best best_at id info
|
||
local -a literals=()
|
||
echo "Writing the CNPG backup paths (cnpg-paths)..."
|
||
command -v yq >/dev/null || fail "yq not found (reads $CNPG_STORES_DIR)"
|
||
if kubectl -n flux-system get configmap cnpg-paths -o name >/dev/null 2>"$OPENBAO_SECRETS_DIR/.err"; then
|
||
echo " already present, kept"
|
||
return 0
|
||
fi
|
||
grep -q NotFound "$OPENBAO_SECRETS_DIR/.err" \
|
||
|| { cat "$OPENBAO_SECRETS_DIR/.err" >&2; fail "could not read the cnpg-paths ConfigMap"; }
|
||
|
||
gen=g$(date -u +%Y%m%d%H%M%S)
|
||
dirs=$(s3_dirs "$CNPG_BUCKET" "") || fail "could not list $CNPG_BUCKET"
|
||
while IFS=$'\t' read -r name extra; do
|
||
best=''
|
||
best_at=''
|
||
for p in $dirs; do
|
||
[[ "$p" == "$name" || "$p" =~ ^${name}-g[0-9]{14}$ || " $extra " == *" $p "* ]] || continue
|
||
listing=$(s3_dirs "$CNPG_BUCKET" "$p/$name/base/") \
|
||
|| fail "could not list $CNPG_BUCKET/$p/$name/base/"
|
||
# Newest completed backup: barman uploads backup.info with
|
||
# status=STARTED when a backup begins and rewrites it with DONE or
|
||
# FAILED when it ends.
|
||
newest=''
|
||
for id in $(sort -r <<<"$listing"); do
|
||
[[ "$id" > "$best_at" ]] || break
|
||
info=$(s3_request GET "$CNPG_BUCKET/$p/$name/base/$id/backup.info") \
|
||
|| fail "could not read $CNPG_BUCKET/$p/$name/base/$id/backup.info"
|
||
if grep -qx 'status=DONE' <<<"$info"; then
|
||
newest=$id
|
||
break
|
||
fi
|
||
echo " $name: skipping $p/$name/base/$id ($(grep '^status=' <<<"$info" || echo 'no status'))"
|
||
done
|
||
if [[ -n "$newest" ]]; then
|
||
best=$p
|
||
best_at=$newest
|
||
fi
|
||
done
|
||
[[ -n "$best" ]] \
|
||
|| fail "no completed base backup of $name in $CNPG_BUCKET (looked under $name, $name-g<timestamp>${extra:+, $extra})"
|
||
var=CNPG_$(tr 'a-z-' 'A-Z_' <<<"$name")
|
||
literals+=("--from-literal=${var}_SOURCE=s3://$CNPG_BUCKET/$best"
|
||
"--from-literal=${var}_ARCHIVE=s3://$archive_bucket/$name-$gen")
|
||
echo " $name: recovers from $CNPG_BUCKET/$best (base backup $best_at), archives to $archive_bucket/$name-$gen"
|
||
done < <(yq -r 'select(.kind == "ObjectStore" and .metadata.annotations["cnpg.renner.uno/server-name"] != null)
|
||
| [.metadata.annotations["cnpg.renner.uno/server-name"],
|
||
(.metadata.annotations["cnpg.renner.uno/source-prefixes"] // "")] | @tsv' \
|
||
"$CNPG_STORES_DIR"/*.yaml)
|
||
[[ ${#literals[@]} -gt 0 ]] \
|
||
|| fail "no source ObjectStore (cnpg.renner.uno/server-name) in $CNPG_STORES_DIR"
|
||
kubectl -n flux-system create configmap cnpg-paths "${literals[@]}" >/dev/null \
|
||
|| fail "could not create the cnpg-paths ConfigMap"
|
||
ok
|
||
}
|
||
|
||
# The restore test's archive bucket. Created when missing; on a rerun it
|
||
# already exists, which S3 reports as an error.
|
||
create_cnpg_test_bucket() {
|
||
local out
|
||
out=$(s3_request PUT "$CNPG_TEST_BUCKET" 2>&1) && return 0
|
||
[[ "$out" == *BucketAlreadyOwnedByYou* ]] && return 0
|
||
echo "$out" >&2
|
||
fail "could not create $CNPG_TEST_BUCKET"
|
||
}
|
||
|
||
# Empties and deletes the restore test's archive bucket. Each pass deletes the
|
||
# first page of keys and lists again, so no continuation token is needed.
|
||
delete_cnpg_test_bucket() {
|
||
local xml key
|
||
while :; do
|
||
if ! xml=$(s3_request GET "$CNPG_TEST_BUCKET?list-type=2" 2>&1); then
|
||
[[ "$xml" == *NoSuchBucket* ]] && return 0
|
||
echo "$xml" >&2
|
||
echo " WARNING: $CNPG_TEST_BUCKET not emptied — check the console" >&2
|
||
return 0
|
||
fi
|
||
while read -r key; do
|
||
[[ -n "$key" ]] || continue
|
||
s3_request DELETE "$CNPG_TEST_BUCKET/$key" >/dev/null \
|
||
|| { echo " WARNING: $CNPG_TEST_BUCKET/$key not deleted — check the console" >&2; return 0; }
|
||
done < <(awk '{
|
||
while (match($0, /<Key>[^<]*<\/Key>/)) {
|
||
print substr($0, RSTART + 5, RLENGTH - 11)
|
||
$0 = substr($0, RSTART + RLENGTH)
|
||
}
|
||
}' <<<"$xml")
|
||
[[ "$xml" == *"<IsTruncated>true</IsTruncated>"* ]] || break
|
||
done
|
||
if s3_request DELETE "$CNPG_TEST_BUCKET" >/dev/null; then
|
||
echo " deleted bucket $CNPG_TEST_BUCKET"
|
||
else
|
||
echo " WARNING: bucket $CNPG_TEST_BUCKET not deleted — check the console" >&2
|
||
fi
|
||
}
|
||
|
||
# Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase
|
||
# against it unchanged — same shape as the vmtest phase below, and for the same
|
||
# reason: a rehearsal that runs different code proves less than one that does
|
||
# not.
|
||
#
|
||
# WHY THIS EXISTS, given vmtest already rehearses a rebuild locally.
|
||
#
|
||
# The one thing vmtest cannot test is the registry fallback. The nodes now carry
|
||
# machine.registries.mirrors with gitea.renner.uno first and ghcr.io second, and
|
||
# the whole point of the second endpoint is the window where the first does not
|
||
# exist. At home it always exists — the production cluster is running and
|
||
# answers on the LAN — so containerd never reaches the fallback and it stays
|
||
# unproven. From Hetzner, Gitea does not answer at all: it has no external
|
||
# ingress.
|
||
#
|
||
# WHAT IT COSTS, because this one bills by the hour.
|
||
#
|
||
# Teardown is by LABEL, not by remembered ID, and runs from the single EXIT
|
||
# trap. A run that dies between `server create` and its own bookkeeping is
|
||
# therefore still cleanable — by the next run, or by `bootstrap.sh hcloudclean`,
|
||
# which does nothing but the sweep. Anything this phase creates carries
|
||
# HCLOUD_LABEL so that sweep can find it without being told.
|
||
#
|
||
# WHAT IT IS.
|
||
#
|
||
# The production config on new hardware: cluster.yaml, site-hcloud.yaml (what
|
||
# the Hetzner site is), containerd-layer-retention.yaml and hcloud-N.yaml (each
|
||
# machine). Nothing stands in for the home network. Flux is a separate run,
|
||
# FLUX_SOURCE=restore-test bootstrap.sh flux, because it needs the OpenBao seal
|
||
# key. Ceph replicates over the pod network there: Hetzner networks are layer 3
|
||
# (restore-test/flux-root.yaml).
|
||
#
|
||
# Containment: an outbound Hetzner firewall on every node admits udp and icmp
|
||
# anywhere and tcp anywhere except the addresses of proxy.renner.uno, the public
|
||
# edge in front of production.
|
||
# Talos version and Image Factory schematic, read out of the installer image in
|
||
# cluster.yaml. Both throwaway-cluster phases need them and neither may state
|
||
# them: a second copy is a copy that goes stale, and the day it did the test
|
||
# would quietly stop booting the image a rebuild installs.
|
||
talos_image_coords() {
|
||
command -v yq >/dev/null || fail "yq not found"
|
||
local image
|
||
image=$(yq -r '.machine.install.image' "$TALOS_DIR/patches/cluster.yaml")
|
||
VM_SCHEMATIC=${image#*metal-installer/}
|
||
VM_SCHEMATIC=${VM_SCHEMATIC%%:*}
|
||
VM_VERSION=${image##*:}
|
||
[[ ${#VM_SCHEMATIC} -eq 64 && "$VM_VERSION" == v* ]] \
|
||
|| fail "could not read a schematic and version out of '$image'"
|
||
}
|
||
|
||
HCLOUD_LABEL="managed-by=bootstrap-hcloudtest"
|
||
|
||
# The disks a machine has are stated in its Talos patch and read from there, so
|
||
# the provisioner carries no second copy. Emits one size in whole GB per raw
|
||
# volume the patch declares off the system disk, in order.
|
||
hcloud_volumes_for_node() {
|
||
local patch="$TALOS_DIR/patches/$1.yaml" size gib
|
||
[[ -f "$patch" ]] || fail "$patch not found — the machine config is the list"
|
||
while read -r size; do
|
||
[[ -n "$size" && "$size" != "null" ]] || continue
|
||
# Talos states GiB; the API bills GB. Round up, so the volume can never
|
||
# come in under the minSize the machine config asked for.
|
||
gib=${size%GiB}
|
||
[[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as <n>GiB"
|
||
echo $(( (gib * 1074 + 999) / 1000 ))
|
||
done < <(yq -r 'select(.kind == "RawVolumeConfig" and .provisioning.diskSelector.match != "system_disk") | .provisioning.minSize' "$patch")
|
||
}
|
||
|
||
# Deletes everything this phase created, and NOTHING else.
|
||
#
|
||
# The project (homelab-backup-verify) also runs the backup-verify CronJob's
|
||
# machines. `hcloud server list -l ""` lists EVERY server, so an unset
|
||
# HCLOUD_LABEL would turn this function into a project wipe: the label is
|
||
# checked before it is used, and every candidate must also carry the name
|
||
# prefix this phase assigns.
|
||
HCLOUD_NAME_PREFIX="hcloud"
|
||
# Outlives the run: the test cluster's talosconfig and kubeconfig.
|
||
HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest"
|
||
# The public edge in front of production. The test cluster must not reach it.
|
||
HCLOUD_PROXY_HOST="proxy.renner.uno"
|
||
# Hetzner's limit on effective rules per firewall and direction.
|
||
HCLOUD_FW_RULE_LIMIT=500
|
||
# Hetzner's limit on prefixes in one rule (not documented; 101 is rejected).
|
||
HCLOUD_FW_RULE_PREFIXES=100
|
||
|
||
# The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when
|
||
# missing: a throwaway server in rescue mode writes the Image Factory hcloud
|
||
# image to its disk, which is then snapshotted. Matched by version and
|
||
# schematic. Called directly, not in $(...), so that fail stops the run.
|
||
hcloud_talos_snapshot() {
|
||
local labels="os=talos,talos-version=$VM_VERSION,talos-schematic=${VM_SCHEMATIC:0:16}"
|
||
local builder="$HCLOUD_NAME_PREFIX-imgbuild" key_dir addr ids listing
|
||
listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \
|
||
|| fail "could not list the Talos snapshots"
|
||
mapfile -t ids <<<"$listing"
|
||
if [[ -n "${ids[0]:-}" ]]; then
|
||
HCLOUD_IMAGE=${ids[0]}
|
||
return 0
|
||
fi
|
||
echo " no snapshot for $VM_VERSION/${VM_SCHEMATIC:0:16}, building one"
|
||
key_dir=$(mktemp -d)
|
||
ssh-keygen -q -t ed25519 -N '' -C "$builder" -f "$key_dir/key"
|
||
if hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=name | grep -qx "$builder"; then
|
||
hcloud ssh-key delete "$builder" >/dev/null || fail "could not replace the stale image builder key"
|
||
fi
|
||
hcloud ssh-key create --name "$builder" --label "$HCLOUD_LABEL" \
|
||
--public-key-from-file "$key_dir/key.pub" >/dev/null || fail "could not upload the image builder key"
|
||
hcloud server create --name "$builder" --type cx23 --image debian-12 \
|
||
--location "${HCLOUD_LOCATION:-fsn1}" --ssh-key "$builder" --label "$HCLOUD_LABEL" >/dev/null \
|
||
|| fail "could not create the image builder"
|
||
# From here a failure leaves a billed server: remove it before failing.
|
||
builder_fail() {
|
||
hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2
|
||
hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2
|
||
rm -rf "$key_dir"
|
||
fail "$1"
|
||
}
|
||
hcloud server enable-rescue "$builder" --ssh-key "$builder" >/dev/null || builder_fail "could not enable rescue on $builder"
|
||
hcloud server reset "$builder" >/dev/null || builder_fail "could not reset $builder into rescue"
|
||
addr=$(hcloud server ip "$builder") || builder_fail "could not read the address of $builder"
|
||
for i in $(seq 1 30); do
|
||
ssh -i "$key_dir/key" -o BatchMode=yes -o ConnectTimeout=10 \
|
||
-o UserKnownHostsFile="$key_dir/known_hosts" -o StrictHostKeyChecking=accept-new \
|
||
"root@$addr" 'grep -q rescue /etc/hostname || hostname | grep -q rescue' && break
|
||
sleep 10
|
||
[[ $i -eq 30 ]] && builder_fail "$builder never came up in rescue"
|
||
done
|
||
ssh -i "$key_dir/key" -o BatchMode=yes -o UserKnownHostsFile="$key_dir/known_hosts" "root@$addr" \
|
||
"set -eo pipefail; curl -fsSL https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz | xz -d | dd of=/dev/sda bs=4M conv=fsync" \
|
||
|| builder_fail "writing the Talos image on $builder failed"
|
||
hcloud server poweroff "$builder" >/dev/null || builder_fail "could not power off $builder"
|
||
hcloud server create-image "$builder" --type snapshot \
|
||
--description "talos $VM_VERSION ${VM_SCHEMATIC:0:16}" --label "$labels" \
|
||
|| builder_fail "could not snapshot $builder"
|
||
listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \
|
||
|| builder_fail "could not list the Talos snapshots"
|
||
mapfile -t ids <<<"$listing"
|
||
[[ -n "${ids[0]:-}" ]] || builder_fail "the snapshot of $builder is not listed under $labels"
|
||
HCLOUD_IMAGE=${ids[0]}
|
||
hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2
|
||
hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2
|
||
rm -rf "$key_dir"
|
||
}
|
||
|
||
# One address as a string of bits: 32 for IPv4, 128 for IPv6. Returns 1, with
|
||
# the reason on stderr, for anything else.
|
||
ip_to_bits() {
|
||
local addr=$1 bits="" part v i head tail
|
||
local -a parts=() hparts=() tparts=()
|
||
if [[ "$addr" == *:* ]]; then
|
||
if [[ "$addr" == *::* ]]; then
|
||
head=${addr%%::*}; tail=${addr#*::}
|
||
[[ -n "$head" ]] && IFS=: read -r -a hparts <<<"$head"
|
||
[[ -n "$tail" ]] && IFS=: read -r -a tparts <<<"$tail"
|
||
parts=("${hparts[@]}")
|
||
for (( i = ${#hparts[@]} + ${#tparts[@]}; i < 8; i++ )); do parts+=(0); done
|
||
parts+=("${tparts[@]}")
|
||
else
|
||
IFS=: read -r -a parts <<<"$addr"
|
||
fi
|
||
[[ ${#parts[@]} -eq 8 ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; }
|
||
for part in "${parts[@]}"; do
|
||
[[ "$part" =~ ^[0-9a-fA-F]{1,4}$ ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; }
|
||
v=$(( 16#$part ))
|
||
for (( i = 15; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done
|
||
done
|
||
else
|
||
IFS=. read -r -a parts <<<"$addr"
|
||
[[ ${#parts[@]} -eq 4 ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
|
||
for part in "${parts[@]}"; do
|
||
[[ "$part" =~ ^[0-9]{1,3}$ ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
|
||
v=$(( 10#$part ))
|
||
(( v <= 255 )) || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
|
||
for (( i = 7; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done
|
||
done
|
||
fi
|
||
printf '%s' "$bits"
|
||
}
|
||
|
||
# A prefix given as bits, in CIDR notation. $2 is the address length (32|128).
|
||
bits_to_cidr() {
|
||
local p=$1 width=$2 full out="" i
|
||
full=$p
|
||
while (( ${#full} < width )); do full+="0"; done
|
||
if (( width == 32 )); then
|
||
for (( i = 0; i < 32; i += 8 )); do out+="${out:+.}$(( 2#${full:i:8} ))"; done
|
||
else
|
||
for (( i = 0; i < 128; i += 16 )); do out+="${out:+:}$(printf '%x' $(( 2#${full:i:16} )))"; done
|
||
fi
|
||
printf '%s/%s\n' "$out" "${#p}"
|
||
}
|
||
|
||
# cidr_complement <address>...: the prefixes that, with the given addresses
|
||
# of one family, cover 0.0.0.0/0 or ::/0 exactly once. Returns 1, with the
|
||
# reason on stderr, on a malformed or mixed-family list.
|
||
cidr_complement() {
|
||
local width=32 a bits
|
||
local -a excl=()
|
||
[[ "$1" == *:* ]] && width=128
|
||
for a in "$@"; do
|
||
[[ ( "$a" == *:* && $width -eq 128 ) || ( "$a" != *:* && $width -eq 32 ) ]] \
|
||
|| { echo "cidr_complement: '$a' is not of the same family as '$1'" >&2; return 1; }
|
||
bits=$(ip_to_bits "$a") || return 1
|
||
excl+=("$bits")
|
||
done
|
||
_cidr_walk() {
|
||
local p=$1 x hit=0
|
||
for x in "${excl[@]}"; do
|
||
[[ "$x" == "$p"* ]] && { hit=1; break; }
|
||
done
|
||
if (( hit == 0 )); then
|
||
bits_to_cidr "$p" "$width"
|
||
return 0
|
||
fi
|
||
(( ${#p} == width )) && return 0
|
||
_cidr_walk "${p}0"
|
||
_cidr_walk "${p}1"
|
||
}
|
||
_cidr_walk ""
|
||
}
|
||
|
||
# Addresses of HCLOUD_PROXY_HOST for one record type (A|AAAA), over DoH: the
|
||
# operator's LAN resolver intercepts plain DNS. Returns 1, with the reason on
|
||
# stderr, when the lookup fails.
|
||
hcloud_proxy_addrs() {
|
||
local type=$1 num answer
|
||
case "$type" in
|
||
A) num=1 ;;
|
||
AAAA) num=28 ;;
|
||
*) echo "hcloud_proxy_addrs: unknown type $type" >&2; return 1 ;;
|
||
esac
|
||
answer=$(curl -sS --max-time 10 -H 'accept: application/dns-json' \
|
||
"https://cloudflare-dns.com/dns-query?name=$HCLOUD_PROXY_HOST&type=$type") \
|
||
|| { echo "could not resolve $HCLOUD_PROXY_HOST $type over DoH" >&2; return 1; }
|
||
jq -e '.Status == 0' >/dev/null <<<"$answer" \
|
||
|| { echo "DoH lookup of $HCLOUD_PROXY_HOST $type failed: $answer" >&2; return 1; }
|
||
jq -r --argjson t "$num" '.Answer[]? | select(.type == $t) | .data' <<<"$answer" | sort -u
|
||
}
|
||
|
||
# Outbound firewall for the nodes: udp and icmp anywhere, tcp anywhere except
|
||
# HCLOUD_PROXY_HOST. Attached before an earlier run's egress firewall is
|
||
# removed, so no node is ever without one.
|
||
hcloud_apply_egress() {
|
||
local fw="hcloudtest-node-egress-$$" v4 v6 c4 c6 n count fw_id fw_name srv_id
|
||
local -a a4=() a6=() tcp_dst=()
|
||
v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)"
|
||
v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)"
|
||
[[ -n "$v4" ]] && mapfile -t a4 <<<"$v4"
|
||
[[ -n "$v6" ]] && mapfile -t a6 <<<"$v6"
|
||
(( ${#a4[@]} + ${#a6[@]} > 0 )) || fail "$HCLOUD_PROXY_HOST resolved to no address — refusing to build the egress firewall"
|
||
echo " blocking tcp to $HCLOUD_PROXY_HOST: ${a4[*]} ${a6[*]}"
|
||
if (( ${#a4[@]} > 0 )); then
|
||
c4=$(cidr_complement "${a4[@]}") || fail "could not compute the IPv4 complement of ${a4[*]}"
|
||
else
|
||
c4=0.0.0.0/0
|
||
fi
|
||
if (( ${#a6[@]} > 0 )); then
|
||
c6=$(cidr_complement "${a6[@]}") || fail "could not compute the IPv6 complement of ${a6[*]}"
|
||
else
|
||
c6=::/0
|
||
fi
|
||
mapfile -t tcp_dst <<<"$c4"$'\n'"$c6"
|
||
# Effective rules are counted per destination prefix; 0.0.0.0/0 and ::/0
|
||
# carry udp and icmp.
|
||
count=$(printf '%s\n' 0.0.0.0/0 ::/0 "${tcp_dst[@]}" | sort -u | grep -c .)
|
||
(( count < HCLOUD_FW_RULE_LIMIT )) \
|
||
|| fail "the egress firewall needs $count effective rules, Hetzner allows $HCLOUD_FW_RULE_LIMIT per firewall"
|
||
hcloud firewall create --name "$fw" --label "$HCLOUD_LABEL" >/dev/null || fail "could not create $fw"
|
||
hcloud firewall add-rule "$fw" --direction out --protocol udp --port any \
|
||
--destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw"
|
||
hcloud firewall add-rule "$fw" --direction out --protocol icmp \
|
||
--destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw"
|
||
for (( n = 0; n < ${#tcp_dst[@]}; n += HCLOUD_FW_RULE_PREFIXES )); do
|
||
hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \
|
||
--destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]:n:HCLOUD_FW_RULE_PREFIXES}")" \
|
||
--description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw"
|
||
done
|
||
for n in "${HCLOUD_NODES[@]}"; do
|
||
hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \
|
||
|| fail "could not attach $fw to $n"
|
||
done
|
||
while read -r fw_id fw_name; do
|
||
[[ -n "$fw_id" && "$fw_name" == hcloudtest-node-egress-* && "$fw_name" != "$fw" ]] || continue
|
||
while read -r srv_id; do
|
||
[[ -n "$srv_id" ]] || continue
|
||
hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \
|
||
|| fail "could not detach $fw_name from server $srv_id"
|
||
done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id')
|
||
hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name"
|
||
echo " removed $fw_name"
|
||
done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
|
||
echo " $fw: $count effective rules"
|
||
}
|
||
|
||
# Runs restore-test/egress-check.yaml in the cluster KUBECONFIG names and fails
|
||
# unless every expectation holds.
|
||
hcloud_egress_check() {
|
||
local v4 v6 addrs phase=""
|
||
step 0 "Checking egress containment from a pod..."
|
||
v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)"
|
||
v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)"
|
||
addrs=$(printf '%s\n' "$v4" "$v6" | awk 'NF' | paste -sd' ' -)
|
||
[[ -n "$addrs" ]] || fail "$HCLOUD_PROXY_HOST resolved to no address"
|
||
kubectl delete pod egress-check --ignore-not-found --wait=true >/dev/null \
|
||
|| fail "could not remove the previous egress-check pod"
|
||
sed "s|PROXY_ADDRS_PLACEHOLDER|$addrs|" "$REPO_ROOT/kubernetes/restore-test/egress-check.yaml" \
|
||
| kubectl apply -f - >/dev/null || fail "could not start the egress check"
|
||
for i in $(seq 1 48); do
|
||
phase=$(kubectl get pod egress-check -o jsonpath='{.status.phase}') || phase=""
|
||
[[ "$phase" == Succeeded || "$phase" == Failed ]] && break
|
||
sleep 5
|
||
done
|
||
if [[ "$phase" != Succeeded && "$phase" != Failed ]]; then
|
||
kubectl describe pod egress-check | sed 's/^/ /' || true
|
||
fail "the egress check never finished (phase '${phase:-none}') — do not start Flux"
|
||
fi
|
||
kubectl logs egress-check | sed 's/^/ /' || true
|
||
[[ "$phase" == "Succeeded" ]] || fail "egress containment does not hold — do not start Flux"
|
||
kubectl delete pod egress-check --wait=false >/dev/null
|
||
ok
|
||
}
|
||
|
||
cleanup_hcloudtest() {
|
||
command -v hcloud >/dev/null || return 0
|
||
[[ -n "${HCLOUD_TOKEN:-}" ]] || return 0
|
||
[[ -n "${HCLOUD_LABEL:-}" ]] \
|
||
|| { echo " REFUSING to sweep: HCLOUD_LABEL is empty, which would match everything" >&2; return 0; }
|
||
|
||
local id name
|
||
# Servers first: a firewall still attached to one cannot be deleted.
|
||
while read -r id name; do
|
||
[[ -n "$id" ]] || continue
|
||
case "$name" in
|
||
"$HCLOUD_NAME_PREFIX"-*) ;;
|
||
*) echo " REFUSING to delete $name ($id): carries the label but not the name prefix" >&2; continue ;;
|
||
esac
|
||
echo " deleting server $name"
|
||
hcloud server delete "$id" >/dev/null \
|
||
|| echo " WARNING: server $name not deleted — check the console" >&2
|
||
done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
|
||
|
||
for id in $(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
||
echo " deleting firewall $id"
|
||
hcloud firewall delete "$id" >/dev/null \
|
||
|| echo " WARNING: firewall $id not deleted — check the console" >&2
|
||
done
|
||
for id in $(hcloud volume list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
||
echo " deleting volume $id"
|
||
hcloud volume delete "$id" >/dev/null \
|
||
|| echo " WARNING: volume $id not deleted — check the console" >&2
|
||
done
|
||
for id in $(hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
||
echo " deleting ssh key $id"
|
||
hcloud ssh-key delete "$id" >/dev/null \
|
||
|| echo " WARNING: ssh key $id not deleted — check the console" >&2
|
||
done
|
||
# Last: a network with a server still attached cannot be deleted, and a
|
||
# leftover one takes its name with it, so the next run cannot recreate it.
|
||
for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
||
echo " deleting network $id"
|
||
hcloud network delete "$id" >/dev/null \
|
||
|| echo " WARNING: network $id not deleted — check the console" >&2
|
||
done
|
||
}
|
||
|
||
# The restore test's source, as restore-test/flux-root.yaml names it: sets
|
||
# restore_url and restore_repo (<owner>/<repo>).
|
||
restore_test_repo() {
|
||
command -v yq >/dev/null || fail "yq not found (reads restore-test/flux-root.yaml)"
|
||
restore_url=$(yq -r 'select(.kind == "GitRepository" and .metadata.name == "restore-test") | .spec.url' \
|
||
"$REPO_ROOT/kubernetes/restore-test/flux-root.yaml")
|
||
[[ "$restore_url" =~ ^ssh://git@github\.com/([^/]+)/([^/]+)\.git$ ]] \
|
||
|| fail "restore-test/flux-root.yaml names '$restore_url', not ssh://git@github.com/<owner>/<repo>.git"
|
||
restore_repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}"
|
||
}
|
||
|
||
# Sweep only. For the case where a previous run was killed hard enough that its
|
||
# own trap never ran, and for a cron job to call before it starts.
|
||
if [[ "$PHASE" == "hcloudclean" ]]; then
|
||
echo -e "${YELLOW}=== Phase: Hetzner cleanup ===${NC}"
|
||
command -v hcloud >/dev/null || fail "hcloud not found"
|
||
command -v bao >/dev/null || fail "bao not found (the Hetzner token is read from OpenBao)"
|
||
require_secrets hcloud-token
|
||
HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN
|
||
cleanup_hcloudtest
|
||
require_secrets s3-access s3-secret
|
||
delete_cnpg_test_bucket
|
||
# The restore test's read-only deploy keys on the GitHub mirror.
|
||
if command -v gh >/dev/null; then
|
||
restore_test_repo
|
||
for key_id in $(gh api "repos/$restore_repo/keys" \
|
||
--jq '.[] | select(.title | startswith("restore-test ")) | .id'); do
|
||
echo " deleting deploy key $key_id"
|
||
gh api -X DELETE "repos/$restore_repo/keys/$key_id" \
|
||
|| echo " WARNING: deploy key $key_id not deleted — check the repo settings" >&2
|
||
done
|
||
else
|
||
echo " WARNING: gh not found — restore-test deploy keys on the GitHub mirror not checked" >&2
|
||
fi
|
||
ok
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$PHASE" == "hcloudtest" ]]; then
|
||
echo -e "${YELLOW}=== Phase: Hetzner rebuild test ===${NC}"
|
||
command -v hcloud >/dev/null || fail "hcloud not found — https://github.com/hetznercloud/cli"
|
||
command -v talosctl >/dev/null || fail "talosctl not found"
|
||
command -v yq >/dev/null || fail "yq not found"
|
||
command -v jq >/dev/null || fail "jq not found (needed to read the SAN pool and server attachments)"
|
||
command -v curl >/dev/null || fail "curl not found (needed to learn this host's public address)"
|
||
command -v bao >/dev/null || fail "bao not found (the Hetzner token is read from OpenBao)"
|
||
command -v kubectl >/dev/null || fail "kubectl not found (runs the egress check)"
|
||
|
||
# Three control planes, all carrying workloads — the production shape. One
|
||
# machine is a node, not a cluster: etcd has no quorum to lose and nothing
|
||
# schedules elsewhere, so a rebuild that only ever ran on a single box
|
||
# proves nothing about the one that has three.
|
||
HCLOUD_NODES=(hcloud-1 hcloud-2 hcloud-3)
|
||
for n in "${HCLOUD_NODES[@]}"; do
|
||
[[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
|
||
done
|
||
[[ -f "$TALOS_DIR/patches/site-hcloud.yaml" ]] || fail "site-hcloud.yaml not found"
|
||
|
||
require_secrets hcloud-token ghcr-pull
|
||
HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN
|
||
|
||
# No sweep before creating: machines from an earlier run are the input to
|
||
# this one, adopted below. The names are fixed, so duplicates cannot
|
||
# accumulate. `hcloudclean` is what deletes.
|
||
|
||
step 0 "Looking up the Talos snapshot..."
|
||
talos_image_coords
|
||
HCLOUD_IMAGE=""
|
||
hcloud_talos_snapshot
|
||
[[ -n "$HCLOUD_IMAGE" ]] || fail "no Talos snapshot"
|
||
echo " snapshot $HCLOUD_IMAGE"
|
||
|
||
# The Talos API is unauthenticated in maintenance mode, which is how
|
||
# apply-config reaches a node that has no config yet. On a public address
|
||
# that has to be closed to everyone but this host for the minutes it takes.
|
||
step 0 "Creating a firewall for this host only..."
|
||
# Both families, and each with its own prefix length. The first version
|
||
# asked ifconfig.me once and appended /32: this host answered over IPv6, and
|
||
# Hetzner rejected `2003:d6:…:86d2/32` because /32 of that address is not a
|
||
# network boundary. A single-address rule is /32 for v4 and /128 for v6.
|
||
#
|
||
# Both are added rather than whichever happened to answer, because the rule
|
||
# has to admit the address talosctl actually leaves from, and that is chosen
|
||
# per connection by the resolver — not by whatever this lookup returned.
|
||
# No `local` here: this phase runs in the script body, not in a function,
|
||
# and `local` is a runtime error there — one `bash -n` does not catch.
|
||
# One family failing is ordinary — a host without IPv6 is not broken — so
|
||
# neither attempt aborts the run on its own. Their stderr is kept rather
|
||
# than discarded, and reported if BOTH fail: at that point the reason is the
|
||
# diagnosis, and "could not determine this host's address" without it sends
|
||
# you looking at the wrong thing.
|
||
fw_sources=()
|
||
fw_err=$(mktemp)
|
||
v4=$(curl -4 -fsS --max-time 10 https://ifconfig.me 2>>"$fw_err") && [[ -n "$v4" ]] && fw_sources+=("$v4/32")
|
||
v6=$(curl -6 -fsS --max-time 10 https://ifconfig.me 2>>"$fw_err") && [[ -n "$v6" ]] && fw_sources+=("$v6/128")
|
||
if [[ ${#fw_sources[@]} -eq 0 ]]; then
|
||
cat "$fw_err" >&2
|
||
rm -f "$fw_err"
|
||
fail "could not determine this host's public address over either IPv4 or IPv6"
|
||
fi
|
||
rm -f "$fw_err"
|
||
fw_src=$(IFS=,; echo "${fw_sources[*]}")
|
||
|
||
hcloud firewall create --name "hcloudtest-$$" --label "$HCLOUD_LABEL" >/dev/null \
|
||
|| fail "could not create the firewall"
|
||
hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 50000 \
|
||
--source-ips "$fw_src" --description "Talos API, this host only" >/dev/null \
|
||
|| fail "could not add the Talos API rule"
|
||
hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 6443 \
|
||
--source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \
|
||
|| fail "could not add the Kubernetes API rule"
|
||
echo " open to $fw_src only"
|
||
# Servers held from an earlier run get this firewall first; only then are
|
||
# the earlier runs' inbound firewalls detached and deleted, so no server is
|
||
# ever without one. Egress firewalls are replaced by hcloud_apply_egress.
|
||
while read -r srv; do
|
||
[[ -n "$srv" ]] || continue
|
||
hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$srv" >/dev/null \
|
||
|| fail "could not attach this run's firewall to $srv"
|
||
done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name)
|
||
while read -r fw_id fw_name; do
|
||
[[ -n "$fw_id" && "$fw_name" != "hcloudtest-$$" && "$fw_name" != hcloudtest-node-egress-* ]] || continue
|
||
while read -r srv_id; do
|
||
[[ -n "$srv_id" ]] || continue
|
||
hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \
|
||
|| fail "could not detach $fw_name from server $srv_id"
|
||
done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id')
|
||
hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name"
|
||
echo " removed $fw_name"
|
||
done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
|
||
ok
|
||
|
||
# Default type cpx52 (12 vCPU, 24 GB, 480 GB local disk): the hcloud-N
|
||
# patches put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system
|
||
# disk. cax* are ARM and do not match the amd64 schematic.
|
||
#
|
||
# Two segments only, unlike the homelab's split in
|
||
# NETWORK-traffic-classes.md: here the node address carries management,
|
||
# cluster-internal and external alike, and Ceph replication has its own.
|
||
#
|
||
# The order is load-bearing: it decides which kernel interface each segment
|
||
# becomes, and the machine config names those interfaces.
|
||
HCLOUD_SEGMENTS=(cluster storage)
|
||
# Segment ranges are read from what the cluster declares, never invented
|
||
# here — the disks' rule applied to networks:
|
||
# cluster the subnet the site patch pins node addresses to.
|
||
# storage the SAN pool the ceph-san attachment allocates from, widened
|
||
# to the containing /24 because the fabric wants a subnet.
|
||
# The fabric reserves the first address of a network range as its gateway.
|
||
# For the storage net that is the SAN pool's first address — a collision
|
||
# that matters only if cross-node SAN traffic on this site is ever
|
||
# attempted, and is left unresolved until it is.
|
||
seg_range() {
|
||
case "$1" in
|
||
cluster)
|
||
yq -r 'select(.machine) | .machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;;
|
||
storage)
|
||
local pool
|
||
pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range')
|
||
[[ "$pool" == *.0/2[4-9] ]] \
|
||
|| fail "cannot derive a /24 from the SAN pool '$pool'"
|
||
printf '%s/24' "${pool%.*}.0" ;;
|
||
esac
|
||
}
|
||
# Read from the location rather than stated a second time.
|
||
HCLOUD_ZONE=$(hcloud location describe "${HCLOUD_LOCATION:-fsn1}" -o format='{{.NetworkZone}}') \
|
||
|| fail "could not read the network zone of ${HCLOUD_LOCATION:-fsn1}"
|
||
[[ -n "$HCLOUD_ZONE" ]] \
|
||
|| fail "${HCLOUD_LOCATION:-fsn1} reported no network zone"
|
||
|
||
step 0 "Creating ${#HCLOUD_SEGMENTS[@]} private network(s) in $HCLOUD_ZONE..."
|
||
for seg in "${HCLOUD_SEGMENTS[@]}"; do
|
||
net="hcloudtest-$seg"
|
||
want_range=$(seg_range "$seg")
|
||
[[ -n "$want_range" && "$want_range" != null* ]] \
|
||
|| fail "could not derive the $seg range from the cluster config"
|
||
# Reused only if the range matches. A stale one fails later, at the
|
||
# attachment, naming the wrong thing.
|
||
if have_range=$(hcloud network describe "$net" -o format='{{.IPRange}}'); then
|
||
if [[ "$have_range" == "$want_range" ]]; then
|
||
echo " $net (already there)"
|
||
continue
|
||
fi
|
||
echo " $net carries $have_range, this run wants $want_range — replacing"
|
||
hcloud network delete "$net" >/dev/null \
|
||
|| fail "could not replace $net — detach its servers first, or run hcloudclean"
|
||
fi
|
||
hcloud network create --name "$net" --ip-range "$want_range" \
|
||
--label "$HCLOUD_LABEL" >/dev/null \
|
||
|| fail "could not create network $net"
|
||
hcloud network add-subnet "$net" --network-zone "$HCLOUD_ZONE" \
|
||
--type cloud --ip-range "$want_range" >/dev/null \
|
||
|| fail "could not add a subnet to $net"
|
||
echo " $net $want_range"
|
||
done
|
||
ok
|
||
|
||
# From here on machines exist and are billed; on_exit says so.
|
||
HCLOUD_PROVISIONED=1
|
||
step 0 "Creating ${#HCLOUD_NODES[@]} server(s)..."
|
||
NODE_ADDRS=()
|
||
# Servers this test already holds. Capacity is the binding constraint, so a
|
||
# machine in hand is worth keeping across runs.
|
||
#
|
||
# Listed by label: a same-named server without it falls through to the
|
||
# create path and fails on "name already used", rather than being adopted
|
||
# and wiped.
|
||
existing_servers=$(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name)
|
||
for n in "${HCLOUD_NODES[@]}"; do
|
||
if grep -qx "$n" <<<"$existing_servers"; then
|
||
# Rebuilt, not reused as-is: a configured Talos rejects a fresh
|
||
# apply-config, and `talosctl reset` needs the previous run's PKI,
|
||
# which is gone. Rebuilding from the snapshot returns the node to
|
||
# maintenance mode, reachable without any old secret.
|
||
echo " $n: reusing the server we already hold"
|
||
hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \
|
||
|| fail "could not rebuild $n back into maintenance mode"
|
||
# This run's firewall was attached above, before the old ones went.
|
||
else
|
||
# Retried, because "error during placement (resource_unavailable)"
|
||
# is the provider being momentarily out of that type in that
|
||
# location, not a wrong request. It happened on the first cx43 run
|
||
# and the same type in the same location created fine minutes later.
|
||
# For a test meant to run on a schedule that distinction matters:
|
||
# without the retry every capacity blip is a red run someone has to
|
||
# come and look at.
|
||
create_err=""
|
||
for attempt in $(seq 1 30); do
|
||
create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \
|
||
--type "${HCLOUD_TYPE:-cpx52}" --location "${HCLOUD_LOCATION:-fsn1}" \
|
||
--firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break
|
||
grep -qi 'resource_unavailable\|placement' <<<"$create_err" \
|
||
|| fail "could not create $n: ${create_err//$'\n'/ | }"
|
||
[[ $attempt -eq 1 ]] && echo " $n: no capacity yet, retrying"
|
||
sleep 20
|
||
[[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cpx52} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes"
|
||
done
|
||
fi
|
||
addr=$(hcloud server ip "$n") || fail "could not read the address of $n"
|
||
[[ -n "$addr" ]] || fail "$n has no public address"
|
||
|
||
# Attached in HCLOUD_SEGMENTS order; the interface names follow from it.
|
||
# An adopted server keeps its attachments through a rebuild, and
|
||
# attaching twice is an error, so an existing one is skipped.
|
||
for seg in "${HCLOUD_SEGMENTS[@]}"; do
|
||
net="hcloudtest-$seg"
|
||
net_id=$(hcloud network describe "$net" -o json | jq -r '.id')
|
||
if hcloud server describe "$n" -o json \
|
||
| jq -e --argjson id "$net_id" 'any(.private_net[]; .network == $id)' >/dev/null; then
|
||
continue
|
||
fi
|
||
hcloud server attach-to-network "$n" --network "$net" >/dev/null \
|
||
|| fail "could not attach $n to $net"
|
||
done
|
||
|
||
# The OSD device. Every node carries one, which is the production shape
|
||
# — k8s-1/2/3 all hold storage — and is only possible here because
|
||
# Hetzner attaches volumes to any server, control plane included. The
|
||
# qemu provisioner cannot, which is why that site has to split into one
|
||
# control plane and three workers.
|
||
#
|
||
#
|
||
# Neither formatted nor mounted. Rook needs the raw device, and an
|
||
# --automount volume arrives with a filesystem on it that makes Ceph
|
||
# skip the disk as already in use.
|
||
#
|
||
# Recreated rather than reused, even on an adopted server: a volume from
|
||
# the previous run carries a bluestore signature, and Rook adopts a
|
||
# device that already looks like an OSD. Volumes are not the scarce
|
||
# resource here; servers are.
|
||
if hcloud volume describe "$n-osd" >/dev/null; then
|
||
echo " $n: discarding the previous OSD volume"
|
||
# An unattached leftover is ordinary after an interrupted run, so
|
||
# this failure is tolerated while the delete below is not.
|
||
hcloud volume detach "$n-osd" >/dev/null || true
|
||
hcloud volume delete "$n-osd" >/dev/null \
|
||
|| fail "could not delete the previous OSD volume for $n"
|
||
fi
|
||
yq -e 'select(.kind == "RawVolumeConfig" and .name == "osd-data")' "$TALOS_DIR/patches/$n.yaml" >/dev/null \
|
||
|| fail "$n declares no osd-data RawVolumeConfig — a node with no OSD device is not this cluster's shape"
|
||
# A raw volume on the system disk needs no Hetzner volume.
|
||
vol_size=$(hcloud_volumes_for_node "$n")
|
||
if [[ -n "$vol_size" ]]; then
|
||
hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \
|
||
--label "$HCLOUD_LABEL" >/dev/null \
|
||
|| fail "could not create the OSD volume for $n"
|
||
echo " $n at $addr, with a ${vol_size}GB OSD volume"
|
||
else
|
||
echo " $n at $addr, OSD on the system disk"
|
||
fi
|
||
NODE_ADDRS+=("$addr")
|
||
done
|
||
ok
|
||
|
||
# The nodes have to reach each other, and until now the firewall admitted
|
||
# only this host. One node did not care; three do — etcd peers on 2380, the
|
||
# apiserver, kubelet on 10250 and Cilium's VXLAN all run between them, and
|
||
# with the rules above alone etcd simply never forms a quorum.
|
||
#
|
||
# Added after creation because the addresses do not exist before it. Rules
|
||
# apply to an attached firewall immediately, so the ordering costs nothing.
|
||
#
|
||
# All ports between the nodes rather than an enumerated list: this is a
|
||
# throwaway cluster whose members are each other's only peers, and guessing
|
||
# the full port set of Talos plus Cilium is how a rebuild fails on the one
|
||
# port nobody wrote down. The outside still sees only 50000 and 6443, and
|
||
# only from this host.
|
||
step 0 "Opening the cluster to itself..."
|
||
node_src=$(IFS=,; printf '%s' "${NODE_ADDRS[*]/%//32}")
|
||
for proto in tcp udp; do
|
||
hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol "$proto" \
|
||
--port any --source-ips "$node_src" \
|
||
--description "intra-cluster $proto" >/dev/null \
|
||
|| fail "could not open $proto between the nodes"
|
||
done
|
||
echo " $node_src"
|
||
ok
|
||
|
||
# Before any node is configured.
|
||
step 0 "Closing tcp egress to $HCLOUD_PROXY_HOST (Hetzner firewall)..."
|
||
hcloud_apply_egress
|
||
ok
|
||
|
||
# Handover: the variables the vmtest phase sets, and HCLOUDTEST.
|
||
SITE=hcloud
|
||
NODES=("${HCLOUD_NODES[@]}")
|
||
WORKERS=()
|
||
ALL_NODES=("${NODES[@]}")
|
||
K8S_API_HOST="${NODE_ADDRS[0]}"
|
||
HCLOUDTEST=1
|
||
|
||
# Isolation from the production cluster. Both halves are required: the
|
||
# _PATH variables are derived before any phase runs, and the talos phase
|
||
# writes to those.
|
||
mkdir -p "$HCLOUD_STATE_DIR"
|
||
chmod 700 "$HCLOUD_STATE_DIR"
|
||
export TALOSCONFIG="$HCLOUD_STATE_DIR/talosconfig"
|
||
export KUBECONFIG="$HCLOUD_STATE_DIR/kubeconfig"
|
||
TALOSCONFIG_PATH="$TALOSCONFIG"
|
||
KUBECONFIG_PATH="$KUBECONFIG"
|
||
|
||
PHASE=talos
|
||
fi
|
||
|
||
# Provision throwaway QEMU VMs, then run the Talos phase below against
|
||
# them unchanged. Deliberately not a copy of that phase: it sets the site
|
||
# variables and falls through, because a rehearsal that runs different code
|
||
# proves less than one that does not. Everything it changes is listed here — if
|
||
# that list grows, the test is drifting away from what it claims to exercise.
|
||
#
|
||
# What it cannot cover, and what therefore still has to be true on hardware:
|
||
# Ceph (the provisioner gives extra disks to workers only), the mgmt and v30iot
|
||
# VLANs and the Layer2 VIP (one flat NIC per VM), and Flux (its own phase).
|
||
#
|
||
# Needs root for the QEMU provisioner (CNI and KVM), and only for that step.
|
||
if [[ "$PHASE" == "vmtest" ]]; then
|
||
echo -e "${YELLOW}=== Phase: VM rebuild test ===${NC}"
|
||
command -v talosctl >/dev/null || fail "talosctl not found"
|
||
command -v yq >/dev/null || fail "yq not found"
|
||
command -v jq >/dev/null || fail "jq not found (needed to read OpenBao's seal status)"
|
||
# The node patches are the list; the VM count follows from them rather than
|
||
# being stated twice. vm-1 is the control plane, the rest are the workers
|
||
# that carry an OSD device — see the comment on NODES below for why.
|
||
VM_CP_NODES=(vm-1)
|
||
VM_WORKER_NODES=(vm-2 vm-3 vm-4)
|
||
VM_CPS=${#VM_CP_NODES[@]}
|
||
VM_WORKER_COUNT=${#VM_WORKER_NODES[@]}
|
||
for n in "${VM_CP_NODES[@]}" "${VM_WORKER_NODES[@]}"; do
|
||
[[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
|
||
done
|
||
[[ -f "$TALOS_DIR/patches/site-qemu.yaml" ]] || fail "site-qemu.yaml not found"
|
||
|
||
# Talos version and Image Factory schematic, from the installer image in
|
||
# cluster.yaml — see talos_image_coords above for why neither phase states
|
||
# them itself.
|
||
talos_image_coords
|
||
|
||
# sun_path caps a unix socket path at 108 bytes and the provisioner puts a
|
||
# qemu monitor socket under the state directory, so this has to stay short.
|
||
# A path under the repo or a long mktemp dir silently fails to boot.
|
||
# Overridable because in-cluster the VM disks belong on the mounted scratch
|
||
# volume, not on the container's own writable layer. Keep whatever is set
|
||
# SHORT for the reason above.
|
||
VM_STATE="${VM_STATE:-/tmp/bootstrap-vmtest}"
|
||
[[ ${#VM_STATE} -le 50 ]] \
|
||
|| fail "VM_STATE '$VM_STATE' is ${#VM_STATE} bytes; keep it under 50 so the qemu monitor socket fits in sun_path's 108"
|
||
# cleanup_vmtest runs `sudo rm -rf "$VM_STATE"`, so refuse anything that is
|
||
# not a nested absolute path.
|
||
[[ $VM_STATE == /*/?* ]] \
|
||
|| fail "VM_STATE '$VM_STATE' must be an absolute path at least two levels deep"
|
||
VM_CLUSTER=bootstrap-vmtest
|
||
VM_DIR=$(mktemp -d)
|
||
TALOSCTL_BIN=$(command -v talosctl)
|
||
|
||
# Hard isolation from the production cluster: talosctl and kubectl both
|
||
# honour these, and the phase below writes to them rather than to $HOME.
|
||
export TALOSCONFIG="$VM_DIR/talosconfig"
|
||
export KUBECONFIG="$VM_DIR/kubeconfig"
|
||
TALOSCONFIG_PATH="$TALOSCONFIG"
|
||
KUBECONFIG_PATH="$KUBECONFIG"
|
||
|
||
# The provisioner needs root for KVM and for the bridges and taps it builds.
|
||
# On a workstation that means sudo; in the Job that runs this in-cluster the
|
||
# process is already root and sudo is not installed, so the elevation is a
|
||
# variable rather than a literal.
|
||
SUDO_KEEPALIVE=""
|
||
if [[ $EUID -eq 0 ]]; then
|
||
SUDO=""
|
||
else
|
||
SUDO="sudo -E"
|
||
# A full run outlasts sudo's timestamp, and the teardown at the end then
|
||
# prompts for a password with nobody watching — the VMs survive and the
|
||
# next run collides with them. Take it once here and keep the timestamp
|
||
# warm for as long as the phase lasts.
|
||
sudo -v || fail "the QEMU provisioner needs root (CNI and KVM)"
|
||
( while sudo -n true; do sleep 50; done ) &
|
||
SUDO_KEEPALIVE=$!
|
||
fi
|
||
|
||
cleanup_vmtest() {
|
||
[[ -n "$SUDO_KEEPALIVE" ]] && kill "$SUDO_KEEPALIVE" >/dev/null || true
|
||
# Diagnosing anything inside the VMs is impossible once they are gone,
|
||
# and the teardown fires the moment a run ends — including when it ends
|
||
# badly, which is exactly when the VMs are worth looking at. VMTEST_KEEP
|
||
# holds them, and the pod, until someone deletes the Job.
|
||
if [[ "${VMTEST_KEEP:-0}" == "1" ]]; then
|
||
echo -e "${YELLOW}VMTEST_KEEP=1 — leaving the VMs up.${NC}"
|
||
echo " talosctl --talosconfig $TALOSCONFIG -n 10.5.0.2 ..."
|
||
echo " delete the Job when finished; nothing else reclaims this."
|
||
sleep infinity
|
||
fi
|
||
echo -n "Destroying throwaway VMs... "
|
||
$SUDO "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
|
||
# destroy removes the cluster's own directory but leaves the state root.
|
||
$SUDO rm -rf "$VM_STATE"
|
||
rm -rf "$VM_DIR"
|
||
echo "done"
|
||
}
|
||
# No `trap ... EXIT` here — on_exit calls cleanup_vmtest once it is defined.
|
||
|
||
# A run killed hard never reaches its EXIT trap, so the previous cluster can
|
||
# still be holding the provisioner's CIDR. Clear it first rather than
|
||
# failing on a collision that looks like a networking problem.
|
||
if [[ -d "$VM_STATE/$VM_CLUSTER" ]]; then
|
||
echo -n "Removing a cluster left behind by an earlier run... "
|
||
$SUDO "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
|
||
$SUDO rm -rf "$VM_STATE"
|
||
echo "done"
|
||
fi
|
||
|
||
step 0 "Creating $VM_CPS control plane and $VM_WORKER_COUNT workers in maintenance mode..."
|
||
# maintenance means the provisioner applies no configuration of its own, so
|
||
# every byte of machine config below comes from talos/patches.
|
||
#
|
||
# The control plane gets 6GiB. At 2560MiB it ran with ~250MiB available and
|
||
# its page cache squeezed to 386MiB, so every etcd fsync hit a system in
|
||
# permanent reclaim: linearized reads took 5-20s against an expected 100ms,
|
||
# the apiserver became unreachable, and the run stalled with no error to
|
||
# read. That failure says nothing about whether the repo can rebuild a
|
||
# cluster, so it is bought off with memory rather than tuned around.
|
||
#
|
||
# The workers get 8GiB for the same reason. At 4GiB a storage node sat at
|
||
# 99% of its memory requests once it carried an OSD, a mon and a mgr, and
|
||
# what could not be scheduled any more was kube-multus-ds and whereabouts —
|
||
# the CNI itself. Without Multus the OSDs came up with no ceph-san
|
||
# interface and crash-looped on "unable to find any IPv4 address in
|
||
# networks 10.99.0.0/29", which reads like an addressing bug and is a
|
||
# memory one. Sizing the VMs to the workload they actually run is cheaper
|
||
# than shrinking the Ceph requests, which would test something other than
|
||
# what production runs.
|
||
#
|
||
# Workers get 4 CPUs, one constraint later: with memory solved they sat at
|
||
# 69% of CPU requests, and what could no longer schedule was the
|
||
# rook-ceph-operator itself. Nothing was left driving the cluster — two
|
||
# OSDs crash-looped unattended and the PVCs waited on a provisioner whose
|
||
# operator was Pending.
|
||
#
|
||
# The disks are sized the same way. At 10GiB the system disk yielded an
|
||
# 8.36GB /var, which the infrastructure image set filled to 82%: kubelet
|
||
# tainted all four nodes node.kubernetes.io/disk-pressure, no pod could be
|
||
# scheduled any more, and Ceph's own monitor sat Pending while the symptom
|
||
# surfaced three layers away as a Cilium HelmRelease stuck InProgress. The
|
||
# second disk is the Ceph OSD; at 8GiB, three of them replicated 3x left
|
||
# 8GiB usable, less than OpenBao's two PVCs alone ask for. Both are raw
|
||
# sparse files, so the larger geometry costs only what is actually written.
|
||
#
|
||
# The control plane gets 16GiB. 6 was not enough and neither was 8: at both
|
||
# it reached 99% of allocatable memory once the infrastructure layer was up,
|
||
# and a control plane with nothing left over does not stay a control plane.
|
||
# The kubelet admits the node-critical CNI DaemonSets by evicting whatever
|
||
# ranks below them, and kube-apiserver — a static pod at
|
||
# system-cluster-critical — is what ranks below them:
|
||
#
|
||
# message: Pod was preempted by Kubelet to accommodate a critical pod.
|
||
# reason: Preempting
|
||
#
|
||
# The priority ordering is not the defect and must not be tuned away. It is
|
||
# deliberate: a node without its CNI cannot run anything, so the CNI wins.
|
||
# What the eviction reports is the shortfall underneath it, and the only
|
||
# honest fix is to remove the shortfall. Production does this by having
|
||
# ~95GiB per node; the throwaway cluster does it here.
|
||
#
|
||
# This is not a spare-capacity argument for the workers. vm-1 is the only
|
||
# control plane and still takes workload, so it carries the static pods, the
|
||
# CNI, and its share of the infrastructure layer at once — which is the
|
||
# asymmetry with production, where every node is a control plane and none of
|
||
# them is small.
|
||
$SUDO "$TALOSCTL_BIN" cluster create qemu \
|
||
--name "$VM_CLUSTER" \
|
||
--state "$VM_STATE" \
|
||
--presets iso,maintenance \
|
||
--controlplanes "$VM_CPS" \
|
||
--workers "$VM_WORKER_COUNT" \
|
||
--talos-version "$VM_VERSION" \
|
||
--schematic-id "$VM_SCHEMATIC" \
|
||
--memory-controlplanes 16GiB \
|
||
--memory-workers 8GiB \
|
||
--cpus-controlplanes 2 \
|
||
--cpus-workers 4 \
|
||
--disks virtio:40GiB,virtio:24GiB \
|
||
--talosconfig-destination "$TALOSCONFIG" \
|
||
|| fail "could not create the throwaway VMs"
|
||
$SUDO chown -R "$(id -u):$(id -g)" "$VM_STATE" "$TALOSCONFIG"
|
||
ok
|
||
|
||
# The provisioner hands out its CIDR sequentially from .2; .1 is the
|
||
# gateway and the load balancer it puts in front of the API. If that ever
|
||
# changes, the apply loop below reports which address stopped answering
|
||
# rather than failing silently.
|
||
# One control plane and three workers, not three control planes: the
|
||
# provisioner gives the second disk only to workers ("disks after the first
|
||
# one are added only to worker machines"), and an OSD needs it. Three of
|
||
# them because the CephCluster runs three mons with allowMultiplePerNode
|
||
# false over pools that are replica 3 across failureDomain host.
|
||
NODES=("${VM_CP_NODES[@]}")
|
||
WORKERS=("${VM_WORKER_NODES[@]}")
|
||
ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
|
||
NODE_ADDRS=()
|
||
for i in $(seq 0 $(( ${#ALL_NODES[@]} - 1 ))); do
|
||
NODE_ADDRS+=("10.5.0.$(( i + 2 ))")
|
||
done
|
||
SITE=qemu
|
||
K8S_API_HOST=10.5.0.1
|
||
VMTEST=1
|
||
|
||
# Run the real phases, not a reimplementation of them.
|
||
#
|
||
# Default is the Talos phase alone: that needs nothing but the machines, so
|
||
# it is the part that can run unattended. VMTEST_FULL=1 continues into flux
|
||
# and wait-apps — the whole bootstrap against VMs — which additionally needs
|
||
# GITEA_TOKEN and a FLUX_BRANCH to write to, because `flux bootstrap`
|
||
# commits its sync manifests into the branch it reconciles.
|
||
if [[ "${VMTEST_FULL:-0}" == "1" ]]; then
|
||
[[ "$FLUX_BRANCH" != "main" ]] \
|
||
|| fail "VMTEST_FULL would have flux bootstrap commit into main — point FLUX_BRANCH at a throwaway branch that already exists"
|
||
echo -e "${YELLOW}Continuing into the flux phase against branch ${FLUX_BRANCH}.${NC}"
|
||
PHASE=all
|
||
else
|
||
PHASE=talos
|
||
fi
|
||
fi
|
||
|
||
# Before anything destructive, and after the vmtest block has decided which
|
||
# phases will run. A rebuild that will need these has no business wiping disks
|
||
# first, only to discover an hour later that no YubiKey is plugged in or a
|
||
# token is unset —
|
||
# by then the old cluster is gone and the new one cannot be finished.
|
||
#
|
||
# In the Job this is where the file-waiting branch blocks, which is the right
|
||
# moment too: the operator has just started the run and is watching it.
|
||
if phase_runs flux; then
|
||
require_secrets
|
||
fi
|
||
|
||
if phase_runs talos; then
|
||
echo -e "${YELLOW}=== Phase: Talos bootstrap ===${NC}"
|
||
command -v talosctl >/dev/null || fail "talosctl not found"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v helm >/dev/null || fail "helm not found (needed to install the CNI)"
|
||
command -v yq >/dev/null || fail "yq not found (needed to read the Cilium HelmRelease)"
|
||
[[ -f "$TALOS_DIR/patches/cluster.yaml" ]] || fail "cluster.yaml patch not found"
|
||
[[ -f "$TALOS_DIR/patches/containerd-layer-retention.yaml" ]] || fail "containerd-layer-retention.yaml patch not found"
|
||
[[ -f "$CILIUM_HR" ]] || fail "cilium helmrelease not found at $CILIUM_HR"
|
||
|
||
step 0 "Generating Talos configs..."
|
||
TALOS_GEN_DIR=$(mktemp -d)
|
||
talosctl gen config homelab "https://${K8S_API_HOST}:6443" \
|
||
--output "$TALOS_GEN_DIR" \
|
||
--force
|
||
mkdir -p "$(dirname "$TALOSCONFIG_PATH")"
|
||
cp "$TALOS_GEN_DIR/talosconfig" "$TALOSCONFIG_PATH"
|
||
|
||
# Lets a node pull the images it needs before the registry that normally
|
||
# serves them exists.
|
||
#
|
||
# Several apps need an image while the cluster is still being built —
|
||
# paperless's fetch-bridge-cert init container, zigbee2mqtt's seed-data init
|
||
# container, the trmnl renderer, and the CNI plugins every pod depends on.
|
||
# They are referenced at gitea.renner.uno, which is correct for normal
|
||
# operation and useless during a rebuild, because Gitea runs inside the
|
||
# cluster being rebuilt.
|
||
#
|
||
# `mirrors` gives containerd an ordered list to try. Gitea is named first, so
|
||
# nothing changes once the cluster is up: pulls stay local and never leave
|
||
# the network. ghcr.io is second and only answers when the first cannot,
|
||
# which is exactly the rebuild window. The manifests therefore never name an
|
||
# external registry — the fallback is a property of the node, not of the
|
||
# deployment.
|
||
#
|
||
# Containerd keeps the repository path when it falls through, which is why
|
||
# the images are published under konairius/ on BOTH registries
|
||
# (.woodpecker/build-ci-images.yml). Verified equal: the same build pushes
|
||
# one digest to both paths.
|
||
#
|
||
# The ghcr namespace is private, so it needs a credential — and the usual
|
||
# place for one, an imagePullSecret, fails here twice over: it would have to
|
||
# exist in five namespaces before any of them exists, and it would come from
|
||
# OpenBao, which needs Ceph, which needs these very images.
|
||
# machine.registries.config is containerd's own auth, in place before the
|
||
# first pod is scheduled and applying to every pull on the node regardless
|
||
# of namespace. Field names follow .docker/config.json
|
||
# (pkg/machinery/config/types/v1alpha1, RegistryAuthConfig).
|
||
#
|
||
# The patch is written into the in-memory secrets dir, never into the repo
|
||
# or the generated config directory, and dies with the Job like the material
|
||
# it carries.
|
||
registry_auth_patch() {
|
||
local out="$OPENBAO_SECRETS_DIR/registry-auth.yaml" token
|
||
token=$(read_secret ghcr-pull) || fail "could not read the GHCR pull token"
|
||
( umask 077; cat > "$out" <<EOF
|
||
machine:
|
||
registries:
|
||
mirrors:
|
||
gitea.renner.uno:
|
||
endpoints:
|
||
- https://gitea.renner.uno
|
||
- https://ghcr.io
|
||
config:
|
||
ghcr.io:
|
||
auth:
|
||
username: "${GHCR_PULL_USER}"
|
||
password: "${token}"
|
||
EOF
|
||
) || fail "could not write the registry auth patch"
|
||
printf '%s' "$out"
|
||
}
|
||
# OUTSIDE the command substitution below, and that placement is the fix for
|
||
# a bug this had: require_secrets prints "All secrets present." on stdout,
|
||
# and calling it inside registry_auth_patch made that line part of the
|
||
# captured path. talosctl then reported
|
||
# open All secrets present. | /tmp/…/registry-auth.yaml: no such file
|
||
# and retried 60 times while the node sat in maintenance. A function whose
|
||
# value is its stdout must be the only thing writing there.
|
||
#
|
||
# This phase can run on its own (`bootstrap.sh talos`), in which case
|
||
# nothing has demanded the secrets yet — the full require_secrets is gated
|
||
# on the flux phase. Ask for the one this needs, and only that one: pulling
|
||
# in the whole list would make an unattended talos run depend on the OpenBao
|
||
# seal key, which never reaches CI.
|
||
require_secrets ghcr-pull
|
||
REGISTRY_PATCH=$(registry_auth_patch)
|
||
[[ -s "$REGISTRY_PATCH" ]] \
|
||
|| fail "the registry auth patch was not written to a readable file: '$REGISTRY_PATCH'"
|
||
|
||
step 0 "Applying Talos configs to nodes..."
|
||
apply_node_config() {
|
||
local node=$1 base_config=$2 fixed_addr=${3:-}
|
||
local fqdn="${node}.${DOMAIN}"
|
||
echo " ${fixed_addr:-$fqdn}:"
|
||
for i in $(seq 1 60); do
|
||
# A fixed address skips DNS. Real hardware has a name to resolve;
|
||
# a throwaway VM has an address and nothing else.
|
||
if [[ -n "$fixed_addr" ]]; then
|
||
node_ip="$fixed_addr"
|
||
else
|
||
node_ip=$(resolve_node_ip "$fqdn")
|
||
fi
|
||
if [[ -z "$node_ip" ]]; then
|
||
echo " attempt $i/60: DNS returned no IP"
|
||
sleep 10
|
||
continue
|
||
fi
|
||
if err=$(talosctl apply-config --insecure -n "$node_ip" \
|
||
--file "$base_config" \
|
||
--config-patch "@$TALOS_DIR/patches/cluster.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/site-${SITE}.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/containerd-layer-retention.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/${node}.yaml" \
|
||
--config-patch "@$REGISTRY_PATCH" 2>&1); then
|
||
echo -e " applied via $node_ip ${GREEN}(OK)${NC}"
|
||
return 0
|
||
fi
|
||
echo " attempt $i/60: $node_ip - ${err//$'\n'/ | }"
|
||
sleep 10
|
||
done
|
||
fail "Timeout waiting for ${fixed_addr:-$fqdn}"
|
||
}
|
||
|
||
# One loop over ALL_NODES rather than one per role, so NODE_ADDRS can be a
|
||
# single list in the same order. Control planes come first in ALL_NODES,
|
||
# which is what decides the base config each node gets.
|
||
for idx in "${!ALL_NODES[@]}"; do
|
||
if [[ $idx -lt ${#NODES[@]} ]]; then
|
||
base="$TALOS_GEN_DIR/controlplane.yaml"
|
||
else
|
||
base="$TALOS_GEN_DIR/worker.yaml"
|
||
fi
|
||
apply_node_config "${ALL_NODES[$idx]}" "$base" "${NODE_ADDRS[$idx]:-}"
|
||
done
|
||
|
||
echo -n "Configuring talosctl... "
|
||
# `gen config` leaves talosconfig endpoints empty; they are the Talos API
|
||
# addresses and unrelated to cluster.controlPlane.endpoint.
|
||
if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
|
||
talosctl config endpoint "${NODE_ADDRS[@]}"
|
||
talosctl config node "${NODE_ADDRS[0]}"
|
||
else
|
||
talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
|
||
talosctl config node "${NODES[0]}.${DOMAIN}"
|
||
fi
|
||
ok
|
||
|
||
step 0 "Waiting for every node to come back configured..."
|
||
# Asks each machine directly, by address, and expects it to answer with the
|
||
# hostname its patch assigns. That is what has to be true before etcd is
|
||
# bootstrapped: the config took, the node rebooted into it, and the machine
|
||
# at that address is the one we think it is.
|
||
#
|
||
# This used to wait on `talosctl get members`, which was wrong twice over.
|
||
# It queried through the configured endpoint list rather than each node, so
|
||
# apid refused it with `PermissionDenied: no request forwarding` — the same
|
||
# thing that broke the bootstrap call. And membership comes from the
|
||
# discovery service, because the Kubernetes registry is disabled: a cold
|
||
# rebuild would have been waiting on discovery.talos.dev being reachable,
|
||
# before the cluster it is trying to rebuild even exists.
|
||
#
|
||
# Each node installs to disk and reboots first, so the window is generous.
|
||
for idx in "${!ALL_NODES[@]}"; do
|
||
want="${ALL_NODES[$idx]}"
|
||
addr="${NODE_ADDRS[$idx]:-${want}.${DOMAIN}}"
|
||
echo -n " $want ($addr)... "
|
||
node_err=""
|
||
for i in $(seq 1 60); do
|
||
got=$(talosctl -n "$addr" -e "$addr" get hostname \
|
||
-o jsonpath='{.spec.hostname}' 2>"$TALOS_GEN_DIR/hostname.err") || true
|
||
node_err=$(cat "$TALOS_GEN_DIR/hostname.err")
|
||
[[ "$got" == "$want" ]] && break
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
echo
|
||
[[ -n "$node_err" ]] && echo " last talosctl error: $node_err" >&2
|
||
fail "$addr never came back as $want (last answer: '${got:-none}')"
|
||
fi
|
||
done
|
||
ok
|
||
done
|
||
|
||
echo -n "Bootstrapping Kubernetes... "
|
||
# -n AND -e on one control plane, deliberately. etcd bootstrap is the one
|
||
# call apid refuses to forward: reached through any other node it fails with
|
||
# `PermissionDenied: no request forwarding`. Without them talosctl picks an
|
||
# endpoint from the configured list — with more than one control plane, or a
|
||
# list that includes workers, that is a coin flip.
|
||
if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
|
||
bootstrap_target="${NODE_ADDRS[0]}"
|
||
else
|
||
bootstrap_target="${NODES[0]}.${DOMAIN}"
|
||
fi
|
||
# Re-running against an already-bootstrapped cluster is not an error — that
|
||
# is why this does not simply fail on a non-zero exit. Anything that is not
|
||
# "already exists" is shown rather than swallowed, because a silent failure
|
||
# here surfaces much later as an API that never answers.
|
||
#
|
||
# And it RETRIES, because "bootstrap is not available yet" is a moment, not
|
||
# a fault. The wait above only establishes that the node answers with its
|
||
# hostname; etcd accepting the bootstrap call comes later, and how much
|
||
# later depends on the machine. On qemu the gap is short enough to miss. On
|
||
# a Hetzner cloud server it is not: the single attempt failed with
|
||
# FailedPrecondition, the run carried on regardless, and then sat out its
|
||
# full timeout waiting for an apiserver that could never start because etcd
|
||
# had never been bootstrapped.
|
||
bootstrap_err=""
|
||
for i in $(seq 1 60); do
|
||
if bootstrap_err=$(talosctl bootstrap -n "$bootstrap_target" -e "$bootstrap_target" 2>&1); then
|
||
break
|
||
fi
|
||
if grep -qi 'already\|AlreadyExists' <<<"$bootstrap_err"; then
|
||
bootstrap_err=""
|
||
break
|
||
fi
|
||
# Not yet ready is the expected answer for the first attempts; anything
|
||
# else is reported once and then retried anyway, since the alternative
|
||
# is failing on a condition that may still clear.
|
||
if ! grep -qi 'not available yet\|connection refused\|FailedPrecondition' <<<"$bootstrap_err"; then
|
||
[[ $i -eq 1 ]] && echo -e "\n${YELLOW}bootstrap reported:${NC} ${bootstrap_err//$'\n'/ | }"
|
||
fi
|
||
sleep 10
|
||
[[ $i -eq 60 ]] && { echo; echo "${bootstrap_err//$'\n'/ | }" >&2; fail "etcd never accepted the bootstrap call"; }
|
||
done
|
||
ok
|
||
|
||
echo -n "Getting kubeconfig... "
|
||
kubeconfig_err=""
|
||
for i in $(seq 1 30); do
|
||
mkdir -p "$(dirname "$KUBECONFIG_PATH")"
|
||
if kubeconfig_err=$(talosctl kubeconfig "$KUBECONFIG_PATH" --force 2>&1); then
|
||
sed -i "s|server:.*|server: https://${K8S_API_HOST}:6443|" "$KUBECONFIG_PATH"
|
||
break
|
||
fi
|
||
sleep 5
|
||
# Held back while retrying — the first attempts fail by design — but
|
||
# printed if it never works, because that message is the diagnosis.
|
||
[[ $i -eq 30 ]] && { echo; echo "$kubeconfig_err" >&2; fail "Couldn't get kubeconfig"; }
|
||
done
|
||
ok
|
||
|
||
echo -n "Waiting for the Kubernetes API to answer... "
|
||
# Having a kubeconfig is not the same as having an apiserver. `talosctl
|
||
# kubeconfig` succeeds as soon as Talos can hand out credentials, which is
|
||
# before kube-apiserver is serving on the endpoint the file points at — so
|
||
# the very next command could fail with an EOF against a port nothing is
|
||
# listening on yet. On hardware the gap usually closes on its own; on a
|
||
# scratch cluster it does not, which is how this was found.
|
||
# `get --raw /version` needs the server; `kubectl version` would answer from
|
||
# the client alone and pass against nothing at all.
|
||
api_err=""
|
||
for i in $(seq 1 60); do
|
||
api_err=$(kubectl get --raw /version 2>&1) && break
|
||
sleep 5
|
||
if [[ $i -eq 60 ]]; then
|
||
echo "$api_err" >&2
|
||
fail "Kubernetes API at https://${K8S_API_HOST}:6443 never answered"
|
||
fi
|
||
done
|
||
ok
|
||
|
||
step 0 "Installing Cilium (CNI)..."
|
||
# Must run before the node-Ready wait below: with `cni: none` the kubelet
|
||
# reports NotReady until a CNI is present, so that loop would time out.
|
||
#
|
||
# Chart repo, version constraint and values all come out of the HelmRelease
|
||
# so this cannot drift from what Flux will reconcile minutes later. helm
|
||
# accepts the semver range verbatim and resolves it the same way Flux does.
|
||
#
|
||
# serviceMonitor is the one override, and it is a bootstrap constraint
|
||
# rather than a value: its CRD arrives with the monitoring stack much
|
||
# later, and the object would fail to apply. Nothing else is overridden,
|
||
# which is what lets helm-controller adopt this release instead of
|
||
# replacing it.
|
||
CILIUM_VALUES=$(mktemp)
|
||
yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES"
|
||
grep -q 'policyMapMax' "$CILIUM_VALUES" \
|
||
|| fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR"
|
||
# The site's part, from the cilium-site ConfigMap in the site patch: the
|
||
# HelmRelease reads it through valuesFrom, under its own values.
|
||
CILIUM_SITE_VALUES=$(mktemp)
|
||
yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "cilium-site") | .contents' \
|
||
"$TALOS_DIR/patches/site-${SITE}.yaml" \
|
||
| yq -r '.data["values.yaml"]' > "$CILIUM_SITE_VALUES"
|
||
grep -q 'routingMode' "$CILIUM_SITE_VALUES" \
|
||
|| fail "no cilium-site values (routingMode) in talos/patches/site-${SITE}.yaml"
|
||
|
||
# Cilium's cni.confPath is a subdirectory of /etc/cni/net.d that Multus
|
||
# creates (#832). /etc/cni is read-only to the kubelet on Talos, so the
|
||
# agent's DirectoryOrCreate mount cannot make it, and on an empty cluster
|
||
# Multus is not there yet. A hostNetwork pod per node creates it; there is
|
||
# no pod network before Cilium.
|
||
cni_conf_dir=$(yq -r '.cni.confPath' "$CILIUM_VALUES")
|
||
if [[ "$cni_conf_dir" == /etc/cni/net.d/* ]]; then
|
||
step 0 "Creating $cni_conf_dir on every node..."
|
||
# kube-controller-manager creates kube-system/default shortly after the API answers.
|
||
for i in $(seq 1 30); do
|
||
kubectl -n kube-system get serviceaccount default >/dev/null && break
|
||
sleep 5
|
||
[[ $i -eq 30 ]] && fail "kube-system/default ServiceAccount never appeared"
|
||
done
|
||
# A pod bound to a node that has not registered yet is garbage collected.
|
||
for node in "${ALL_NODES[@]}"; do
|
||
for i in $(seq 1 60); do
|
||
kubectl get node "$node" >/dev/null && break
|
||
sleep 5
|
||
[[ $i -eq 60 ]] && fail "$node never registered with the API server"
|
||
done
|
||
done
|
||
for node in "${ALL_NODES[@]}"; do
|
||
kubectl -n kube-system delete pod "cni-confdir-$node" --ignore-not-found --wait=true >/dev/null \
|
||
|| fail "could not remove an earlier cni-confdir pod on $node"
|
||
kubectl apply -f - >/dev/null <<EOF || fail "could not create the cni-confdir pod on $node"
|
||
apiVersion: v1
|
||
kind: Pod
|
||
metadata:
|
||
name: cni-confdir-$node
|
||
namespace: kube-system
|
||
spec:
|
||
nodeName: $node
|
||
hostNetwork: true
|
||
restartPolicy: Never
|
||
tolerations:
|
||
- operator: Exists
|
||
containers:
|
||
- name: mkdir
|
||
image: docker.io/library/busybox:1.37
|
||
command: ["mkdir", "-p", "/host/${cni_conf_dir#/etc/cni/net.d/}"]
|
||
volumeMounts:
|
||
- name: netd
|
||
mountPath: /host
|
||
volumes:
|
||
- name: netd
|
||
hostPath:
|
||
path: /etc/cni/net.d
|
||
type: Directory
|
||
EOF
|
||
done
|
||
for node in "${ALL_NODES[@]}"; do
|
||
kubectl -n kube-system wait --for=jsonpath='{.status.phase}'=Succeeded \
|
||
"pod/cni-confdir-$node" --timeout=5m >/dev/null \
|
||
|| fail "creating $cni_conf_dir on $node did not succeed (kubectl -n kube-system describe pod cni-confdir-$node)"
|
||
kubectl -n kube-system delete pod "cni-confdir-$node" >/dev/null
|
||
done
|
||
ok
|
||
fi
|
||
|
||
helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null
|
||
helm repo update cilium >/dev/null
|
||
helm upgrade --install cilium cilium/cilium \
|
||
--version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \
|
||
--namespace kube-system \
|
||
-f "$CILIUM_SITE_VALUES" \
|
||
-f "$CILIUM_VALUES" \
|
||
--set prometheus.serviceMonitor.enabled=false \
|
||
--wait --timeout 10m
|
||
rm -f "$CILIUM_VALUES" "$CILIUM_SITE_VALUES"
|
||
ok
|
||
|
||
# Multus is the only writer of /etc/cni/net.d (#832); without it the kubelet
|
||
# reports "cni plugin not initialized" and nothing schedules, Flux included.
|
||
# Applied from the manifests Flux owns, which adopts them.
|
||
step 0 "Installing Multus (the CNI entry point)..."
|
||
kubectl apply -k "$REPO_ROOT/kubernetes/infrastructure/multus" >/dev/null \
|
||
|| fail "could not apply kubernetes/infrastructure/multus"
|
||
kubectl -n kube-system rollout status daemonset/kube-multus-ds-subdir --timeout=10m >/dev/null \
|
||
|| fail "Multus never became ready (kubectl -n kube-system get pods -l app=multus)"
|
||
ok
|
||
|
||
step 0 "Waiting for K8s nodes..."
|
||
expected=${#ALL_NODES[@]}
|
||
# Before the loop, because 0 == 0 would satisfy its exit condition on the
|
||
# first pass and report a complete Talos phase for a cluster with no nodes
|
||
# in it. `expected` comes from ALL_NODES, which a provisioning phase hands
|
||
# over — an empty one is a bug in that phase, not a cluster that is ready.
|
||
[[ "$expected" -gt 0 ]] \
|
||
|| fail "ALL_NODES is empty, so this wait would pass against any cluster, including one with no nodes"
|
||
nodes_err=""
|
||
for i in $(seq 1 60); do
|
||
# The STATUS column, matched whole. `grep -c Ready` counted NotReady
|
||
# too — the substring is in it — so this waited for nodes to register
|
||
# rather than to become ready, and passed while every one of them was
|
||
# still NotReady.
|
||
nodes_out=$(kubectl get nodes --no-headers 2>"$TALOS_GEN_DIR/nodes.err") || true
|
||
nodes_err=$(cat "$TALOS_GEN_DIR/nodes.err")
|
||
ready=$(awk '$2 == "Ready"' <<<"$nodes_out" | grep -c . || true)
|
||
if [[ "$ready" -eq "$expected" ]]; then
|
||
# Say what was accepted. Silence on success made this step
|
||
# unauditable: a run that printed "Waiting for K8s nodes..." and
|
||
# then "Talos phase complete" was indistinguishable from one that
|
||
# matched 0 against 0, and the log gave no way to tell which had
|
||
# happened. A count that is also the pass condition is the one
|
||
# number worth printing.
|
||
echo "$ready/$expected Ready"
|
||
break
|
||
fi
|
||
sleep 5
|
||
if [[ $i -eq 60 ]]; then
|
||
echo
|
||
[[ -n "$nodes_err" ]] && echo "last kubectl error: $nodes_err" >&2
|
||
echo "${nodes_out:- (no nodes listed)}" >&2
|
||
fail "Not all nodes ready (got $ready/$expected)"
|
||
fi
|
||
done
|
||
|
||
# The nodes' resolver asks k8s-gateway first (ResolverConfig in the site
|
||
# patch), and CoreDNS forwards external names to the node. Until k8s-gateway
|
||
# answers, every lookup waits out the node's 2 s read timeout and CoreDNS
|
||
# gives up first: pods resolve nothing, Flux included. So it is installed
|
||
# here, from the values Flux will reconcile (the HelmRelease, under the
|
||
# site's k8s-gateway-site values, as valuesFrom merges them), and Flux
|
||
# adopts the release.
|
||
step 0 "Installing k8s-gateway (the nodes' first resolver)..."
|
||
k8sgw_values=$(mktemp)
|
||
k8sgw_site_values=$(mktemp)
|
||
yq 'select(.kind == "HelmRelease") | .spec.values' "$K8SGW_HR" > "$k8sgw_values"
|
||
yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "k8s-gateway-site") | .contents' \
|
||
"$TALOS_DIR/patches/site-${SITE}.yaml" \
|
||
| yq -r '.data["values.yaml"]' > "$k8sgw_site_values"
|
||
grep -q 'dnsPolicy' "$k8sgw_site_values" \
|
||
|| fail "no k8s-gateway-site values (dnsPolicy) in talos/patches/site-${SITE}.yaml"
|
||
helm upgrade --install k8s-gateway \
|
||
"$(yq -r 'select(.kind == "OCIRepository") | .spec.url' "$K8SGW_HR")" \
|
||
--namespace kube-system \
|
||
-f "$k8sgw_site_values" \
|
||
-f "$k8sgw_values" \
|
||
--wait --timeout 5m >/dev/null \
|
||
|| fail "k8s-gateway did not come up (kubectl -n kube-system get pods -l app.kubernetes.io/name=k8s-gateway)"
|
||
rm -f "$k8sgw_values" "$k8sgw_site_values"
|
||
ok
|
||
|
||
# Stash configs for subsequent phases / export-access step. Skipped under
|
||
# vmtest and hcloudtest: those are a throwaway cluster's credentials, and a
|
||
# later production `bootstrap.sh flux` copies access-staging/ over the
|
||
# operator's kubeconfig and talosconfig.
|
||
if [[ "${HCLOUDTEST:-0}" == "1" ]]; then
|
||
hcloud_egress_check
|
||
echo -e "${GREEN}Talos phase complete.${NC} Access files in $HCLOUD_STATE_DIR/ (not staged)"
|
||
elif [[ "${VMTEST:-0}" == "1" ]]; then
|
||
# Nodes being Ready says nothing about storage. The CephCluster picks
|
||
# OSD nodes by a label out of the machine config and their device by the
|
||
# partition label RawVolumeConfig creates, so assert both here — running
|
||
# it by hand afterwards races the teardown, and a test that does not
|
||
# check its own premise is worth little.
|
||
step 0 "Checking the OSD prerequisites Ceph selects on..."
|
||
for idx in $(seq "${#NODES[@]}" $(( ${#ALL_NODES[@]} - 1 ))); do
|
||
vm_node="${ALL_NODES[$idx]}"
|
||
vm_addr="${NODE_ADDRS[$idx]}"
|
||
vols=$(talosctl -n "$vm_addr" -e "$vm_addr" get discoveredvolumes 2>&1) \
|
||
|| fail "could not read volumes from $vm_node ($vm_addr): ${vols//$'\n'/ | }"
|
||
awk '$NF == "r-osd-data" {found=1} END {exit !found}' <<<"$vols" \
|
||
|| fail "$vm_node has no r-osd-data partition — RawVolumeConfig did not take"
|
||
kubectl get node "$vm_node" \
|
||
-o jsonpath='{.metadata.labels.storage\.renner\.uno/ceph-osd}' \
|
||
| grep -q true \
|
||
|| fail "$vm_node is missing the storage.renner.uno/ceph-osd label"
|
||
echo -e " $vm_node: partition and label ${GREEN}(OK)${NC}"
|
||
done
|
||
|
||
echo -e "${GREEN}VM rebuild test passed.${NC} ${#ALL_NODES[@]} nodes Ready on the repo's own config."
|
||
else
|
||
mkdir -p "$ACCESS_STAGING"
|
||
cp "$TALOSCONFIG_PATH" "$ACCESS_STAGING/talosconfig"
|
||
cp "$KUBECONFIG_PATH" "$ACCESS_STAGING/kubeconfig"
|
||
echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
|
||
fi
|
||
fi
|
||
|
||
# Helper: load staged access files into home dir (for later CI steps)
|
||
# Bring OpenBao up far enough that External Secrets can read from it.
|
||
#
|
||
# This was eight commands typed by hand, and it sits in the middle of the cold
|
||
# rebuild: infrastructure-crds waits on the ClusterSecretStore, which needs an
|
||
# unsealed OpenBao with Kubernetes auth configured, and every app Kustomization
|
||
# waits on infrastructure-crds. A rebuild could not reach a reconciled state
|
||
# without someone at a keyboard.
|
||
#
|
||
# The seal key stays where it was — the operator's `pass` store — and is read
|
||
# here at run time. It is never written to the repo and never reaches CI, which
|
||
# is the reason this was left manual in the first place. It reaches the cluster
|
||
# as the openbao-seal-key Secret, created by the flux phase before Flux starts
|
||
# OpenBao, and from then on every replica unseals itself.
|
||
#
|
||
# Everything below is idempotent: it is safe to re-run against an OpenBao that
|
||
# is already unsealed and configured, which is what makes it usable as part of
|
||
# `bootstrap.sh all` rather than a one-shot.
|
||
|
||
bao_exec() {
|
||
local pod=$1; shift
|
||
kubectl -n "$OPENBAO_NS" exec -i "$pod" -- "$@"
|
||
}
|
||
|
||
# Where the OpenBao raft snapshots already are: inside the Velero backup, in the
|
||
# kopia repository the data mover writes to. Nothing new is produced for the
|
||
# rebuild — the nightly snapshot CronJob and the openbao-daily Velero Schedule
|
||
# have been doing this all along.
|
||
#
|
||
# Reading them needs three things, all of which live OUTSIDE the cluster, which
|
||
# is the point: bucket credentials from HOMELAB_SECRETS_DIR (their own key, not
|
||
# Velero's),
|
||
# kopia, and the repository password. The last is not a secret — Velero
|
||
# hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
|
||
# the offsite backup's confidentiality rests on bucket access alone.
|
||
#
|
||
# This is what breaks the circle. Velero's own credentials come from an
|
||
# ExternalSecret, which needs the ClusterSecretStore, which needs an unsealed
|
||
# OpenBao — so Velero cannot be the thing that restores OpenBao.
|
||
OPENBAO_KOPIA_BUCKET="renner-uno-k8s-velero"
|
||
OPENBAO_KOPIA_PREFIX="kopia/openbao/"
|
||
OPENBAO_S3_ENDPOINT="$S3_ENDPOINT"
|
||
OPENBAO_S3_REGION="$S3_REGION"
|
||
# Velero's hardcoded backup repository password (pkg/repository/keys/keys.go).
|
||
OPENBAO_KOPIA_PASSWORD="static-passw0rd"
|
||
|
||
restore_openbao() {
|
||
local pod=$1
|
||
step 0 "Restoring OpenBao from the offsite backup..."
|
||
command -v kopia >/dev/null || fail "kopia not found — needed to read the Velero backup repository without Velero"
|
||
|
||
local work kopia_err
|
||
work=$(mktemp -d)
|
||
export KOPIA_CONFIG_PATH="$work/repository.config"
|
||
export KOPIA_CACHE_DIRECTORY="$work/cache"
|
||
export KOPIA_LOG_DIR="$work/logs"
|
||
export KOPIA_CHECK_FOR_UPDATES=false
|
||
export KOPIA_PASSWORD="$OPENBAO_KOPIA_PASSWORD"
|
||
|
||
# Assigned first, not interpolated into the argument list. `fail` inside
|
||
# $( ) exits only the subshell, and a failed substitution in an argument does
|
||
# not stop the outer command — so a missing key produced an EMPTY
|
||
# --access-key and an authentication error instead of the real reason.
|
||
local s3_access s3_secret
|
||
s3_access=$(read_secret s3-access) || fail "no s3-access secret"
|
||
s3_secret=$(read_secret s3-secret) || fail "no s3-secret secret"
|
||
|
||
# Nothing here is expected to fail, so nothing here discards its error. This
|
||
# is the path that only runs during a genuine disaster; it is the last place
|
||
# to be guessing why something did not work.
|
||
echo -n " connecting to the backup repository (read-only)... "
|
||
if ! kopia_err=$(kopia repository connect s3 \
|
||
--bucket "$OPENBAO_KOPIA_BUCKET" \
|
||
--prefix "$OPENBAO_KOPIA_PREFIX" \
|
||
--endpoint "$OPENBAO_S3_ENDPOINT" \
|
||
--region "$OPENBAO_S3_REGION" \
|
||
--access-key "$s3_access" \
|
||
--secret-access-key "$s3_secret" \
|
||
--readonly 2>&1); then
|
||
echo; echo "$kopia_err" >&2
|
||
fail "could not open the backup repository at $OPENBAO_S3_ENDPOINT/$OPENBAO_KOPIA_BUCKET/$OPENBAO_KOPIA_PREFIX"
|
||
fi
|
||
ok
|
||
|
||
# Newest kopia snapshot, then the newest .snap inside it. The PVC holds a
|
||
# week of them, so the last line is the most recent.
|
||
local ksnap snapfile listing
|
||
listing=$(kopia snapshot list --all 2>&1) \
|
||
|| { echo "$listing" >&2; fail "could not list snapshots in the backup repository"; }
|
||
ksnap=$(tail -1 <<<"$listing" | awk '{print $4}')
|
||
[[ -n "$ksnap" ]] \
|
||
|| { echo "$listing" >&2; fail "could not read a snapshot id out of the listing above"; }
|
||
snapfile=$(kopia ls "$ksnap" 2>&1 | grep -o 'openbao-raft-[0-9-]*\.snap' | sort | tail -1) || true
|
||
[[ -n "$snapfile" ]] || fail "no openbao-raft-*.snap in kopia snapshot $ksnap"
|
||
echo " newest snapshot: $snapfile"
|
||
|
||
if ! kopia_err=$(kopia restore "$ksnap/$snapfile" "$work/restore.snap" 2>&1); then
|
||
echo "$kopia_err" >&2
|
||
fail "could not extract $snapfile from the backup"
|
||
fi
|
||
[[ -s "$work/restore.snap" ]] || fail "$snapfile came out empty"
|
||
echo " extracted $(stat -c%s "$work/restore.snap") bytes"
|
||
|
||
# A snapshot can only be loaded into a running OpenBao, so the fresh one is
|
||
# initialised first. Under the static seal, init hands out a recovery key
|
||
# rather than an unseal key and the server unseals itself; the only thing
|
||
# used from its output is the root token, for the next few seconds, and
|
||
# then it stops existing: restoring replaces the keyring with the
|
||
# snapshot's, after which the operator's root token
|
||
# (~/.config/openbao/root-token.age) and recovery key apply.
|
||
#
|
||
# No -force. Without it OpenBao checks that the snapshot was written under
|
||
# the seal key this server runs with, and refuses a mismatch by name
|
||
# ("possibly the snapshot is using a different autoseal key"). That check
|
||
# is the one guard against restoring under the wrong seal-key Secret;
|
||
# -force would skip it and leave a server that can never unseal.
|
||
#
|
||
# The init output is not echoed on a parse failure: it carries the root
|
||
# token and the recovery key of a server that is about to be overwritten,
|
||
# but a log line is a log line.
|
||
echo -n " initialising the empty server (throwaway recovery key)... "
|
||
local init_json root
|
||
init_json=$(bao_exec "$pod" bao operator init -recovery-shares=1 -recovery-threshold=1 -format=json 2>&1) \
|
||
|| { echo; echo "$init_json" >&2; fail "could not initialise OpenBao"; }
|
||
root=$(jq -r '.root_token // empty' <<<"$init_json")
|
||
[[ -n "$root" ]] || fail "init output carried no root token"
|
||
ok
|
||
|
||
# Sealed after init means the static seal is not working — the Secret is
|
||
# missing or malformed — and the restore below cannot run against it.
|
||
echo -n " waiting for it to unseal itself"
|
||
for i in $(seq 1 30); do
|
||
if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
|
||
| jq -e '.sealed == false' >/dev/null; then
|
||
echo; break
|
||
fi
|
||
printf '.'
|
||
sleep 2
|
||
if [[ $i -eq 30 ]]; then
|
||
echo
|
||
cat "$OPENBAO_SECRETS_DIR/.baoerr" >&2
|
||
fail "$pod did not unseal itself after init — check the openbao-seal-key Secret and the pod log"
|
||
fi
|
||
done
|
||
|
||
echo -n " loading the snapshot... "
|
||
kubectl -n "$OPENBAO_NS" cp "$work/restore.snap" "$pod:/tmp/restore.snap" >/dev/null \
|
||
|| fail "could not copy the snapshot into $pod"
|
||
local out
|
||
out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore /tmp/restore.snap 2>&1) \
|
||
|| { echo; echo "$out" >&2; fail "snapshot restore rejected"; }
|
||
ok
|
||
|
||
rm -rf "$work"
|
||
# The server seals itself against the keyring that came with the snapshot
|
||
# and unseals again from the static seal — the same key the snapshot was
|
||
# written under. bring_up_openbao waits for that on every replica.
|
||
echo " restored — the server re-seals and unseals itself from the seal key"
|
||
}
|
||
|
||
bring_up_openbao() {
|
||
step 0 "Bringing OpenBao up..."
|
||
command -v jq >/dev/null || fail "jq not found (needed to read OpenBao's seal status)"
|
||
|
||
# OpenBao is deployed by the infrastructure Kustomization. What this step
|
||
# depends on is therefore that Flux has applied it — not that some number of
|
||
# minutes have gone by. A guessed duration is not a condition: the first
|
||
# version failed after ten minutes on a slow cluster, and raising the number
|
||
# only moves the guess.
|
||
#
|
||
# So the loop ends on one of two facts, both read from the cluster: the
|
||
# StatefulSet exists, or Flux reports the Kustomization has failed. The
|
||
# elapsed-time bound below is a backstop against hanging forever, not the
|
||
# mechanism — it is deliberately far longer than any real reconcile.
|
||
local pods=() ks_state ks_bad=0
|
||
echo -n " waiting for Flux to apply the openbao StatefulSet"
|
||
for i in $(seq 1 720); do
|
||
if kubectl -n "$OPENBAO_NS" get statefulset openbao >/dev/null 2>&1; then
|
||
echo
|
||
break
|
||
fi
|
||
ks_state=$(ks_ready_condition infrastructure)
|
||
# Only a persistent failure counts. The first version matched any message
|
||
# containing "error" or "failed", which fires on `failed to download
|
||
# artifact from source-controller` — a routine transient in the minutes
|
||
# after bootstrap that Flux retries on its own, and one that aborted the
|
||
# rebuild. A wait built on cluster facts is no better than a guessed
|
||
# timeout if it reacts to a fact that is not the one it means.
|
||
if [[ "${ks_state%%|*}" == "False" ]] && grep -qiE 'failed|error' <<<"${ks_state#*|}"; then
|
||
(( ks_bad++ )) || true
|
||
else
|
||
ks_bad=0
|
||
fi
|
||
if (( ks_bad >= 30 )); then
|
||
echo
|
||
echo " infrastructure Ready=${ks_state%%|*} — ${ks_state#*|}" >&2
|
||
fail "the infrastructure Kustomization has been failing for five minutes — OpenBao will not appear"
|
||
fi
|
||
(( i % 18 == 0 )) && { echo; echo " infrastructure Ready=${ks_state%%|*} — ${ks_state#*|}"; echo -n " waiting"; }
|
||
printf '.'
|
||
sleep 10
|
||
[[ $i -eq 720 ]] && { echo; fail "openbao StatefulSet never appeared, and Flux never reported a failure — something is wedged"; }
|
||
done
|
||
|
||
# The StatefulSet exists; its pods follow.
|
||
for i in $(seq 1 60); do
|
||
mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
|
||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
||
[[ ${#pods[@]} -gt 0 ]] && break
|
||
sleep 10
|
||
[[ $i -eq 60 ]] && fail "the openbao StatefulSet exists but produced no pods"
|
||
done
|
||
echo " found ${#pods[@]} pod(s): ${pods[*]}"
|
||
|
||
# A pod object existing is not a pod that can answer. Waiting only for the
|
||
# name meant `bao status` ran against a container still pulling or in Init,
|
||
# its error was thrown away, and an empty answer parsed as "not initialised"
|
||
# — which routes into the restore branch, the one branch that must never be
|
||
# entered by mistake. Wait for the container to be running first.
|
||
echo -n " waiting for ${pods[0]} to answer"
|
||
local status="" status_err=""
|
||
# Bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to
|
||
# provision its volume.
|
||
for i in $(seq 1 180); do
|
||
if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then
|
||
echo; break
|
||
fi
|
||
status_err=$(cat "$OPENBAO_SECRETS_DIR/.baoerr")
|
||
# `bao status` exits 2 when sealed and still prints valid JSON — that is
|
||
# an answer, not a failure.
|
||
#
|
||
# Ask whether the field is PRESENT, not whether it is true. `jq -e
|
||
# '.initialized'` tests the value, so a brand-new OpenBao answering
|
||
# {"initialized": false, "sealed": true} — a complete and correct
|
||
# answer, and the only one a cold rebuild can give — read as no answer
|
||
# at all. The wait then ran its full 90 rounds against a healthy pod and
|
||
# failed with "never answered bao status", one step after the cluster
|
||
# had finally come up on its own.
|
||
if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi
|
||
printf '.'
|
||
sleep 10
|
||
if [[ $i -eq 180 ]]; then
|
||
echo
|
||
echo " last error: $status_err" >&2
|
||
fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised"
|
||
fi
|
||
done
|
||
|
||
# `bao status` reports Initialized either way, so it distinguishes "needs a
|
||
# key" from "needs initialising". Those must not be confused: initialising an
|
||
# OpenBao that already holds data is how the data gets lost.
|
||
# Read presence separately from value. jq's `//` treats false as absent, so
|
||
# `.initialized // empty` on the one answer a rebuild always gives —
|
||
# {"initialized": false} — yields an empty string, and the guard below then
|
||
# reports that OpenBao's state could not be read at all. Same mistake as the
|
||
# `jq -e` in the wait above, one line apart: false is a value, not a gap.
|
||
local initialized
|
||
initialized=$(jq -r 'if has("initialized") then (.initialized | tostring) else "" end' <<<"$status")
|
||
[[ -n "$initialized" ]] \
|
||
|| { echo "$status" >&2; fail "could not read initialised state from bao status"; }
|
||
|
||
if [[ "$initialized" != "true" ]]; then
|
||
# An uninitialised OpenBao is restored from the offsite snapshot, never
|
||
# initialised fresh: initialising over an existing keyring is how the
|
||
# data is lost, and a genuinely new OpenBao is a decision rather than a
|
||
# step (MIGRATION-sops-openbao.md).
|
||
restore_openbao "${pods[0]}"
|
||
fi
|
||
|
||
# Wait for every replica the StatefulSet wants, not the ones that happen to
|
||
# exist right now. On a cold start only openbao-0 exists when this runs:
|
||
# podManagementPolicy is OrderedReady, so openbao-1 is created once
|
||
# openbao-0 passes its readiness probe and openbao-2 after that. Each
|
||
# unseals itself from the static seal; nothing here types a key.
|
||
#
|
||
# Waiting for the whole chain rather than the first pod matters because the
|
||
# openbao Service sets publishNotReadyAddresses — the chart does this so
|
||
# replicas can reach each other to join raft before any of them is Ready —
|
||
# so a replica that is still starting is a live backend on the service IP.
|
||
# Clients balanced onto it get `dial tcp <svc>:8200: connect: no route to
|
||
# host`, and the failure surfaces nowhere near here: external-secrets'
|
||
# ClusterSecretStore flaps, every ExternalSecret reports "could not get
|
||
# secret data from provider", and infrastructure-crds — which waits on those
|
||
# — never goes Ready, taking all twenty app Kustomizations with it.
|
||
local want
|
||
want=$(kubectl -n "$OPENBAO_NS" get statefulset openbao -o jsonpath='{.spec.replicas}') \
|
||
|| fail "could not read the openbao StatefulSet's replica count"
|
||
[[ "$want" =~ ^[0-9]+$ ]] || fail "openbao StatefulSet reported a non-numeric replica count: '$want'"
|
||
echo " waiting for $want replica(s) to unseal themselves"
|
||
|
||
local unsealed=0 round pod
|
||
local -A seen=()
|
||
for round in $(seq 1 180); do
|
||
mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
|
||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||
unsealed=0
|
||
for pod in "${pods[@]}"; do
|
||
if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
|
||
| jq -e '.sealed == false' >/dev/null; then
|
||
[[ -n "${seen[$pod]:-}" ]] || { echo -e " $pod: ${GREEN}unsealed${NC}"; seen[$pod]=1; }
|
||
unsealed=$(( unsealed + 1 ))
|
||
fi
|
||
done
|
||
(( unsealed >= want )) && break
|
||
sleep 10
|
||
if [[ $round -eq 180 ]]; then
|
||
# A replica that is running and still sealed did not get a working
|
||
# seal. Say which, with its own answer, rather than "timed out".
|
||
for pod in "${pods[@]}"; do
|
||
[[ -n "${seen[$pod]:-}" ]] && continue
|
||
echo " $pod:" >&2
|
||
bao_exec "$pod" bao status 2>&1 | sed 's/^/ /' >&2 || true
|
||
done
|
||
fail "only $unsealed of $want OpenBao replicas unsealed after 30 minutes — a sealed one means the static seal failed (openbao-seal-key Secret, pod log)"
|
||
fi
|
||
done
|
||
ok
|
||
}
|
||
|
||
load_staged_access() {
|
||
[[ -f "$ACCESS_STAGING/talosconfig" && -f "$ACCESS_STAGING/kubeconfig" ]] \
|
||
|| fail "access-staging/ missing talosconfig or kubeconfig (run talos phase first)"
|
||
# To the paths talosctl and kubectl will actually read, not to $HOME. An
|
||
# operator with KUBECONFIG pointing somewhere else — an ordinary setup — got
|
||
# the staged config written where nothing would look at it, and every step
|
||
# after this ran against whatever cluster the old KUBECONFIG named. Wrong
|
||
# target, no error.
|
||
mkdir -p "$(dirname "$TALOSCONFIG_PATH")" "$(dirname "$KUBECONFIG_PATH")"
|
||
cp "$ACCESS_STAGING/talosconfig" "$TALOSCONFIG_PATH"
|
||
cp "$ACCESS_STAGING/kubeconfig" "$KUBECONFIG_PATH"
|
||
}
|
||
|
||
if phase_runs flux; then
|
||
echo -e "${YELLOW}=== Phase: FluxCD bootstrap ===${NC}"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v flux >/dev/null || fail "flux not found"
|
||
# In the in-cluster Job neither the SOPS key nor the Gitea token is present,
|
||
# and neither should be baked in. Both are streamed into the secrets
|
||
# directory when the run asks for them, the same way the seal key is.
|
||
# Unconditional: the same collection step on a laptop and in the Job, so the
|
||
# recovery path is the one that gets exercised.
|
||
require_secrets
|
||
SOPS_KEY="$OPENBAO_SECRETS_DIR/sops-age.key"
|
||
[[ -f "$SOPS_KEY" ]] || fail "SOPS age key not found at $SOPS_KEY"
|
||
# One collected token, handed to whichever variable the chosen source reads.
|
||
# `flux bootstrap gitea` and `flux bootstrap github` each take theirs from
|
||
# the environment.
|
||
case "$FLUX_SOURCE" in
|
||
gitea) GITEA_TOKEN=$(read_secret git-token); export GITEA_TOKEN ;;
|
||
github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;;
|
||
restore-test)
|
||
GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN
|
||
command -v gh >/dev/null || fail "gh not found (registers the read-only deploy key)"
|
||
command -v jq >/dev/null || fail "jq not found (reads the DoH answer for the egress check)"
|
||
# The throwaway cluster carries production secrets, and this phase
|
||
# copies access files over KUBECONFIG_PATH and TALOSCONFIG_PATH:
|
||
# neither may be the operator's production file.
|
||
[[ "$KUBECONFIG_PATH" != "$HOME/.kube/config" && "$TALOSCONFIG_PATH" != "$HOME/.talos/config" ]] \
|
||
|| fail "FLUX_SOURCE=restore-test needs KUBECONFIG and TALOSCONFIG pointed at the test cluster ($HCLOUD_STATE_DIR)"
|
||
;;
|
||
*) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github|restore-test)" ;;
|
||
esac
|
||
# The restore test reads its access from KUBECONFIG/TALOSCONFIG, which
|
||
# hcloudtest writes; access-staging/ may hold an older run's files.
|
||
[[ "$PHASE" == "flux" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access
|
||
if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
|
||
# Whatever the paths say, the cluster itself must be the test cluster,
|
||
# before anything below writes a secret to it.
|
||
node_names=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') \
|
||
|| fail "cannot list the nodes of the target cluster"
|
||
[[ -n "$node_names" ]] || fail "the target cluster has no nodes"
|
||
for n in $node_names; do
|
||
[[ "$n" == "$HCLOUD_NAME_PREFIX"-* ]] \
|
||
|| fail "node '$n' is not a $HCLOUD_NAME_PREFIX-* test node — refusing the restore test against this cluster"
|
||
done
|
||
hcloud_egress_check
|
||
fi
|
||
|
||
echo -n "Creating SOPS age secret... "
|
||
kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f -
|
||
kubectl create secret generic sops-age -n flux-system \
|
||
--from-file=age.agekey="$SOPS_KEY" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
ok
|
||
|
||
# The static seal key, before Flux brings OpenBao up: a pod whose env
|
||
# references a Secret that does not exist is never started, and OpenBao
|
||
# reads the key at start. Left alone when the cluster already holds the
|
||
# same value. A DIFFERENT value is refused rather than applied: the data
|
||
# is encrypted under whichever key OpenBao started with, and swapping the
|
||
# Secret would leave every replica sealed for good at its next restart.
|
||
# That is a migration, not a bootstrap step.
|
||
#
|
||
# --from-literal, not --from-file: the stored value ends with a
|
||
# newline, the substitution strips it, and OpenBao rejects a key that is
|
||
# not exactly 32 bytes of base64. The key sits in this process's argument
|
||
# list for the length of the call, as it did in the pod's before.
|
||
echo -n "Creating OpenBao seal key secret... "
|
||
seal_key=$(read_secret seal-key) || fail "could not read the seal key"
|
||
kubectl create namespace "$OPENBAO_NS" --dry-run=client -o yaml | kubectl apply -f -
|
||
if existing=$(kubectl -n "$OPENBAO_NS" get secret openbao-seal-key \
|
||
-o go-template='{{index .data "key" | base64decode}}' 2>"$OPENBAO_SECRETS_DIR/.err"); then
|
||
[[ "$existing" == "$seal_key" ]] \
|
||
|| fail "openbao-seal-key in $OPENBAO_NS holds a different key than pass — not replacing it (OpenBao's data is written under the key it started with)"
|
||
echo "already present"
|
||
elif grep -q NotFound "$OPENBAO_SECRETS_DIR/.err"; then
|
||
kubectl -n "$OPENBAO_NS" create secret generic openbao-seal-key \
|
||
--from-literal=key="$seal_key" >/dev/null \
|
||
|| fail "could not create the openbao-seal-key Secret"
|
||
ok
|
||
else
|
||
echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
|
||
fail "could not read the openbao-seal-key Secret"
|
||
fi
|
||
unset seal_key existing
|
||
|
||
# Before Flux: cnpg-object-stores fails without it rather than render empty
|
||
# paths, and the paths must exist before any Cluster bootstraps.
|
||
if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
|
||
create_cnpg_test_bucket
|
||
write_cnpg_paths "$CNPG_TEST_BUCKET"
|
||
else
|
||
write_cnpg_paths "$CNPG_BUCKET"
|
||
fi
|
||
|
||
if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
|
||
# The production tree through kubernetes/restore-test/flux-root.yaml,
|
||
# whose patches are the test's whole deviation from production. Nothing
|
||
# is committed anywhere: the source is the GitHub mirror, read through a
|
||
# deploy key GitHub itself holds to read-only, so the test cluster
|
||
# cannot push (image automation) even with production's manifests.
|
||
step 0 "Bootstrapping FluxCD for the restore test (restore-test/flux-root.yaml)..."
|
||
flux install --components-extra=image-reflector-controller,image-automation-controller \
|
||
|| fail "flux install failed"
|
||
restore_test_repo
|
||
deploy_key="$OPENBAO_SECRETS_DIR/restore-test-git"
|
||
rm -f "$deploy_key" "$deploy_key.pub"
|
||
ssh-keygen -q -t ed25519 -N '' -C "restore-test" -f "$deploy_key"
|
||
gh api -X POST "repos/$restore_repo/keys" \
|
||
-f title="restore-test $(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||
-f key="$(cat "$deploy_key.pub")" -F read_only=true >/dev/null \
|
||
|| fail "could not register the read-only deploy key on $restore_repo"
|
||
flux create secret git restore-test-git \
|
||
--url="$restore_url" \
|
||
--private-key-file="$deploy_key" >/dev/null \
|
||
|| fail "could not create the restore-test-git Secret"
|
||
# On FLUX_BRANCH, so a branch can be tried here before it is merged.
|
||
# shellcheck disable=SC2016 # $b is a jq variable
|
||
yq -y --arg b "$FLUX_BRANCH" \
|
||
'if .kind == "GitRepository" and .metadata.name == "restore-test" then .spec.ref.branch = $b else . end' \
|
||
"$REPO_ROOT/kubernetes/restore-test/flux-root.yaml" | kubectl apply -f - >/dev/null \
|
||
|| fail "could not apply restore-test/flux-root.yaml"
|
||
echo " source: $restore_url, branch $FLUX_BRANCH"
|
||
ok
|
||
elif [[ "$FLUX_SOURCE" == "github" ]]; then
|
||
step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..."
|
||
echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}"
|
||
echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}"
|
||
# GITHUB_TOKEN is read from the environment by `flux bootstrap github`.
|
||
#
|
||
# --secret-name, and this is what makes the break-glass path survive its
|
||
# own success. The default is `flux-system`, which is also the target of
|
||
# the flux-system-git-auth ExternalSecret in post-crds — that one carries
|
||
# the GITEA credentials, which is correct for the steady state where the
|
||
# source is in-cluster Gitea.
|
||
#
|
||
# Sharing the name means the moment infrastructure-crds reconciles and
|
||
# ESO resolves, the GitHub token is overwritten with Gitea credentials
|
||
# while the GitRepository still points at github.com. Flux then stops
|
||
# being able to read its own source:
|
||
#
|
||
# GitOperationFailed: unable to list remote for
|
||
# 'https://github.com/.../Homelab.git': authentication required:
|
||
# Invalid username or token
|
||
#
|
||
# The rebuild destroys its git source at the exact moment it starts
|
||
# working, which is why it stayed hidden: no earlier run ever got the
|
||
# ExternalSecrets to resolve at all.
|
||
#
|
||
# A separate name keeps the two credentials apart. `flux-system` still
|
||
# receives the Gitea credentials and is simply unused until the source is
|
||
# switched back to Gitea — the manual step the warning above names.
|
||
flux bootstrap github \
|
||
--owner="$GITHUB_OWNER" \
|
||
--repository="$GITHUB_REPO" \
|
||
--branch="$FLUX_BRANCH" \
|
||
--path="$FLUX_PATH" \
|
||
--secret-name=flux-system-breakglass \
|
||
--components-extra=image-reflector-controller,image-automation-controller \
|
||
--personal \
|
||
--token-auth --timeout 15m 2>&1
|
||
else
|
||
step 0 "Bootstrapping FluxCD from in-cluster Gitea..."
|
||
flux bootstrap gitea \
|
||
--hostname="$GITEA_HOST" \
|
||
--owner="$GITEA_OWNER" \
|
||
--repository="$GITEA_REPO" \
|
||
--branch="$FLUX_BRANCH" \
|
||
--path="$FLUX_PATH" \
|
||
--components-extra=image-reflector-controller,image-automation-controller \
|
||
--personal \
|
||
--token-auth --timeout 15m 2>&1
|
||
fi
|
||
|
||
bring_up_openbao
|
||
|
||
step 0 "Waiting for infrastructure..."
|
||
for i in $(seq 1 60); do
|
||
# The Ready condition, not the printed table: `grep -c True` used to
|
||
# match SUSPENDED=True as well, so a Kustomization Flux is not
|
||
# reconciling at all read as ready and let the run proceed.
|
||
ready=$(ks_ready_condition infrastructure)
|
||
if [[ "${ready%%|*}" == "True" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
echo -e "${YELLOW}Warning: infrastructure not ready after 10m, continuing...${NC}"
|
||
flux get kustomizations 2>&1 | sed 's/^/ /' | head -12
|
||
fi
|
||
done
|
||
|
||
step 0 "Waiting for Ceph cluster..."
|
||
for i in $(seq 1 60); do
|
||
health=$(kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)
|
||
if [[ "$health" == "Ready" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
# Advisory, so it continues — but a warning that says only "not ready"
|
||
# sends the reader back to the cluster for information this step
|
||
# already had.
|
||
echo -e "${YELLOW}Warning: Ceph not ready after 10m (phase: ${health:-none}), continuing...${NC}"
|
||
kubectl -n rook-ceph get cephcluster -o wide 2>&1 | sed 's/^/ /'
|
||
kubectl -n rook-ceph get pods --no-headers 2>&1 \
|
||
| awk '$3 != "Running" && $3 != "Completed"' | head -10 | sed 's/^/ /'
|
||
fi
|
||
done
|
||
|
||
step 0 "Waiting for Ceph Object Store..."
|
||
for i in $(seq 1 60); do
|
||
phase=$(kubectl get cephobjectstore -n rook-ceph migration -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||
if [[ "$phase" == "Ready" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
echo -e "${YELLOW}Warning: Object Store not ready after 10m (phase: ${phase:-none}), continuing...${NC}"
|
||
kubectl -n rook-ceph get cephobjectstore 2>&1 | sed 's/^/ /'
|
||
fi
|
||
done
|
||
fi
|
||
|
||
if phase_runs wait-apps; then
|
||
echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v flux >/dev/null || fail "flux not found"
|
||
# The restore test reads its access from KUBECONFIG/TALOSCONFIG.
|
||
[[ "$PHASE" == "wait-apps" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access
|
||
|
||
# Wait for a Flux Kustomization to become Ready=True. Prints a status line
|
||
# whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a
|
||
# silent-but-progressing reconcile is still visible in CI output. Returns
|
||
# 0 on Ready, 1 on timeout (caller decides whether to fail or continue).
|
||
wait_kustomization() {
|
||
local name="$1" label="$2" max_iters="$3"
|
||
local interval=10 elapsed=0 prev_status=""
|
||
local timeout_min=$(( max_iters * interval / 60 ))
|
||
step 0 "Waiting for ${label} (timeout ${timeout_min}m)..."
|
||
for _ in $(seq 1 "$max_iters"); do
|
||
local cond ready msg status
|
||
cond=$(ks_ready_condition "$name")
|
||
ready="${cond%%|*}"
|
||
msg="${cond#*|}"
|
||
if [[ "$ready" == "True" ]]; then
|
||
printf " [%4ds] Ready ✓\n" "$elapsed"
|
||
return 0
|
||
fi
|
||
status="${ready:-?}: ${msg:-(no status yet)}"
|
||
# Print on first iteration, on state change, or as a 60-s heartbeat.
|
||
# 240 chars is enough to see the full Flux stalled-resource message.
|
||
if [[ "$prev_status" != "$status" ]] || (( elapsed > 0 && elapsed % 60 == 0 )); then
|
||
printf " [%4ds] %s\n" "$elapsed" "${status:0:240}"
|
||
prev_status="$status"
|
||
fi
|
||
sleep "$interval"
|
||
elapsed=$(( elapsed + interval ))
|
||
done
|
||
printf " [%4ds] TIMEOUT after %sm — full status follows\n" "$elapsed" "$timeout_min"
|
||
# Dump the full untruncated status + Events for the offending resources
|
||
# so the post-mortem doesn't need a second cluster trip.
|
||
flux get kustomization "$name" 2>&1 | sed 's/^/ /' || true
|
||
kubectl get events --sort-by=.lastTimestamp -A 2>/dev/null \
|
||
| grep -i -E "warning|error" | tail -20 | sed 's/^/ /' || true
|
||
echo -e "${YELLOW}Warning: ${label} not Ready, continuing anyway${NC}"
|
||
# 0, not 1. Called bare under `set -e`, a non-zero return ends the script
|
||
# immediately — so this said "continuing anyway" and then stopped, and
|
||
# every wait after it never ran. The message was the truth; the return
|
||
# value contradicted it.
|
||
return 0
|
||
}
|
||
|
||
# Each app KS now owns its CNPG database (+ any data PVCs) directly — the
|
||
# per-app `seed-<app>` KSes that used to bootstrap them were folded into the
|
||
# app KSes when the seed layer was retired. Each wait below gates on the app
|
||
# KS, whose readiness includes its CNPG cluster coming up (bootstrap.recovery)
|
||
# and, through the restore KS it depends on, its volumes restored by Velero.
|
||
# Timeouts stay generous to cover first-time cluster bootstrap on fresh Ceph.
|
||
# apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up
|
||
# is genuinely 5–10 min on a fresh OSD-wiped cluster).
|
||
wait_kustomization gitea "Gitea (incl. data-gitea-0 restore)" 360
|
||
wait_kustomization apps "apps (after restore-gitea)" 360
|
||
wait_kustomization influxdb "InfluxDB (HR install + token Jobs)" 600
|
||
wait_kustomization woodpecker "Woodpecker CI (incl. wp-pg CNPG backend)" 600
|
||
wait_kustomization authentik "Authentik (incl. ak-pg CNPG backend)" 600
|
||
wait_kustomization monitoring "kube-prometheus-stack (Loki + Prom + AM + Grafana)" 180
|
||
wait_kustomization music-assistant "Music Assistant (incl. data PVC)" 300
|
||
wait_kustomization paperless "Paperless (incl. pprl-pg CNPG + data PVCs)" 600
|
||
wait_kustomization home-assistant "Home Assistant (HAOS backup → native restore)" 180
|
||
wait_kustomization audiobookshelf "Audiobookshelf (incl. library/metadata PVCs)" 900
|
||
wait_kustomization navidrome "Navidrome (incl. data + music library PVCs)" 600
|
||
wait_kustomization immich "Immich (incl. im-pg CNPG + library PVC)" 1800
|
||
|
||
# Wait for cert-manager to finish issuing every Certificate. Kustomization
|
||
# Ready only means the Ingress + Certificate manifests are applied; cert-
|
||
# manager then runs an async ACME DNS-01 challenge per Certificate (~30s–2m
|
||
# via Cloudflare). e2e's TLS check fails on the ingress-nginx default fake
|
||
# cert if any Certificate is still Issuing when the test runs.
|
||
wait_certificates() {
|
||
local max_iters=90 interval=10 elapsed=0 prev_status=""
|
||
local timeout_min=$(( max_iters * interval / 60 ))
|
||
step 0 "Waiting for all cert-manager Certificates Ready (timeout ${timeout_min}m)..."
|
||
for _ in $(seq 1 "$max_iters"); do
|
||
local total ready not_ready_names status
|
||
total=$(kubectl get certificate -A --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
||
ready=$(kubectl get certificate -A --no-headers 2>/dev/null | awk '$3=="True"' | wc -l | tr -d ' ')
|
||
if [[ "$total" -gt 0 && "$total" == "$ready" ]]; then
|
||
printf " [%4ds] All %s Certificates Ready ✓\n" "$elapsed" "$total"
|
||
return 0
|
||
fi
|
||
not_ready_names=$(kubectl get certificate -A --no-headers 2>/dev/null \
|
||
| awk '$3!="True" {print $1"/"$2}' | head -5 | tr '\n' ' ')
|
||
status="${ready:-0}/${total:-?} Ready, waiting: ${not_ready_names:-(none)}"
|
||
if [[ "$prev_status" != "$status" ]] || (( elapsed > 0 && elapsed % 60 == 0 )); then
|
||
printf " [%4ds] %s\n" "$elapsed" "${status:0:140}"
|
||
prev_status="$status"
|
||
fi
|
||
sleep "$interval"
|
||
elapsed=$(( elapsed + interval ))
|
||
done
|
||
printf " [%4ds] TIMEOUT after %sm\n" "$elapsed" "$timeout_min"
|
||
echo -e "${YELLOW}Warning: not all Certificates Ready, continuing anyway${NC}"
|
||
kubectl get certificate -A --no-headers | awk '$3!="True"' | head -10
|
||
# 0 for the same reason as wait_kustomization: under `set -e` a non-zero
|
||
# return here ends the run despite the message saying otherwise.
|
||
return 0
|
||
}
|
||
wait_certificates
|
||
|
||
echo ""
|
||
echo -e "${YELLOW}=== Verification ===${NC}"
|
||
# Whole-field matches. `grep -c Ready` counts NotReady too — the substring is
|
||
# in it — so this block reported every node healthy on a cluster where none
|
||
# was. `grep -c True` on HelmReleases matched the SUSPENDED column and any
|
||
# message containing the word.
|
||
#
|
||
# awk also avoids grep's exit 1 on zero matches, which under pipefail ended
|
||
# the run here with no output at all.
|
||
echo -n "Nodes: "
|
||
kubectl get nodes --no-headers | awk '$2 == "Ready"' | wc -l | tr -d '\n'
|
||
echo "/${#ALL_NODES[@]}"
|
||
|
||
# Ready count against a denominator: "12 ready" reads as success whether the
|
||
# cluster has 12 HelmReleases or 40. Both numbers come from the objects, not
|
||
# from `flux get`'s table, whose columns shift when REVISION is empty.
|
||
echo -n "HelmReleases: "
|
||
hr_total=$(kubectl get helmrelease --all-namespaces --no-headers | wc -l)
|
||
hr_ready=$(kubectl get helmrelease --all-namespaces -o jsonpath=\
|
||
'{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \
|
||
| grep -c '^True$') || true
|
||
echo "${hr_ready:-0}/${hr_total} ready"
|
||
|
||
echo -n "Ceph: "
|
||
kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.ceph.health}' 2>/dev/null || echo "pending"
|
||
echo ""
|
||
|
||
echo -n "Ingress (local): "
|
||
curl -sk --connect-timeout 5 --resolve status.renner.uno:443:10.2.99.86 https://status.renner.uno -o /dev/null -w "%{http_code}" 2>/dev/null || echo "N/A"
|
||
echo ""
|
||
fi
|