claude-botandClaude Opus 5.5
222d0263a4
Navidrome (Subsonic API) as the Spotify replacement for phone apps. - Library PVC on ceph-filesystem-bulk (RWX, 300Gi, mounted read-only); SQLite data on ceph-block (RWO). - Web UI behind a dedicated navidrome-gate Authentik proxy outpost; Navidrome trusts X-Authentik-Username (ExtAuth) and auto-creates the user on first SSO login (first user = admin). - /rest and /share routed straight to Navidrome with the identity header stripped, so Subsonic clients use their Navidrome password. - Namespace baseline CNP, gate egress CNP, Flux image automation, Velero daily schedule (09:15), bootstrap wait. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2028 lines
103 KiB
Bash
Executable File
2028 lines
103 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# K8s Cluster Bootstrap — split into phases so CI can export access as early as possible.
|
||
#
|
||
# Usage:
|
||
# ./bootstrap.sh # run all phases locally
|
||
# ./bootstrap.sh talos # Talos apply + K8s bootstrap + kubeconfig (writes access-staging/)
|
||
# ./bootstrap.sh flux # reads access-staging/, installs flux, waits for infrastructure
|
||
# ./bootstrap.sh wait-apps # waits for apps/authentik/paperless to reconcile
|
||
# ./bootstrap.sh vmtest # throwaway QEMU VMs, then the talos phase
|
||
# # against them unchanged. Needs root for the
|
||
# # provisioner and leaves nothing behind.
|
||
# ./bootstrap.sh hcloudtest # the same rehearsal on Hetzner Cloud, which is
|
||
# # the only place the registry fallback can be
|
||
# # tested: gitea.renner.uno has no external
|
||
# # ingress, so a node there genuinely cannot
|
||
# # reach it. BILLS BY THE HOUR — teardown runs
|
||
# # from the EXIT trap and is by label.
|
||
# ./bootstrap.sh hcloudclean # that teardown on its own, for a run that was
|
||
# # killed before its trap could fire.
|
||
#
|
||
# Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls
|
||
# from is itself Flux-managed, so a cold rebuild cannot pull from it until the
|
||
# cluster already exists. Set FLUX_SOURCE=github to bootstrap Flux from the
|
||
# external GitHub mirror instead — the only way out of that loop:
|
||
# FLUX_SOURCE=github GITHUB_TOKEN=<pat> ./bootstrap.sh flux
|
||
# The mirror is kept current server-side by Gitea's native push-mirror. Full
|
||
# cold-rebuild procedure: kubernetes/RUNBOOK-cold-rebuild.md.
|
||
#
|
||
# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt
|
||
|
||
TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
|
||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||
# Cilium's single source of truth. Talos ships `cni: none` (talos/patches/
|
||
# cluster.yaml) and nothing else, so the CNI has to be installed here, between
|
||
# `talosctl bootstrap` and Flux — Flux's own controllers are pods and cannot
|
||
# reconcile the network they need in order to start. Everything the install
|
||
# needs is read out of this file at run time so there is no second copy of a
|
||
# version or a value to keep in step. See PROPOSAL-cilium-single-owner.md.
|
||
CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml"
|
||
ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging"
|
||
NODES=(k8s-1 k8s-2 k8s-3)
|
||
# No dedicated workers since the k8s-edge VM was retired (2026-07-02) —
|
||
# hardware-bound services live on the bare-metal edge-1 Arch host
|
||
# (ansible/docs/edge-host.md).
|
||
WORKERS=()
|
||
ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
|
||
DOMAIN="srv.renner.uno"
|
||
DNS_SERVER="10.2.40.1" # UDM (old PVE DNS VIP 10.2.40.53 decommissioned)
|
||
# Cluster API endpoint — Talos VIP (10.2.40.60) elected via etcd, fronted by
|
||
# UniFi DNS A record k8s.renner.uno. Defined in talos/patches/k8s-{1,2,3}.yaml.
|
||
K8S_API_HOST="k8s.renner.uno"
|
||
# Which site patch the nodes get: talos/patches/site-$SITE.yaml. The three
|
||
# variables above and this one are everything the vmtest phase has to change in
|
||
# order to run this same code against throwaway VMs — if that list ever grows,
|
||
# the test is drifting away from the thing it claims to exercise.
|
||
SITE="${SITE:-homelab}"
|
||
# Where this script writes the cluster access files. These are the paths
|
||
# talosctl and kubectl read from anyway, so unset they resolve to exactly the
|
||
# old behaviour. Set, the whole run stays out of $HOME — which is what lets the
|
||
# vmtest phase drive a throwaway cluster without replacing the operator's
|
||
# production access with a VM's.
|
||
TALOSCONFIG_PATH="${TALOSCONFIG:-$HOME/.talos/config}"
|
||
KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}"
|
||
case "$KUBECONFIG_PATH" in
|
||
*:*) echo "KUBECONFIG holds multiple paths; set it to a single file" >&2; exit 1 ;;
|
||
esac
|
||
# Addresses to apply to, in the order of NODES. Empty means resolve the node
|
||
# FQDNs, which is right for real hardware on a network that has DNS for it.
|
||
NODE_ADDRS=()
|
||
GITEA_HOST="gitea.renner.uno"
|
||
GITEA_OWNER="gitea_admin"
|
||
GITEA_REPO="homelab"
|
||
# Flux git source: "gitea" (default, in-cluster) or "github" (break-glass mirror
|
||
# for a cold rebuild — see the usage note above and RUNBOOK-cold-rebuild.md).
|
||
FLUX_SOURCE="${FLUX_SOURCE:-gitea}"
|
||
# GitHub break-glass mirror (github.com/konairius/Homelab), kept current by
|
||
# Gitea's native push-mirror. Overridable in case the mirror moves.
|
||
GITHUB_OWNER="${GITHUB_OWNER:-konairius}"
|
||
GITHUB_REPO="${GITHUB_REPO:-Homelab}"
|
||
# Owner of the private ghcr.io namespace the rebuild-critical images are
|
||
# mirrored into, and the username half of the node registry credential. Matches
|
||
# GHCR_USER in .woodpecker/build-ci-images.yml, which pushes them.
|
||
GHCR_PULL_USER="${GHCR_PULL_USER:-konairius}"
|
||
FLUX_PATH="kubernetes/clusters/homelab"
|
||
# Branch Flux reconciles from, and that `flux bootstrap` commits its own sync
|
||
# manifests into. master for a real rebuild.
|
||
#
|
||
# Overridable so the phase can be rehearsed without writing to master — point it
|
||
# at a throwaway branch, which has to exist already: `flux bootstrap` clones the
|
||
# branch and fails with `couldn't find remote ref` rather than creating it.
|
||
#
|
||
# git push origin origin/master:refs/heads/rehearsal-1
|
||
# FLUX_BRANCH=rehearsal-1 ./bootstrap.sh flux
|
||
#
|
||
# Nothing else about the phase changes, which is the point. Delete the branch
|
||
# afterwards.
|
||
FLUX_BRANCH="${FLUX_BRANCH:-master}"
|
||
SOPS_KEY="$HOME/.config/sops/age/keys.txt"
|
||
|
||
PHASE="${1:-all}"
|
||
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m'
|
||
|
||
is_tty() { [[ -t 1 ]]; }
|
||
|
||
step() {
|
||
local _ignored=$1
|
||
local label="$2"
|
||
if is_tty; then
|
||
printf "${BLUE}==>${NC} %s\n" "$label"
|
||
else
|
||
printf "==> %s\n" "$label"
|
||
fi
|
||
}
|
||
|
||
# One EXIT trap for the whole script. Handlers register here rather than calling
|
||
# `trap ... EXIT` themselves, because a second registration replaces the first
|
||
# instead of chaining — which is how a secrets-cleanup handler silently disabled
|
||
# the VM teardown.
|
||
SECRETS_TMPDIR=""
|
||
on_exit() {
|
||
[[ -n "$SECRETS_TMPDIR" ]] && rm -rf "$SECRETS_TMPDIR"
|
||
declare -F cleanup_vmtest >/dev/null && cleanup_vmtest
|
||
# No teardown here: provisioning is its own phase and the machines it leaves
|
||
# behind are its product. `hcloudclean` is what deletes them.
|
||
# Only when this run actually booked machines. The function is always
|
||
# defined, so testing for it fired on the qemu phase too.
|
||
if [[ "${HCLOUD_PROVISIONED:-0}" == "1" ]]; then
|
||
echo -e "${YELLOW}Hetzner machines are UP and billed until deleted.${NC}"
|
||
echo " ./kubernetes/bootstrap.sh hcloudclean # when finished"
|
||
fi
|
||
return 0
|
||
}
|
||
trap on_exit EXIT
|
||
|
||
fail() { echo -e "\n${RED}FAILED:${NC} $1"; exit 1; }
|
||
ok() { echo -e "${GREEN}OK${NC}"; }
|
||
|
||
# Reads the Ready condition of a Flux Kustomization, without going through the
|
||
# table `flux get` prints.
|
||
#
|
||
# That table is NAME REVISION SUSPENDED READY MESSAGE, and every wait below used
|
||
# to pick READY as the fourth field. REVISION is empty until the first
|
||
# successful reconcile — precisely the state these waits exist to catch — so in
|
||
# the failure case the columns shift left and the fourth field lands on the
|
||
# message. Matching it against "False" then never fires, and a persistently
|
||
# failing Kustomization reads as merely slow. That cost a full two-hour run:
|
||
# infrastructure had been failing its server-side dry-run from the first minute,
|
||
# and the counter meant to catch it never moved.
|
||
#
|
||
# Echoes "<status>|<message>". An empty status means the object does not exist
|
||
# yet; kubectl's own error is carried through as the message rather than
|
||
# discarded, so "not created yet" stays distinguishable from "lookup broke".
|
||
ks_ready_condition() {
|
||
local name="$1" out jp
|
||
jp='{.status.conditions[?(@.type=="Ready")].status}'
|
||
jp+='|{.status.conditions[?(@.type=="Ready")].message}'
|
||
if out=$(kubectl -n flux-system get kustomization "$name" -o jsonpath="$jp" 2>&1); then
|
||
printf '%s' "$out"
|
||
else
|
||
printf '|%s' "$out"
|
||
fi
|
||
}
|
||
|
||
resolve_node_ip() {
|
||
local fqdn=$1
|
||
dig +short "$fqdn" A @"$DNS_SERVER" | head -1
|
||
}
|
||
|
||
phase_runs() {
|
||
[[ "$PHASE" == "all" || "$PHASE" == "$1" ]]
|
||
}
|
||
OPENBAO_NS="openbao"
|
||
|
||
# Where the secrets come from. On a workstation, the operator's pass store. In
|
||
# the in-cluster Job there is no pass and no GPG key — and there should not be:
|
||
# the pass store opens everything, and it stays on the operator's machine. Set
|
||
# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
|
||
# operator streams in when the run asks for them:
|
||
#
|
||
# pass show homelab/openbao/seal-key \
|
||
# | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
|
||
#
|
||
# They exist in the pod's filesystem for the length of the run and are never
|
||
# written to a Secret in this cluster, so they do not land in its etcd or in
|
||
# any backup of it. The seal key alone becomes a Secret in the cluster being
|
||
# built, which is what lets that OpenBao unseal itself.
|
||
OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}"
|
||
|
||
# Every secret the run needs, named once, in one place. The list is here and
|
||
# nowhere else on purpose: when it was spread over the call sites, each was read
|
||
# at the moment it was needed, so a missing one surfaced twenty minutes into a
|
||
# run — and a second copy of the same path in a runbook or a wrapper is a copy
|
||
# that goes stale.
|
||
#
|
||
# The git token for `flux bootstrap`, chosen by the source it will authenticate
|
||
# against — which is the whole point and is what the previous expression got
|
||
# wrong. It read ${GITHUB_TOKEN:-${GITEA_TOKEN:-}}, taking whichever variable
|
||
# happened to be set: on a workstation with GITEA_TOKEN exported and no
|
||
# GITHUB_TOKEN, a break-glass run handed a Gitea token to the GitHub API and
|
||
# died on `401 Bad credentials` after the cluster was already built.
|
||
#
|
||
# `gh auth token` as the GitHub fallback because that is the credential the
|
||
# operator already has on this machine, and it is what the vmtest secret feeder
|
||
# has always used. Emptiness is not checked here — require_secrets rejects an
|
||
# empty result and names what is missing.
|
||
#
|
||
# gh is called plainly, with its stderr left alone. Swallowing it would turn a
|
||
# failed login into an empty string, and require_secrets would then report
|
||
# "came back empty" for a credential that exists but could not be read — a zero
|
||
# that cannot be told apart from a fault. Its non-zero exit propagates, which is
|
||
# what makes require_secrets print the reason instead.
|
||
git_token_for_source() {
|
||
if [[ "$FLUX_SOURCE" == "github" ]]; then
|
||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||
printf '%s' "$GITHUB_TOKEN"
|
||
else
|
||
gh auth token
|
||
fi
|
||
else
|
||
printf '%s' "${GITEA_TOKEN:-}"
|
||
fi
|
||
}
|
||
|
||
# Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
|
||
OPENBAO_REQUIRED_SECRETS=(
|
||
"sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
|
||
# Which token depends on FLUX_SOURCE, and that is not a detail: in a real
|
||
# cold rebuild the in-cluster Gitea cannot serve the cluster being rebuilt,
|
||
# so the break-glass path pulls from the GitHub mirror. A run that demands a
|
||
# Gitea token is a run that cannot rehearse the disaster it exists for.
|
||
# The variable names are the ones the usage header and RUNBOOK-cold-rebuild
|
||
# tell the operator to export. An invented third name was worse than a typo:
|
||
# `${VAR:?}` inside eval terminates the shell outright, so the documented
|
||
# recovery command exited with no message and no cleanup at all.
|
||
#
|
||
# Emptiness is not tested here on purpose — require_secrets rejects an empty
|
||
# file and names what is missing.
|
||
"git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
|
||
"s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
|
||
"s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
|
||
# The static seal key. The snapshots are encrypted under it, so it has to
|
||
# be the one the backed-up cluster ran with — a fresh key restores nothing.
|
||
"seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
|
||
# Read by the node registry auth (see registry_auth_patch): the images an app
|
||
# needs while the cluster is still being built are mirrored to a private
|
||
# ghcr.io namespace, and a node has to be able to pull them before anything
|
||
# in the cluster exists to hold a credential. Demanded up front like the
|
||
# rest, because a rebuild that discovers it missing has already wiped the
|
||
# disks.
|
||
"ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
|
||
# Only the hcloudtest and hcloudclean phases ask for this one, by name, so a
|
||
# rebuild of the real cluster never demands it.
|
||
#
|
||
# Note the path: homelab/hetzner/, spelled correctly, while the two restore
|
||
# keys above live under homelab/hetztner/. Two folders, one typo, both real
|
||
# — do not "fix" one of these paths without moving the entry first.
|
||
"hcloud-token|Hetzner Cloud API token for the throwaway test cluster|pass show homelab/hetzner/cloud-api-secret"
|
||
)
|
||
|
||
# Collects all of them up front, into one directory, so everything after this
|
||
# point reads files and takes the same code path whether the run is on a laptop
|
||
# during a real recovery or in the in-cluster Job.
|
||
#
|
||
# The only difference between those two is where the material comes from, and it
|
||
# is confined to this function: on a laptop the commands in the list produce it
|
||
# from pass; in the Job the operator streams it in. Everything downstream cannot
|
||
# tell the difference — which is the point, because the recovery path must not be
|
||
# a path that only gets exercised during a recovery.
|
||
# require_secrets [name...]
|
||
#
|
||
# With no arguments, demands every entry in OPENBAO_REQUIRED_SECRETS. With
|
||
# names, demands only those — which is what lets the talos phase stand alone.
|
||
#
|
||
# That phase used to need nothing but the machines, and `bootstrap.sh talos` is
|
||
# documented as the part that can run unattended. Node registry auth changed
|
||
# that: the machine config now carries a pull credential, so the phase needs
|
||
# exactly one secret. Demanding all six there would drag in the OpenBao seal
|
||
# key, which deliberately never reaches CI — and demanding none, as before,
|
||
# left the phase to discover the missing file halfway through `talosctl gen`.
|
||
require_secrets() {
|
||
local entry name what how
|
||
local -a wanted=()
|
||
if [[ $# -gt 0 ]]; then
|
||
for entry in "${OPENBAO_REQUIRED_SECRETS[@]}"; do
|
||
for name in "$@"; do
|
||
[[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry")
|
||
done
|
||
done
|
||
[[ ${#wanted[@]} -eq $# ]] \
|
||
|| fail "require_secrets asked for [$*] but the list defines only: ${wanted[*]%%|*}"
|
||
else
|
||
wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
|
||
fi
|
||
|
||
# Which branch: fetch from pass, or wait for someone to stream files in.
|
||
#
|
||
# The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
|
||
# FUNCTION sets it. Once a subset call had run — hcloudtest asks for
|
||
# ghcr-pull and hcloud-token before anything else — every later call found
|
||
# the variable populated and took the wait branch, printing streaming
|
||
# instructions for a Job that does not exist on a workstation where pass was
|
||
# available all along. The run sat there while three nodes stayed in
|
||
# maintenance.
|
||
#
|
||
# SECRETS_SELF_FETCH records which world we are in, and it is decided once:
|
||
# an empty OPENBAO_SECRETS_DIR on entry means pass, an inherited one means
|
||
# the Job. Later calls follow that decision instead of re-deriving it from
|
||
# state this function has since changed.
|
||
if [[ -z "$OPENBAO_SECRETS_DIR" || "${SECRETS_SELF_FETCH:-0}" == "1" ]]; then
|
||
if [[ -z "$OPENBAO_SECRETS_DIR" ]]; then
|
||
OPENBAO_SECRETS_DIR=$(mktemp -d)
|
||
chmod 700 "$OPENBAO_SECRETS_DIR"
|
||
fi
|
||
SECRETS_SELF_FETCH=1
|
||
# Registered by name, not with a second `trap ... EXIT`: a second one
|
||
# REPLACES the first, and the first is the vmtest teardown. Setting it
|
||
# here left four VMs, their bridges and the sudo keepalive running after
|
||
# a workstation rehearsal — silently, and only on the laptop path, since
|
||
# in-cluster this branch is not taken.
|
||
SECRETS_TMPDIR="$OPENBAO_SECRETS_DIR"
|
||
echo -n "Collecting secrets... "
|
||
for entry in "${wanted[@]}"; do
|
||
name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
|
||
# Already collected by an earlier call — a subset call fetches only
|
||
# what it asked for, so the full call that follows must fill the
|
||
# gaps rather than re-run every command. `pass show` on a locked
|
||
# store prompts, and prompting twice for the same key is how an
|
||
# unattended run turns into a hung one.
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
|
||
if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name" 2>"$OPENBAO_SECRETS_DIR/.err"; then
|
||
echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
|
||
fail "could not obtain the $what with: $how"
|
||
fi
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
|
||
|| fail "the $what came back empty from: $how"
|
||
done
|
||
ok
|
||
return 0
|
||
fi
|
||
|
||
mkdir -p "$OPENBAO_SECRETS_DIR"
|
||
local entry name what how missing
|
||
for i in $(seq 1 180); do
|
||
missing=()
|
||
for entry in "${wanted[@]}"; do
|
||
name=${entry%%|*}
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] || missing+=("$entry")
|
||
done
|
||
[[ ${#missing[@]} -eq 0 ]] && { echo "All secrets present."; return 0; }
|
||
if [[ $i -eq 1 ]]; then
|
||
echo -e "${YELLOW}Waiting for ${#missing[@]} secret(s). Stream each in with:${NC}" >&2
|
||
for entry in "${missing[@]}"; do
|
||
name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
|
||
echo " # $what" >&2
|
||
echo " $how | kubectl -n $OPENBAO_NS_HINT exec -i \$POD -- sh -c 'cat > $OPENBAO_SECRETS_DIR/$name'" >&2
|
||
done
|
||
fi
|
||
sleep 10
|
||
[[ $i -eq 180 ]] && fail "still missing: ${missing[*]%%|*}"
|
||
done
|
||
}
|
||
OPENBAO_NS_HINT=vmtest
|
||
|
||
# Reads one secret by its name in the list above. In the Job that is the file the
|
||
# operator streamed in; on a workstation it runs the command the list names,
|
||
# which for these is `pass show <path>`.
|
||
#
|
||
# The list is the only place the paths appear. They used to sit here as
|
||
# constants as well, which is how the same value ends up defined twice and one
|
||
# copy goes stale.
|
||
read_secret() {
|
||
local name=$1
|
||
[[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
|
||
|| fail "$name is not in $OPENBAO_SECRETS_DIR — require_secrets runs first and should have caught this"
|
||
cat "$OPENBAO_SECRETS_DIR/$name"
|
||
}
|
||
|
||
# Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase
|
||
# against it unchanged — same shape as the vmtest phase below, and for the same
|
||
# reason: a rehearsal that runs different code proves less than one that does
|
||
# not.
|
||
#
|
||
# WHY THIS EXISTS, given vmtest already rehearses a rebuild locally.
|
||
#
|
||
# The one thing vmtest cannot test is the registry fallback. The nodes now carry
|
||
# machine.registries.mirrors with gitea.renner.uno first and ghcr.io second, and
|
||
# the whole point of the second endpoint is the window where the first does not
|
||
# exist. At home it always exists — the production cluster is running and
|
||
# answers on the LAN — so containerd never reaches the fallback and it stays
|
||
# unproven. From Hetzner, Gitea does not answer at all: it has no external
|
||
# ingress.
|
||
#
|
||
# WHAT IT COSTS, because this one bills by the hour.
|
||
#
|
||
# Teardown is by LABEL, not by remembered ID, and runs from the single EXIT
|
||
# trap. A run that dies between `server create` and its own bookkeeping is
|
||
# therefore still cleanable — by the next run, or by `bootstrap.sh hcloudclean`,
|
||
# which does nothing but the sweep. Anything this phase creates carries
|
||
# HCLOUD_LABEL so that sweep can find it without being told.
|
||
#
|
||
# WHAT IT DOES NOT COVER.
|
||
#
|
||
# Everything downstream of Multus. The provider is not the limit — a cloud
|
||
# server takes up to 16 volumes and 3 private networks — but ceph-san binds to
|
||
# the kernel link name enp2s0f1np1, and that name cannot be manufactured here
|
||
# without assuming a Hetzner private network passes VLAN tags. See
|
||
# talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase
|
||
# needs the OpenBao seal key, which deliberately never reaches CI, and this
|
||
# test is meant to run unattended.
|
||
# Talos version and Image Factory schematic, read out of the installer image in
|
||
# cluster.yaml. Both throwaway-cluster phases need them and neither may state
|
||
# them: a second copy is a copy that goes stale, and the day it did the test
|
||
# would quietly stop booting the image a rebuild installs.
|
||
talos_image_coords() {
|
||
command -v yq >/dev/null || fail "yq not found"
|
||
local image
|
||
image=$(yq -r '.machine.install.image' "$TALOS_DIR/patches/cluster.yaml")
|
||
VM_SCHEMATIC=${image#*metal-installer/}
|
||
VM_SCHEMATIC=${VM_SCHEMATIC%%:*}
|
||
VM_VERSION=${image##*:}
|
||
[[ ${#VM_SCHEMATIC} -eq 64 && "$VM_VERSION" == v* ]] \
|
||
|| fail "could not read a schematic and version out of '$image'"
|
||
}
|
||
|
||
HCLOUD_LABEL="managed-by=bootstrap-hcloudtest"
|
||
|
||
# The disks a machine has are stated in its Talos patch and read from there, so
|
||
# the provisioner carries no second copy. Emits one size in whole GB per raw
|
||
# volume the patch declares, in order.
|
||
hcloud_volumes_for_node() {
|
||
local patch="$TALOS_DIR/patches/$1.yaml" size gib
|
||
[[ -f "$patch" ]] || fail "$patch not found — the machine config is the list"
|
||
while read -r size; do
|
||
[[ -n "$size" && "$size" != "null" ]] || continue
|
||
# Talos states GiB; the API bills GB. Round up, so the volume can never
|
||
# come in under the minSize the machine config asked for.
|
||
gib=${size%GiB}
|
||
[[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as <n>GiB"
|
||
echo $(( (gib * 1074 + 999) / 1000 ))
|
||
done < <(yq -r 'select(.kind == "RawVolumeConfig") | .provisioning.minSize' "$patch")
|
||
}
|
||
|
||
# Deletes everything this phase created, and NOTHING else.
|
||
#
|
||
# That distinction is not theoretical: the project this runs in also holds
|
||
# netbird.renner.uno, which has been up for months and is the VPN way in. The
|
||
# label is the only thing separating a throwaway server from it — and
|
||
# `hcloud server list -l ""` lists EVERY server, so an unset HCLOUD_LABEL would
|
||
# turn this function into a project wipe. Hence the guard: the label is checked
|
||
# before it is used, and every candidate must also carry the name prefix this
|
||
# phase assigns. Two independent conditions, because one of them is a variable.
|
||
#
|
||
# A separate Hetzner project, with a token scoped to it, would make this
|
||
# structural rather than defensive. Worth doing before this runs unattended.
|
||
HCLOUD_NAME_PREFIX="hcloud"
|
||
|
||
cleanup_hcloudtest() {
|
||
command -v hcloud >/dev/null || return 0
|
||
[[ -n "${HCLOUD_TOKEN:-}" ]] || return 0
|
||
[[ -n "${HCLOUD_LABEL:-}" ]] \
|
||
|| { echo " REFUSING to sweep: HCLOUD_LABEL is empty, which would match everything" >&2; return 0; }
|
||
|
||
local id name
|
||
# Servers first: a firewall still attached to one cannot be deleted.
|
||
while read -r id name; do
|
||
[[ -n "$id" ]] || continue
|
||
case "$name" in
|
||
"$HCLOUD_NAME_PREFIX"-*) ;;
|
||
*) echo " REFUSING to delete $name ($id): carries the label but not the name prefix" >&2; continue ;;
|
||
esac
|
||
echo " deleting server $name"
|
||
hcloud server delete "$id" >/dev/null \
|
||
|| echo " WARNING: server $name not deleted — check the console" >&2
|
||
done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
|
||
|
||
for id in $(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
||
echo " deleting firewall $id"
|
||
hcloud firewall delete "$id" >/dev/null \
|
||
|| echo " WARNING: firewall $id not deleted — check the console" >&2
|
||
done
|
||
for id in $(hcloud volume list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
||
echo " deleting volume $id"
|
||
hcloud volume delete "$id" >/dev/null \
|
||
|| echo " WARNING: volume $id not deleted — check the console" >&2
|
||
done
|
||
# Last: a network with a server still attached cannot be deleted, and a
|
||
# leftover one takes its name with it, so the next run cannot recreate it.
|
||
for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
||
echo " deleting network $id"
|
||
hcloud network delete "$id" >/dev/null \
|
||
|| echo " WARNING: network $id not deleted — check the console" >&2
|
||
done
|
||
}
|
||
|
||
# Sweep only. For the case where a previous run was killed hard enough that its
|
||
# own trap never ran, and for a cron job to call before it starts.
|
||
if [[ "$PHASE" == "hcloudclean" ]]; then
|
||
echo -e "${YELLOW}=== Phase: Hetzner cleanup ===${NC}"
|
||
command -v hcloud >/dev/null || fail "hcloud not found"
|
||
require_secrets hcloud-token
|
||
HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN
|
||
cleanup_hcloudtest
|
||
ok
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$PHASE" == "hcloudtest" ]]; then
|
||
echo -e "${YELLOW}=== Phase: Hetzner rebuild test ===${NC}"
|
||
command -v hcloud >/dev/null || fail "hcloud not found — https://github.com/hetznercloud/cli"
|
||
command -v talosctl >/dev/null || fail "talosctl not found"
|
||
command -v yq >/dev/null || fail "yq not found"
|
||
command -v jq >/dev/null || fail "jq not found (needed to read the SAN pool and server attachments)"
|
||
command -v curl >/dev/null || fail "curl not found (needed to learn this host's public address)"
|
||
|
||
# Three control planes, all carrying workloads — the production shape. One
|
||
# machine is a node, not a cluster: etcd has no quorum to lose and nothing
|
||
# schedules elsewhere, so a rebuild that only ever ran on a single box
|
||
# proves nothing about the one that has three.
|
||
HCLOUD_NODES=(hcloud-1 hcloud-2 hcloud-3)
|
||
for n in "${HCLOUD_NODES[@]}"; do
|
||
[[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
|
||
done
|
||
[[ -f "$TALOS_DIR/patches/site-hcloud.yaml" ]] || fail "site-hcloud.yaml not found"
|
||
|
||
require_secrets hcloud-token ghcr-pull
|
||
HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN
|
||
|
||
# No sweep before creating: machines from an earlier run are the input to
|
||
# this one, adopted below. The names are fixed, so duplicates cannot
|
||
# accumulate. `hcloudclean` is what deletes.
|
||
|
||
# The Talos snapshot is a prerequisite, not something this builds: making it
|
||
# means booting a server into rescue and writing the raw image, which is a
|
||
# one-off and needs a different set of permissions. Fail with the recipe
|
||
# rather than silently booting Debian and timing out on the Talos API.
|
||
step 0 "Looking up the Talos snapshot..."
|
||
talos_image_coords
|
||
HCLOUD_IMAGE=$(hcloud image list --type snapshot -l os=talos -o noheader -o columns=id | head -1)
|
||
[[ -n "$HCLOUD_IMAGE" ]] || fail "no snapshot labelled os=talos. Create one once — boot any server into rescue, then:
|
||
wget -O /tmp/talos.raw.xz https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz
|
||
xz -d -c /tmp/talos.raw.xz | dd of=/dev/sda && sync && shutdown -h now
|
||
then snapshot that server in the console and label the snapshot os=talos.
|
||
The schematic and version above are this cluster's, read from cluster.yaml."
|
||
echo " snapshot $HCLOUD_IMAGE"
|
||
|
||
# The Talos API is unauthenticated in maintenance mode, which is how
|
||
# apply-config reaches a node that has no config yet. On a public address
|
||
# that has to be closed to everyone but this host for the minutes it takes.
|
||
step 0 "Creating a firewall for this host only..."
|
||
# Both families, and each with its own prefix length. The first version
|
||
# asked ifconfig.me once and appended /32: this host answered over IPv6, and
|
||
# Hetzner rejected `2003:d6:…:86d2/32` because /32 of that address is not a
|
||
# network boundary. A single-address rule is /32 for v4 and /128 for v6.
|
||
#
|
||
# Both are added rather than whichever happened to answer, because the rule
|
||
# has to admit the address talosctl actually leaves from, and that is chosen
|
||
# per connection by the resolver — not by whatever this lookup returned.
|
||
# No `local` here: this phase runs in the script body, not in a function,
|
||
# and `local` is a runtime error there — one `bash -n` does not catch.
|
||
# One family failing is ordinary — a host without IPv6 is not broken — so
|
||
# neither attempt aborts the run on its own. Their stderr is kept rather
|
||
# than discarded, and reported if BOTH fail: at that point the reason is the
|
||
# diagnosis, and "could not determine this host's address" without it sends
|
||
# you looking at the wrong thing.
|
||
fw_sources=()
|
||
fw_err=$(mktemp)
|
||
v4=$(curl -4 -fsS --max-time 10 https://ifconfig.me 2>>"$fw_err") && [[ -n "$v4" ]] && fw_sources+=("$v4/32")
|
||
v6=$(curl -6 -fsS --max-time 10 https://ifconfig.me 2>>"$fw_err") && [[ -n "$v6" ]] && fw_sources+=("$v6/128")
|
||
if [[ ${#fw_sources[@]} -eq 0 ]]; then
|
||
cat "$fw_err" >&2
|
||
rm -f "$fw_err"
|
||
fail "could not determine this host's public address over either IPv4 or IPv6"
|
||
fi
|
||
rm -f "$fw_err"
|
||
fw_src=$(IFS=,; echo "${fw_sources[*]}")
|
||
|
||
hcloud firewall create --name "hcloudtest-$$" --label "$HCLOUD_LABEL" >/dev/null \
|
||
|| fail "could not create the firewall"
|
||
hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 50000 \
|
||
--source-ips "$fw_src" --description "Talos API, this host only" >/dev/null \
|
||
|| fail "could not add the Talos API rule"
|
||
hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 6443 \
|
||
--source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \
|
||
|| fail "could not add the Kubernetes API rule"
|
||
echo " open to $fw_src only"
|
||
ok
|
||
|
||
# cx43 — 16 GB, 8 vCPU. Sized from what the workload needs, not from the
|
||
# Talos guide, which names cx22 as a minimal example (and that type no
|
||
# longer exists; `server create` rejects it outright).
|
||
#
|
||
# cx23 was tried first, on the reasoning that it is cx22's successor. It has
|
||
# 4 GB, 3.2 GB of it allocatable, and the three nodes sat at 59-78% of that
|
||
# with the infrastructure skeleton alone — no Ceph OSDs, no OpenBao running,
|
||
# not one app. Limits were already at 134-207% of allocatable and both
|
||
# kube-controller-manager and kube-scheduler had restarted twice.
|
||
#
|
||
# The local rehearsal is the reference: it runs a 16GiB control plane and
|
||
# three 8GiB workers, 40GiB in total, and needed every step of the way up
|
||
# from 6GiB to get there. Three nodes here carry the same workload plus a
|
||
# control plane each, so 48GiB total is the comparable figure.
|
||
#
|
||
# Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which
|
||
# would not match the amd64 schematic this cluster installs. That leaves the
|
||
# cx*/cpx*2 families in the German locations.
|
||
# Two segments only, unlike the homelab's split in
|
||
# NETWORK-traffic-classes.md: here the node address carries management,
|
||
# cluster-internal and external alike, and Ceph replication has its own.
|
||
#
|
||
# The order is load-bearing: it decides which kernel interface each segment
|
||
# becomes, and the machine config names those interfaces.
|
||
HCLOUD_SEGMENTS=(cluster storage)
|
||
# Segment ranges are read from what the cluster declares, never invented
|
||
# here — the disks' rule applied to networks:
|
||
# cluster the subnet the site patch pins node addresses to.
|
||
# storage the SAN pool the ceph-san attachment allocates from, widened
|
||
# to the containing /24 because the fabric wants a subnet.
|
||
# The fabric reserves the first address of a network range as its gateway.
|
||
# For the storage net that is the SAN pool's first address — a collision
|
||
# that matters only if cross-node SAN traffic on this site is ever
|
||
# attempted, and is left unresolved until it is.
|
||
seg_range() {
|
||
case "$1" in
|
||
cluster)
|
||
yq -r '.machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;;
|
||
storage)
|
||
local pool
|
||
pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range')
|
||
[[ "$pool" == *.0/2[4-9] ]] \
|
||
|| fail "cannot derive a /24 from the SAN pool '$pool'"
|
||
printf '%s/24' "${pool%.*}.0" ;;
|
||
esac
|
||
}
|
||
# Read from the location rather than stated a second time.
|
||
HCLOUD_ZONE=$(hcloud location describe "${HCLOUD_LOCATION:-fsn1}" -o format='{{.NetworkZone}}') \
|
||
|| fail "could not read the network zone of ${HCLOUD_LOCATION:-fsn1}"
|
||
[[ -n "$HCLOUD_ZONE" ]] \
|
||
|| fail "${HCLOUD_LOCATION:-fsn1} reported no network zone"
|
||
|
||
step 0 "Creating ${#HCLOUD_SEGMENTS[@]} private network(s) in $HCLOUD_ZONE..."
|
||
for seg in "${HCLOUD_SEGMENTS[@]}"; do
|
||
net="hcloudtest-$seg"
|
||
want_range=$(seg_range "$seg")
|
||
[[ -n "$want_range" && "$want_range" != null* ]] \
|
||
|| fail "could not derive the $seg range from the cluster config"
|
||
# Reused only if the range matches. A stale one fails later, at the
|
||
# attachment, naming the wrong thing.
|
||
if have_range=$(hcloud network describe "$net" -o format='{{.IPRange}}'); then
|
||
if [[ "$have_range" == "$want_range" ]]; then
|
||
echo " $net (already there)"
|
||
continue
|
||
fi
|
||
echo " $net carries $have_range, this run wants $want_range — replacing"
|
||
hcloud network delete "$net" >/dev/null \
|
||
|| fail "could not replace $net — detach its servers first, or run hcloudclean"
|
||
fi
|
||
hcloud network create --name "$net" --ip-range "$want_range" \
|
||
--label "$HCLOUD_LABEL" >/dev/null \
|
||
|| fail "could not create network $net"
|
||
hcloud network add-subnet "$net" --network-zone "$HCLOUD_ZONE" \
|
||
--type cloud --ip-range "$want_range" >/dev/null \
|
||
|| fail "could not add a subnet to $net"
|
||
echo " $net $want_range"
|
||
done
|
||
ok
|
||
|
||
# From here on machines exist and are billed; on_exit says so.
|
||
HCLOUD_PROVISIONED=1
|
||
step 0 "Creating ${#HCLOUD_NODES[@]} server(s)..."
|
||
NODE_ADDRS=()
|
||
# Servers this test already holds. Capacity is the binding constraint, so a
|
||
# machine in hand is worth keeping across runs.
|
||
#
|
||
# Listed by label: a same-named server without it falls through to the
|
||
# create path and fails on "name already used", rather than being adopted
|
||
# and wiped.
|
||
existing_servers=$(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name)
|
||
for n in "${HCLOUD_NODES[@]}"; do
|
||
if grep -qx "$n" <<<"$existing_servers"; then
|
||
# Rebuilt, not reused as-is: a configured Talos rejects a fresh
|
||
# apply-config, and `talosctl reset` needs the previous run's PKI,
|
||
# which is gone. Rebuilding from the snapshot returns the node to
|
||
# maintenance mode, reachable without any old secret.
|
||
echo " $n: reusing the server we already hold"
|
||
hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \
|
||
|| fail "could not rebuild $n back into maintenance mode"
|
||
# The firewall is named per run (hcloudtest-$$), so an adopted
|
||
# server is still attached to the previous run's. Attach this run's
|
||
# too — without it the Talos API is unreachable once the old one is
|
||
# cleaned up, and the addresses in it are last run's anyway.
|
||
hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$n" >/dev/null \
|
||
|| fail "could not attach this run's firewall to $n"
|
||
else
|
||
# Retried, because "error during placement (resource_unavailable)"
|
||
# is the provider being momentarily out of that type in that
|
||
# location, not a wrong request. It happened on the first cx43 run
|
||
# and the same type in the same location created fine minutes later.
|
||
# For a test meant to run on a schedule that distinction matters:
|
||
# without the retry every capacity blip is a red run someone has to
|
||
# come and look at.
|
||
create_err=""
|
||
for attempt in $(seq 1 30); do
|
||
create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \
|
||
--type "${HCLOUD_TYPE:-cx43}" --location "${HCLOUD_LOCATION:-fsn1}" \
|
||
--firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break
|
||
grep -qi 'resource_unavailable\|placement' <<<"$create_err" \
|
||
|| fail "could not create $n: ${create_err//$'\n'/ | }"
|
||
[[ $attempt -eq 1 ]] && echo " $n: no capacity yet, retrying"
|
||
sleep 20
|
||
[[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cx43} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes"
|
||
done
|
||
fi
|
||
addr=$(hcloud server ip "$n") || fail "could not read the address of $n"
|
||
[[ -n "$addr" ]] || fail "$n has no public address"
|
||
|
||
# Attached in HCLOUD_SEGMENTS order; the interface names follow from it.
|
||
# An adopted server keeps its attachments through a rebuild, and
|
||
# attaching twice is an error, so an existing one is skipped.
|
||
for seg in "${HCLOUD_SEGMENTS[@]}"; do
|
||
net="hcloudtest-$seg"
|
||
net_id=$(hcloud network describe "$net" -o json | jq -r '.id')
|
||
if hcloud server describe "$n" -o json \
|
||
| jq -e --argjson id "$net_id" 'any(.private_net[]; .network == $id)' >/dev/null; then
|
||
continue
|
||
fi
|
||
hcloud server attach-to-network "$n" --network "$net" >/dev/null \
|
||
|| fail "could not attach $n to $net"
|
||
done
|
||
|
||
# The OSD device. Every node carries one, which is the production shape
|
||
# — k8s-1/2/3 all hold storage — and is only possible here because
|
||
# Hetzner attaches volumes to any server, control plane included. The
|
||
# qemu provisioner cannot, which is why that site has to split into one
|
||
# control plane and three workers.
|
||
#
|
||
#
|
||
# Neither formatted nor mounted. Rook needs the raw device, and an
|
||
# --automount volume arrives with a filesystem on it that makes Ceph
|
||
# skip the disk as already in use.
|
||
#
|
||
# Recreated rather than reused, even on an adopted server: a volume from
|
||
# the previous run carries a bluestore signature, and Rook adopts a
|
||
# device that already looks like an OSD. Volumes are not the scarce
|
||
# resource here; servers are.
|
||
if hcloud volume describe "$n-osd" >/dev/null; then
|
||
echo " $n: discarding the previous OSD volume"
|
||
# An unattached leftover is ordinary after an interrupted run, so
|
||
# this failure is tolerated while the delete below is not.
|
||
hcloud volume detach "$n-osd" >/dev/null || true
|
||
hcloud volume delete "$n-osd" >/dev/null \
|
||
|| fail "could not delete the previous OSD volume for $n"
|
||
fi
|
||
vol_size=$(hcloud_volumes_for_node "$n")
|
||
[[ -n "$vol_size" ]] || fail "$n declares no RawVolumeConfig — a node with no OSD device is not this cluster's shape"
|
||
hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \
|
||
--label "$HCLOUD_LABEL" >/dev/null \
|
||
|| fail "could not create the OSD volume for $n"
|
||
|
||
echo " $n at $addr, with a ${vol_size}GB OSD volume"
|
||
NODE_ADDRS+=("$addr")
|
||
done
|
||
ok
|
||
|
||
# The nodes have to reach each other, and until now the firewall admitted
|
||
# only this host. One node did not care; three do — etcd peers on 2380, the
|
||
# apiserver, kubelet on 10250 and Cilium's VXLAN all run between them, and
|
||
# with the rules above alone etcd simply never forms a quorum.
|
||
#
|
||
# Added after creation because the addresses do not exist before it. Rules
|
||
# apply to an attached firewall immediately, so the ordering costs nothing.
|
||
#
|
||
# All ports between the nodes rather than an enumerated list: this is a
|
||
# throwaway cluster whose members are each other's only peers, and guessing
|
||
# the full port set of Talos plus Cilium is how a rebuild fails on the one
|
||
# port nobody wrote down. The outside still sees only 50000 and 6443, and
|
||
# only from this host.
|
||
step 0 "Opening the cluster to itself..."
|
||
node_src=$(IFS=,; printf '%s' "${NODE_ADDRS[*]/%//32}")
|
||
for proto in tcp udp; do
|
||
hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol "$proto" \
|
||
--port any --source-ips "$node_src" \
|
||
--description "intra-cluster $proto" >/dev/null \
|
||
|| fail "could not open $proto between the nodes"
|
||
done
|
||
echo " $node_src"
|
||
ok
|
||
|
||
# Handover, and this list is the whole of it — the same four variables the
|
||
# vmtest phase sets. If it grows, the test is drifting away from production.
|
||
SITE=hcloud
|
||
NODES=("${HCLOUD_NODES[@]}")
|
||
WORKERS=()
|
||
ALL_NODES=("${NODES[@]}")
|
||
K8S_API_HOST="${NODE_ADDRS[0]}"
|
||
|
||
# Isolation from the production cluster. Both halves are required: the
|
||
# _PATH variables are derived before any phase runs, and the talos phase
|
||
# writes to those.
|
||
HCLOUD_DIR=$(mktemp -d)
|
||
export TALOSCONFIG="$HCLOUD_DIR/talosconfig"
|
||
export KUBECONFIG="$HCLOUD_DIR/kubeconfig"
|
||
TALOSCONFIG_PATH="$TALOSCONFIG"
|
||
KUBECONFIG_PATH="$KUBECONFIG"
|
||
|
||
PHASE=talos
|
||
fi
|
||
|
||
# Provision throwaway QEMU VMs, then run the Talos phase below against
|
||
# them unchanged. Deliberately not a copy of that phase: it sets the site
|
||
# variables and falls through, because a rehearsal that runs different code
|
||
# proves less than one that does not. Everything it changes is listed here — if
|
||
# that list grows, the test is drifting away from what it claims to exercise.
|
||
#
|
||
# What it cannot cover, and what therefore still has to be true on hardware:
|
||
# Ceph (the provisioner gives extra disks to workers only), the mgmt and v30iot
|
||
# VLANs and the Layer2 VIP (one flat NIC per VM), and Flux (its own phase).
|
||
#
|
||
# Needs root for the QEMU provisioner (CNI and KVM), and only for that step.
|
||
if [[ "$PHASE" == "vmtest" ]]; then
|
||
echo -e "${YELLOW}=== Phase: VM rebuild test ===${NC}"
|
||
command -v talosctl >/dev/null || fail "talosctl not found"
|
||
command -v yq >/dev/null || fail "yq not found"
|
||
command -v jq >/dev/null || fail "jq not found (needed to read OpenBao's seal status)"
|
||
# The node patches are the list; the VM count follows from them rather than
|
||
# being stated twice. vm-1 is the control plane, the rest are the workers
|
||
# that carry an OSD device — see the comment on NODES below for why.
|
||
VM_CP_NODES=(vm-1)
|
||
VM_WORKER_NODES=(vm-2 vm-3 vm-4)
|
||
VM_CPS=${#VM_CP_NODES[@]}
|
||
VM_WORKER_COUNT=${#VM_WORKER_NODES[@]}
|
||
for n in "${VM_CP_NODES[@]}" "${VM_WORKER_NODES[@]}"; do
|
||
[[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
|
||
done
|
||
[[ -f "$TALOS_DIR/patches/site-qemu.yaml" ]] || fail "site-qemu.yaml not found"
|
||
|
||
# Talos version and Image Factory schematic, from the installer image in
|
||
# cluster.yaml — see talos_image_coords above for why neither phase states
|
||
# them itself.
|
||
talos_image_coords
|
||
|
||
# sun_path caps a unix socket path at 108 bytes and the provisioner puts a
|
||
# qemu monitor socket under the state directory, so this has to stay short.
|
||
# A path under the repo or a long mktemp dir silently fails to boot.
|
||
# Overridable because in-cluster the VM disks belong on the mounted scratch
|
||
# volume, not on the container's own writable layer. Keep whatever is set
|
||
# SHORT for the reason above.
|
||
VM_STATE="${VM_STATE:-/tmp/bootstrap-vmtest}"
|
||
[[ ${#VM_STATE} -le 50 ]] \
|
||
|| fail "VM_STATE '$VM_STATE' is ${#VM_STATE} bytes; keep it under 50 so the qemu monitor socket fits in sun_path's 108"
|
||
# cleanup_vmtest runs `sudo rm -rf "$VM_STATE"`, so refuse anything that is
|
||
# not a nested absolute path.
|
||
[[ $VM_STATE == /*/?* ]] \
|
||
|| fail "VM_STATE '$VM_STATE' must be an absolute path at least two levels deep"
|
||
VM_CLUSTER=bootstrap-vmtest
|
||
VM_DIR=$(mktemp -d)
|
||
TALOSCTL_BIN=$(command -v talosctl)
|
||
|
||
# Hard isolation from the production cluster: talosctl and kubectl both
|
||
# honour these, and the phase below writes to them rather than to $HOME.
|
||
export TALOSCONFIG="$VM_DIR/talosconfig"
|
||
export KUBECONFIG="$VM_DIR/kubeconfig"
|
||
TALOSCONFIG_PATH="$TALOSCONFIG"
|
||
KUBECONFIG_PATH="$KUBECONFIG"
|
||
|
||
# The provisioner needs root for KVM and for the bridges and taps it builds.
|
||
# On a workstation that means sudo; in the Job that runs this in-cluster the
|
||
# process is already root and sudo is not installed, so the elevation is a
|
||
# variable rather than a literal.
|
||
SUDO_KEEPALIVE=""
|
||
if [[ $EUID -eq 0 ]]; then
|
||
SUDO=""
|
||
else
|
||
SUDO="sudo -E"
|
||
# A full run outlasts sudo's timestamp, and the teardown at the end then
|
||
# prompts for a password with nobody watching — the VMs survive and the
|
||
# next run collides with them. Take it once here and keep the timestamp
|
||
# warm for as long as the phase lasts.
|
||
sudo -v || fail "the QEMU provisioner needs root (CNI and KVM)"
|
||
( while sudo -n true; do sleep 50; done ) &
|
||
SUDO_KEEPALIVE=$!
|
||
fi
|
||
|
||
cleanup_vmtest() {
|
||
[[ -n "$SUDO_KEEPALIVE" ]] && kill "$SUDO_KEEPALIVE" >/dev/null || true
|
||
# Diagnosing anything inside the VMs is impossible once they are gone,
|
||
# and the teardown fires the moment a run ends — including when it ends
|
||
# badly, which is exactly when the VMs are worth looking at. VMTEST_KEEP
|
||
# holds them, and the pod, until someone deletes the Job.
|
||
if [[ "${VMTEST_KEEP:-0}" == "1" ]]; then
|
||
echo -e "${YELLOW}VMTEST_KEEP=1 — leaving the VMs up.${NC}"
|
||
echo " talosctl --talosconfig $TALOSCONFIG -n 10.5.0.2 ..."
|
||
echo " delete the Job when finished; nothing else reclaims this."
|
||
sleep infinity
|
||
fi
|
||
echo -n "Destroying throwaway VMs... "
|
||
$SUDO "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
|
||
# destroy removes the cluster's own directory but leaves the state root.
|
||
$SUDO rm -rf "$VM_STATE"
|
||
rm -rf "$VM_DIR"
|
||
echo "done"
|
||
}
|
||
# No `trap ... EXIT` here — on_exit calls cleanup_vmtest once it is defined.
|
||
|
||
# A run killed hard never reaches its EXIT trap, so the previous cluster can
|
||
# still be holding the provisioner's CIDR. Clear it first rather than
|
||
# failing on a collision that looks like a networking problem.
|
||
if [[ -d "$VM_STATE/$VM_CLUSTER" ]]; then
|
||
echo -n "Removing a cluster left behind by an earlier run... "
|
||
$SUDO "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
|
||
$SUDO rm -rf "$VM_STATE"
|
||
echo "done"
|
||
fi
|
||
|
||
step 0 "Creating $VM_CPS control plane and $VM_WORKER_COUNT workers in maintenance mode..."
|
||
# maintenance means the provisioner applies no configuration of its own, so
|
||
# every byte of machine config below comes from talos/patches.
|
||
#
|
||
# The control plane gets 6GiB. At 2560MiB it ran with ~250MiB available and
|
||
# its page cache squeezed to 386MiB, so every etcd fsync hit a system in
|
||
# permanent reclaim: linearized reads took 5-20s against an expected 100ms,
|
||
# the apiserver became unreachable, and the run stalled with no error to
|
||
# read. That failure says nothing about whether the repo can rebuild a
|
||
# cluster, so it is bought off with memory rather than tuned around.
|
||
#
|
||
# The workers get 8GiB for the same reason. At 4GiB a storage node sat at
|
||
# 99% of its memory requests once it carried an OSD, a mon and a mgr, and
|
||
# what could not be scheduled any more was kube-multus-ds and whereabouts —
|
||
# the CNI itself. Without Multus the OSDs came up with no ceph-san
|
||
# interface and crash-looped on "unable to find any IPv4 address in
|
||
# networks 10.99.0.0/29", which reads like an addressing bug and is a
|
||
# memory one. Sizing the VMs to the workload they actually run is cheaper
|
||
# than shrinking the Ceph requests, which would test something other than
|
||
# what production runs.
|
||
#
|
||
# Workers get 4 CPUs, one constraint later: with memory solved they sat at
|
||
# 69% of CPU requests, and what could no longer schedule was the
|
||
# rook-ceph-operator itself. Nothing was left driving the cluster — two
|
||
# OSDs crash-looped unattended and the PVCs waited on a provisioner whose
|
||
# operator was Pending.
|
||
#
|
||
# The disks are sized the same way. At 10GiB the system disk yielded an
|
||
# 8.36GB /var, which the infrastructure image set filled to 82%: kubelet
|
||
# tainted all four nodes node.kubernetes.io/disk-pressure, no pod could be
|
||
# scheduled any more, and Ceph's own monitor sat Pending while the symptom
|
||
# surfaced three layers away as a Cilium HelmRelease stuck InProgress. The
|
||
# second disk is the Ceph OSD; at 8GiB, three of them replicated 3x left
|
||
# 8GiB usable, less than OpenBao's two PVCs alone ask for. Both are raw
|
||
# sparse files, so the larger geometry costs only what is actually written.
|
||
#
|
||
# The control plane gets 16GiB. 6 was not enough and neither was 8: at both
|
||
# it reached 99% of allocatable memory once the infrastructure layer was up,
|
||
# and a control plane with nothing left over does not stay a control plane.
|
||
# The kubelet admits the node-critical CNI DaemonSets by evicting whatever
|
||
# ranks below them, and kube-apiserver — a static pod at
|
||
# system-cluster-critical — is what ranks below them:
|
||
#
|
||
# message: Pod was preempted by Kubelet to accommodate a critical pod.
|
||
# reason: Preempting
|
||
#
|
||
# The priority ordering is not the defect and must not be tuned away. It is
|
||
# deliberate: a node without its CNI cannot run anything, so the CNI wins.
|
||
# What the eviction reports is the shortfall underneath it, and the only
|
||
# honest fix is to remove the shortfall. Production does this by having
|
||
# ~95GiB per node; the throwaway cluster does it here.
|
||
#
|
||
# This is not a spare-capacity argument for the workers. vm-1 is the only
|
||
# control plane and still takes workload, so it carries the static pods, the
|
||
# CNI, and its share of the infrastructure layer at once — which is the
|
||
# asymmetry with production, where every node is a control plane and none of
|
||
# them is small.
|
||
$SUDO "$TALOSCTL_BIN" cluster create qemu \
|
||
--name "$VM_CLUSTER" \
|
||
--state "$VM_STATE" \
|
||
--presets iso,maintenance \
|
||
--controlplanes "$VM_CPS" \
|
||
--workers "$VM_WORKER_COUNT" \
|
||
--talos-version "$VM_VERSION" \
|
||
--schematic-id "$VM_SCHEMATIC" \
|
||
--memory-controlplanes 16GiB \
|
||
--memory-workers 8GiB \
|
||
--cpus-controlplanes 2 \
|
||
--cpus-workers 4 \
|
||
--disks virtio:40GiB,virtio:24GiB \
|
||
--talosconfig-destination "$TALOSCONFIG" \
|
||
|| fail "could not create the throwaway VMs"
|
||
$SUDO chown -R "$(id -u):$(id -g)" "$VM_STATE" "$TALOSCONFIG"
|
||
ok
|
||
|
||
# The provisioner hands out its CIDR sequentially from .2; .1 is the
|
||
# gateway and the load balancer it puts in front of the API. If that ever
|
||
# changes, the apply loop below reports which address stopped answering
|
||
# rather than failing silently.
|
||
# One control plane and three workers, not three control planes: the
|
||
# provisioner gives the second disk only to workers ("disks after the first
|
||
# one are added only to worker machines"), and an OSD needs it. Three of
|
||
# them because the CephCluster runs three mons with allowMultiplePerNode
|
||
# false over pools that are replica 3 across failureDomain host.
|
||
NODES=("${VM_CP_NODES[@]}")
|
||
WORKERS=("${VM_WORKER_NODES[@]}")
|
||
ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
|
||
NODE_ADDRS=()
|
||
for i in $(seq 0 $(( ${#ALL_NODES[@]} - 1 ))); do
|
||
NODE_ADDRS+=("10.5.0.$(( i + 2 ))")
|
||
done
|
||
SITE=qemu
|
||
K8S_API_HOST=10.5.0.1
|
||
VMTEST=1
|
||
|
||
# Run the real phases, not a reimplementation of them.
|
||
#
|
||
# Default is the Talos phase alone: that needs nothing but the machines, so
|
||
# it is the part that can run unattended. VMTEST_FULL=1 continues into flux
|
||
# and wait-apps — the whole bootstrap against VMs — which additionally needs
|
||
# GITEA_TOKEN and a FLUX_BRANCH to write to, because `flux bootstrap`
|
||
# commits its sync manifests into the branch it reconciles.
|
||
if [[ "${VMTEST_FULL:-0}" == "1" ]]; then
|
||
[[ "$FLUX_BRANCH" != "master" ]] \
|
||
|| fail "VMTEST_FULL would have flux bootstrap commit into master — point FLUX_BRANCH at a throwaway branch that already exists"
|
||
echo -e "${YELLOW}Continuing into the flux phase against branch ${FLUX_BRANCH}.${NC}"
|
||
PHASE=all
|
||
else
|
||
PHASE=talos
|
||
fi
|
||
fi
|
||
|
||
# Before anything destructive, and after the vmtest block has decided which
|
||
# phases will run. A rebuild that will need these has no business wiping disks
|
||
# first, only to discover an hour later that pass is locked or a token is unset —
|
||
# by then the old cluster is gone and the new one cannot be finished.
|
||
#
|
||
# In the Job this is where the file-waiting branch blocks, which is the right
|
||
# moment too: the operator has just started the run and is watching it.
|
||
if phase_runs flux; then
|
||
require_secrets
|
||
fi
|
||
|
||
if phase_runs talos; then
|
||
echo -e "${YELLOW}=== Phase: Talos bootstrap ===${NC}"
|
||
command -v talosctl >/dev/null || fail "talosctl not found"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v helm >/dev/null || fail "helm not found (needed to install the CNI)"
|
||
command -v yq >/dev/null || fail "yq not found (needed to read the Cilium HelmRelease)"
|
||
[[ -f "$TALOS_DIR/patches/cluster.yaml" ]] || fail "cluster.yaml patch not found"
|
||
[[ -f "$TALOS_DIR/patches/containerd-layer-retention.yaml" ]] || fail "containerd-layer-retention.yaml patch not found"
|
||
[[ -f "$CILIUM_HR" ]] || fail "cilium helmrelease not found at $CILIUM_HR"
|
||
|
||
step 0 "Generating Talos configs..."
|
||
TALOS_GEN_DIR=$(mktemp -d)
|
||
talosctl gen config homelab "https://${K8S_API_HOST}:6443" \
|
||
--output "$TALOS_GEN_DIR" \
|
||
--force
|
||
mkdir -p "$(dirname "$TALOSCONFIG_PATH")"
|
||
cp "$TALOS_GEN_DIR/talosconfig" "$TALOSCONFIG_PATH"
|
||
|
||
# Lets a node pull the images it needs before the registry that normally
|
||
# serves them exists.
|
||
#
|
||
# Several apps need an image while the cluster is still being built —
|
||
# paperless's fetch-bridge-cert init container, zigbee2mqtt's seed-data init
|
||
# container, the trmnl renderer, and the CNI plugins every pod depends on.
|
||
# They are referenced at gitea.renner.uno, which is correct for normal
|
||
# operation and useless during a rebuild, because Gitea runs inside the
|
||
# cluster being rebuilt.
|
||
#
|
||
# `mirrors` gives containerd an ordered list to try. Gitea is named first, so
|
||
# nothing changes once the cluster is up: pulls stay local and never leave
|
||
# the network. ghcr.io is second and only answers when the first cannot,
|
||
# which is exactly the rebuild window. The manifests therefore never name an
|
||
# external registry — the fallback is a property of the node, not of the
|
||
# deployment.
|
||
#
|
||
# Containerd keeps the repository path when it falls through, which is why
|
||
# the images are published under konairius/ on BOTH registries
|
||
# (.woodpecker/build-ci-images.yml). Verified equal: the same build pushes
|
||
# one digest to both paths.
|
||
#
|
||
# The ghcr namespace is private, so it needs a credential — and the usual
|
||
# place for one, an imagePullSecret, fails here twice over: it would have to
|
||
# exist in five namespaces before any of them exists, and it would come from
|
||
# OpenBao, which needs Ceph, which needs these very images.
|
||
# machine.registries.config is containerd's own auth, in place before the
|
||
# first pod is scheduled and applying to every pull on the node regardless
|
||
# of namespace. Field names follow .docker/config.json
|
||
# (pkg/machinery/config/types/v1alpha1, RegistryAuthConfig).
|
||
#
|
||
# The patch is written into the in-memory secrets dir, never into the repo
|
||
# or the generated config directory, and dies with the Job like the material
|
||
# it carries.
|
||
registry_auth_patch() {
|
||
local out="$OPENBAO_SECRETS_DIR/registry-auth.yaml" token
|
||
token=$(read_secret ghcr-pull) || fail "could not read the GHCR pull token"
|
||
( umask 077; cat > "$out" <<EOF
|
||
machine:
|
||
registries:
|
||
mirrors:
|
||
gitea.renner.uno:
|
||
endpoints:
|
||
- https://gitea.renner.uno
|
||
- https://ghcr.io
|
||
config:
|
||
ghcr.io:
|
||
auth:
|
||
username: "${GHCR_PULL_USER}"
|
||
password: "${token}"
|
||
EOF
|
||
) || fail "could not write the registry auth patch"
|
||
printf '%s' "$out"
|
||
}
|
||
# OUTSIDE the command substitution below, and that placement is the fix for
|
||
# a bug this had: require_secrets prints "All secrets present." on stdout,
|
||
# and calling it inside registry_auth_patch made that line part of the
|
||
# captured path. talosctl then reported
|
||
# open All secrets present. | /tmp/…/registry-auth.yaml: no such file
|
||
# and retried 60 times while the node sat in maintenance. A function whose
|
||
# value is its stdout must be the only thing writing there.
|
||
#
|
||
# This phase can run on its own (`bootstrap.sh talos`), in which case
|
||
# nothing has demanded the secrets yet — the full require_secrets is gated
|
||
# on the flux phase. Ask for the one this needs, and only that one: pulling
|
||
# in the whole list would make an unattended talos run depend on the OpenBao
|
||
# seal key, which never reaches CI.
|
||
require_secrets ghcr-pull
|
||
REGISTRY_PATCH=$(registry_auth_patch)
|
||
[[ -s "$REGISTRY_PATCH" ]] \
|
||
|| fail "the registry auth patch was not written to a readable file: '$REGISTRY_PATCH'"
|
||
|
||
step 0 "Applying Talos configs to nodes..."
|
||
apply_node_config() {
|
||
local node=$1 base_config=$2 fixed_addr=${3:-}
|
||
local fqdn="${node}.${DOMAIN}"
|
||
echo " ${fixed_addr:-$fqdn}:"
|
||
for i in $(seq 1 60); do
|
||
# A fixed address skips DNS. Real hardware has a name to resolve;
|
||
# a throwaway VM has an address and nothing else.
|
||
if [[ -n "$fixed_addr" ]]; then
|
||
node_ip="$fixed_addr"
|
||
else
|
||
node_ip=$(resolve_node_ip "$fqdn")
|
||
fi
|
||
if [[ -z "$node_ip" ]]; then
|
||
echo " attempt $i/60: DNS returned no IP"
|
||
sleep 10
|
||
continue
|
||
fi
|
||
if err=$(talosctl apply-config --insecure -n "$node_ip" \
|
||
--file "$base_config" \
|
||
--config-patch "@$TALOS_DIR/patches/cluster.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/site-${SITE}.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/containerd-layer-retention.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/${node}.yaml" \
|
||
--config-patch "@$REGISTRY_PATCH" 2>&1); then
|
||
echo -e " applied via $node_ip ${GREEN}(OK)${NC}"
|
||
return 0
|
||
fi
|
||
echo " attempt $i/60: $node_ip - ${err//$'\n'/ | }"
|
||
sleep 10
|
||
done
|
||
fail "Timeout waiting for ${fixed_addr:-$fqdn}"
|
||
}
|
||
|
||
# One loop over ALL_NODES rather than one per role, so NODE_ADDRS can be a
|
||
# single list in the same order. Control planes come first in ALL_NODES,
|
||
# which is what decides the base config each node gets.
|
||
for idx in "${!ALL_NODES[@]}"; do
|
||
if [[ $idx -lt ${#NODES[@]} ]]; then
|
||
base="$TALOS_GEN_DIR/controlplane.yaml"
|
||
else
|
||
base="$TALOS_GEN_DIR/worker.yaml"
|
||
fi
|
||
apply_node_config "${ALL_NODES[$idx]}" "$base" "${NODE_ADDRS[$idx]:-}"
|
||
done
|
||
|
||
echo -n "Configuring talosctl... "
|
||
# `gen config` leaves talosconfig endpoints empty; they are the Talos API
|
||
# addresses and unrelated to cluster.controlPlane.endpoint.
|
||
if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
|
||
talosctl config endpoint "${NODE_ADDRS[@]}"
|
||
talosctl config node "${NODE_ADDRS[0]}"
|
||
else
|
||
talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
|
||
talosctl config node "${NODES[0]}.${DOMAIN}"
|
||
fi
|
||
ok
|
||
|
||
step 0 "Waiting for every node to come back configured..."
|
||
# Asks each machine directly, by address, and expects it to answer with the
|
||
# hostname its patch assigns. That is what has to be true before etcd is
|
||
# bootstrapped: the config took, the node rebooted into it, and the machine
|
||
# at that address is the one we think it is.
|
||
#
|
||
# This used to wait on `talosctl get members`, which was wrong twice over.
|
||
# It queried through the configured endpoint list rather than each node, so
|
||
# apid refused it with `PermissionDenied: no request forwarding` — the same
|
||
# thing that broke the bootstrap call. And membership comes from the
|
||
# discovery service, because the Kubernetes registry is disabled: a cold
|
||
# rebuild would have been waiting on discovery.talos.dev being reachable,
|
||
# before the cluster it is trying to rebuild even exists.
|
||
#
|
||
# Each node installs to disk and reboots first, so the window is generous.
|
||
for idx in "${!ALL_NODES[@]}"; do
|
||
want="${ALL_NODES[$idx]}"
|
||
addr="${NODE_ADDRS[$idx]:-${want}.${DOMAIN}}"
|
||
echo -n " $want ($addr)... "
|
||
node_err=""
|
||
for i in $(seq 1 60); do
|
||
got=$(talosctl -n "$addr" -e "$addr" get hostname \
|
||
-o jsonpath='{.spec.hostname}' 2>"$TALOS_GEN_DIR/hostname.err") || true
|
||
node_err=$(cat "$TALOS_GEN_DIR/hostname.err")
|
||
[[ "$got" == "$want" ]] && break
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
echo
|
||
[[ -n "$node_err" ]] && echo " last talosctl error: $node_err" >&2
|
||
fail "$addr never came back as $want (last answer: '${got:-none}')"
|
||
fi
|
||
done
|
||
ok
|
||
done
|
||
|
||
echo -n "Bootstrapping Kubernetes... "
|
||
# -n AND -e on one control plane, deliberately. etcd bootstrap is the one
|
||
# call apid refuses to forward: reached through any other node it fails with
|
||
# `PermissionDenied: no request forwarding`. Without them talosctl picks an
|
||
# endpoint from the configured list — with more than one control plane, or a
|
||
# list that includes workers, that is a coin flip.
|
||
if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
|
||
bootstrap_target="${NODE_ADDRS[0]}"
|
||
else
|
||
bootstrap_target="${NODES[0]}.${DOMAIN}"
|
||
fi
|
||
# Re-running against an already-bootstrapped cluster is not an error — that
|
||
# is why this does not simply fail on a non-zero exit. Anything that is not
|
||
# "already exists" is shown rather than swallowed, because a silent failure
|
||
# here surfaces much later as an API that never answers.
|
||
#
|
||
# And it RETRIES, because "bootstrap is not available yet" is a moment, not
|
||
# a fault. The wait above only establishes that the node answers with its
|
||
# hostname; etcd accepting the bootstrap call comes later, and how much
|
||
# later depends on the machine. On qemu the gap is short enough to miss. On
|
||
# a Hetzner cloud server it is not: the single attempt failed with
|
||
# FailedPrecondition, the run carried on regardless, and then sat out its
|
||
# full timeout waiting for an apiserver that could never start because etcd
|
||
# had never been bootstrapped.
|
||
bootstrap_err=""
|
||
for i in $(seq 1 60); do
|
||
if bootstrap_err=$(talosctl bootstrap -n "$bootstrap_target" -e "$bootstrap_target" 2>&1); then
|
||
break
|
||
fi
|
||
if grep -qi 'already\|AlreadyExists' <<<"$bootstrap_err"; then
|
||
bootstrap_err=""
|
||
break
|
||
fi
|
||
# Not yet ready is the expected answer for the first attempts; anything
|
||
# else is reported once and then retried anyway, since the alternative
|
||
# is failing on a condition that may still clear.
|
||
if ! grep -qi 'not available yet\|connection refused\|FailedPrecondition' <<<"$bootstrap_err"; then
|
||
[[ $i -eq 1 ]] && echo -e "\n${YELLOW}bootstrap reported:${NC} ${bootstrap_err//$'\n'/ | }"
|
||
fi
|
||
sleep 10
|
||
[[ $i -eq 60 ]] && { echo; echo "${bootstrap_err//$'\n'/ | }" >&2; fail "etcd never accepted the bootstrap call"; }
|
||
done
|
||
ok
|
||
|
||
echo -n "Getting kubeconfig... "
|
||
kubeconfig_err=""
|
||
for i in $(seq 1 30); do
|
||
mkdir -p "$(dirname "$KUBECONFIG_PATH")"
|
||
if kubeconfig_err=$(talosctl kubeconfig "$KUBECONFIG_PATH" --force 2>&1); then
|
||
sed -i "s|server:.*|server: https://${K8S_API_HOST}:6443|" "$KUBECONFIG_PATH"
|
||
break
|
||
fi
|
||
sleep 5
|
||
# Held back while retrying — the first attempts fail by design — but
|
||
# printed if it never works, because that message is the diagnosis.
|
||
[[ $i -eq 30 ]] && { echo; echo "$kubeconfig_err" >&2; fail "Couldn't get kubeconfig"; }
|
||
done
|
||
ok
|
||
|
||
echo -n "Waiting for the Kubernetes API to answer... "
|
||
# Having a kubeconfig is not the same as having an apiserver. `talosctl
|
||
# kubeconfig` succeeds as soon as Talos can hand out credentials, which is
|
||
# before kube-apiserver is serving on the endpoint the file points at — so
|
||
# the very next command could fail with an EOF against a port nothing is
|
||
# listening on yet. On hardware the gap usually closes on its own; on a
|
||
# scratch cluster it does not, which is how this was found.
|
||
# `get --raw /version` needs the server; `kubectl version` would answer from
|
||
# the client alone and pass against nothing at all.
|
||
api_err=""
|
||
for i in $(seq 1 60); do
|
||
api_err=$(kubectl get --raw /version 2>&1) && break
|
||
sleep 5
|
||
if [[ $i -eq 60 ]]; then
|
||
echo "$api_err" >&2
|
||
fail "Kubernetes API at https://${K8S_API_HOST}:6443 never answered"
|
||
fi
|
||
done
|
||
ok
|
||
|
||
step 0 "Installing Cilium (CNI)..."
|
||
# Must run before the node-Ready wait below: with `cni: none` the kubelet
|
||
# reports NotReady until a CNI is present, so that loop would time out.
|
||
#
|
||
# Chart repo, version constraint and values all come out of the HelmRelease
|
||
# so this cannot drift from what Flux will reconcile minutes later. helm
|
||
# accepts the semver range verbatim and resolves it the same way Flux does.
|
||
#
|
||
# serviceMonitor is the one override, and it is a bootstrap constraint
|
||
# rather than a value: its CRD arrives with the monitoring stack much
|
||
# later, and the object would fail to apply. Nothing else is overridden,
|
||
# which is what lets helm-controller adopt this release instead of
|
||
# replacing it.
|
||
CILIUM_VALUES=$(mktemp)
|
||
yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES"
|
||
grep -q 'policyMapMax' "$CILIUM_VALUES" \
|
||
|| fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR"
|
||
|
||
helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null
|
||
helm repo update cilium >/dev/null
|
||
helm upgrade --install cilium cilium/cilium \
|
||
--version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \
|
||
--namespace kube-system \
|
||
-f "$CILIUM_VALUES" \
|
||
--set prometheus.serviceMonitor.enabled=false \
|
||
--wait --timeout 10m
|
||
rm -f "$CILIUM_VALUES"
|
||
ok
|
||
|
||
step 0 "Waiting for K8s nodes..."
|
||
expected=${#ALL_NODES[@]}
|
||
# Before the loop, because 0 == 0 would satisfy its exit condition on the
|
||
# first pass and report a complete Talos phase for a cluster with no nodes
|
||
# in it. `expected` comes from ALL_NODES, which a provisioning phase hands
|
||
# over — an empty one is a bug in that phase, not a cluster that is ready.
|
||
[[ "$expected" -gt 0 ]] \
|
||
|| fail "ALL_NODES is empty, so this wait would pass against any cluster, including one with no nodes"
|
||
nodes_err=""
|
||
for i in $(seq 1 60); do
|
||
# The STATUS column, matched whole. `grep -c Ready` counted NotReady
|
||
# too — the substring is in it — so this waited for nodes to register
|
||
# rather than to become ready, and passed while every one of them was
|
||
# still NotReady.
|
||
nodes_out=$(kubectl get nodes --no-headers 2>"$TALOS_GEN_DIR/nodes.err") || true
|
||
nodes_err=$(cat "$TALOS_GEN_DIR/nodes.err")
|
||
ready=$(awk '$2 == "Ready"' <<<"$nodes_out" | grep -c . || true)
|
||
if [[ "$ready" -eq "$expected" ]]; then
|
||
# Say what was accepted. Silence on success made this step
|
||
# unauditable: a run that printed "Waiting for K8s nodes..." and
|
||
# then "Talos phase complete" was indistinguishable from one that
|
||
# matched 0 against 0, and the log gave no way to tell which had
|
||
# happened. A count that is also the pass condition is the one
|
||
# number worth printing.
|
||
echo "$ready/$expected Ready"
|
||
break
|
||
fi
|
||
sleep 5
|
||
if [[ $i -eq 60 ]]; then
|
||
echo
|
||
[[ -n "$nodes_err" ]] && echo "last kubectl error: $nodes_err" >&2
|
||
echo "${nodes_out:- (no nodes listed)}" >&2
|
||
fail "Not all nodes ready (got $ready/$expected)"
|
||
fi
|
||
done
|
||
|
||
# Stash configs for subsequent phases / export-access step. Skipped under
|
||
# vmtest: those are a throwaway cluster's credentials, and writing them here
|
||
# would leave the flux phase pointed at VMs that no longer exist.
|
||
if [[ "${VMTEST:-0}" == "1" ]]; then
|
||
# Nodes being Ready says nothing about storage. The CephCluster picks
|
||
# OSD nodes by a label out of the machine config and their device by the
|
||
# partition label RawVolumeConfig creates, so assert both here — running
|
||
# it by hand afterwards races the teardown, and a test that does not
|
||
# check its own premise is worth little.
|
||
step 0 "Checking the OSD prerequisites Ceph selects on..."
|
||
for idx in $(seq "${#NODES[@]}" $(( ${#ALL_NODES[@]} - 1 ))); do
|
||
vm_node="${ALL_NODES[$idx]}"
|
||
vm_addr="${NODE_ADDRS[$idx]}"
|
||
vols=$(talosctl -n "$vm_addr" -e "$vm_addr" get discoveredvolumes 2>&1) \
|
||
|| fail "could not read volumes from $vm_node ($vm_addr): ${vols//$'\n'/ | }"
|
||
awk '$NF == "r-osd-data" {found=1} END {exit !found}' <<<"$vols" \
|
||
|| fail "$vm_node has no r-osd-data partition — RawVolumeConfig did not take"
|
||
kubectl get node "$vm_node" \
|
||
-o jsonpath='{.metadata.labels.storage\.renner\.uno/ceph-osd}' \
|
||
| grep -q true \
|
||
|| fail "$vm_node is missing the storage.renner.uno/ceph-osd label"
|
||
echo -e " $vm_node: partition and label ${GREEN}(OK)${NC}"
|
||
done
|
||
|
||
echo -e "${GREEN}VM rebuild test passed.${NC} ${#ALL_NODES[@]} nodes Ready on the repo's own config."
|
||
else
|
||
mkdir -p "$ACCESS_STAGING"
|
||
cp "$TALOSCONFIG_PATH" "$ACCESS_STAGING/talosconfig"
|
||
cp "$KUBECONFIG_PATH" "$ACCESS_STAGING/kubeconfig"
|
||
echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
|
||
fi
|
||
fi
|
||
|
||
# Helper: load staged access files into home dir (for later CI steps)
|
||
# Bring OpenBao up far enough that External Secrets can read from it.
|
||
#
|
||
# This was eight commands typed by hand, and it sits in the middle of the cold
|
||
# rebuild: infrastructure-crds waits on the ClusterSecretStore, which needs an
|
||
# unsealed OpenBao with Kubernetes auth configured, and every app Kustomization
|
||
# waits on infrastructure-crds. A rebuild could not reach a reconciled state
|
||
# without someone at a keyboard.
|
||
#
|
||
# The seal key stays where it was — the operator's `pass` store — and is read
|
||
# here at run time. It is never written to the repo and never reaches CI, which
|
||
# is the reason this was left manual in the first place. It reaches the cluster
|
||
# as the openbao-seal-key Secret, created by the flux phase before Flux starts
|
||
# OpenBao, and from then on every replica unseals itself.
|
||
#
|
||
# Everything below is idempotent: it is safe to re-run against an OpenBao that
|
||
# is already unsealed and configured, which is what makes it usable as part of
|
||
# `bootstrap.sh all` rather than a one-shot.
|
||
|
||
bao_exec() {
|
||
local pod=$1; shift
|
||
kubectl -n "$OPENBAO_NS" exec -i "$pod" -- "$@"
|
||
}
|
||
|
||
# Where the OpenBao raft snapshots already are: inside the Velero backup, in the
|
||
# kopia repository the data mover writes to. Nothing new is produced for the
|
||
# rebuild — the nightly snapshot CronJob and the openbao-daily Velero Schedule
|
||
# have been doing this all along.
|
||
#
|
||
# Reading them needs three things, all of which live OUTSIDE the cluster, which
|
||
# is the point: bucket credentials from pass (their own key, not Velero's),
|
||
# kopia, and the repository password. The last is not a secret — Velero
|
||
# hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
|
||
# the offsite backup's confidentiality rests on bucket access alone.
|
||
#
|
||
# This is what breaks the circle. Velero's own credentials come from an
|
||
# ExternalSecret, which needs the ClusterSecretStore, which needs an unsealed
|
||
# OpenBao — so Velero cannot be the thing that restores OpenBao.
|
||
OPENBAO_KOPIA_BUCKET="renner-uno-k8s-velero"
|
||
OPENBAO_KOPIA_PREFIX="kopia/openbao/"
|
||
OPENBAO_S3_ENDPOINT="fsn1.your-objectstorage.com"
|
||
OPENBAO_S3_REGION="fsn1"
|
||
# Velero's hardcoded backup repository password (pkg/repository/keys/keys.go).
|
||
OPENBAO_KOPIA_PASSWORD="static-passw0rd"
|
||
|
||
restore_openbao() {
|
||
local pod=$1
|
||
step 0 "Restoring OpenBao from the offsite backup..."
|
||
command -v kopia >/dev/null || fail "kopia not found — needed to read the Velero backup repository without Velero"
|
||
|
||
local work kopia_err
|
||
work=$(mktemp -d)
|
||
export KOPIA_CONFIG_PATH="$work/repository.config"
|
||
export KOPIA_CACHE_DIRECTORY="$work/cache"
|
||
export KOPIA_LOG_DIR="$work/logs"
|
||
export KOPIA_CHECK_FOR_UPDATES=false
|
||
export KOPIA_PASSWORD="$OPENBAO_KOPIA_PASSWORD"
|
||
|
||
# Assigned first, not interpolated into the argument list. `fail` inside
|
||
# $( ) exits only the subshell, and a failed substitution in an argument does
|
||
# not stop the outer command — so a missing key produced an EMPTY
|
||
# --access-key and an authentication error instead of the real reason.
|
||
local s3_access s3_secret
|
||
s3_access=$(read_secret s3-access) || fail "no s3-access secret"
|
||
s3_secret=$(read_secret s3-secret) || fail "no s3-secret secret"
|
||
|
||
# Nothing here is expected to fail, so nothing here discards its error. This
|
||
# is the path that only runs during a genuine disaster; it is the last place
|
||
# to be guessing why something did not work.
|
||
echo -n " connecting to the backup repository (read-only)... "
|
||
if ! kopia_err=$(kopia repository connect s3 \
|
||
--bucket "$OPENBAO_KOPIA_BUCKET" \
|
||
--prefix "$OPENBAO_KOPIA_PREFIX" \
|
||
--endpoint "$OPENBAO_S3_ENDPOINT" \
|
||
--region "$OPENBAO_S3_REGION" \
|
||
--access-key "$s3_access" \
|
||
--secret-access-key "$s3_secret" \
|
||
--readonly 2>&1); then
|
||
echo; echo "$kopia_err" >&2
|
||
fail "could not open the backup repository at $OPENBAO_S3_ENDPOINT/$OPENBAO_KOPIA_BUCKET/$OPENBAO_KOPIA_PREFIX"
|
||
fi
|
||
ok
|
||
|
||
# Newest kopia snapshot, then the newest .snap inside it. The PVC holds a
|
||
# week of them, so the last line is the most recent.
|
||
local ksnap snapfile listing
|
||
listing=$(kopia snapshot list --all 2>&1) \
|
||
|| { echo "$listing" >&2; fail "could not list snapshots in the backup repository"; }
|
||
ksnap=$(tail -1 <<<"$listing" | awk '{print $4}')
|
||
[[ -n "$ksnap" ]] \
|
||
|| { echo "$listing" >&2; fail "could not read a snapshot id out of the listing above"; }
|
||
snapfile=$(kopia ls "$ksnap" 2>&1 | grep -o 'openbao-raft-[0-9-]*\.snap' | sort | tail -1) || true
|
||
[[ -n "$snapfile" ]] || fail "no openbao-raft-*.snap in kopia snapshot $ksnap"
|
||
echo " newest snapshot: $snapfile"
|
||
|
||
if ! kopia_err=$(kopia restore "$ksnap/$snapfile" "$work/restore.snap" 2>&1); then
|
||
echo "$kopia_err" >&2
|
||
fail "could not extract $snapfile from the backup"
|
||
fi
|
||
[[ -s "$work/restore.snap" ]] || fail "$snapfile came out empty"
|
||
echo " extracted $(stat -c%s "$work/restore.snap") bytes"
|
||
|
||
# A snapshot can only be loaded into a running OpenBao, so the fresh one is
|
||
# initialised first. Under the static seal, init hands out a recovery key
|
||
# rather than an unseal key and the server unseals itself; the only thing
|
||
# used from its output is the root token, for the next few seconds, and
|
||
# then it stops existing: restoring replaces the keyring with the
|
||
# snapshot's, after which the root token and recovery key from pass apply.
|
||
#
|
||
# No -force. Without it OpenBao checks that the snapshot was written under
|
||
# the seal key this server runs with, and refuses a mismatch by name
|
||
# ("possibly the snapshot is using a different autoseal key"). That check
|
||
# is the one guard against restoring under the wrong seal-key Secret;
|
||
# -force would skip it and leave a server that can never unseal.
|
||
#
|
||
# The init output is not echoed on a parse failure: it carries the root
|
||
# token and the recovery key of a server that is about to be overwritten,
|
||
# but a log line is a log line.
|
||
echo -n " initialising the empty server (throwaway recovery key)... "
|
||
local init_json root
|
||
init_json=$(bao_exec "$pod" bao operator init -recovery-shares=1 -recovery-threshold=1 -format=json 2>&1) \
|
||
|| { echo; echo "$init_json" >&2; fail "could not initialise OpenBao"; }
|
||
root=$(jq -r '.root_token // empty' <<<"$init_json")
|
||
[[ -n "$root" ]] || fail "init output carried no root token"
|
||
ok
|
||
|
||
# Sealed after init means the static seal is not working — the Secret is
|
||
# missing or malformed — and the restore below cannot run against it.
|
||
echo -n " waiting for it to unseal itself"
|
||
for i in $(seq 1 30); do
|
||
if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
|
||
| jq -e '.sealed == false' >/dev/null; then
|
||
echo; break
|
||
fi
|
||
printf '.'
|
||
sleep 2
|
||
if [[ $i -eq 30 ]]; then
|
||
echo
|
||
cat "$OPENBAO_SECRETS_DIR/.baoerr" >&2
|
||
fail "$pod did not unseal itself after init — check the openbao-seal-key Secret and the pod log"
|
||
fi
|
||
done
|
||
|
||
echo -n " loading the snapshot... "
|
||
kubectl -n "$OPENBAO_NS" cp "$work/restore.snap" "$pod:/tmp/restore.snap" >/dev/null \
|
||
|| fail "could not copy the snapshot into $pod"
|
||
local out
|
||
out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore /tmp/restore.snap 2>&1) \
|
||
|| { echo; echo "$out" >&2; fail "snapshot restore rejected"; }
|
||
ok
|
||
|
||
rm -rf "$work"
|
||
# The server seals itself against the keyring that came with the snapshot
|
||
# and unseals again from the static seal — the same key the snapshot was
|
||
# written under. bring_up_openbao waits for that on every replica.
|
||
echo " restored — the server re-seals and unseals itself from the seal key"
|
||
}
|
||
|
||
bring_up_openbao() {
|
||
step 0 "Bringing OpenBao up..."
|
||
command -v jq >/dev/null || fail "jq not found (needed to read OpenBao's seal status)"
|
||
|
||
# OpenBao is deployed by the infrastructure Kustomization. What this step
|
||
# depends on is therefore that Flux has applied it — not that some number of
|
||
# minutes have gone by. A guessed duration is not a condition: the first
|
||
# version failed after ten minutes on a slow cluster, and raising the number
|
||
# only moves the guess.
|
||
#
|
||
# So the loop ends on one of two facts, both read from the cluster: the
|
||
# StatefulSet exists, or Flux reports the Kustomization has failed. The
|
||
# elapsed-time bound below is a backstop against hanging forever, not the
|
||
# mechanism — it is deliberately far longer than any real reconcile.
|
||
local pods=() ks_state ks_bad=0
|
||
echo -n " waiting for Flux to apply the openbao StatefulSet"
|
||
for i in $(seq 1 720); do
|
||
if kubectl -n "$OPENBAO_NS" get statefulset openbao >/dev/null 2>&1; then
|
||
echo
|
||
break
|
||
fi
|
||
ks_state=$(ks_ready_condition infrastructure)
|
||
# Only a persistent failure counts. The first version matched any message
|
||
# containing "error" or "failed", which fires on `failed to download
|
||
# artifact from source-controller` — a routine transient in the minutes
|
||
# after bootstrap that Flux retries on its own, and one that aborted the
|
||
# rebuild. A wait built on cluster facts is no better than a guessed
|
||
# timeout if it reacts to a fact that is not the one it means.
|
||
if [[ "${ks_state%%|*}" == "False" ]] && grep -qiE 'failed|error' <<<"${ks_state#*|}"; then
|
||
(( ks_bad++ )) || true
|
||
else
|
||
ks_bad=0
|
||
fi
|
||
if (( ks_bad >= 30 )); then
|
||
echo
|
||
echo " infrastructure Ready=${ks_state%%|*} — ${ks_state#*|}" >&2
|
||
fail "the infrastructure Kustomization has been failing for five minutes — OpenBao will not appear"
|
||
fi
|
||
(( i % 18 == 0 )) && { echo; echo " infrastructure Ready=${ks_state%%|*} — ${ks_state#*|}"; echo -n " waiting"; }
|
||
printf '.'
|
||
sleep 10
|
||
[[ $i -eq 720 ]] && { echo; fail "openbao StatefulSet never appeared, and Flux never reported a failure — something is wedged"; }
|
||
done
|
||
|
||
# The StatefulSet exists; its pods follow.
|
||
for i in $(seq 1 60); do
|
||
mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
|
||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
|
||
[[ ${#pods[@]} -gt 0 ]] && break
|
||
sleep 10
|
||
[[ $i -eq 60 ]] && fail "the openbao StatefulSet exists but produced no pods"
|
||
done
|
||
echo " found ${#pods[@]} pod(s): ${pods[*]}"
|
||
|
||
# A pod object existing is not a pod that can answer. Waiting only for the
|
||
# name meant `bao status` ran against a container still pulling or in Init,
|
||
# its error was thrown away, and an empty answer parsed as "not initialised"
|
||
# — which routes into the restore branch, the one branch that must never be
|
||
# entered by mistake. Wait for the container to be running first.
|
||
echo -n " waiting for ${pods[0]} to answer"
|
||
local status="" status_err=""
|
||
for i in $(seq 1 90); do
|
||
if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then
|
||
echo; break
|
||
fi
|
||
status_err=$(cat "$OPENBAO_SECRETS_DIR/.baoerr")
|
||
# `bao status` exits 2 when sealed and still prints valid JSON — that is
|
||
# an answer, not a failure.
|
||
#
|
||
# Ask whether the field is PRESENT, not whether it is true. `jq -e
|
||
# '.initialized'` tests the value, so a brand-new OpenBao answering
|
||
# {"initialized": false, "sealed": true} — a complete and correct
|
||
# answer, and the only one a cold rebuild can give — read as no answer
|
||
# at all. The wait then ran its full 90 rounds against a healthy pod and
|
||
# failed with "never answered bao status", one step after the cluster
|
||
# had finally come up on its own.
|
||
if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi
|
||
printf '.'
|
||
sleep 10
|
||
if [[ $i -eq 90 ]]; then
|
||
echo
|
||
echo " last error: $status_err" >&2
|
||
fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised"
|
||
fi
|
||
done
|
||
|
||
# `bao status` reports Initialized either way, so it distinguishes "needs a
|
||
# key" from "needs initialising". Those must not be confused: initialising an
|
||
# OpenBao that already holds data is how the data gets lost.
|
||
# Read presence separately from value. jq's `//` treats false as absent, so
|
||
# `.initialized // empty` on the one answer a rebuild always gives —
|
||
# {"initialized": false} — yields an empty string, and the guard below then
|
||
# reports that OpenBao's state could not be read at all. Same mistake as the
|
||
# `jq -e` in the wait above, one line apart: false is a value, not a gap.
|
||
local initialized
|
||
initialized=$(jq -r 'if has("initialized") then (.initialized | tostring) else "" end' <<<"$status")
|
||
[[ -n "$initialized" ]] \
|
||
|| { echo "$status" >&2; fail "could not read initialised state from bao status"; }
|
||
|
||
if [[ "$initialized" != "true" ]]; then
|
||
# An uninitialised OpenBao is restored from the offsite snapshot, never
|
||
# initialised fresh: initialising over an existing keyring is how the
|
||
# data is lost, and a genuinely new OpenBao is a decision rather than a
|
||
# step (MIGRATION-sops-openbao.md).
|
||
restore_openbao "${pods[0]}"
|
||
fi
|
||
|
||
# Wait for every replica the StatefulSet wants, not the ones that happen to
|
||
# exist right now. On a cold start only openbao-0 exists when this runs:
|
||
# podManagementPolicy is OrderedReady, so openbao-1 is created once
|
||
# openbao-0 passes its readiness probe and openbao-2 after that. Each
|
||
# unseals itself from the static seal; nothing here types a key.
|
||
#
|
||
# Waiting for the whole chain rather than the first pod matters because the
|
||
# openbao Service sets publishNotReadyAddresses — the chart does this so
|
||
# replicas can reach each other to join raft before any of them is Ready —
|
||
# so a replica that is still starting is a live backend on the service IP.
|
||
# Clients balanced onto it get `dial tcp <svc>:8200: connect: no route to
|
||
# host`, and the failure surfaces nowhere near here: external-secrets'
|
||
# ClusterSecretStore flaps, every ExternalSecret reports "could not get
|
||
# secret data from provider", and infrastructure-crds — which waits on those
|
||
# — never goes Ready, taking all twenty app Kustomizations with it.
|
||
local want
|
||
want=$(kubectl -n "$OPENBAO_NS" get statefulset openbao -o jsonpath='{.spec.replicas}') \
|
||
|| fail "could not read the openbao StatefulSet's replica count"
|
||
[[ "$want" =~ ^[0-9]+$ ]] || fail "openbao StatefulSet reported a non-numeric replica count: '$want'"
|
||
echo " waiting for $want replica(s) to unseal themselves"
|
||
|
||
local unsealed=0 round pod
|
||
local -A seen=()
|
||
for round in $(seq 1 180); do
|
||
mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
|
||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||
unsealed=0
|
||
for pod in "${pods[@]}"; do
|
||
if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
|
||
| jq -e '.sealed == false' >/dev/null; then
|
||
[[ -n "${seen[$pod]:-}" ]] || { echo -e " $pod: ${GREEN}unsealed${NC}"; seen[$pod]=1; }
|
||
unsealed=$(( unsealed + 1 ))
|
||
fi
|
||
done
|
||
(( unsealed >= want )) && break
|
||
sleep 10
|
||
if [[ $round -eq 180 ]]; then
|
||
# A replica that is running and still sealed did not get a working
|
||
# seal. Say which, with its own answer, rather than "timed out".
|
||
for pod in "${pods[@]}"; do
|
||
[[ -n "${seen[$pod]:-}" ]] && continue
|
||
echo " $pod:" >&2
|
||
bao_exec "$pod" bao status 2>&1 | sed 's/^/ /' >&2 || true
|
||
done
|
||
fail "only $unsealed of $want OpenBao replicas unsealed after 30 minutes — a sealed one means the static seal failed (openbao-seal-key Secret, pod log)"
|
||
fi
|
||
done
|
||
ok
|
||
}
|
||
|
||
load_staged_access() {
|
||
[[ -f "$ACCESS_STAGING/talosconfig" && -f "$ACCESS_STAGING/kubeconfig" ]] \
|
||
|| fail "access-staging/ missing talosconfig or kubeconfig (run talos phase first)"
|
||
# To the paths talosctl and kubectl will actually read, not to $HOME. An
|
||
# operator with KUBECONFIG pointing somewhere else — an ordinary setup — got
|
||
# the staged config written where nothing would look at it, and every step
|
||
# after this ran against whatever cluster the old KUBECONFIG named. Wrong
|
||
# target, no error.
|
||
mkdir -p "$(dirname "$TALOSCONFIG_PATH")" "$(dirname "$KUBECONFIG_PATH")"
|
||
cp "$ACCESS_STAGING/talosconfig" "$TALOSCONFIG_PATH"
|
||
cp "$ACCESS_STAGING/kubeconfig" "$KUBECONFIG_PATH"
|
||
}
|
||
|
||
if phase_runs flux; then
|
||
echo -e "${YELLOW}=== Phase: FluxCD bootstrap ===${NC}"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v flux >/dev/null || fail "flux not found"
|
||
# In the in-cluster Job neither the SOPS key nor the Gitea token is present,
|
||
# and neither should be baked in. Both are streamed into the secrets
|
||
# directory when the run asks for them, the same way the seal key is.
|
||
# Unconditional: the same collection step on a laptop and in the Job, so the
|
||
# recovery path is the one that gets exercised.
|
||
require_secrets
|
||
SOPS_KEY="$OPENBAO_SECRETS_DIR/sops-age.key"
|
||
[[ -f "$SOPS_KEY" ]] || fail "SOPS age key not found at $SOPS_KEY"
|
||
# One collected token, handed to whichever variable the chosen source reads.
|
||
# `flux bootstrap gitea` and `flux bootstrap github` each take theirs from
|
||
# the environment.
|
||
case "$FLUX_SOURCE" in
|
||
gitea) GITEA_TOKEN=$(read_secret git-token); export GITEA_TOKEN ;;
|
||
github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;;
|
||
*) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github)" ;;
|
||
esac
|
||
[[ "$PHASE" == "flux" ]] && load_staged_access
|
||
|
||
echo -n "Creating SOPS age secret... "
|
||
kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f -
|
||
kubectl create secret generic sops-age -n flux-system \
|
||
--from-file=age.agekey="$SOPS_KEY" \
|
||
--dry-run=client -o yaml | kubectl apply -f -
|
||
ok
|
||
|
||
# The static seal key, before Flux brings OpenBao up: a pod whose env
|
||
# references a Secret that does not exist is never started, and OpenBao
|
||
# reads the key at start. Left alone when the cluster already holds the
|
||
# same value. A DIFFERENT value is refused rather than applied: the data
|
||
# is encrypted under whichever key OpenBao started with, and swapping the
|
||
# Secret would leave every replica sealed for good at its next restart.
|
||
# That is a migration, not a bootstrap step.
|
||
#
|
||
# --from-literal, not --from-file: `pass show` ends its output with a
|
||
# newline, the substitution strips it, and OpenBao rejects a key that is
|
||
# not exactly 32 bytes of base64. The key sits in this process's argument
|
||
# list for the length of the call, as it did in the pod's before.
|
||
echo -n "Creating OpenBao seal key secret... "
|
||
seal_key=$(read_secret seal-key) || fail "could not read the seal key"
|
||
kubectl create namespace "$OPENBAO_NS" --dry-run=client -o yaml | kubectl apply -f -
|
||
if existing=$(kubectl -n "$OPENBAO_NS" get secret openbao-seal-key \
|
||
-o go-template='{{index .data "key" | base64decode}}' 2>"$OPENBAO_SECRETS_DIR/.err"); then
|
||
[[ "$existing" == "$seal_key" ]] \
|
||
|| fail "openbao-seal-key in $OPENBAO_NS holds a different key than pass — not replacing it (OpenBao's data is written under the key it started with)"
|
||
echo "already present"
|
||
elif grep -q NotFound "$OPENBAO_SECRETS_DIR/.err"; then
|
||
kubectl -n "$OPENBAO_NS" create secret generic openbao-seal-key \
|
||
--from-literal=key="$seal_key" >/dev/null \
|
||
|| fail "could not create the openbao-seal-key Secret"
|
||
ok
|
||
else
|
||
echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
|
||
fail "could not read the openbao-seal-key Secret"
|
||
fi
|
||
unset seal_key existing
|
||
|
||
if [[ "$FLUX_SOURCE" == "github" ]]; then
|
||
step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..."
|
||
echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}"
|
||
echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}"
|
||
# GITHUB_TOKEN is read from the environment by `flux bootstrap github`.
|
||
#
|
||
# --secret-name, and this is what makes the break-glass path survive its
|
||
# own success. The default is `flux-system`, which is also the target of
|
||
# the flux-system-git-auth ExternalSecret in post-crds — that one carries
|
||
# the GITEA credentials, which is correct for the steady state where the
|
||
# source is in-cluster Gitea.
|
||
#
|
||
# Sharing the name means the moment infrastructure-crds reconciles and
|
||
# ESO resolves, the GitHub token is overwritten with Gitea credentials
|
||
# while the GitRepository still points at github.com. Flux then stops
|
||
# being able to read its own source:
|
||
#
|
||
# GitOperationFailed: unable to list remote for
|
||
# 'https://github.com/.../Homelab.git': authentication required:
|
||
# Invalid username or token
|
||
#
|
||
# The rebuild destroys its git source at the exact moment it starts
|
||
# working, which is why it stayed hidden: no earlier run ever got the
|
||
# ExternalSecrets to resolve at all.
|
||
#
|
||
# A separate name keeps the two credentials apart. `flux-system` still
|
||
# receives the Gitea credentials and is simply unused until the source is
|
||
# switched back to Gitea — the manual step the warning above names.
|
||
flux bootstrap github \
|
||
--owner="$GITHUB_OWNER" \
|
||
--repository="$GITHUB_REPO" \
|
||
--branch="$FLUX_BRANCH" \
|
||
--path="$FLUX_PATH" \
|
||
--secret-name=flux-system-breakglass \
|
||
--components-extra=image-reflector-controller,image-automation-controller \
|
||
--personal \
|
||
--token-auth --timeout 15m 2>&1
|
||
else
|
||
step 0 "Bootstrapping FluxCD from in-cluster Gitea..."
|
||
flux bootstrap gitea \
|
||
--hostname="$GITEA_HOST" \
|
||
--owner="$GITEA_OWNER" \
|
||
--repository="$GITEA_REPO" \
|
||
--branch="$FLUX_BRANCH" \
|
||
--path="$FLUX_PATH" \
|
||
--components-extra=image-reflector-controller,image-automation-controller \
|
||
--personal \
|
||
--token-auth --timeout 15m 2>&1
|
||
fi
|
||
|
||
bring_up_openbao
|
||
|
||
step 0 "Waiting for infrastructure..."
|
||
for i in $(seq 1 60); do
|
||
# The Ready condition, not the printed table: `grep -c True` used to
|
||
# match SUSPENDED=True as well, so a Kustomization Flux is not
|
||
# reconciling at all read as ready and let the run proceed.
|
||
ready=$(ks_ready_condition infrastructure)
|
||
if [[ "${ready%%|*}" == "True" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
echo -e "${YELLOW}Warning: infrastructure not ready after 10m, continuing...${NC}"
|
||
flux get kustomizations 2>&1 | sed 's/^/ /' | head -12
|
||
fi
|
||
done
|
||
|
||
step 0 "Waiting for Ceph cluster..."
|
||
for i in $(seq 1 60); do
|
||
health=$(kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)
|
||
if [[ "$health" == "Ready" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
# Advisory, so it continues — but a warning that says only "not ready"
|
||
# sends the reader back to the cluster for information this step
|
||
# already had.
|
||
echo -e "${YELLOW}Warning: Ceph not ready after 10m (phase: ${health:-none}), continuing...${NC}"
|
||
kubectl -n rook-ceph get cephcluster -o wide 2>&1 | sed 's/^/ /'
|
||
kubectl -n rook-ceph get pods --no-headers 2>&1 \
|
||
| awk '$3 != "Running" && $3 != "Completed"' | head -10 | sed 's/^/ /'
|
||
fi
|
||
done
|
||
|
||
step 0 "Waiting for Ceph Object Store..."
|
||
for i in $(seq 1 60); do
|
||
phase=$(kubectl get cephobjectstore -n rook-ceph migration -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||
if [[ "$phase" == "Ready" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
if [[ $i -eq 60 ]]; then
|
||
echo -e "${YELLOW}Warning: Object Store not ready after 10m (phase: ${phase:-none}), continuing...${NC}"
|
||
kubectl -n rook-ceph get cephobjectstore 2>&1 | sed 's/^/ /'
|
||
fi
|
||
done
|
||
fi
|
||
|
||
if phase_runs wait-apps; then
|
||
echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v flux >/dev/null || fail "flux not found"
|
||
[[ "$PHASE" == "wait-apps" ]] && load_staged_access
|
||
|
||
# Wait for a Flux Kustomization to become Ready=True. Prints a status line
|
||
# whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a
|
||
# silent-but-progressing reconcile is still visible in CI output. Returns
|
||
# 0 on Ready, 1 on timeout (caller decides whether to fail or continue).
|
||
wait_kustomization() {
|
||
local name="$1" label="$2" max_iters="$3"
|
||
local interval=10 elapsed=0 prev_status=""
|
||
local timeout_min=$(( max_iters * interval / 60 ))
|
||
step 0 "Waiting for ${label} (timeout ${timeout_min}m)..."
|
||
for _ in $(seq 1 "$max_iters"); do
|
||
local cond ready msg status
|
||
cond=$(ks_ready_condition "$name")
|
||
ready="${cond%%|*}"
|
||
msg="${cond#*|}"
|
||
if [[ "$ready" == "True" ]]; then
|
||
printf " [%4ds] Ready ✓\n" "$elapsed"
|
||
return 0
|
||
fi
|
||
status="${ready:-?}: ${msg:-(no status yet)}"
|
||
# Print on first iteration, on state change, or as a 60-s heartbeat.
|
||
# 240 chars is enough to see the full Flux stalled-resource message.
|
||
if [[ "$prev_status" != "$status" ]] || (( elapsed > 0 && elapsed % 60 == 0 )); then
|
||
printf " [%4ds] %s\n" "$elapsed" "${status:0:240}"
|
||
prev_status="$status"
|
||
fi
|
||
sleep "$interval"
|
||
elapsed=$(( elapsed + interval ))
|
||
done
|
||
printf " [%4ds] TIMEOUT after %sm — full status follows\n" "$elapsed" "$timeout_min"
|
||
# Dump the full untruncated status + Events for the offending resources
|
||
# so the post-mortem doesn't need a second cluster trip.
|
||
flux get kustomization "$name" 2>&1 | sed 's/^/ /' || true
|
||
kubectl get events --sort-by=.lastTimestamp -A 2>/dev/null \
|
||
| grep -i -E "warning|error" | tail -20 | sed 's/^/ /' || true
|
||
echo -e "${YELLOW}Warning: ${label} not Ready, continuing anyway${NC}"
|
||
# 0, not 1. Called bare under `set -e`, a non-zero return ends the script
|
||
# immediately — so this said "continuing anyway" and then stopped, and
|
||
# every wait after it never ran. The message was the truth; the return
|
||
# value contradicted it.
|
||
return 0
|
||
}
|
||
|
||
# Each app KS now owns its CNPG database (+ any data PVCs) directly — the
|
||
# per-app `seed-<app>` KSes that used to bootstrap them were folded into the
|
||
# app KSes when the seed layer was retired. Each wait below gates on the app
|
||
# KS, whose readiness includes its CNPG cluster coming up (initdb on a fresh
|
||
# cluster; a cold-rebuild data restore is the separate RECOVERY.md runbook).
|
||
# Timeouts stay generous to cover first-time cluster bootstrap on fresh Ceph.
|
||
# apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up
|
||
# is genuinely 5–10 min on a fresh OSD-wiped cluster).
|
||
wait_kustomization apps "apps" 90
|
||
wait_kustomization gitea "Gitea (incl. data-gitea-0 PVC)" 90
|
||
wait_kustomization influxdb "InfluxDB (HR install + token Jobs)" 600
|
||
wait_kustomization woodpecker "Woodpecker CI (incl. wp-pg CNPG backend)" 600
|
||
wait_kustomization authentik "Authentik (incl. ak-pg CNPG backend)" 600
|
||
wait_kustomization monitoring "kube-prometheus-stack (Loki + Prom + AM + Grafana)" 180
|
||
wait_kustomization music-assistant "Music Assistant (incl. data PVC)" 300
|
||
wait_kustomization paperless "Paperless (incl. pprl-pg CNPG + data PVCs)" 600
|
||
wait_kustomization home-assistant "Home Assistant (HAOS backup → native restore)" 180
|
||
wait_kustomization audiobookshelf "Audiobookshelf (incl. library/metadata PVCs)" 900
|
||
wait_kustomization navidrome "Navidrome (incl. data + music library PVCs)" 600
|
||
wait_kustomization immich "Immich (incl. im-pg CNPG + library PVC)" 1800
|
||
|
||
# Wait for cert-manager to finish issuing every Certificate. Kustomization
|
||
# Ready only means the Ingress + Certificate manifests are applied; cert-
|
||
# manager then runs an async ACME DNS-01 challenge per Certificate (~30s–2m
|
||
# via Cloudflare). e2e's TLS check fails on the ingress-nginx default fake
|
||
# cert if any Certificate is still Issuing when the test runs.
|
||
wait_certificates() {
|
||
local max_iters=90 interval=10 elapsed=0 prev_status=""
|
||
local timeout_min=$(( max_iters * interval / 60 ))
|
||
step 0 "Waiting for all cert-manager Certificates Ready (timeout ${timeout_min}m)..."
|
||
for _ in $(seq 1 "$max_iters"); do
|
||
local total ready not_ready_names status
|
||
total=$(kubectl get certificate -A --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
||
ready=$(kubectl get certificate -A --no-headers 2>/dev/null | awk '$3=="True"' | wc -l | tr -d ' ')
|
||
if [[ "$total" -gt 0 && "$total" == "$ready" ]]; then
|
||
printf " [%4ds] All %s Certificates Ready ✓\n" "$elapsed" "$total"
|
||
return 0
|
||
fi
|
||
not_ready_names=$(kubectl get certificate -A --no-headers 2>/dev/null \
|
||
| awk '$3!="True" {print $1"/"$2}' | head -5 | tr '\n' ' ')
|
||
status="${ready:-0}/${total:-?} Ready, waiting: ${not_ready_names:-(none)}"
|
||
if [[ "$prev_status" != "$status" ]] || (( elapsed > 0 && elapsed % 60 == 0 )); then
|
||
printf " [%4ds] %s\n" "$elapsed" "${status:0:140}"
|
||
prev_status="$status"
|
||
fi
|
||
sleep "$interval"
|
||
elapsed=$(( elapsed + interval ))
|
||
done
|
||
printf " [%4ds] TIMEOUT after %sm\n" "$elapsed" "$timeout_min"
|
||
echo -e "${YELLOW}Warning: not all Certificates Ready, continuing anyway${NC}"
|
||
kubectl get certificate -A --no-headers | awk '$3!="True"' | head -10
|
||
# 0 for the same reason as wait_kustomization: under `set -e` a non-zero
|
||
# return here ends the run despite the message saying otherwise.
|
||
return 0
|
||
}
|
||
wait_certificates
|
||
|
||
echo ""
|
||
echo -e "${YELLOW}=== Verification ===${NC}"
|
||
# Whole-field matches. `grep -c Ready` counts NotReady too — the substring is
|
||
# in it — so this block reported every node healthy on a cluster where none
|
||
# was. `grep -c True` on HelmReleases matched the SUSPENDED column and any
|
||
# message containing the word.
|
||
#
|
||
# awk also avoids grep's exit 1 on zero matches, which under pipefail ended
|
||
# the run here with no output at all.
|
||
echo -n "Nodes: "
|
||
kubectl get nodes --no-headers | awk '$2 == "Ready"' | wc -l | tr -d '\n'
|
||
echo "/${#ALL_NODES[@]}"
|
||
|
||
# Ready count against a denominator: "12 ready" reads as success whether the
|
||
# cluster has 12 HelmReleases or 40. Both numbers come from the objects, not
|
||
# from `flux get`'s table, whose columns shift when REVISION is empty.
|
||
echo -n "HelmReleases: "
|
||
hr_total=$(kubectl get helmrelease --all-namespaces --no-headers | wc -l)
|
||
hr_ready=$(kubectl get helmrelease --all-namespaces -o jsonpath=\
|
||
'{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \
|
||
| grep -c '^True$') || true
|
||
echo "${hr_ready:-0}/${hr_total} ready"
|
||
|
||
echo -n "Ceph: "
|
||
kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.ceph.health}' 2>/dev/null || echo "pending"
|
||
echo ""
|
||
|
||
echo -n "Ingress (local): "
|
||
curl -sk --connect-timeout 5 --resolve status.renner.uno:443:10.2.99.86 https://status.renner.uno -o /dev/null -w "%{http_code}" 2>/dev/null || echo "N/A"
|
||
echo ""
|
||
fi
|