|
|
|
@@ -15,10 +15,17 @@ set -euo pipefail
|
|
|
|
|
# # the only place the registry fallback can be
|
|
|
|
|
# # tested: gitea.renner.uno has no external
|
|
|
|
|
# # ingress, so a node there genuinely cannot
|
|
|
|
|
# # reach it. BILLS BY THE HOUR — teardown runs
|
|
|
|
|
# # from the EXIT trap and is by label.
|
|
|
|
|
# ./bootstrap.sh hcloudclean # that teardown on its own, for a run that was
|
|
|
|
|
# # killed before its trap could fire.
|
|
|
|
|
# # reach it. BILLS BY THE HOUR — the machines
|
|
|
|
|
# # stay up until hcloudclean. Access files land in
|
|
|
|
|
# # ~/.local/state/homelab-hcloudtest/.
|
|
|
|
|
# ./bootstrap.sh hcloudclean # deletes everything hcloudtest created, by
|
|
|
|
|
# # label, and the restore test's deploy keys.
|
|
|
|
|
#
|
|
|
|
|
# Restore test on the hcloudtest cluster (production tree, read-only source,
|
|
|
|
|
# kubernetes/restore-test/flux-root.yaml):
|
|
|
|
|
# KUBECONFIG=~/.local/state/homelab-hcloudtest/kubeconfig \
|
|
|
|
|
# TALOSCONFIG=~/.local/state/homelab-hcloudtest/talosconfig \
|
|
|
|
|
# FLUX_SOURCE=restore-test ./bootstrap.sh flux
|
|
|
|
|
#
|
|
|
|
|
# Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls
|
|
|
|
|
# from is itself Flux-managed, so a cold rebuild cannot pull from it until the
|
|
|
|
@@ -177,6 +184,10 @@ resolve_node_ip() {
|
|
|
|
|
phase_runs() {
|
|
|
|
|
[[ "$PHASE" == "all" || "$PHASE" == "$1" ]]
|
|
|
|
|
}
|
|
|
|
|
# The restore test targets the cluster bootstrap.sh hcloudtest builds.
|
|
|
|
|
if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wait-apps" ]]; then
|
|
|
|
|
fail "FLUX_SOURCE=restore-test runs the flux and wait-apps phases only"
|
|
|
|
|
fi
|
|
|
|
|
OPENBAO_NS="openbao"
|
|
|
|
|
|
|
|
|
|
# Where the secrets come from. On a workstation, the operator's pass store. In
|
|
|
|
@@ -218,7 +229,7 @@ OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}"
|
|
|
|
|
# that cannot be told apart from a fault. Its non-zero exit propagates, which is
|
|
|
|
|
# what makes require_secrets print the reason instead.
|
|
|
|
|
git_token_for_source() {
|
|
|
|
|
if [[ "$FLUX_SOURCE" == "github" ]]; then
|
|
|
|
|
if [[ "$FLUX_SOURCE" == "github" || "$FLUX_SOURCE" == "restore-test" ]]; then
|
|
|
|
|
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
|
|
|
|
printf '%s' "$GITHUB_TOKEN"
|
|
|
|
|
else
|
|
|
|
@@ -256,13 +267,13 @@ OPENBAO_REQUIRED_SECRETS=(
|
|
|
|
|
# rest, because a rebuild that discovers it missing has already wiped the
|
|
|
|
|
# disks.
|
|
|
|
|
"ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
|
|
|
|
|
# Only the hcloudtest and hcloudclean phases ask for this one, by name, so a
|
|
|
|
|
# rebuild of the real cluster never demands it.
|
|
|
|
|
#
|
|
|
|
|
# Note the path: homelab/hetzner/, spelled correctly, while the two restore
|
|
|
|
|
# keys above live under homelab/hetztner/. Two folders, one typo, both real
|
|
|
|
|
# — do not "fix" one of these paths without moving the entry first.
|
|
|
|
|
"hcloud-token|Hetzner Cloud API token for the throwaway test cluster|pass show homelab/hetzner/cloud-api-secret"
|
|
|
|
|
)
|
|
|
|
|
# Asked for by name only, never by a bare require_secrets: a rebuild of the
|
|
|
|
|
# real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
|
|
|
|
|
# Hetzner project homelab-backup-verify, which holds nothing but throwaway
|
|
|
|
|
# machines.
|
|
|
|
|
HCLOUD_REQUIRED_SECRETS=(
|
|
|
|
|
"hcloud-token|Hetzner Cloud API token for the throwaway test cluster|bao kv get -mount=homelab -field=token infrastructure/hcloud-backup-verify"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Collects all of them up front, into one directory, so everything after this
|
|
|
|
@@ -289,7 +300,7 @@ require_secrets() {
|
|
|
|
|
local entry name what how
|
|
|
|
|
local -a wanted=()
|
|
|
|
|
if [[ $# -gt 0 ]]; then
|
|
|
|
|
for entry in "${OPENBAO_REQUIRED_SECRETS[@]}"; do
|
|
|
|
|
for entry in "${OPENBAO_REQUIRED_SECRETS[@]}" "${HCLOUD_REQUIRED_SECRETS[@]}"; do
|
|
|
|
|
for name in "$@"; do
|
|
|
|
|
[[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry")
|
|
|
|
|
done
|
|
|
|
@@ -429,8 +440,12 @@ s3_dirs() {
|
|
|
|
|
}' <<<"$xml"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# The bucket every CNPG cluster recovers from (cnpg-object-stores/).
|
|
|
|
|
# The bucket every CNPG cluster recovers from (cnpg-object-stores/). The
|
|
|
|
|
# restore test archives to CNPG_TEST_BUCKET instead: a test incarnation with a
|
|
|
|
|
# base backup in the production bucket would be the newest there, and the next
|
|
|
|
|
# real recovery would read the test's data.
|
|
|
|
|
CNPG_BUCKET="renner-uno-k8s-cnpg"
|
|
|
|
|
CNPG_TEST_BUCKET="renner-uno-k8s-cnpg-rtest"
|
|
|
|
|
CNPG_STORES_DIR="$REPO_ROOT/kubernetes/cnpg-object-stores"
|
|
|
|
|
|
|
|
|
|
# cnpg-paths, the ConfigMap cnpg-object-stores takes its paths from. Written
|
|
|
|
@@ -488,6 +503,46 @@ write_cnpg_paths() {
|
|
|
|
|
ok
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# The restore test's archive bucket. Created when missing; on a rerun it
|
|
|
|
|
# already exists, which S3 reports as an error.
|
|
|
|
|
create_cnpg_test_bucket() {
|
|
|
|
|
local out
|
|
|
|
|
out=$(s3_request PUT "$CNPG_TEST_BUCKET" 2>&1) && return 0
|
|
|
|
|
[[ "$out" == *BucketAlreadyOwnedByYou* ]] && return 0
|
|
|
|
|
echo "$out" >&2
|
|
|
|
|
fail "could not create $CNPG_TEST_BUCKET"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Empties and deletes the restore test's archive bucket. Each pass deletes the
|
|
|
|
|
# first page of keys and lists again, so no continuation token is needed.
|
|
|
|
|
delete_cnpg_test_bucket() {
|
|
|
|
|
local xml key
|
|
|
|
|
while :; do
|
|
|
|
|
if ! xml=$(s3_request GET "$CNPG_TEST_BUCKET?list-type=2" 2>&1); then
|
|
|
|
|
[[ "$xml" == *NoSuchBucket* ]] && return 0
|
|
|
|
|
echo "$xml" >&2
|
|
|
|
|
echo " WARNING: $CNPG_TEST_BUCKET not emptied — check the console" >&2
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
while read -r key; do
|
|
|
|
|
[[ -n "$key" ]] || continue
|
|
|
|
|
s3_request DELETE "$CNPG_TEST_BUCKET/$key" >/dev/null \
|
|
|
|
|
|| { echo " WARNING: $CNPG_TEST_BUCKET/$key not deleted — check the console" >&2; return 0; }
|
|
|
|
|
done < <(awk '{
|
|
|
|
|
while (match($0, /<Key>[^<]*<\/Key>/)) {
|
|
|
|
|
print substr($0, RSTART + 5, RLENGTH - 11)
|
|
|
|
|
$0 = substr($0, RSTART + RLENGTH)
|
|
|
|
|
}
|
|
|
|
|
}' <<<"$xml")
|
|
|
|
|
[[ "$xml" == *"<IsTruncated>true</IsTruncated>"* ]] || break
|
|
|
|
|
done
|
|
|
|
|
if s3_request DELETE "$CNPG_TEST_BUCKET" >/dev/null; then
|
|
|
|
|
echo " deleted bucket $CNPG_TEST_BUCKET"
|
|
|
|
|
else
|
|
|
|
|
echo " WARNING: bucket $CNPG_TEST_BUCKET not deleted — check the console" >&2
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase
|
|
|
|
|
# against it unchanged — same shape as the vmtest phase below, and for the same
|
|
|
|
|
# reason: a rehearsal that runs different code proves less than one that does
|
|
|
|
@@ -511,15 +566,18 @@ write_cnpg_paths() {
|
|
|
|
|
# which does nothing but the sweep. Anything this phase creates carries
|
|
|
|
|
# HCLOUD_LABEL so that sweep can find it without being told.
|
|
|
|
|
#
|
|
|
|
|
# WHAT IT DOES NOT COVER.
|
|
|
|
|
# WHAT IT IS.
|
|
|
|
|
#
|
|
|
|
|
# Everything downstream of Multus. The provider is not the limit — a cloud
|
|
|
|
|
# server takes up to 16 volumes and 3 private networks — but ceph-san binds to
|
|
|
|
|
# the kernel link name enp2s0f1np1, and that name cannot be manufactured here
|
|
|
|
|
# without assuming a Hetzner private network passes VLAN tags. See
|
|
|
|
|
# talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase
|
|
|
|
|
# needs the OpenBao seal key, which deliberately never reaches CI, and this
|
|
|
|
|
# test is meant to run unattended.
|
|
|
|
|
# The production config on new hardware: cluster.yaml, site-hcloud.yaml (what
|
|
|
|
|
# the Hetzner site is), containerd-layer-retention.yaml and hcloud-N.yaml (each
|
|
|
|
|
# machine). Nothing stands in for the home network. Flux is a separate run,
|
|
|
|
|
# FLUX_SOURCE=restore-test bootstrap.sh flux, because it needs the OpenBao seal
|
|
|
|
|
# key. Ceph replicates over the pod network there: Hetzner networks are layer 3
|
|
|
|
|
# (restore-test/flux-root.yaml).
|
|
|
|
|
#
|
|
|
|
|
# Containment: an outbound Hetzner firewall on every node admits udp and icmp
|
|
|
|
|
# anywhere and tcp anywhere except the addresses of proxy.renner.uno, the public
|
|
|
|
|
# edge in front of production.
|
|
|
|
|
# Talos version and Image Factory schematic, read out of the installer image in
|
|
|
|
|
# cluster.yaml. Both throwaway-cluster phases need them and neither may state
|
|
|
|
|
# them: a second copy is a copy that goes stale, and the day it did the test
|
|
|
|
@@ -539,7 +597,7 @@ HCLOUD_LABEL="managed-by=bootstrap-hcloudtest"
|
|
|
|
|
|
|
|
|
|
# The disks a machine has are stated in its Talos patch and read from there, so
|
|
|
|
|
# the provisioner carries no second copy. Emits one size in whole GB per raw
|
|
|
|
|
# volume the patch declares, in order.
|
|
|
|
|
# volume the patch declares off the system disk, in order.
|
|
|
|
|
hcloud_volumes_for_node() {
|
|
|
|
|
local patch="$TALOS_DIR/patches/$1.yaml" size gib
|
|
|
|
|
[[ -f "$patch" ]] || fail "$patch not found — the machine config is the list"
|
|
|
|
@@ -550,22 +608,260 @@ hcloud_volumes_for_node() {
|
|
|
|
|
gib=${size%GiB}
|
|
|
|
|
[[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as <n>GiB"
|
|
|
|
|
echo $(( (gib * 1074 + 999) / 1000 ))
|
|
|
|
|
done < <(yq -r 'select(.kind == "RawVolumeConfig") | .provisioning.minSize' "$patch")
|
|
|
|
|
done < <(yq -r 'select(.kind == "RawVolumeConfig" and .provisioning.diskSelector.match != "system_disk") | .provisioning.minSize' "$patch")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Deletes everything this phase created, and NOTHING else.
|
|
|
|
|
#
|
|
|
|
|
# That distinction is not theoretical: the project this runs in also holds
|
|
|
|
|
# netbird.renner.uno, which has been up for months and is the VPN way in. The
|
|
|
|
|
# label is the only thing separating a throwaway server from it — and
|
|
|
|
|
# `hcloud server list -l ""` lists EVERY server, so an unset HCLOUD_LABEL would
|
|
|
|
|
# turn this function into a project wipe. Hence the guard: the label is checked
|
|
|
|
|
# before it is used, and every candidate must also carry the name prefix this
|
|
|
|
|
# phase assigns. Two independent conditions, because one of them is a variable.
|
|
|
|
|
#
|
|
|
|
|
# A separate Hetzner project, with a token scoped to it, would make this
|
|
|
|
|
# structural rather than defensive. Worth doing before this runs unattended.
|
|
|
|
|
# The project (homelab-backup-verify) also runs the backup-verify CronJob's
|
|
|
|
|
# machines. `hcloud server list -l ""` lists EVERY server, so an unset
|
|
|
|
|
# HCLOUD_LABEL would turn this function into a project wipe: the label is
|
|
|
|
|
# checked before it is used, and every candidate must also carry the name
|
|
|
|
|
# prefix this phase assigns.
|
|
|
|
|
HCLOUD_NAME_PREFIX="hcloud"
|
|
|
|
|
# Outlives the run: the test cluster's talosconfig and kubeconfig.
|
|
|
|
|
HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest"
|
|
|
|
|
# The public edge in front of production. The test cluster must not reach it.
|
|
|
|
|
HCLOUD_PROXY_HOST="proxy.renner.uno"
|
|
|
|
|
# Hetzner's limit on effective rules per firewall and direction.
|
|
|
|
|
HCLOUD_FW_RULE_LIMIT=500
|
|
|
|
|
# Hetzner's limit on prefixes in one rule (not documented; 101 is rejected).
|
|
|
|
|
HCLOUD_FW_RULE_PREFIXES=100
|
|
|
|
|
|
|
|
|
|
# The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when
|
|
|
|
|
# missing: a throwaway server in rescue mode writes the Image Factory hcloud
|
|
|
|
|
# image to its disk, which is then snapshotted. Matched by version and
|
|
|
|
|
# schematic. Called directly, not in $(...), so that fail stops the run.
|
|
|
|
|
hcloud_talos_snapshot() {
|
|
|
|
|
local labels="os=talos,talos-version=$VM_VERSION,talos-schematic=${VM_SCHEMATIC:0:16}"
|
|
|
|
|
local builder="$HCLOUD_NAME_PREFIX-imgbuild" key_dir addr ids listing
|
|
|
|
|
listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \
|
|
|
|
|
|| fail "could not list the Talos snapshots"
|
|
|
|
|
mapfile -t ids <<<"$listing"
|
|
|
|
|
if [[ -n "${ids[0]:-}" ]]; then
|
|
|
|
|
HCLOUD_IMAGE=${ids[0]}
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
echo " no snapshot for $VM_VERSION/${VM_SCHEMATIC:0:16}, building one"
|
|
|
|
|
key_dir=$(mktemp -d)
|
|
|
|
|
ssh-keygen -q -t ed25519 -N '' -C "$builder" -f "$key_dir/key"
|
|
|
|
|
if hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=name | grep -qx "$builder"; then
|
|
|
|
|
hcloud ssh-key delete "$builder" >/dev/null || fail "could not replace the stale image builder key"
|
|
|
|
|
fi
|
|
|
|
|
hcloud ssh-key create --name "$builder" --label "$HCLOUD_LABEL" \
|
|
|
|
|
--public-key-from-file "$key_dir/key.pub" >/dev/null || fail "could not upload the image builder key"
|
|
|
|
|
hcloud server create --name "$builder" --type cx23 --image debian-12 \
|
|
|
|
|
--location "${HCLOUD_LOCATION:-fsn1}" --ssh-key "$builder" --label "$HCLOUD_LABEL" >/dev/null \
|
|
|
|
|
|| fail "could not create the image builder"
|
|
|
|
|
# From here a failure leaves a billed server: remove it before failing.
|
|
|
|
|
builder_fail() {
|
|
|
|
|
hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2
|
|
|
|
|
hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2
|
|
|
|
|
rm -rf "$key_dir"
|
|
|
|
|
fail "$1"
|
|
|
|
|
}
|
|
|
|
|
hcloud server enable-rescue "$builder" --ssh-key "$builder" >/dev/null || builder_fail "could not enable rescue on $builder"
|
|
|
|
|
hcloud server reset "$builder" >/dev/null || builder_fail "could not reset $builder into rescue"
|
|
|
|
|
addr=$(hcloud server ip "$builder") || builder_fail "could not read the address of $builder"
|
|
|
|
|
for i in $(seq 1 30); do
|
|
|
|
|
ssh -i "$key_dir/key" -o BatchMode=yes -o ConnectTimeout=10 \
|
|
|
|
|
-o UserKnownHostsFile="$key_dir/known_hosts" -o StrictHostKeyChecking=accept-new \
|
|
|
|
|
"root@$addr" 'grep -q rescue /etc/hostname || hostname | grep -q rescue' && break
|
|
|
|
|
sleep 10
|
|
|
|
|
[[ $i -eq 30 ]] && builder_fail "$builder never came up in rescue"
|
|
|
|
|
done
|
|
|
|
|
ssh -i "$key_dir/key" -o BatchMode=yes -o UserKnownHostsFile="$key_dir/known_hosts" "root@$addr" \
|
|
|
|
|
"set -eo pipefail; curl -fsSL https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz | xz -d | dd of=/dev/sda bs=4M conv=fsync" \
|
|
|
|
|
|| builder_fail "writing the Talos image on $builder failed"
|
|
|
|
|
hcloud server poweroff "$builder" >/dev/null || builder_fail "could not power off $builder"
|
|
|
|
|
hcloud server create-image "$builder" --type snapshot \
|
|
|
|
|
--description "talos $VM_VERSION ${VM_SCHEMATIC:0:16}" --label "$labels" \
|
|
|
|
|
|| builder_fail "could not snapshot $builder"
|
|
|
|
|
listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \
|
|
|
|
|
|| builder_fail "could not list the Talos snapshots"
|
|
|
|
|
mapfile -t ids <<<"$listing"
|
|
|
|
|
[[ -n "${ids[0]:-}" ]] || builder_fail "the snapshot of $builder is not listed under $labels"
|
|
|
|
|
HCLOUD_IMAGE=${ids[0]}
|
|
|
|
|
hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2
|
|
|
|
|
hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2
|
|
|
|
|
rm -rf "$key_dir"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# One address as a string of bits: 32 for IPv4, 128 for IPv6. Returns 1, with
|
|
|
|
|
# the reason on stderr, for anything else.
|
|
|
|
|
ip_to_bits() {
|
|
|
|
|
local addr=$1 bits="" part v i head tail
|
|
|
|
|
local -a parts=() hparts=() tparts=()
|
|
|
|
|
if [[ "$addr" == *:* ]]; then
|
|
|
|
|
if [[ "$addr" == *::* ]]; then
|
|
|
|
|
head=${addr%%::*}; tail=${addr#*::}
|
|
|
|
|
[[ -n "$head" ]] && IFS=: read -r -a hparts <<<"$head"
|
|
|
|
|
[[ -n "$tail" ]] && IFS=: read -r -a tparts <<<"$tail"
|
|
|
|
|
parts=("${hparts[@]}")
|
|
|
|
|
for (( i = ${#hparts[@]} + ${#tparts[@]}; i < 8; i++ )); do parts+=(0); done
|
|
|
|
|
parts+=("${tparts[@]}")
|
|
|
|
|
else
|
|
|
|
|
IFS=: read -r -a parts <<<"$addr"
|
|
|
|
|
fi
|
|
|
|
|
[[ ${#parts[@]} -eq 8 ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; }
|
|
|
|
|
for part in "${parts[@]}"; do
|
|
|
|
|
[[ "$part" =~ ^[0-9a-fA-F]{1,4}$ ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; }
|
|
|
|
|
v=$(( 16#$part ))
|
|
|
|
|
for (( i = 15; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done
|
|
|
|
|
done
|
|
|
|
|
else
|
|
|
|
|
IFS=. read -r -a parts <<<"$addr"
|
|
|
|
|
[[ ${#parts[@]} -eq 4 ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
|
|
|
|
|
for part in "${parts[@]}"; do
|
|
|
|
|
[[ "$part" =~ ^[0-9]{1,3}$ ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
|
|
|
|
|
v=$(( 10#$part ))
|
|
|
|
|
(( v <= 255 )) || { echo "cannot read IPv4 address '$addr'" >&2; return 1; }
|
|
|
|
|
for (( i = 7; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done
|
|
|
|
|
done
|
|
|
|
|
fi
|
|
|
|
|
printf '%s' "$bits"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# A prefix given as bits, in CIDR notation. $2 is the address length (32|128).
|
|
|
|
|
bits_to_cidr() {
|
|
|
|
|
local p=$1 width=$2 full out="" i
|
|
|
|
|
full=$p
|
|
|
|
|
while (( ${#full} < width )); do full+="0"; done
|
|
|
|
|
if (( width == 32 )); then
|
|
|
|
|
for (( i = 0; i < 32; i += 8 )); do out+="${out:+.}$(( 2#${full:i:8} ))"; done
|
|
|
|
|
else
|
|
|
|
|
for (( i = 0; i < 128; i += 16 )); do out+="${out:+:}$(printf '%x' $(( 2#${full:i:16} )))"; done
|
|
|
|
|
fi
|
|
|
|
|
printf '%s/%s\n' "$out" "${#p}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# cidr_complement <address>...: the prefixes that, with the given addresses
|
|
|
|
|
# of one family, cover 0.0.0.0/0 or ::/0 exactly once. Returns 1, with the
|
|
|
|
|
# reason on stderr, on a malformed or mixed-family list.
|
|
|
|
|
cidr_complement() {
|
|
|
|
|
local width=32 a bits
|
|
|
|
|
local -a excl=()
|
|
|
|
|
[[ "$1" == *:* ]] && width=128
|
|
|
|
|
for a in "$@"; do
|
|
|
|
|
[[ ( "$a" == *:* && $width -eq 128 ) || ( "$a" != *:* && $width -eq 32 ) ]] \
|
|
|
|
|
|| { echo "cidr_complement: '$a' is not of the same family as '$1'" >&2; return 1; }
|
|
|
|
|
bits=$(ip_to_bits "$a") || return 1
|
|
|
|
|
excl+=("$bits")
|
|
|
|
|
done
|
|
|
|
|
_cidr_walk() {
|
|
|
|
|
local p=$1 x hit=0
|
|
|
|
|
for x in "${excl[@]}"; do
|
|
|
|
|
[[ "$x" == "$p"* ]] && { hit=1; break; }
|
|
|
|
|
done
|
|
|
|
|
if (( hit == 0 )); then
|
|
|
|
|
bits_to_cidr "$p" "$width"
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
(( ${#p} == width )) && return 0
|
|
|
|
|
_cidr_walk "${p}0"
|
|
|
|
|
_cidr_walk "${p}1"
|
|
|
|
|
}
|
|
|
|
|
_cidr_walk ""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Addresses of HCLOUD_PROXY_HOST for one record type (A|AAAA), over DoH: the
|
|
|
|
|
# operator's LAN resolver intercepts plain DNS. Returns 1, with the reason on
|
|
|
|
|
# stderr, when the lookup fails.
|
|
|
|
|
hcloud_proxy_addrs() {
|
|
|
|
|
local type=$1 num answer
|
|
|
|
|
case "$type" in
|
|
|
|
|
A) num=1 ;;
|
|
|
|
|
AAAA) num=28 ;;
|
|
|
|
|
*) echo "hcloud_proxy_addrs: unknown type $type" >&2; return 1 ;;
|
|
|
|
|
esac
|
|
|
|
|
answer=$(curl -sS --max-time 10 -H 'accept: application/dns-json' \
|
|
|
|
|
"https://cloudflare-dns.com/dns-query?name=$HCLOUD_PROXY_HOST&type=$type") \
|
|
|
|
|
|| { echo "could not resolve $HCLOUD_PROXY_HOST $type over DoH" >&2; return 1; }
|
|
|
|
|
jq -e '.Status == 0' >/dev/null <<<"$answer" \
|
|
|
|
|
|| { echo "DoH lookup of $HCLOUD_PROXY_HOST $type failed: $answer" >&2; return 1; }
|
|
|
|
|
jq -r --argjson t "$num" '.Answer[]? | select(.type == $t) | .data' <<<"$answer" | sort -u
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Outbound firewall for the nodes: udp and icmp anywhere, tcp anywhere except
|
|
|
|
|
# HCLOUD_PROXY_HOST. Attached before an earlier run's egress firewall is
|
|
|
|
|
# removed, so no node is ever without one.
|
|
|
|
|
hcloud_apply_egress() {
|
|
|
|
|
local fw="hcloudtest-node-egress-$$" v4 v6 c4 c6 n count fw_id fw_name srv_id
|
|
|
|
|
local -a a4=() a6=() tcp_dst=()
|
|
|
|
|
v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)"
|
|
|
|
|
v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)"
|
|
|
|
|
[[ -n "$v4" ]] && mapfile -t a4 <<<"$v4"
|
|
|
|
|
[[ -n "$v6" ]] && mapfile -t a6 <<<"$v6"
|
|
|
|
|
(( ${#a4[@]} + ${#a6[@]} > 0 )) || fail "$HCLOUD_PROXY_HOST resolved to no address — refusing to build the egress firewall"
|
|
|
|
|
echo " blocking tcp to $HCLOUD_PROXY_HOST: ${a4[*]} ${a6[*]}"
|
|
|
|
|
if (( ${#a4[@]} > 0 )); then
|
|
|
|
|
c4=$(cidr_complement "${a4[@]}") || fail "could not compute the IPv4 complement of ${a4[*]}"
|
|
|
|
|
else
|
|
|
|
|
c4=0.0.0.0/0
|
|
|
|
|
fi
|
|
|
|
|
if (( ${#a6[@]} > 0 )); then
|
|
|
|
|
c6=$(cidr_complement "${a6[@]}") || fail "could not compute the IPv6 complement of ${a6[*]}"
|
|
|
|
|
else
|
|
|
|
|
c6=::/0
|
|
|
|
|
fi
|
|
|
|
|
mapfile -t tcp_dst <<<"$c4"$'\n'"$c6"
|
|
|
|
|
# Effective rules are counted per destination prefix; 0.0.0.0/0 and ::/0
|
|
|
|
|
# carry udp and icmp.
|
|
|
|
|
count=$(printf '%s\n' 0.0.0.0/0 ::/0 "${tcp_dst[@]}" | sort -u | grep -c .)
|
|
|
|
|
(( count < HCLOUD_FW_RULE_LIMIT )) \
|
|
|
|
|
|| fail "the egress firewall needs $count effective rules, Hetzner allows $HCLOUD_FW_RULE_LIMIT per firewall"
|
|
|
|
|
hcloud firewall create --name "$fw" --label "$HCLOUD_LABEL" >/dev/null || fail "could not create $fw"
|
|
|
|
|
hcloud firewall add-rule "$fw" --direction out --protocol udp --port any \
|
|
|
|
|
--destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw"
|
|
|
|
|
hcloud firewall add-rule "$fw" --direction out --protocol icmp \
|
|
|
|
|
--destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw"
|
|
|
|
|
for (( n = 0; n < ${#tcp_dst[@]}; n += HCLOUD_FW_RULE_PREFIXES )); do
|
|
|
|
|
hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \
|
|
|
|
|
--destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]:n:HCLOUD_FW_RULE_PREFIXES}")" \
|
|
|
|
|
--description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw"
|
|
|
|
|
done
|
|
|
|
|
for n in "${HCLOUD_NODES[@]}"; do
|
|
|
|
|
hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \
|
|
|
|
|
|| fail "could not attach $fw to $n"
|
|
|
|
|
done
|
|
|
|
|
while read -r fw_id fw_name; do
|
|
|
|
|
[[ -n "$fw_id" && "$fw_name" == hcloudtest-node-egress-* && "$fw_name" != "$fw" ]] || continue
|
|
|
|
|
while read -r srv_id; do
|
|
|
|
|
[[ -n "$srv_id" ]] || continue
|
|
|
|
|
hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \
|
|
|
|
|
|| fail "could not detach $fw_name from server $srv_id"
|
|
|
|
|
done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id')
|
|
|
|
|
hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name"
|
|
|
|
|
echo " removed $fw_name"
|
|
|
|
|
done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
|
|
|
|
|
echo " $fw: $count effective rules"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Runs restore-test/egress-check.yaml in the cluster KUBECONFIG names and fails
|
|
|
|
|
# unless every expectation holds.
|
|
|
|
|
hcloud_egress_check() {
|
|
|
|
|
local addrs phase=""
|
|
|
|
|
step 0 "Checking egress containment from a pod..."
|
|
|
|
|
addrs=$( { hcloud_proxy_addrs A; hcloud_proxy_addrs AAAA; } | paste -sd' ' -) \
|
|
|
|
|
|| fail "could not resolve $HCLOUD_PROXY_HOST"
|
|
|
|
|
[[ -n "$addrs" ]] || fail "$HCLOUD_PROXY_HOST resolved to no address"
|
|
|
|
|
kubectl delete pod egress-check --ignore-not-found --wait=true >/dev/null
|
|
|
|
|
sed "s|PROXY_ADDRS_PLACEHOLDER|$addrs|" "$REPO_ROOT/kubernetes/restore-test/egress-check.yaml" \
|
|
|
|
|
| kubectl apply -f - >/dev/null || fail "could not start the egress check"
|
|
|
|
|
for i in $(seq 1 48); do
|
|
|
|
|
phase=$(kubectl get pod egress-check -o jsonpath='{.status.phase}') || phase=""
|
|
|
|
|
[[ "$phase" == Succeeded || "$phase" == Failed ]] && break
|
|
|
|
|
sleep 5
|
|
|
|
|
done
|
|
|
|
|
if [[ "$phase" != Succeeded && "$phase" != Failed ]]; then
|
|
|
|
|
kubectl describe pod egress-check | sed 's/^/ /' || true
|
|
|
|
|
fail "the egress check never finished (phase '${phase:-none}') — do not start Flux"
|
|
|
|
|
fi
|
|
|
|
|
kubectl logs egress-check | sed 's/^/ /' || true
|
|
|
|
|
[[ "$phase" == "Succeeded" ]] || fail "egress containment does not hold — do not start Flux"
|
|
|
|
|
kubectl delete pod egress-check --wait=false >/dev/null
|
|
|
|
|
ok
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cleanup_hcloudtest() {
|
|
|
|
|
command -v hcloud >/dev/null || return 0
|
|
|
|
@@ -596,6 +892,11 @@ cleanup_hcloudtest() {
|
|
|
|
|
hcloud volume delete "$id" >/dev/null \
|
|
|
|
|
|| echo " WARNING: volume $id not deleted — check the console" >&2
|
|
|
|
|
done
|
|
|
|
|
for id in $(hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
|
|
|
|
echo " deleting ssh key $id"
|
|
|
|
|
hcloud ssh-key delete "$id" >/dev/null \
|
|
|
|
|
|| echo " WARNING: ssh key $id not deleted — check the console" >&2
|
|
|
|
|
done
|
|
|
|
|
# Last: a network with a server still attached cannot be deleted, and a
|
|
|
|
|
# leftover one takes its name with it, so the next run cannot recreate it.
|
|
|
|
|
for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do
|
|
|
|
@@ -605,6 +906,17 @@ cleanup_hcloudtest() {
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# The restore test's source, as restore-test/flux-root.yaml names it: sets
|
|
|
|
|
# restore_url and restore_repo (<owner>/<repo>).
|
|
|
|
|
restore_test_repo() {
|
|
|
|
|
command -v yq >/dev/null || fail "yq not found (reads restore-test/flux-root.yaml)"
|
|
|
|
|
restore_url=$(yq -r 'select(.kind == "GitRepository" and .metadata.name == "restore-test") | .spec.url' \
|
|
|
|
|
"$REPO_ROOT/kubernetes/restore-test/flux-root.yaml")
|
|
|
|
|
[[ "$restore_url" =~ ^ssh://git@github\.com/([^/]+)/([^/]+)\.git$ ]] \
|
|
|
|
|
|| fail "restore-test/flux-root.yaml names '$restore_url', not ssh://git@github.com/<owner>/<repo>.git"
|
|
|
|
|
restore_repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Sweep only. For the case where a previous run was killed hard enough that its
|
|
|
|
|
# own trap never ran, and for a cron job to call before it starts.
|
|
|
|
|
if [[ "$PHASE" == "hcloudclean" ]]; then
|
|
|
|
@@ -613,6 +925,20 @@ if [[ "$PHASE" == "hcloudclean" ]]; then
|
|
|
|
|
require_secrets hcloud-token
|
|
|
|
|
HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN
|
|
|
|
|
cleanup_hcloudtest
|
|
|
|
|
require_secrets s3-access s3-secret
|
|
|
|
|
delete_cnpg_test_bucket
|
|
|
|
|
# The restore test's read-only deploy keys on the GitHub mirror.
|
|
|
|
|
if command -v gh >/dev/null; then
|
|
|
|
|
restore_test_repo
|
|
|
|
|
for key_id in $(gh api "repos/$restore_repo/keys" \
|
|
|
|
|
--jq '.[] | select(.title | startswith("restore-test ")) | .id'); do
|
|
|
|
|
echo " deleting deploy key $key_id"
|
|
|
|
|
gh api -X DELETE "repos/$restore_repo/keys/$key_id" \
|
|
|
|
|
|| echo " WARNING: deploy key $key_id not deleted — check the repo settings" >&2
|
|
|
|
|
done
|
|
|
|
|
else
|
|
|
|
|
echo " WARNING: gh not found — restore-test deploy keys on the GitHub mirror not checked" >&2
|
|
|
|
|
fi
|
|
|
|
|
ok
|
|
|
|
|
exit 0
|
|
|
|
|
fi
|
|
|
|
@@ -624,6 +950,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
command -v yq >/dev/null || fail "yq not found"
|
|
|
|
|
command -v jq >/dev/null || fail "jq not found (needed to read the SAN pool and server attachments)"
|
|
|
|
|
command -v curl >/dev/null || fail "curl not found (needed to learn this host's public address)"
|
|
|
|
|
command -v bao >/dev/null || fail "bao not found (the Hetzner token is read from OpenBao)"
|
|
|
|
|
command -v kubectl >/dev/null || fail "kubectl not found (runs the egress check)"
|
|
|
|
|
|
|
|
|
|
# Three control planes, all carrying workloads — the production shape. One
|
|
|
|
|
# machine is a node, not a cluster: etcd has no quorum to lose and nothing
|
|
|
|
@@ -642,18 +970,11 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
# this one, adopted below. The names are fixed, so duplicates cannot
|
|
|
|
|
# accumulate. `hcloudclean` is what deletes.
|
|
|
|
|
|
|
|
|
|
# The Talos snapshot is a prerequisite, not something this builds: making it
|
|
|
|
|
# means booting a server into rescue and writing the raw image, which is a
|
|
|
|
|
# one-off and needs a different set of permissions. Fail with the recipe
|
|
|
|
|
# rather than silently booting Debian and timing out on the Talos API.
|
|
|
|
|
step 0 "Looking up the Talos snapshot..."
|
|
|
|
|
talos_image_coords
|
|
|
|
|
HCLOUD_IMAGE=$(hcloud image list --type snapshot -l os=talos -o noheader -o columns=id | head -1)
|
|
|
|
|
[[ -n "$HCLOUD_IMAGE" ]] || fail "no snapshot labelled os=talos. Create one once — boot any server into rescue, then:
|
|
|
|
|
wget -O /tmp/talos.raw.xz https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz
|
|
|
|
|
xz -d -c /tmp/talos.raw.xz | dd of=/dev/sda && sync && shutdown -h now
|
|
|
|
|
then snapshot that server in the console and label the snapshot os=talos.
|
|
|
|
|
The schematic and version above are this cluster's, read from cluster.yaml."
|
|
|
|
|
HCLOUD_IMAGE=""
|
|
|
|
|
hcloud_talos_snapshot
|
|
|
|
|
[[ -n "$HCLOUD_IMAGE" ]] || fail "no Talos snapshot"
|
|
|
|
|
echo " snapshot $HCLOUD_IMAGE"
|
|
|
|
|
|
|
|
|
|
# The Talos API is unauthenticated in maintenance mode, which is how
|
|
|
|
@@ -696,26 +1017,30 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
--source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \
|
|
|
|
|
|| fail "could not add the Kubernetes API rule"
|
|
|
|
|
echo " open to $fw_src only"
|
|
|
|
|
# Servers held from an earlier run get this firewall first; only then are
|
|
|
|
|
# the earlier runs' inbound firewalls detached and deleted, so no server is
|
|
|
|
|
# ever without one. Egress firewalls are replaced by hcloud_apply_egress.
|
|
|
|
|
while read -r srv; do
|
|
|
|
|
[[ -n "$srv" ]] || continue
|
|
|
|
|
hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$srv" >/dev/null \
|
|
|
|
|
|| fail "could not attach this run's firewall to $srv"
|
|
|
|
|
done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name)
|
|
|
|
|
while read -r fw_id fw_name; do
|
|
|
|
|
[[ -n "$fw_id" && "$fw_name" != "hcloudtest-$$" && "$fw_name" != hcloudtest-node-egress-* ]] || continue
|
|
|
|
|
while read -r srv_id; do
|
|
|
|
|
[[ -n "$srv_id" ]] || continue
|
|
|
|
|
hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \
|
|
|
|
|
|| fail "could not detach $fw_name from server $srv_id"
|
|
|
|
|
done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id')
|
|
|
|
|
hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name"
|
|
|
|
|
echo " removed $fw_name"
|
|
|
|
|
done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name)
|
|
|
|
|
ok
|
|
|
|
|
|
|
|
|
|
# cx43 — 16 GB, 8 vCPU. Sized from what the workload needs, not from the
|
|
|
|
|
# Talos guide, which names cx22 as a minimal example (and that type no
|
|
|
|
|
# longer exists; `server create` rejects it outright).
|
|
|
|
|
# Default type cpx52 (12 vCPU, 24 GB, 480 GB local disk): the hcloud-N
|
|
|
|
|
# patches put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system
|
|
|
|
|
# disk. cax* are ARM and do not match the amd64 schematic.
|
|
|
|
|
#
|
|
|
|
|
# cx23 was tried first, on the reasoning that it is cx22's successor. It has
|
|
|
|
|
# 4 GB, 3.2 GB of it allocatable, and the three nodes sat at 59-78% of that
|
|
|
|
|
# with the infrastructure skeleton alone — no Ceph OSDs, no OpenBao running,
|
|
|
|
|
# not one app. Limits were already at 134-207% of allocatable and both
|
|
|
|
|
# kube-controller-manager and kube-scheduler had restarted twice.
|
|
|
|
|
#
|
|
|
|
|
# The local rehearsal is the reference: it runs a 16GiB control plane and
|
|
|
|
|
# three 8GiB workers, 40GiB in total, and needed every step of the way up
|
|
|
|
|
# from 6GiB to get there. Three nodes here carry the same workload plus a
|
|
|
|
|
# control plane each, so 48GiB total is the comparable figure.
|
|
|
|
|
#
|
|
|
|
|
# Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which
|
|
|
|
|
# would not match the amd64 schematic this cluster installs. That leaves the
|
|
|
|
|
# cx*/cpx*2 families in the German locations.
|
|
|
|
|
# Two segments only, unlike the homelab's split in
|
|
|
|
|
# NETWORK-traffic-classes.md: here the node address carries management,
|
|
|
|
|
# cluster-internal and external alike, and Ceph replication has its own.
|
|
|
|
@@ -735,7 +1060,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
seg_range() {
|
|
|
|
|
case "$1" in
|
|
|
|
|
cluster)
|
|
|
|
|
yq -r '.machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;;
|
|
|
|
|
yq -r 'select(.machine) | .machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;;
|
|
|
|
|
storage)
|
|
|
|
|
local pool
|
|
|
|
|
pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range')
|
|
|
|
@@ -797,12 +1122,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
echo " $n: reusing the server we already hold"
|
|
|
|
|
hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \
|
|
|
|
|
|| fail "could not rebuild $n back into maintenance mode"
|
|
|
|
|
# The firewall is named per run (hcloudtest-$$), so an adopted
|
|
|
|
|
# server is still attached to the previous run's. Attach this run's
|
|
|
|
|
# too — without it the Talos API is unreachable once the old one is
|
|
|
|
|
# cleaned up, and the addresses in it are last run's anyway.
|
|
|
|
|
hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$n" >/dev/null \
|
|
|
|
|
|| fail "could not attach this run's firewall to $n"
|
|
|
|
|
# This run's firewall was attached above, before the old ones went.
|
|
|
|
|
else
|
|
|
|
|
# Retried, because "error during placement (resource_unavailable)"
|
|
|
|
|
# is the provider being momentarily out of that type in that
|
|
|
|
@@ -814,13 +1134,13 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
create_err=""
|
|
|
|
|
for attempt in $(seq 1 30); do
|
|
|
|
|
create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \
|
|
|
|
|
--type "${HCLOUD_TYPE:-cx43}" --location "${HCLOUD_LOCATION:-fsn1}" \
|
|
|
|
|
--type "${HCLOUD_TYPE:-cpx52}" --location "${HCLOUD_LOCATION:-fsn1}" \
|
|
|
|
|
--firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break
|
|
|
|
|
grep -qi 'resource_unavailable\|placement' <<<"$create_err" \
|
|
|
|
|
|| fail "could not create $n: ${create_err//$'\n'/ | }"
|
|
|
|
|
[[ $attempt -eq 1 ]] && echo " $n: no capacity yet, retrying"
|
|
|
|
|
sleep 20
|
|
|
|
|
[[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cx43} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes"
|
|
|
|
|
[[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cpx52} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes"
|
|
|
|
|
done
|
|
|
|
|
fi
|
|
|
|
|
addr=$(hcloud server ip "$n") || fail "could not read the address of $n"
|
|
|
|
@@ -863,13 +1183,18 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
hcloud volume delete "$n-osd" >/dev/null \
|
|
|
|
|
|| fail "could not delete the previous OSD volume for $n"
|
|
|
|
|
fi
|
|
|
|
|
yq -e 'select(.kind == "RawVolumeConfig" and .name == "osd-data")' "$TALOS_DIR/patches/$n.yaml" >/dev/null \
|
|
|
|
|
|| fail "$n declares no osd-data RawVolumeConfig — a node with no OSD device is not this cluster's shape"
|
|
|
|
|
# A raw volume on the system disk needs no Hetzner volume.
|
|
|
|
|
vol_size=$(hcloud_volumes_for_node "$n")
|
|
|
|
|
[[ -n "$vol_size" ]] || fail "$n declares no RawVolumeConfig — a node with no OSD device is not this cluster's shape"
|
|
|
|
|
hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \
|
|
|
|
|
--label "$HCLOUD_LABEL" >/dev/null \
|
|
|
|
|
|| fail "could not create the OSD volume for $n"
|
|
|
|
|
|
|
|
|
|
echo " $n at $addr, with a ${vol_size}GB OSD volume"
|
|
|
|
|
if [[ -n "$vol_size" ]]; then
|
|
|
|
|
hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \
|
|
|
|
|
--label "$HCLOUD_LABEL" >/dev/null \
|
|
|
|
|
|| fail "could not create the OSD volume for $n"
|
|
|
|
|
echo " $n at $addr, with a ${vol_size}GB OSD volume"
|
|
|
|
|
else
|
|
|
|
|
echo " $n at $addr, OSD on the system disk"
|
|
|
|
|
fi
|
|
|
|
|
NODE_ADDRS+=("$addr")
|
|
|
|
|
done
|
|
|
|
|
ok
|
|
|
|
@@ -898,20 +1223,26 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
|
|
|
|
echo " $node_src"
|
|
|
|
|
ok
|
|
|
|
|
|
|
|
|
|
# Handover, and this list is the whole of it — the same four variables the
|
|
|
|
|
# vmtest phase sets. If it grows, the test is drifting away from production.
|
|
|
|
|
# Before any node is configured.
|
|
|
|
|
step 0 "Closing tcp egress to $HCLOUD_PROXY_HOST (Hetzner firewall)..."
|
|
|
|
|
hcloud_apply_egress
|
|
|
|
|
ok
|
|
|
|
|
|
|
|
|
|
# Handover: the variables the vmtest phase sets, and HCLOUDTEST.
|
|
|
|
|
SITE=hcloud
|
|
|
|
|
NODES=("${HCLOUD_NODES[@]}")
|
|
|
|
|
WORKERS=()
|
|
|
|
|
ALL_NODES=("${NODES[@]}")
|
|
|
|
|
K8S_API_HOST="${NODE_ADDRS[0]}"
|
|
|
|
|
HCLOUDTEST=1
|
|
|
|
|
|
|
|
|
|
# Isolation from the production cluster. Both halves are required: the
|
|
|
|
|
# _PATH variables are derived before any phase runs, and the talos phase
|
|
|
|
|
# writes to those.
|
|
|
|
|
HCLOUD_DIR=$(mktemp -d)
|
|
|
|
|
export TALOSCONFIG="$HCLOUD_DIR/talosconfig"
|
|
|
|
|
export KUBECONFIG="$HCLOUD_DIR/kubeconfig"
|
|
|
|
|
mkdir -p "$HCLOUD_STATE_DIR"
|
|
|
|
|
chmod 700 "$HCLOUD_STATE_DIR"
|
|
|
|
|
export TALOSCONFIG="$HCLOUD_STATE_DIR/talosconfig"
|
|
|
|
|
export KUBECONFIG="$HCLOUD_STATE_DIR/kubeconfig"
|
|
|
|
|
TALOSCONFIG_PATH="$TALOSCONFIG"
|
|
|
|
|
KUBECONFIG_PATH="$KUBECONFIG"
|
|
|
|
|
|
|
|
|
@@ -1578,9 +1909,13 @@ EOF
|
|
|
|
|
ok
|
|
|
|
|
|
|
|
|
|
# Stash configs for subsequent phases / export-access step. Skipped under
|
|
|
|
|
# vmtest: those are a throwaway cluster's credentials, and writing them here
|
|
|
|
|
# would leave the flux phase pointed at VMs that no longer exist.
|
|
|
|
|
if [[ "${VMTEST:-0}" == "1" ]]; then
|
|
|
|
|
# vmtest and hcloudtest: those are a throwaway cluster's credentials, and a
|
|
|
|
|
# later production `bootstrap.sh flux` copies access-staging/ over the
|
|
|
|
|
# operator's kubeconfig and talosconfig.
|
|
|
|
|
if [[ "${HCLOUDTEST:-0}" == "1" ]]; then
|
|
|
|
|
hcloud_egress_check
|
|
|
|
|
echo -e "${GREEN}Talos phase complete.${NC} Access files in $HCLOUD_STATE_DIR/ (not staged)"
|
|
|
|
|
elif [[ "${VMTEST:-0}" == "1" ]]; then
|
|
|
|
|
# Nodes being Ready says nothing about storage. The CephCluster picks
|
|
|
|
|
# OSD nodes by a label out of the machine config and their device by the
|
|
|
|
|
# partition label RawVolumeConfig creates, so assert both here — running
|
|
|
|
@@ -1829,7 +2164,9 @@ bring_up_openbao() {
|
|
|
|
|
# entered by mistake. Wait for the container to be running first.
|
|
|
|
|
echo -n " waiting for ${pods[0]} to answer"
|
|
|
|
|
local status="" status_err=""
|
|
|
|
|
for i in $(seq 1 90); do
|
|
|
|
|
# Bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to
|
|
|
|
|
# provision its volume.
|
|
|
|
|
for i in $(seq 1 180); do
|
|
|
|
|
if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then
|
|
|
|
|
echo; break
|
|
|
|
|
fi
|
|
|
|
@@ -1847,7 +2184,7 @@ bring_up_openbao() {
|
|
|
|
|
if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi
|
|
|
|
|
printf '.'
|
|
|
|
|
sleep 10
|
|
|
|
|
if [[ $i -eq 90 ]]; then
|
|
|
|
|
if [[ $i -eq 180 ]]; then
|
|
|
|
|
echo
|
|
|
|
|
echo " last error: $status_err" >&2
|
|
|
|
|
fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised"
|
|
|
|
@@ -1956,9 +2293,33 @@ if phase_runs flux; then
|
|
|
|
|
case "$FLUX_SOURCE" in
|
|
|
|
|
gitea) GITEA_TOKEN=$(read_secret git-token); export GITEA_TOKEN ;;
|
|
|
|
|
github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;;
|
|
|
|
|
*) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github)" ;;
|
|
|
|
|
restore-test)
|
|
|
|
|
GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN
|
|
|
|
|
command -v gh >/dev/null || fail "gh not found (registers the read-only deploy key)"
|
|
|
|
|
command -v jq >/dev/null || fail "jq not found (reads the DoH answer for the egress check)"
|
|
|
|
|
# The throwaway cluster carries production secrets, and this phase
|
|
|
|
|
# copies access files over KUBECONFIG_PATH and TALOSCONFIG_PATH:
|
|
|
|
|
# neither may be the operator's production file.
|
|
|
|
|
[[ "$KUBECONFIG_PATH" != "$HOME/.kube/config" && "$TALOSCONFIG_PATH" != "$HOME/.talos/config" ]] \
|
|
|
|
|
|| fail "FLUX_SOURCE=restore-test needs KUBECONFIG and TALOSCONFIG pointed at the test cluster ($HCLOUD_STATE_DIR)"
|
|
|
|
|
;;
|
|
|
|
|
*) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github|restore-test)" ;;
|
|
|
|
|
esac
|
|
|
|
|
[[ "$PHASE" == "flux" ]] && load_staged_access
|
|
|
|
|
# The restore test reads its access from KUBECONFIG/TALOSCONFIG, which
|
|
|
|
|
# hcloudtest writes; access-staging/ may hold an older run's files.
|
|
|
|
|
[[ "$PHASE" == "flux" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access
|
|
|
|
|
if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
|
|
|
|
|
# Whatever the paths say, the cluster itself must be the test cluster,
|
|
|
|
|
# before anything below writes a secret to it.
|
|
|
|
|
node_names=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') \
|
|
|
|
|
|| fail "cannot list the nodes of the target cluster"
|
|
|
|
|
[[ -n "$node_names" ]] || fail "the target cluster has no nodes"
|
|
|
|
|
for n in $node_names; do
|
|
|
|
|
[[ "$n" == "$HCLOUD_NAME_PREFIX"-* ]] \
|
|
|
|
|
|| fail "node '$n' is not a $HCLOUD_NAME_PREFIX-* test node — refusing the restore test against this cluster"
|
|
|
|
|
done
|
|
|
|
|
hcloud_egress_check
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
echo -n "Creating SOPS age secret... "
|
|
|
|
|
kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f -
|
|
|
|
@@ -2000,9 +2361,43 @@ if phase_runs flux; then
|
|
|
|
|
|
|
|
|
|
# Before Flux: cnpg-object-stores fails without it rather than render empty
|
|
|
|
|
# paths, and the paths must exist before any Cluster bootstraps.
|
|
|
|
|
write_cnpg_paths "$CNPG_BUCKET"
|
|
|
|
|
if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
|
|
|
|
|
create_cnpg_test_bucket
|
|
|
|
|
write_cnpg_paths "$CNPG_TEST_BUCKET"
|
|
|
|
|
else
|
|
|
|
|
write_cnpg_paths "$CNPG_BUCKET"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [[ "$FLUX_SOURCE" == "github" ]]; then
|
|
|
|
|
if [[ "$FLUX_SOURCE" == "restore-test" ]]; then
|
|
|
|
|
# The production tree through kubernetes/restore-test/flux-root.yaml,
|
|
|
|
|
# whose patches are the test's whole deviation from production. Nothing
|
|
|
|
|
# is committed anywhere: the source is the GitHub mirror, read through a
|
|
|
|
|
# deploy key GitHub itself holds to read-only, so the test cluster
|
|
|
|
|
# cannot push (image automation) even with production's manifests.
|
|
|
|
|
step 0 "Bootstrapping FluxCD for the restore test (restore-test/flux-root.yaml)..."
|
|
|
|
|
flux install --components-extra=image-reflector-controller,image-automation-controller \
|
|
|
|
|
|| fail "flux install failed"
|
|
|
|
|
restore_test_repo
|
|
|
|
|
deploy_key="$OPENBAO_SECRETS_DIR/restore-test-git"
|
|
|
|
|
rm -f "$deploy_key" "$deploy_key.pub"
|
|
|
|
|
ssh-keygen -q -t ed25519 -N '' -C "restore-test" -f "$deploy_key"
|
|
|
|
|
gh api -X POST "repos/$restore_repo/keys" \
|
|
|
|
|
-f title="restore-test $(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
|
|
|
|
-f key="$(cat "$deploy_key.pub")" -F read_only=true >/dev/null \
|
|
|
|
|
|| fail "could not register the read-only deploy key on $restore_repo"
|
|
|
|
|
flux create secret git restore-test-git \
|
|
|
|
|
--url="$restore_url" \
|
|
|
|
|
--private-key-file="$deploy_key" >/dev/null \
|
|
|
|
|
|| fail "could not create the restore-test-git Secret"
|
|
|
|
|
# On FLUX_BRANCH, so a branch can be tried here before it is merged.
|
|
|
|
|
# shellcheck disable=SC2016 # $b is a jq variable
|
|
|
|
|
yq -y --arg b "$FLUX_BRANCH" \
|
|
|
|
|
'if .kind == "GitRepository" and .metadata.name == "restore-test" then .spec.ref.branch = $b else . end' \
|
|
|
|
|
"$REPO_ROOT/kubernetes/restore-test/flux-root.yaml" | kubectl apply -f - >/dev/null \
|
|
|
|
|
|| fail "could not apply restore-test/flux-root.yaml"
|
|
|
|
|
echo " source: $restore_url, branch $FLUX_BRANCH"
|
|
|
|
|
ok
|
|
|
|
|
elif [[ "$FLUX_SOURCE" == "github" ]]; then
|
|
|
|
|
step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..."
|
|
|
|
|
echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}"
|
|
|
|
|
echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}"
|
|
|
|
@@ -2106,7 +2501,8 @@ if phase_runs wait-apps; then
|
|
|
|
|
echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}"
|
|
|
|
|
command -v kubectl >/dev/null || fail "kubectl not found"
|
|
|
|
|
command -v flux >/dev/null || fail "flux not found"
|
|
|
|
|
[[ "$PHASE" == "wait-apps" ]] && load_staged_access
|
|
|
|
|
# The restore test reads its access from KUBECONFIG/TALOSCONFIG.
|
|
|
|
|
[[ "$PHASE" == "wait-apps" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access
|
|
|
|
|
|
|
|
|
|
# Wait for a Flux Kustomization to become Ready=True. Prints a status line
|
|
|
|
|
# whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a
|
|
|
|
|