diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 8fe80633..386e08ce 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -15,10 +15,17 @@ set -euo pipefail # # the only place the registry fallback can be # # tested: gitea.renner.uno has no external # # ingress, so a node there genuinely cannot -# # reach it. BILLS BY THE HOUR — teardown runs -# # from the EXIT trap and is by label. -# ./bootstrap.sh hcloudclean # that teardown on its own, for a run that was -# # killed before its trap could fire. +# # reach it. BILLS BY THE HOUR — the machines +# # stay up until hcloudclean. Access files land in +# # ~/.local/state/homelab-hcloudtest/. +# ./bootstrap.sh hcloudclean # deletes everything hcloudtest created, by +# # label, and the restore test's deploy keys. +# +# Restore test on the hcloudtest cluster (production tree, read-only source, +# kubernetes/restore-test/flux-root.yaml): +# KUBECONFIG=~/.local/state/homelab-hcloudtest/kubeconfig \ +# TALOSCONFIG=~/.local/state/homelab-hcloudtest/talosconfig \ +# FLUX_SOURCE=restore-test ./bootstrap.sh flux # # Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls # from is itself Flux-managed, so a cold rebuild cannot pull from it until the @@ -177,6 +184,10 @@ resolve_node_ip() { phase_runs() { [[ "$PHASE" == "all" || "$PHASE" == "$1" ]] } +# The restore test targets the cluster bootstrap.sh hcloudtest builds. +if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wait-apps" ]]; then + fail "FLUX_SOURCE=restore-test runs the flux and wait-apps phases only" +fi OPENBAO_NS="openbao" # Where the secrets come from. On a workstation, the operator's pass store. In @@ -218,7 +229,7 @@ OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}" # that cannot be told apart from a fault. Its non-zero exit propagates, which is # what makes require_secrets print the reason instead. git_token_for_source() { - if [[ "$FLUX_SOURCE" == "github" ]]; then + if [[ "$FLUX_SOURCE" == "github" || "$FLUX_SOURCE" == "restore-test" ]]; then if [[ -n "${GITHUB_TOKEN:-}" ]]; then printf '%s' "$GITHUB_TOKEN" else @@ -256,13 +267,13 @@ OPENBAO_REQUIRED_SECRETS=( # rest, because a rebuild that discovers it missing has already wiped the # disks. "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull" - # Only the hcloudtest and hcloudclean phases ask for this one, by name, so a - # rebuild of the real cluster never demands it. - # - # Note the path: homelab/hetzner/, spelled correctly, while the two restore - # keys above live under homelab/hetztner/. Two folders, one typo, both real - # — do not "fix" one of these paths without moving the entry first. - "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|pass show homelab/hetzner/cloud-api-secret" +) +# Asked for by name only, never by a bare require_secrets: a rebuild of the +# real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the +# Hetzner project homelab-backup-verify, which holds nothing but throwaway +# machines. +HCLOUD_REQUIRED_SECRETS=( + "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|bao kv get -mount=homelab -field=token infrastructure/hcloud-backup-verify" ) # Collects all of them up front, into one directory, so everything after this @@ -289,7 +300,7 @@ require_secrets() { local entry name what how local -a wanted=() if [[ $# -gt 0 ]]; then - for entry in "${OPENBAO_REQUIRED_SECRETS[@]}"; do + for entry in "${OPENBAO_REQUIRED_SECRETS[@]}" "${HCLOUD_REQUIRED_SECRETS[@]}"; do for name in "$@"; do [[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry") done @@ -429,8 +440,12 @@ s3_dirs() { }' <<<"$xml" } -# The bucket every CNPG cluster recovers from (cnpg-object-stores/). +# The bucket every CNPG cluster recovers from (cnpg-object-stores/). The +# restore test archives to CNPG_TEST_BUCKET instead: a test incarnation with a +# base backup in the production bucket would be the newest there, and the next +# real recovery would read the test's data. CNPG_BUCKET="renner-uno-k8s-cnpg" +CNPG_TEST_BUCKET="renner-uno-k8s-cnpg-rtest" CNPG_STORES_DIR="$REPO_ROOT/kubernetes/cnpg-object-stores" # cnpg-paths, the ConfigMap cnpg-object-stores takes its paths from. Written @@ -488,6 +503,46 @@ write_cnpg_paths() { ok } +# The restore test's archive bucket. Created when missing; on a rerun it +# already exists, which S3 reports as an error. +create_cnpg_test_bucket() { + local out + out=$(s3_request PUT "$CNPG_TEST_BUCKET" 2>&1) && return 0 + [[ "$out" == *BucketAlreadyOwnedByYou* ]] && return 0 + echo "$out" >&2 + fail "could not create $CNPG_TEST_BUCKET" +} + +# Empties and deletes the restore test's archive bucket. Each pass deletes the +# first page of keys and lists again, so no continuation token is needed. +delete_cnpg_test_bucket() { + local xml key + while :; do + if ! xml=$(s3_request GET "$CNPG_TEST_BUCKET?list-type=2" 2>&1); then + [[ "$xml" == *NoSuchBucket* ]] && return 0 + echo "$xml" >&2 + echo " WARNING: $CNPG_TEST_BUCKET not emptied — check the console" >&2 + return 0 + fi + while read -r key; do + [[ -n "$key" ]] || continue + s3_request DELETE "$CNPG_TEST_BUCKET/$key" >/dev/null \ + || { echo " WARNING: $CNPG_TEST_BUCKET/$key not deleted — check the console" >&2; return 0; } + done < <(awk '{ + while (match($0, /[^<]*<\/Key>/)) { + print substr($0, RSTART + 5, RLENGTH - 11) + $0 = substr($0, RSTART + RLENGTH) + } + }' <<<"$xml") + [[ "$xml" == *"true"* ]] || break + done + if s3_request DELETE "$CNPG_TEST_BUCKET" >/dev/null; then + echo " deleted bucket $CNPG_TEST_BUCKET" + else + echo " WARNING: bucket $CNPG_TEST_BUCKET not deleted — check the console" >&2 + fi +} + # Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase # against it unchanged — same shape as the vmtest phase below, and for the same # reason: a rehearsal that runs different code proves less than one that does @@ -511,15 +566,18 @@ write_cnpg_paths() { # which does nothing but the sweep. Anything this phase creates carries # HCLOUD_LABEL so that sweep can find it without being told. # -# WHAT IT DOES NOT COVER. +# WHAT IT IS. # -# Everything downstream of Multus. The provider is not the limit — a cloud -# server takes up to 16 volumes and 3 private networks — but ceph-san binds to -# the kernel link name enp2s0f1np1, and that name cannot be manufactured here -# without assuming a Hetzner private network passes VLAN tags. See -# talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase -# needs the OpenBao seal key, which deliberately never reaches CI, and this -# test is meant to run unattended. +# The production config on new hardware: cluster.yaml, site-hcloud.yaml (what +# the Hetzner site is), containerd-layer-retention.yaml and hcloud-N.yaml (each +# machine). Nothing stands in for the home network. Flux is a separate run, +# FLUX_SOURCE=restore-test bootstrap.sh flux, because it needs the OpenBao seal +# key. Ceph replicates over the pod network there: Hetzner networks are layer 3 +# (restore-test/flux-root.yaml). +# +# Containment: an outbound Hetzner firewall on every node admits udp and icmp +# anywhere and tcp anywhere except the addresses of proxy.renner.uno, the public +# edge in front of production. # Talos version and Image Factory schematic, read out of the installer image in # cluster.yaml. Both throwaway-cluster phases need them and neither may state # them: a second copy is a copy that goes stale, and the day it did the test @@ -539,7 +597,7 @@ HCLOUD_LABEL="managed-by=bootstrap-hcloudtest" # The disks a machine has are stated in its Talos patch and read from there, so # the provisioner carries no second copy. Emits one size in whole GB per raw -# volume the patch declares, in order. +# volume the patch declares off the system disk, in order. hcloud_volumes_for_node() { local patch="$TALOS_DIR/patches/$1.yaml" size gib [[ -f "$patch" ]] || fail "$patch not found — the machine config is the list" @@ -550,22 +608,260 @@ hcloud_volumes_for_node() { gib=${size%GiB} [[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as GiB" echo $(( (gib * 1074 + 999) / 1000 )) - done < <(yq -r 'select(.kind == "RawVolumeConfig") | .provisioning.minSize' "$patch") + done < <(yq -r 'select(.kind == "RawVolumeConfig" and .provisioning.diskSelector.match != "system_disk") | .provisioning.minSize' "$patch") } # Deletes everything this phase created, and NOTHING else. # -# That distinction is not theoretical: the project this runs in also holds -# netbird.renner.uno, which has been up for months and is the VPN way in. The -# label is the only thing separating a throwaway server from it — and -# `hcloud server list -l ""` lists EVERY server, so an unset HCLOUD_LABEL would -# turn this function into a project wipe. Hence the guard: the label is checked -# before it is used, and every candidate must also carry the name prefix this -# phase assigns. Two independent conditions, because one of them is a variable. -# -# A separate Hetzner project, with a token scoped to it, would make this -# structural rather than defensive. Worth doing before this runs unattended. +# The project (homelab-backup-verify) also runs the backup-verify CronJob's +# machines. `hcloud server list -l ""` lists EVERY server, so an unset +# HCLOUD_LABEL would turn this function into a project wipe: the label is +# checked before it is used, and every candidate must also carry the name +# prefix this phase assigns. HCLOUD_NAME_PREFIX="hcloud" +# Outlives the run: the test cluster's talosconfig and kubeconfig. +HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest" +# The public edge in front of production. The test cluster must not reach it. +HCLOUD_PROXY_HOST="proxy.renner.uno" +# Hetzner's limit on effective rules per firewall and direction. +HCLOUD_FW_RULE_LIMIT=500 +# Hetzner's limit on prefixes in one rule (not documented; 101 is rejected). +HCLOUD_FW_RULE_PREFIXES=100 + +# The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when +# missing: a throwaway server in rescue mode writes the Image Factory hcloud +# image to its disk, which is then snapshotted. Matched by version and +# schematic. Called directly, not in $(...), so that fail stops the run. +hcloud_talos_snapshot() { + local labels="os=talos,talos-version=$VM_VERSION,talos-schematic=${VM_SCHEMATIC:0:16}" + local builder="$HCLOUD_NAME_PREFIX-imgbuild" key_dir addr ids listing + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + if [[ -n "${ids[0]:-}" ]]; then + HCLOUD_IMAGE=${ids[0]} + return 0 + fi + echo " no snapshot for $VM_VERSION/${VM_SCHEMATIC:0:16}, building one" + key_dir=$(mktemp -d) + ssh-keygen -q -t ed25519 -N '' -C "$builder" -f "$key_dir/key" + if hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=name | grep -qx "$builder"; then + hcloud ssh-key delete "$builder" >/dev/null || fail "could not replace the stale image builder key" + fi + hcloud ssh-key create --name "$builder" --label "$HCLOUD_LABEL" \ + --public-key-from-file "$key_dir/key.pub" >/dev/null || fail "could not upload the image builder key" + hcloud server create --name "$builder" --type cx23 --image debian-12 \ + --location "${HCLOUD_LOCATION:-fsn1}" --ssh-key "$builder" --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the image builder" + # From here a failure leaves a billed server: remove it before failing. + builder_fail() { + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" + fail "$1" + } + hcloud server enable-rescue "$builder" --ssh-key "$builder" >/dev/null || builder_fail "could not enable rescue on $builder" + hcloud server reset "$builder" >/dev/null || builder_fail "could not reset $builder into rescue" + addr=$(hcloud server ip "$builder") || builder_fail "could not read the address of $builder" + for i in $(seq 1 30); do + ssh -i "$key_dir/key" -o BatchMode=yes -o ConnectTimeout=10 \ + -o UserKnownHostsFile="$key_dir/known_hosts" -o StrictHostKeyChecking=accept-new \ + "root@$addr" 'grep -q rescue /etc/hostname || hostname | grep -q rescue' && break + sleep 10 + [[ $i -eq 30 ]] && builder_fail "$builder never came up in rescue" + done + ssh -i "$key_dir/key" -o BatchMode=yes -o UserKnownHostsFile="$key_dir/known_hosts" "root@$addr" \ + "set -eo pipefail; curl -fsSL https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz | xz -d | dd of=/dev/sda bs=4M conv=fsync" \ + || builder_fail "writing the Talos image on $builder failed" + hcloud server poweroff "$builder" >/dev/null || builder_fail "could not power off $builder" + hcloud server create-image "$builder" --type snapshot \ + --description "talos $VM_VERSION ${VM_SCHEMATIC:0:16}" --label "$labels" \ + || builder_fail "could not snapshot $builder" + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || builder_fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + [[ -n "${ids[0]:-}" ]] || builder_fail "the snapshot of $builder is not listed under $labels" + HCLOUD_IMAGE=${ids[0]} + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" +} + +# One address as a string of bits: 32 for IPv4, 128 for IPv6. Returns 1, with +# the reason on stderr, for anything else. +ip_to_bits() { + local addr=$1 bits="" part v i head tail + local -a parts=() hparts=() tparts=() + if [[ "$addr" == *:* ]]; then + if [[ "$addr" == *::* ]]; then + head=${addr%%::*}; tail=${addr#*::} + [[ -n "$head" ]] && IFS=: read -r -a hparts <<<"$head" + [[ -n "$tail" ]] && IFS=: read -r -a tparts <<<"$tail" + parts=("${hparts[@]}") + for (( i = ${#hparts[@]} + ${#tparts[@]}; i < 8; i++ )); do parts+=(0); done + parts+=("${tparts[@]}") + else + IFS=: read -r -a parts <<<"$addr" + fi + [[ ${#parts[@]} -eq 8 ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9a-fA-F]{1,4}$ ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + v=$(( 16#$part )) + for (( i = 15; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + else + IFS=. read -r -a parts <<<"$addr" + [[ ${#parts[@]} -eq 4 ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9]{1,3}$ ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + v=$(( 10#$part )) + (( v <= 255 )) || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for (( i = 7; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + fi + printf '%s' "$bits" +} + +# A prefix given as bits, in CIDR notation. $2 is the address length (32|128). +bits_to_cidr() { + local p=$1 width=$2 full out="" i + full=$p + while (( ${#full} < width )); do full+="0"; done + if (( width == 32 )); then + for (( i = 0; i < 32; i += 8 )); do out+="${out:+.}$(( 2#${full:i:8} ))"; done + else + for (( i = 0; i < 128; i += 16 )); do out+="${out:+:}$(printf '%x' $(( 2#${full:i:16} )))"; done + fi + printf '%s/%s\n' "$out" "${#p}" +} + +# cidr_complement
...: the prefixes that, with the given addresses +# of one family, cover 0.0.0.0/0 or ::/0 exactly once. Returns 1, with the +# reason on stderr, on a malformed or mixed-family list. +cidr_complement() { + local width=32 a bits + local -a excl=() + [[ "$1" == *:* ]] && width=128 + for a in "$@"; do + [[ ( "$a" == *:* && $width -eq 128 ) || ( "$a" != *:* && $width -eq 32 ) ]] \ + || { echo "cidr_complement: '$a' is not of the same family as '$1'" >&2; return 1; } + bits=$(ip_to_bits "$a") || return 1 + excl+=("$bits") + done + _cidr_walk() { + local p=$1 x hit=0 + for x in "${excl[@]}"; do + [[ "$x" == "$p"* ]] && { hit=1; break; } + done + if (( hit == 0 )); then + bits_to_cidr "$p" "$width" + return 0 + fi + (( ${#p} == width )) && return 0 + _cidr_walk "${p}0" + _cidr_walk "${p}1" + } + _cidr_walk "" +} + +# Addresses of HCLOUD_PROXY_HOST for one record type (A|AAAA), over DoH: the +# operator's LAN resolver intercepts plain DNS. Returns 1, with the reason on +# stderr, when the lookup fails. +hcloud_proxy_addrs() { + local type=$1 num answer + case "$type" in + A) num=1 ;; + AAAA) num=28 ;; + *) echo "hcloud_proxy_addrs: unknown type $type" >&2; return 1 ;; + esac + answer=$(curl -sS --max-time 10 -H 'accept: application/dns-json' \ + "https://cloudflare-dns.com/dns-query?name=$HCLOUD_PROXY_HOST&type=$type") \ + || { echo "could not resolve $HCLOUD_PROXY_HOST $type over DoH" >&2; return 1; } + jq -e '.Status == 0' >/dev/null <<<"$answer" \ + || { echo "DoH lookup of $HCLOUD_PROXY_HOST $type failed: $answer" >&2; return 1; } + jq -r --argjson t "$num" '.Answer[]? | select(.type == $t) | .data' <<<"$answer" | sort -u +} + +# Outbound firewall for the nodes: udp and icmp anywhere, tcp anywhere except +# HCLOUD_PROXY_HOST. Attached before an earlier run's egress firewall is +# removed, so no node is ever without one. +hcloud_apply_egress() { + local fw="hcloudtest-node-egress-$$" v4 v6 c4 c6 n count fw_id fw_name srv_id + local -a a4=() a6=() tcp_dst=() + v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)" + v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)" + [[ -n "$v4" ]] && mapfile -t a4 <<<"$v4" + [[ -n "$v6" ]] && mapfile -t a6 <<<"$v6" + (( ${#a4[@]} + ${#a6[@]} > 0 )) || fail "$HCLOUD_PROXY_HOST resolved to no address — refusing to build the egress firewall" + echo " blocking tcp to $HCLOUD_PROXY_HOST: ${a4[*]} ${a6[*]}" + if (( ${#a4[@]} > 0 )); then + c4=$(cidr_complement "${a4[@]}") || fail "could not compute the IPv4 complement of ${a4[*]}" + else + c4=0.0.0.0/0 + fi + if (( ${#a6[@]} > 0 )); then + c6=$(cidr_complement "${a6[@]}") || fail "could not compute the IPv6 complement of ${a6[*]}" + else + c6=::/0 + fi + mapfile -t tcp_dst <<<"$c4"$'\n'"$c6" + # Effective rules are counted per destination prefix; 0.0.0.0/0 and ::/0 + # carry udp and icmp. + count=$(printf '%s\n' 0.0.0.0/0 ::/0 "${tcp_dst[@]}" | sort -u | grep -c .) + (( count < HCLOUD_FW_RULE_LIMIT )) \ + || fail "the egress firewall needs $count effective rules, Hetzner allows $HCLOUD_FW_RULE_LIMIT per firewall" + hcloud firewall create --name "$fw" --label "$HCLOUD_LABEL" >/dev/null || fail "could not create $fw" + hcloud firewall add-rule "$fw" --direction out --protocol udp --port any \ + --destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw" + hcloud firewall add-rule "$fw" --direction out --protocol icmp \ + --destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw" + for (( n = 0; n < ${#tcp_dst[@]}; n += HCLOUD_FW_RULE_PREFIXES )); do + hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \ + --destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]:n:HCLOUD_FW_RULE_PREFIXES}")" \ + --description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw" + done + for n in "${HCLOUD_NODES[@]}"; do + hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \ + || fail "could not attach $fw to $n" + done + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" == hcloudtest-node-egress-* && "$fw_name" != "$fw" ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) + echo " $fw: $count effective rules" +} + +# Runs restore-test/egress-check.yaml in the cluster KUBECONFIG names and fails +# unless every expectation holds. +hcloud_egress_check() { + local addrs phase="" + step 0 "Checking egress containment from a pod..." + addrs=$( { hcloud_proxy_addrs A; hcloud_proxy_addrs AAAA; } | paste -sd' ' -) \ + || fail "could not resolve $HCLOUD_PROXY_HOST" + [[ -n "$addrs" ]] || fail "$HCLOUD_PROXY_HOST resolved to no address" + kubectl delete pod egress-check --ignore-not-found --wait=true >/dev/null + sed "s|PROXY_ADDRS_PLACEHOLDER|$addrs|" "$REPO_ROOT/kubernetes/restore-test/egress-check.yaml" \ + | kubectl apply -f - >/dev/null || fail "could not start the egress check" + for i in $(seq 1 48); do + phase=$(kubectl get pod egress-check -o jsonpath='{.status.phase}') || phase="" + [[ "$phase" == Succeeded || "$phase" == Failed ]] && break + sleep 5 + done + if [[ "$phase" != Succeeded && "$phase" != Failed ]]; then + kubectl describe pod egress-check | sed 's/^/ /' || true + fail "the egress check never finished (phase '${phase:-none}') — do not start Flux" + fi + kubectl logs egress-check | sed 's/^/ /' || true + [[ "$phase" == "Succeeded" ]] || fail "egress containment does not hold — do not start Flux" + kubectl delete pod egress-check --wait=false >/dev/null + ok +} cleanup_hcloudtest() { command -v hcloud >/dev/null || return 0 @@ -596,6 +892,11 @@ cleanup_hcloudtest() { hcloud volume delete "$id" >/dev/null \ || echo " WARNING: volume $id not deleted — check the console" >&2 done + for id in $(hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do + echo " deleting ssh key $id" + hcloud ssh-key delete "$id" >/dev/null \ + || echo " WARNING: ssh key $id not deleted — check the console" >&2 + done # Last: a network with a server still attached cannot be deleted, and a # leftover one takes its name with it, so the next run cannot recreate it. for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do @@ -605,6 +906,17 @@ cleanup_hcloudtest() { done } +# The restore test's source, as restore-test/flux-root.yaml names it: sets +# restore_url and restore_repo (/). +restore_test_repo() { + command -v yq >/dev/null || fail "yq not found (reads restore-test/flux-root.yaml)" + restore_url=$(yq -r 'select(.kind == "GitRepository" and .metadata.name == "restore-test") | .spec.url' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml") + [[ "$restore_url" =~ ^ssh://git@github\.com/([^/]+)/([^/]+)\.git$ ]] \ + || fail "restore-test/flux-root.yaml names '$restore_url', not ssh://git@github.com//.git" + restore_repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" +} + # Sweep only. For the case where a previous run was killed hard enough that its # own trap never ran, and for a cron job to call before it starts. if [[ "$PHASE" == "hcloudclean" ]]; then @@ -613,6 +925,20 @@ if [[ "$PHASE" == "hcloudclean" ]]; then require_secrets hcloud-token HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN cleanup_hcloudtest + require_secrets s3-access s3-secret + delete_cnpg_test_bucket + # The restore test's read-only deploy keys on the GitHub mirror. + if command -v gh >/dev/null; then + restore_test_repo + for key_id in $(gh api "repos/$restore_repo/keys" \ + --jq '.[] | select(.title | startswith("restore-test ")) | .id'); do + echo " deleting deploy key $key_id" + gh api -X DELETE "repos/$restore_repo/keys/$key_id" \ + || echo " WARNING: deploy key $key_id not deleted — check the repo settings" >&2 + done + else + echo " WARNING: gh not found — restore-test deploy keys on the GitHub mirror not checked" >&2 + fi ok exit 0 fi @@ -624,6 +950,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then command -v yq >/dev/null || fail "yq not found" command -v jq >/dev/null || fail "jq not found (needed to read the SAN pool and server attachments)" command -v curl >/dev/null || fail "curl not found (needed to learn this host's public address)" + command -v bao >/dev/null || fail "bao not found (the Hetzner token is read from OpenBao)" + command -v kubectl >/dev/null || fail "kubectl not found (runs the egress check)" # Three control planes, all carrying workloads — the production shape. One # machine is a node, not a cluster: etcd has no quorum to lose and nothing @@ -642,18 +970,11 @@ if [[ "$PHASE" == "hcloudtest" ]]; then # this one, adopted below. The names are fixed, so duplicates cannot # accumulate. `hcloudclean` is what deletes. - # The Talos snapshot is a prerequisite, not something this builds: making it - # means booting a server into rescue and writing the raw image, which is a - # one-off and needs a different set of permissions. Fail with the recipe - # rather than silently booting Debian and timing out on the Talos API. step 0 "Looking up the Talos snapshot..." talos_image_coords - HCLOUD_IMAGE=$(hcloud image list --type snapshot -l os=talos -o noheader -o columns=id | head -1) - [[ -n "$HCLOUD_IMAGE" ]] || fail "no snapshot labelled os=talos. Create one once — boot any server into rescue, then: - wget -O /tmp/talos.raw.xz https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz - xz -d -c /tmp/talos.raw.xz | dd of=/dev/sda && sync && shutdown -h now - then snapshot that server in the console and label the snapshot os=talos. - The schematic and version above are this cluster's, read from cluster.yaml." + HCLOUD_IMAGE="" + hcloud_talos_snapshot + [[ -n "$HCLOUD_IMAGE" ]] || fail "no Talos snapshot" echo " snapshot $HCLOUD_IMAGE" # The Talos API is unauthenticated in maintenance mode, which is how @@ -696,26 +1017,30 @@ if [[ "$PHASE" == "hcloudtest" ]]; then --source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \ || fail "could not add the Kubernetes API rule" echo " open to $fw_src only" + # Servers held from an earlier run get this firewall first; only then are + # the earlier runs' inbound firewalls detached and deleted, so no server is + # ever without one. Egress firewalls are replaced by hcloud_apply_egress. + while read -r srv; do + [[ -n "$srv" ]] || continue + hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$srv" >/dev/null \ + || fail "could not attach this run's firewall to $srv" + done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name) + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" != "hcloudtest-$$" && "$fw_name" != hcloudtest-node-egress-* ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) ok - # cx43 — 16 GB, 8 vCPU. Sized from what the workload needs, not from the - # Talos guide, which names cx22 as a minimal example (and that type no - # longer exists; `server create` rejects it outright). + # Default type cpx52 (12 vCPU, 24 GB, 480 GB local disk): the hcloud-N + # patches put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system + # disk. cax* are ARM and do not match the amd64 schematic. # - # cx23 was tried first, on the reasoning that it is cx22's successor. It has - # 4 GB, 3.2 GB of it allocatable, and the three nodes sat at 59-78% of that - # with the infrastructure skeleton alone — no Ceph OSDs, no OpenBao running, - # not one app. Limits were already at 134-207% of allocatable and both - # kube-controller-manager and kube-scheduler had restarted twice. - # - # The local rehearsal is the reference: it runs a 16GiB control plane and - # three 8GiB workers, 40GiB in total, and needed every step of the way up - # from 6GiB to get there. Three nodes here carry the same workload plus a - # control plane each, so 48GiB total is the comparable figure. - # - # Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which - # would not match the amd64 schematic this cluster installs. That leaves the - # cx*/cpx*2 families in the German locations. # Two segments only, unlike the homelab's split in # NETWORK-traffic-classes.md: here the node address carries management, # cluster-internal and external alike, and Ceph replication has its own. @@ -735,7 +1060,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then seg_range() { case "$1" in cluster) - yq -r '.machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; + yq -r 'select(.machine) | .machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; storage) local pool pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range') @@ -797,12 +1122,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $n: reusing the server we already hold" hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \ || fail "could not rebuild $n back into maintenance mode" - # The firewall is named per run (hcloudtest-$$), so an adopted - # server is still attached to the previous run's. Attach this run's - # too — without it the Talos API is unreachable once the old one is - # cleaned up, and the addresses in it are last run's anyway. - hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$n" >/dev/null \ - || fail "could not attach this run's firewall to $n" + # This run's firewall was attached above, before the old ones went. else # Retried, because "error during placement (resource_unavailable)" # is the provider being momentarily out of that type in that @@ -814,13 +1134,13 @@ if [[ "$PHASE" == "hcloudtest" ]]; then create_err="" for attempt in $(seq 1 30); do create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \ - --type "${HCLOUD_TYPE:-cx43}" --location "${HCLOUD_LOCATION:-fsn1}" \ + --type "${HCLOUD_TYPE:-cpx52}" --location "${HCLOUD_LOCATION:-fsn1}" \ --firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break grep -qi 'resource_unavailable\|placement' <<<"$create_err" \ || fail "could not create $n: ${create_err//$'\n'/ | }" [[ $attempt -eq 1 ]] && echo " $n: no capacity yet, retrying" sleep 20 - [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cx43} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" + [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cpx52} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" done fi addr=$(hcloud server ip "$n") || fail "could not read the address of $n" @@ -863,13 +1183,18 @@ if [[ "$PHASE" == "hcloudtest" ]]; then hcloud volume delete "$n-osd" >/dev/null \ || fail "could not delete the previous OSD volume for $n" fi + yq -e 'select(.kind == "RawVolumeConfig" and .name == "osd-data")' "$TALOS_DIR/patches/$n.yaml" >/dev/null \ + || fail "$n declares no osd-data RawVolumeConfig — a node with no OSD device is not this cluster's shape" + # A raw volume on the system disk needs no Hetzner volume. vol_size=$(hcloud_volumes_for_node "$n") - [[ -n "$vol_size" ]] || fail "$n declares no RawVolumeConfig — a node with no OSD device is not this cluster's shape" - hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ - --label "$HCLOUD_LABEL" >/dev/null \ - || fail "could not create the OSD volume for $n" - - echo " $n at $addr, with a ${vol_size}GB OSD volume" + if [[ -n "$vol_size" ]]; then + hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ + --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the OSD volume for $n" + echo " $n at $addr, with a ${vol_size}GB OSD volume" + else + echo " $n at $addr, OSD on the system disk" + fi NODE_ADDRS+=("$addr") done ok @@ -898,20 +1223,26 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $node_src" ok - # Handover, and this list is the whole of it — the same four variables the - # vmtest phase sets. If it grows, the test is drifting away from production. + # Before any node is configured. + step 0 "Closing tcp egress to $HCLOUD_PROXY_HOST (Hetzner firewall)..." + hcloud_apply_egress + ok + + # Handover: the variables the vmtest phase sets, and HCLOUDTEST. SITE=hcloud NODES=("${HCLOUD_NODES[@]}") WORKERS=() ALL_NODES=("${NODES[@]}") K8S_API_HOST="${NODE_ADDRS[0]}" + HCLOUDTEST=1 # Isolation from the production cluster. Both halves are required: the # _PATH variables are derived before any phase runs, and the talos phase # writes to those. - HCLOUD_DIR=$(mktemp -d) - export TALOSCONFIG="$HCLOUD_DIR/talosconfig" - export KUBECONFIG="$HCLOUD_DIR/kubeconfig" + mkdir -p "$HCLOUD_STATE_DIR" + chmod 700 "$HCLOUD_STATE_DIR" + export TALOSCONFIG="$HCLOUD_STATE_DIR/talosconfig" + export KUBECONFIG="$HCLOUD_STATE_DIR/kubeconfig" TALOSCONFIG_PATH="$TALOSCONFIG" KUBECONFIG_PATH="$KUBECONFIG" @@ -1578,9 +1909,13 @@ EOF ok # Stash configs for subsequent phases / export-access step. Skipped under - # vmtest: those are a throwaway cluster's credentials, and writing them here - # would leave the flux phase pointed at VMs that no longer exist. - if [[ "${VMTEST:-0}" == "1" ]]; then + # vmtest and hcloudtest: those are a throwaway cluster's credentials, and a + # later production `bootstrap.sh flux` copies access-staging/ over the + # operator's kubeconfig and talosconfig. + if [[ "${HCLOUDTEST:-0}" == "1" ]]; then + hcloud_egress_check + echo -e "${GREEN}Talos phase complete.${NC} Access files in $HCLOUD_STATE_DIR/ (not staged)" + elif [[ "${VMTEST:-0}" == "1" ]]; then # Nodes being Ready says nothing about storage. The CephCluster picks # OSD nodes by a label out of the machine config and their device by the # partition label RawVolumeConfig creates, so assert both here — running @@ -1829,7 +2164,9 @@ bring_up_openbao() { # entered by mistake. Wait for the container to be running first. echo -n " waiting for ${pods[0]} to answer" local status="" status_err="" - for i in $(seq 1 90); do + # Bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to + # provision its volume. + for i in $(seq 1 180); do if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then echo; break fi @@ -1847,7 +2184,7 @@ bring_up_openbao() { if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi printf '.' sleep 10 - if [[ $i -eq 90 ]]; then + if [[ $i -eq 180 ]]; then echo echo " last error: $status_err" >&2 fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised" @@ -1956,9 +2293,33 @@ if phase_runs flux; then case "$FLUX_SOURCE" in gitea) GITEA_TOKEN=$(read_secret git-token); export GITEA_TOKEN ;; github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;; - *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github)" ;; + restore-test) + GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN + command -v gh >/dev/null || fail "gh not found (registers the read-only deploy key)" + command -v jq >/dev/null || fail "jq not found (reads the DoH answer for the egress check)" + # The throwaway cluster carries production secrets, and this phase + # copies access files over KUBECONFIG_PATH and TALOSCONFIG_PATH: + # neither may be the operator's production file. + [[ "$KUBECONFIG_PATH" != "$HOME/.kube/config" && "$TALOSCONFIG_PATH" != "$HOME/.talos/config" ]] \ + || fail "FLUX_SOURCE=restore-test needs KUBECONFIG and TALOSCONFIG pointed at the test cluster ($HCLOUD_STATE_DIR)" + ;; + *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github|restore-test)" ;; esac - [[ "$PHASE" == "flux" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG, which + # hcloudtest writes; access-staging/ may hold an older run's files. + [[ "$PHASE" == "flux" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # Whatever the paths say, the cluster itself must be the test cluster, + # before anything below writes a secret to it. + node_names=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') \ + || fail "cannot list the nodes of the target cluster" + [[ -n "$node_names" ]] || fail "the target cluster has no nodes" + for n in $node_names; do + [[ "$n" == "$HCLOUD_NAME_PREFIX"-* ]] \ + || fail "node '$n' is not a $HCLOUD_NAME_PREFIX-* test node — refusing the restore test against this cluster" + done + hcloud_egress_check + fi echo -n "Creating SOPS age secret... " kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f - @@ -2000,9 +2361,43 @@ if phase_runs flux; then # Before Flux: cnpg-object-stores fails without it rather than render empty # paths, and the paths must exist before any Cluster bootstraps. - write_cnpg_paths "$CNPG_BUCKET" + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + create_cnpg_test_bucket + write_cnpg_paths "$CNPG_TEST_BUCKET" + else + write_cnpg_paths "$CNPG_BUCKET" + fi - if [[ "$FLUX_SOURCE" == "github" ]]; then + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # The production tree through kubernetes/restore-test/flux-root.yaml, + # whose patches are the test's whole deviation from production. Nothing + # is committed anywhere: the source is the GitHub mirror, read through a + # deploy key GitHub itself holds to read-only, so the test cluster + # cannot push (image automation) even with production's manifests. + step 0 "Bootstrapping FluxCD for the restore test (restore-test/flux-root.yaml)..." + flux install --components-extra=image-reflector-controller,image-automation-controller \ + || fail "flux install failed" + restore_test_repo + deploy_key="$OPENBAO_SECRETS_DIR/restore-test-git" + rm -f "$deploy_key" "$deploy_key.pub" + ssh-keygen -q -t ed25519 -N '' -C "restore-test" -f "$deploy_key" + gh api -X POST "repos/$restore_repo/keys" \ + -f title="restore-test $(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -f key="$(cat "$deploy_key.pub")" -F read_only=true >/dev/null \ + || fail "could not register the read-only deploy key on $restore_repo" + flux create secret git restore-test-git \ + --url="$restore_url" \ + --private-key-file="$deploy_key" >/dev/null \ + || fail "could not create the restore-test-git Secret" + # On FLUX_BRANCH, so a branch can be tried here before it is merged. + # shellcheck disable=SC2016 # $b is a jq variable + yq -y --arg b "$FLUX_BRANCH" \ + 'if .kind == "GitRepository" and .metadata.name == "restore-test" then .spec.ref.branch = $b else . end' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml" | kubectl apply -f - >/dev/null \ + || fail "could not apply restore-test/flux-root.yaml" + echo " source: $restore_url, branch $FLUX_BRANCH" + ok + elif [[ "$FLUX_SOURCE" == "github" ]]; then step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..." echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}" echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}" @@ -2106,7 +2501,8 @@ if phase_runs wait-apps; then echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}" command -v kubectl >/dev/null || fail "kubectl not found" command -v flux >/dev/null || fail "flux not found" - [[ "$PHASE" == "wait-apps" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG. + [[ "$PHASE" == "wait-apps" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access # Wait for a Flux Kustomization to become Ready=True. Prints a status line # whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a diff --git a/kubernetes/restore-test/egress-check.yaml b/kubernetes/restore-test/egress-check.yaml new file mode 100644 index 00000000..d49be521 --- /dev/null +++ b/kubernetes/restore-test/egress-check.yaml @@ -0,0 +1,36 @@ +# Asserts the restore test's containment from a pod, before Flux restores +# production's secrets. Applied by bootstrap.sh (hcloudtest, and +# FLUX_SOURCE=restore-test flux), which substitutes PROXY_ADDRS; exits non-zero +# if any expectation does not hold. +apiVersion: v1 +kind: Pod +metadata: + name: egress-check + namespace: default +spec: + restartPolicy: Never + containers: + - name: check + image: docker.io/nicolaka/netshoot:v0.13 + command: + - /bin/bash + - -c + - | + rc=0 + expect() { # + if [[ "$3" == $2 ]]; then echo "PASS $1: $3"; else echo "FAIL $1: got '$3', want '$2'"; rc=1; fi + } + code() { curl -sS -m 8 -o /dev/null -w '%{http_code} via %{remote_ip}' "$1" 2>&1 || true; } + port() { local out; if out=$(nc -z -w 5 "$1" "$2" 2>&1); then echo open; else echo "closed ${out}"; fi; } + expect "public DNS resolves" "[0-9]*.[0-9]*.[0-9]*.[0-9]*" "$(dig +short example.com A 2>&1 | head -1)" + expect "github.com https reachable" "[23]?? via *" "$(code https://github.com)" + expect "github.com ssh reachable" "open" "$(port github.com 22)" + expect "ghcr.io reachable" "401 via *" "$(code https://ghcr.io/v2/)" + expect "object storage reachable" "[1-4]?? via *" "$(code https://fsn1.your-objectstorage.com)" + for a in $PROXY_ADDRS; do + expect "proxy $a:443 blocked" "closed*" "$(port "$a" 443)" + done + exit $rc + env: + - name: PROXY_ADDRS + value: PROXY_ADDRS_PLACEHOLDER diff --git a/kubernetes/restore-test/flux-root.yaml b/kubernetes/restore-test/flux-root.yaml new file mode 100644 index 00000000..7aacc935 --- /dev/null +++ b/kubernetes/restore-test/flux-root.yaml @@ -0,0 +1,144 @@ +# Flux entry point of the off-site restore test (bootstrap.sh hcloudtest, then +# FLUX_SOURCE=restore-test bootstrap.sh flux). Not reconciled by production. +# +# Applies the production tree unchanged. spec.patches below is the complete +# list of differences between the test cluster and production; each is marked +# source, hardware or safety. +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + url: ssh://git@github.com/konairius/Homelab.git + ref: + branch: master + # Read-only deploy key, created per run by bootstrap.sh. + secretRef: + name: restore-test-git +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + path: ./kubernetes/clusters/homelab + prune: true + sourceRef: + kind: GitRepository + name: restore-test + patches: + # 1. Source: the GitHub mirror, read-only. In-cluster Gitea is restored by + # this test, not available to it. + - target: + kind: GitRepository + name: flux-system + namespace: flux-system + patch: | + - op: replace + path: /spec/url + value: ssh://git@github.com/konairius/Homelab.git + - op: replace + path: /spec/secretRef/name + value: restore-test-git + # 2. Source: the tree's own root sync is suspended; this Kustomization + # replaces it. + - target: + kind: Kustomization + name: flux-system + namespace: flux-system + patch: | + - op: add + path: /spec/suspend + value: true + # 3. Source: every Kustomization of the tree reads restore-test (the branch + # bootstrap.sh sets) and carries the patches below; a target a + # Kustomization does not contain matches nothing. + - target: + kind: Kustomization + patch: | + - op: replace + path: /spec/sourceRef/name + value: restore-test + - op: add + path: /spec/patches + value: + # 3a. Hardware: no Ceph cluster network. Hetzner networks are + # layer 3 and a macvlan child does not cross servers; Ceph + # replicates over the public (pod) network. + - target: + kind: CephCluster + name: rook-ceph + patch: | + - op: remove + path: /spec/network + # 3b. Safety: Velero ReadOnly blocks backups, deletion, GC and + # repository maintenance; restores and backup sync still work. + - target: + kind: HelmRelease + name: velero + patch: | + - op: add + path: /spec/values/configuration/backupStorageLocation/0/accessMode + value: ReadOnly + # CNPG needs no patch: bootstrap.sh points every cluster's archive + # at the test bucket (cnpg-paths), and the source stores carry no + # retentionPolicy, so WAL, base backups and retention all act on + # the test bucket only. + # 3f. Safety: no MariaDB logical dumps. + - target: + group: k8s.mariadb.com + kind: Backup + patch: | + - op: replace + path: /spec/schedule/suspend + value: true + # 3g. Safety: credential rotators would roll production's tokens + # at their issuers. + - target: + group: batch + kind: CronJob + labelSelector: app.kubernetes.io/component=secret-rotator + patch: | + - op: add + path: /spec/suspend + value: true + # 3h. Safety: Gitea gets no egress beyond the cluster. Its restored + # push mirror force-pushes (--mirror) over the GitHub mirror. + - target: + kind: CiliumNetworkPolicy + name: default-deny + namespace: gitea + patch: | + - op: test + path: /spec/egress/5/toEntities + value: [host, remote-node, world] + - op: replace + path: /spec/egress/5/toEntities + value: [host, remote-node] + # 3i. Safety: no writes to production NetBird. netbird-service-sync + # rewrites the public proxy services at netbird.renner.uno to + # this cluster's Traefik, fighting production's own controller. + - target: + kind: Deployment + name: netbird-service-sync + namespace: netbird-service-sync + patch: | + - op: replace + path: /spec/replicas + value: 0 + # 3j. Safety: no routing peers in the production mesh. They would + # announce this cluster's service and LoadBalancer ranges, the + # same ranges production announces. + - target: + kind: DaemonSet + name: netbird-ingress + namespace: ingress + patch: | + - op: add + path: /spec/template/spec/nodeSelector + value: + restore-test.renner.uno/never: "true" diff --git a/kubernetes/talos/patches/hcloud-1.yaml b/kubernetes/talos/patches/hcloud-1.yaml index 298a0251..13a7f6d6 100644 --- a/kubernetes/talos/patches/hcloud-1.yaml +++ b/kubernetes/talos/patches/hcloud-1.yaml @@ -55,15 +55,21 @@ kind: LinkConfig name: ceph-san up: true --- +# Capped so the OSD partition fits beside it on cpx52's 480 GB local disk. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- +# The OSD is a partition on the local system disk; bootstrap.sh creates no +# Hetzner volume for a raw volume matching system_disk. apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - # Two disks: the system disk and the attached volume. Matching on transport - # as the qemu patches do selects nothing here — a Hetzner volume arrives - # over virtio-scsi, and a selector matching zero disks is skipped silently. - match: '!system_disk' - # bootstrap.sh reads this size and creates the volume to match. - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-2.yaml b/kubernetes/talos/patches/hcloud-2.yaml index a32341e7..32d3b3d0 100644 --- a/kubernetes/talos/patches/hcloud-2.yaml +++ b/kubernetes/talos/patches/hcloud-2.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-3.yaml b/kubernetes/talos/patches/hcloud-3.yaml index 05a424b3..cdcceb03 100644 --- a/kubernetes/talos/patches/hcloud-3.yaml +++ b/kubernetes/talos/patches/hcloud-3.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index e2ce3856..fad5a8f0 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -19,7 +19,64 @@ machine: validSubnets: - 10.90.0.0/16 cluster: + # etcd peers on the node segment. Hetzner firewalls do not filter private + # networks; the public addresses admit only this host and the nodes. + etcd: + advertisedSubnets: + - 10.90.0.0/16 # Three control planes that also carry workloads, which is the production # shape. The qemu site cannot mirror it: that provisioner gives a second disk # to workers only, so a Ceph OSD forces 1 control plane and 3 workers there. allowSchedulingOnControlPlanes: true + # The site's part of Cilium's values (see site-homelab.yaml). Hetzner + # networks route only server addresses and only IPv4, so pod traffic between + # nodes is tunnelled (VXLAN) over the node segment. Their gateway drops any + # source address not assigned to the sending server (uRPF), so only the pod + # CIDRs are exempt from masquerading. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: tunnel + tunnelProtocol: vxlan + autoDirectNodeRoutes: false + ipv4NativeRoutingCIDR: 10.244.0.0/16 + ipv6NativeRoutingCIDR: fd10:244::/48 + # The site's upstream for k8s-gateway (its pod resolv.conf): the + # resolvers Hetzner's DHCP hands out. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: None + dnsConfig: + nameservers: + - 185.12.64.1 + - 185.12.64.2 +--- +# Host DNS asks k8s-gateway first, so containerd resolves in-cluster names +# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right +# after the CNI. Hetzner's resolvers after it. This list replaces the +# nameservers from DHCP. +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: 10.96.0.11 + - address: 185.12.64.1 + - address: 185.12.64.2