From 730a7cc24f93828c3fe387287288b188b1cbb374 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sat, 17 Oct 2026 11:49:27 +0200 Subject: [PATCH] feat(restore-test): hcloudtest boots production's config on Hetzner, contained, and Flux restores into it The restore rehearsal: a throwaway three-node cluster on Hetzner Cloud that runs the production tree and restores production's backups into itself. hcloudtest (bootstrap.sh) runs SITE=hcloud on cpx52, and nothing stands in for the home network: - site-hcloud.yaml states the Hetzner site's facts, as site-homelab.yaml does at home: etcd advertised on the node segment 10.90.0.0/16; a cilium-site ConfigMap with tunnel (VXLAN) routing over the node segment and only the pod CIDRs exempt from masquerading (Hetzner networks are layer 3, IPv4-only, and drop packets with a source not assigned to the sending server); a ResolverConfig with k8s-gateway first and a k8s-gateway-site ConfigMap pointing at Hetzner's resolvers. The node range is read from the machine document only, now that the patch holds more than one (run 28). - hcloud-1..3.yaml put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system disk: the project's volume quota cannot hold three OSD volumes. No Hetzner volume is created for a raw volume on the system disk. - The Talos snapshot is built when missing (a rescue-mode server writes the Image Factory image; removed on failure), matched by version and schematic. This host's inbound firewall replaces earlier runs' without a server ever being left without one. The Hetzner token comes from OpenBao (homelab-backup-verify project). Access files land in ~/.local/state/homelab-hcloudtest instead of access-staging. - Containment: an outbound firewall on the nodes, applied before any config, admits udp and icmp anywhere and tcp to the complement of proxy.renner.uno's addresses, resolved over DoH at run time (cidr_complement, in bash; fails above Hetzner's 500 effective rules; one tcp rule per 100 prefixes, Hetzner's undocumented per-rule limit). The egress check pod (restore-test/egress-check.yaml) runs at the end of hcloudtest and again before the restore test writes any secret. FLUX_SOURCE=restore-test bootstrap.sh flux runs against that cluster only (its nodes must all be hcloud-*, and KUBECONFIG/TALOSCONFIG must not be the operator's): flux install, a per-run read-only deploy key on the GitHub mirror, the restore-test-git Secret, and restore-test/flux-root.yaml on FLUX_BRANCH. flux-root.yaml applies the production tree; its patches are the test's whole deviation: the source rewrite, no Ceph cluster network, Velero ReadOnly, no MariaDB dumps, rotators suspended, Gitea without world egress, netbird-service-sync at zero replicas and netbird-ingress on no node (the test cluster rewrote production NetBird's proxy services in run 28). CNPG archives, backs up and prunes as in production: the flux phase writes cnpg-paths with every archive path in renner-uno-k8s-cnpg-rtest, created there and emptied and deleted by hcloudclean. A test incarnation with a base backup in the production bucket would be the newest there, and the next real recovery would read the test's data. hcloudclean also deletes the test's ssh keys and the restore-test deploy keys. OpenBao's status wait is bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to provision its volume. Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 578 ++++++++++++++++++---- kubernetes/restore-test/egress-check.yaml | 36 ++ kubernetes/restore-test/flux-root.yaml | 144 ++++++ kubernetes/talos/patches/hcloud-1.yaml | 18 +- kubernetes/talos/patches/hcloud-2.yaml | 12 +- kubernetes/talos/patches/hcloud-3.yaml | 12 +- kubernetes/talos/patches/site-hcloud.yaml | 57 +++ 7 files changed, 756 insertions(+), 101 deletions(-) create mode 100644 kubernetes/restore-test/egress-check.yaml create mode 100644 kubernetes/restore-test/flux-root.yaml diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 8fe80633..386e08ce 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -15,10 +15,17 @@ set -euo pipefail # # the only place the registry fallback can be # # tested: gitea.renner.uno has no external # # ingress, so a node there genuinely cannot -# # reach it. BILLS BY THE HOUR — teardown runs -# # from the EXIT trap and is by label. -# ./bootstrap.sh hcloudclean # that teardown on its own, for a run that was -# # killed before its trap could fire. +# # reach it. BILLS BY THE HOUR — the machines +# # stay up until hcloudclean. Access files land in +# # ~/.local/state/homelab-hcloudtest/. +# ./bootstrap.sh hcloudclean # deletes everything hcloudtest created, by +# # label, and the restore test's deploy keys. +# +# Restore test on the hcloudtest cluster (production tree, read-only source, +# kubernetes/restore-test/flux-root.yaml): +# KUBECONFIG=~/.local/state/homelab-hcloudtest/kubeconfig \ +# TALOSCONFIG=~/.local/state/homelab-hcloudtest/talosconfig \ +# FLUX_SOURCE=restore-test ./bootstrap.sh flux # # Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls # from is itself Flux-managed, so a cold rebuild cannot pull from it until the @@ -177,6 +184,10 @@ resolve_node_ip() { phase_runs() { [[ "$PHASE" == "all" || "$PHASE" == "$1" ]] } +# The restore test targets the cluster bootstrap.sh hcloudtest builds. +if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wait-apps" ]]; then + fail "FLUX_SOURCE=restore-test runs the flux and wait-apps phases only" +fi OPENBAO_NS="openbao" # Where the secrets come from. On a workstation, the operator's pass store. In @@ -218,7 +229,7 @@ OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}" # that cannot be told apart from a fault. Its non-zero exit propagates, which is # what makes require_secrets print the reason instead. git_token_for_source() { - if [[ "$FLUX_SOURCE" == "github" ]]; then + if [[ "$FLUX_SOURCE" == "github" || "$FLUX_SOURCE" == "restore-test" ]]; then if [[ -n "${GITHUB_TOKEN:-}" ]]; then printf '%s' "$GITHUB_TOKEN" else @@ -256,13 +267,13 @@ OPENBAO_REQUIRED_SECRETS=( # rest, because a rebuild that discovers it missing has already wiped the # disks. "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull" - # Only the hcloudtest and hcloudclean phases ask for this one, by name, so a - # rebuild of the real cluster never demands it. - # - # Note the path: homelab/hetzner/, spelled correctly, while the two restore - # keys above live under homelab/hetztner/. Two folders, one typo, both real - # — do not "fix" one of these paths without moving the entry first. - "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|pass show homelab/hetzner/cloud-api-secret" +) +# Asked for by name only, never by a bare require_secrets: a rebuild of the +# real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the +# Hetzner project homelab-backup-verify, which holds nothing but throwaway +# machines. +HCLOUD_REQUIRED_SECRETS=( + "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|bao kv get -mount=homelab -field=token infrastructure/hcloud-backup-verify" ) # Collects all of them up front, into one directory, so everything after this @@ -289,7 +300,7 @@ require_secrets() { local entry name what how local -a wanted=() if [[ $# -gt 0 ]]; then - for entry in "${OPENBAO_REQUIRED_SECRETS[@]}"; do + for entry in "${OPENBAO_REQUIRED_SECRETS[@]}" "${HCLOUD_REQUIRED_SECRETS[@]}"; do for name in "$@"; do [[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry") done @@ -429,8 +440,12 @@ s3_dirs() { }' <<<"$xml" } -# The bucket every CNPG cluster recovers from (cnpg-object-stores/). +# The bucket every CNPG cluster recovers from (cnpg-object-stores/). The +# restore test archives to CNPG_TEST_BUCKET instead: a test incarnation with a +# base backup in the production bucket would be the newest there, and the next +# real recovery would read the test's data. CNPG_BUCKET="renner-uno-k8s-cnpg" +CNPG_TEST_BUCKET="renner-uno-k8s-cnpg-rtest" CNPG_STORES_DIR="$REPO_ROOT/kubernetes/cnpg-object-stores" # cnpg-paths, the ConfigMap cnpg-object-stores takes its paths from. Written @@ -488,6 +503,46 @@ write_cnpg_paths() { ok } +# The restore test's archive bucket. Created when missing; on a rerun it +# already exists, which S3 reports as an error. +create_cnpg_test_bucket() { + local out + out=$(s3_request PUT "$CNPG_TEST_BUCKET" 2>&1) && return 0 + [[ "$out" == *BucketAlreadyOwnedByYou* ]] && return 0 + echo "$out" >&2 + fail "could not create $CNPG_TEST_BUCKET" +} + +# Empties and deletes the restore test's archive bucket. Each pass deletes the +# first page of keys and lists again, so no continuation token is needed. +delete_cnpg_test_bucket() { + local xml key + while :; do + if ! xml=$(s3_request GET "$CNPG_TEST_BUCKET?list-type=2" 2>&1); then + [[ "$xml" == *NoSuchBucket* ]] && return 0 + echo "$xml" >&2 + echo " WARNING: $CNPG_TEST_BUCKET not emptied — check the console" >&2 + return 0 + fi + while read -r key; do + [[ -n "$key" ]] || continue + s3_request DELETE "$CNPG_TEST_BUCKET/$key" >/dev/null \ + || { echo " WARNING: $CNPG_TEST_BUCKET/$key not deleted — check the console" >&2; return 0; } + done < <(awk '{ + while (match($0, /[^<]*<\/Key>/)) { + print substr($0, RSTART + 5, RLENGTH - 11) + $0 = substr($0, RSTART + RLENGTH) + } + }' <<<"$xml") + [[ "$xml" == *"true"* ]] || break + done + if s3_request DELETE "$CNPG_TEST_BUCKET" >/dev/null; then + echo " deleted bucket $CNPG_TEST_BUCKET" + else + echo " WARNING: bucket $CNPG_TEST_BUCKET not deleted — check the console" >&2 + fi +} + # Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase # against it unchanged — same shape as the vmtest phase below, and for the same # reason: a rehearsal that runs different code proves less than one that does @@ -511,15 +566,18 @@ write_cnpg_paths() { # which does nothing but the sweep. Anything this phase creates carries # HCLOUD_LABEL so that sweep can find it without being told. # -# WHAT IT DOES NOT COVER. +# WHAT IT IS. # -# Everything downstream of Multus. The provider is not the limit — a cloud -# server takes up to 16 volumes and 3 private networks — but ceph-san binds to -# the kernel link name enp2s0f1np1, and that name cannot be manufactured here -# without assuming a Hetzner private network passes VLAN tags. See -# talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase -# needs the OpenBao seal key, which deliberately never reaches CI, and this -# test is meant to run unattended. +# The production config on new hardware: cluster.yaml, site-hcloud.yaml (what +# the Hetzner site is), containerd-layer-retention.yaml and hcloud-N.yaml (each +# machine). Nothing stands in for the home network. Flux is a separate run, +# FLUX_SOURCE=restore-test bootstrap.sh flux, because it needs the OpenBao seal +# key. Ceph replicates over the pod network there: Hetzner networks are layer 3 +# (restore-test/flux-root.yaml). +# +# Containment: an outbound Hetzner firewall on every node admits udp and icmp +# anywhere and tcp anywhere except the addresses of proxy.renner.uno, the public +# edge in front of production. # Talos version and Image Factory schematic, read out of the installer image in # cluster.yaml. Both throwaway-cluster phases need them and neither may state # them: a second copy is a copy that goes stale, and the day it did the test @@ -539,7 +597,7 @@ HCLOUD_LABEL="managed-by=bootstrap-hcloudtest" # The disks a machine has are stated in its Talos patch and read from there, so # the provisioner carries no second copy. Emits one size in whole GB per raw -# volume the patch declares, in order. +# volume the patch declares off the system disk, in order. hcloud_volumes_for_node() { local patch="$TALOS_DIR/patches/$1.yaml" size gib [[ -f "$patch" ]] || fail "$patch not found — the machine config is the list" @@ -550,22 +608,260 @@ hcloud_volumes_for_node() { gib=${size%GiB} [[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as GiB" echo $(( (gib * 1074 + 999) / 1000 )) - done < <(yq -r 'select(.kind == "RawVolumeConfig") | .provisioning.minSize' "$patch") + done < <(yq -r 'select(.kind == "RawVolumeConfig" and .provisioning.diskSelector.match != "system_disk") | .provisioning.minSize' "$patch") } # Deletes everything this phase created, and NOTHING else. # -# That distinction is not theoretical: the project this runs in also holds -# netbird.renner.uno, which has been up for months and is the VPN way in. The -# label is the only thing separating a throwaway server from it — and -# `hcloud server list -l ""` lists EVERY server, so an unset HCLOUD_LABEL would -# turn this function into a project wipe. Hence the guard: the label is checked -# before it is used, and every candidate must also carry the name prefix this -# phase assigns. Two independent conditions, because one of them is a variable. -# -# A separate Hetzner project, with a token scoped to it, would make this -# structural rather than defensive. Worth doing before this runs unattended. +# The project (homelab-backup-verify) also runs the backup-verify CronJob's +# machines. `hcloud server list -l ""` lists EVERY server, so an unset +# HCLOUD_LABEL would turn this function into a project wipe: the label is +# checked before it is used, and every candidate must also carry the name +# prefix this phase assigns. HCLOUD_NAME_PREFIX="hcloud" +# Outlives the run: the test cluster's talosconfig and kubeconfig. +HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest" +# The public edge in front of production. The test cluster must not reach it. +HCLOUD_PROXY_HOST="proxy.renner.uno" +# Hetzner's limit on effective rules per firewall and direction. +HCLOUD_FW_RULE_LIMIT=500 +# Hetzner's limit on prefixes in one rule (not documented; 101 is rejected). +HCLOUD_FW_RULE_PREFIXES=100 + +# The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when +# missing: a throwaway server in rescue mode writes the Image Factory hcloud +# image to its disk, which is then snapshotted. Matched by version and +# schematic. Called directly, not in $(...), so that fail stops the run. +hcloud_talos_snapshot() { + local labels="os=talos,talos-version=$VM_VERSION,talos-schematic=${VM_SCHEMATIC:0:16}" + local builder="$HCLOUD_NAME_PREFIX-imgbuild" key_dir addr ids listing + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + if [[ -n "${ids[0]:-}" ]]; then + HCLOUD_IMAGE=${ids[0]} + return 0 + fi + echo " no snapshot for $VM_VERSION/${VM_SCHEMATIC:0:16}, building one" + key_dir=$(mktemp -d) + ssh-keygen -q -t ed25519 -N '' -C "$builder" -f "$key_dir/key" + if hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=name | grep -qx "$builder"; then + hcloud ssh-key delete "$builder" >/dev/null || fail "could not replace the stale image builder key" + fi + hcloud ssh-key create --name "$builder" --label "$HCLOUD_LABEL" \ + --public-key-from-file "$key_dir/key.pub" >/dev/null || fail "could not upload the image builder key" + hcloud server create --name "$builder" --type cx23 --image debian-12 \ + --location "${HCLOUD_LOCATION:-fsn1}" --ssh-key "$builder" --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the image builder" + # From here a failure leaves a billed server: remove it before failing. + builder_fail() { + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" + fail "$1" + } + hcloud server enable-rescue "$builder" --ssh-key "$builder" >/dev/null || builder_fail "could not enable rescue on $builder" + hcloud server reset "$builder" >/dev/null || builder_fail "could not reset $builder into rescue" + addr=$(hcloud server ip "$builder") || builder_fail "could not read the address of $builder" + for i in $(seq 1 30); do + ssh -i "$key_dir/key" -o BatchMode=yes -o ConnectTimeout=10 \ + -o UserKnownHostsFile="$key_dir/known_hosts" -o StrictHostKeyChecking=accept-new \ + "root@$addr" 'grep -q rescue /etc/hostname || hostname | grep -q rescue' && break + sleep 10 + [[ $i -eq 30 ]] && builder_fail "$builder never came up in rescue" + done + ssh -i "$key_dir/key" -o BatchMode=yes -o UserKnownHostsFile="$key_dir/known_hosts" "root@$addr" \ + "set -eo pipefail; curl -fsSL https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz | xz -d | dd of=/dev/sda bs=4M conv=fsync" \ + || builder_fail "writing the Talos image on $builder failed" + hcloud server poweroff "$builder" >/dev/null || builder_fail "could not power off $builder" + hcloud server create-image "$builder" --type snapshot \ + --description "talos $VM_VERSION ${VM_SCHEMATIC:0:16}" --label "$labels" \ + || builder_fail "could not snapshot $builder" + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || builder_fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + [[ -n "${ids[0]:-}" ]] || builder_fail "the snapshot of $builder is not listed under $labels" + HCLOUD_IMAGE=${ids[0]} + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" +} + +# One address as a string of bits: 32 for IPv4, 128 for IPv6. Returns 1, with +# the reason on stderr, for anything else. +ip_to_bits() { + local addr=$1 bits="" part v i head tail + local -a parts=() hparts=() tparts=() + if [[ "$addr" == *:* ]]; then + if [[ "$addr" == *::* ]]; then + head=${addr%%::*}; tail=${addr#*::} + [[ -n "$head" ]] && IFS=: read -r -a hparts <<<"$head" + [[ -n "$tail" ]] && IFS=: read -r -a tparts <<<"$tail" + parts=("${hparts[@]}") + for (( i = ${#hparts[@]} + ${#tparts[@]}; i < 8; i++ )); do parts+=(0); done + parts+=("${tparts[@]}") + else + IFS=: read -r -a parts <<<"$addr" + fi + [[ ${#parts[@]} -eq 8 ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9a-fA-F]{1,4}$ ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + v=$(( 16#$part )) + for (( i = 15; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + else + IFS=. read -r -a parts <<<"$addr" + [[ ${#parts[@]} -eq 4 ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9]{1,3}$ ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + v=$(( 10#$part )) + (( v <= 255 )) || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for (( i = 7; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + fi + printf '%s' "$bits" +} + +# A prefix given as bits, in CIDR notation. $2 is the address length (32|128). +bits_to_cidr() { + local p=$1 width=$2 full out="" i + full=$p + while (( ${#full} < width )); do full+="0"; done + if (( width == 32 )); then + for (( i = 0; i < 32; i += 8 )); do out+="${out:+.}$(( 2#${full:i:8} ))"; done + else + for (( i = 0; i < 128; i += 16 )); do out+="${out:+:}$(printf '%x' $(( 2#${full:i:16} )))"; done + fi + printf '%s/%s\n' "$out" "${#p}" +} + +# cidr_complement
...: the prefixes that, with the given addresses +# of one family, cover 0.0.0.0/0 or ::/0 exactly once. Returns 1, with the +# reason on stderr, on a malformed or mixed-family list. +cidr_complement() { + local width=32 a bits + local -a excl=() + [[ "$1" == *:* ]] && width=128 + for a in "$@"; do + [[ ( "$a" == *:* && $width -eq 128 ) || ( "$a" != *:* && $width -eq 32 ) ]] \ + || { echo "cidr_complement: '$a' is not of the same family as '$1'" >&2; return 1; } + bits=$(ip_to_bits "$a") || return 1 + excl+=("$bits") + done + _cidr_walk() { + local p=$1 x hit=0 + for x in "${excl[@]}"; do + [[ "$x" == "$p"* ]] && { hit=1; break; } + done + if (( hit == 0 )); then + bits_to_cidr "$p" "$width" + return 0 + fi + (( ${#p} == width )) && return 0 + _cidr_walk "${p}0" + _cidr_walk "${p}1" + } + _cidr_walk "" +} + +# Addresses of HCLOUD_PROXY_HOST for one record type (A|AAAA), over DoH: the +# operator's LAN resolver intercepts plain DNS. Returns 1, with the reason on +# stderr, when the lookup fails. +hcloud_proxy_addrs() { + local type=$1 num answer + case "$type" in + A) num=1 ;; + AAAA) num=28 ;; + *) echo "hcloud_proxy_addrs: unknown type $type" >&2; return 1 ;; + esac + answer=$(curl -sS --max-time 10 -H 'accept: application/dns-json' \ + "https://cloudflare-dns.com/dns-query?name=$HCLOUD_PROXY_HOST&type=$type") \ + || { echo "could not resolve $HCLOUD_PROXY_HOST $type over DoH" >&2; return 1; } + jq -e '.Status == 0' >/dev/null <<<"$answer" \ + || { echo "DoH lookup of $HCLOUD_PROXY_HOST $type failed: $answer" >&2; return 1; } + jq -r --argjson t "$num" '.Answer[]? | select(.type == $t) | .data' <<<"$answer" | sort -u +} + +# Outbound firewall for the nodes: udp and icmp anywhere, tcp anywhere except +# HCLOUD_PROXY_HOST. Attached before an earlier run's egress firewall is +# removed, so no node is ever without one. +hcloud_apply_egress() { + local fw="hcloudtest-node-egress-$$" v4 v6 c4 c6 n count fw_id fw_name srv_id + local -a a4=() a6=() tcp_dst=() + v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)" + v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)" + [[ -n "$v4" ]] && mapfile -t a4 <<<"$v4" + [[ -n "$v6" ]] && mapfile -t a6 <<<"$v6" + (( ${#a4[@]} + ${#a6[@]} > 0 )) || fail "$HCLOUD_PROXY_HOST resolved to no address — refusing to build the egress firewall" + echo " blocking tcp to $HCLOUD_PROXY_HOST: ${a4[*]} ${a6[*]}" + if (( ${#a4[@]} > 0 )); then + c4=$(cidr_complement "${a4[@]}") || fail "could not compute the IPv4 complement of ${a4[*]}" + else + c4=0.0.0.0/0 + fi + if (( ${#a6[@]} > 0 )); then + c6=$(cidr_complement "${a6[@]}") || fail "could not compute the IPv6 complement of ${a6[*]}" + else + c6=::/0 + fi + mapfile -t tcp_dst <<<"$c4"$'\n'"$c6" + # Effective rules are counted per destination prefix; 0.0.0.0/0 and ::/0 + # carry udp and icmp. + count=$(printf '%s\n' 0.0.0.0/0 ::/0 "${tcp_dst[@]}" | sort -u | grep -c .) + (( count < HCLOUD_FW_RULE_LIMIT )) \ + || fail "the egress firewall needs $count effective rules, Hetzner allows $HCLOUD_FW_RULE_LIMIT per firewall" + hcloud firewall create --name "$fw" --label "$HCLOUD_LABEL" >/dev/null || fail "could not create $fw" + hcloud firewall add-rule "$fw" --direction out --protocol udp --port any \ + --destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw" + hcloud firewall add-rule "$fw" --direction out --protocol icmp \ + --destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw" + for (( n = 0; n < ${#tcp_dst[@]}; n += HCLOUD_FW_RULE_PREFIXES )); do + hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \ + --destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]:n:HCLOUD_FW_RULE_PREFIXES}")" \ + --description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw" + done + for n in "${HCLOUD_NODES[@]}"; do + hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \ + || fail "could not attach $fw to $n" + done + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" == hcloudtest-node-egress-* && "$fw_name" != "$fw" ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) + echo " $fw: $count effective rules" +} + +# Runs restore-test/egress-check.yaml in the cluster KUBECONFIG names and fails +# unless every expectation holds. +hcloud_egress_check() { + local addrs phase="" + step 0 "Checking egress containment from a pod..." + addrs=$( { hcloud_proxy_addrs A; hcloud_proxy_addrs AAAA; } | paste -sd' ' -) \ + || fail "could not resolve $HCLOUD_PROXY_HOST" + [[ -n "$addrs" ]] || fail "$HCLOUD_PROXY_HOST resolved to no address" + kubectl delete pod egress-check --ignore-not-found --wait=true >/dev/null + sed "s|PROXY_ADDRS_PLACEHOLDER|$addrs|" "$REPO_ROOT/kubernetes/restore-test/egress-check.yaml" \ + | kubectl apply -f - >/dev/null || fail "could not start the egress check" + for i in $(seq 1 48); do + phase=$(kubectl get pod egress-check -o jsonpath='{.status.phase}') || phase="" + [[ "$phase" == Succeeded || "$phase" == Failed ]] && break + sleep 5 + done + if [[ "$phase" != Succeeded && "$phase" != Failed ]]; then + kubectl describe pod egress-check | sed 's/^/ /' || true + fail "the egress check never finished (phase '${phase:-none}') — do not start Flux" + fi + kubectl logs egress-check | sed 's/^/ /' || true + [[ "$phase" == "Succeeded" ]] || fail "egress containment does not hold — do not start Flux" + kubectl delete pod egress-check --wait=false >/dev/null + ok +} cleanup_hcloudtest() { command -v hcloud >/dev/null || return 0 @@ -596,6 +892,11 @@ cleanup_hcloudtest() { hcloud volume delete "$id" >/dev/null \ || echo " WARNING: volume $id not deleted — check the console" >&2 done + for id in $(hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do + echo " deleting ssh key $id" + hcloud ssh-key delete "$id" >/dev/null \ + || echo " WARNING: ssh key $id not deleted — check the console" >&2 + done # Last: a network with a server still attached cannot be deleted, and a # leftover one takes its name with it, so the next run cannot recreate it. for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do @@ -605,6 +906,17 @@ cleanup_hcloudtest() { done } +# The restore test's source, as restore-test/flux-root.yaml names it: sets +# restore_url and restore_repo (/). +restore_test_repo() { + command -v yq >/dev/null || fail "yq not found (reads restore-test/flux-root.yaml)" + restore_url=$(yq -r 'select(.kind == "GitRepository" and .metadata.name == "restore-test") | .spec.url' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml") + [[ "$restore_url" =~ ^ssh://git@github\.com/([^/]+)/([^/]+)\.git$ ]] \ + || fail "restore-test/flux-root.yaml names '$restore_url', not ssh://git@github.com//.git" + restore_repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" +} + # Sweep only. For the case where a previous run was killed hard enough that its # own trap never ran, and for a cron job to call before it starts. if [[ "$PHASE" == "hcloudclean" ]]; then @@ -613,6 +925,20 @@ if [[ "$PHASE" == "hcloudclean" ]]; then require_secrets hcloud-token HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN cleanup_hcloudtest + require_secrets s3-access s3-secret + delete_cnpg_test_bucket + # The restore test's read-only deploy keys on the GitHub mirror. + if command -v gh >/dev/null; then + restore_test_repo + for key_id in $(gh api "repos/$restore_repo/keys" \ + --jq '.[] | select(.title | startswith("restore-test ")) | .id'); do + echo " deleting deploy key $key_id" + gh api -X DELETE "repos/$restore_repo/keys/$key_id" \ + || echo " WARNING: deploy key $key_id not deleted — check the repo settings" >&2 + done + else + echo " WARNING: gh not found — restore-test deploy keys on the GitHub mirror not checked" >&2 + fi ok exit 0 fi @@ -624,6 +950,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then command -v yq >/dev/null || fail "yq not found" command -v jq >/dev/null || fail "jq not found (needed to read the SAN pool and server attachments)" command -v curl >/dev/null || fail "curl not found (needed to learn this host's public address)" + command -v bao >/dev/null || fail "bao not found (the Hetzner token is read from OpenBao)" + command -v kubectl >/dev/null || fail "kubectl not found (runs the egress check)" # Three control planes, all carrying workloads — the production shape. One # machine is a node, not a cluster: etcd has no quorum to lose and nothing @@ -642,18 +970,11 @@ if [[ "$PHASE" == "hcloudtest" ]]; then # this one, adopted below. The names are fixed, so duplicates cannot # accumulate. `hcloudclean` is what deletes. - # The Talos snapshot is a prerequisite, not something this builds: making it - # means booting a server into rescue and writing the raw image, which is a - # one-off and needs a different set of permissions. Fail with the recipe - # rather than silently booting Debian and timing out on the Talos API. step 0 "Looking up the Talos snapshot..." talos_image_coords - HCLOUD_IMAGE=$(hcloud image list --type snapshot -l os=talos -o noheader -o columns=id | head -1) - [[ -n "$HCLOUD_IMAGE" ]] || fail "no snapshot labelled os=talos. Create one once — boot any server into rescue, then: - wget -O /tmp/talos.raw.xz https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz - xz -d -c /tmp/talos.raw.xz | dd of=/dev/sda && sync && shutdown -h now - then snapshot that server in the console and label the snapshot os=talos. - The schematic and version above are this cluster's, read from cluster.yaml." + HCLOUD_IMAGE="" + hcloud_talos_snapshot + [[ -n "$HCLOUD_IMAGE" ]] || fail "no Talos snapshot" echo " snapshot $HCLOUD_IMAGE" # The Talos API is unauthenticated in maintenance mode, which is how @@ -696,26 +1017,30 @@ if [[ "$PHASE" == "hcloudtest" ]]; then --source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \ || fail "could not add the Kubernetes API rule" echo " open to $fw_src only" + # Servers held from an earlier run get this firewall first; only then are + # the earlier runs' inbound firewalls detached and deleted, so no server is + # ever without one. Egress firewalls are replaced by hcloud_apply_egress. + while read -r srv; do + [[ -n "$srv" ]] || continue + hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$srv" >/dev/null \ + || fail "could not attach this run's firewall to $srv" + done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name) + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" != "hcloudtest-$$" && "$fw_name" != hcloudtest-node-egress-* ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) ok - # cx43 — 16 GB, 8 vCPU. Sized from what the workload needs, not from the - # Talos guide, which names cx22 as a minimal example (and that type no - # longer exists; `server create` rejects it outright). + # Default type cpx52 (12 vCPU, 24 GB, 480 GB local disk): the hcloud-N + # patches put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system + # disk. cax* are ARM and do not match the amd64 schematic. # - # cx23 was tried first, on the reasoning that it is cx22's successor. It has - # 4 GB, 3.2 GB of it allocatable, and the three nodes sat at 59-78% of that - # with the infrastructure skeleton alone — no Ceph OSDs, no OpenBao running, - # not one app. Limits were already at 134-207% of allocatable and both - # kube-controller-manager and kube-scheduler had restarted twice. - # - # The local rehearsal is the reference: it runs a 16GiB control plane and - # three 8GiB workers, 40GiB in total, and needed every step of the way up - # from 6GiB to get there. Three nodes here carry the same workload plus a - # control plane each, so 48GiB total is the comparable figure. - # - # Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which - # would not match the amd64 schematic this cluster installs. That leaves the - # cx*/cpx*2 families in the German locations. # Two segments only, unlike the homelab's split in # NETWORK-traffic-classes.md: here the node address carries management, # cluster-internal and external alike, and Ceph replication has its own. @@ -735,7 +1060,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then seg_range() { case "$1" in cluster) - yq -r '.machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; + yq -r 'select(.machine) | .machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; storage) local pool pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range') @@ -797,12 +1122,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $n: reusing the server we already hold" hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \ || fail "could not rebuild $n back into maintenance mode" - # The firewall is named per run (hcloudtest-$$), so an adopted - # server is still attached to the previous run's. Attach this run's - # too — without it the Talos API is unreachable once the old one is - # cleaned up, and the addresses in it are last run's anyway. - hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$n" >/dev/null \ - || fail "could not attach this run's firewall to $n" + # This run's firewall was attached above, before the old ones went. else # Retried, because "error during placement (resource_unavailable)" # is the provider being momentarily out of that type in that @@ -814,13 +1134,13 @@ if [[ "$PHASE" == "hcloudtest" ]]; then create_err="" for attempt in $(seq 1 30); do create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \ - --type "${HCLOUD_TYPE:-cx43}" --location "${HCLOUD_LOCATION:-fsn1}" \ + --type "${HCLOUD_TYPE:-cpx52}" --location "${HCLOUD_LOCATION:-fsn1}" \ --firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break grep -qi 'resource_unavailable\|placement' <<<"$create_err" \ || fail "could not create $n: ${create_err//$'\n'/ | }" [[ $attempt -eq 1 ]] && echo " $n: no capacity yet, retrying" sleep 20 - [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cx43} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" + [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cpx52} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" done fi addr=$(hcloud server ip "$n") || fail "could not read the address of $n" @@ -863,13 +1183,18 @@ if [[ "$PHASE" == "hcloudtest" ]]; then hcloud volume delete "$n-osd" >/dev/null \ || fail "could not delete the previous OSD volume for $n" fi + yq -e 'select(.kind == "RawVolumeConfig" and .name == "osd-data")' "$TALOS_DIR/patches/$n.yaml" >/dev/null \ + || fail "$n declares no osd-data RawVolumeConfig — a node with no OSD device is not this cluster's shape" + # A raw volume on the system disk needs no Hetzner volume. vol_size=$(hcloud_volumes_for_node "$n") - [[ -n "$vol_size" ]] || fail "$n declares no RawVolumeConfig — a node with no OSD device is not this cluster's shape" - hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ - --label "$HCLOUD_LABEL" >/dev/null \ - || fail "could not create the OSD volume for $n" - - echo " $n at $addr, with a ${vol_size}GB OSD volume" + if [[ -n "$vol_size" ]]; then + hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ + --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the OSD volume for $n" + echo " $n at $addr, with a ${vol_size}GB OSD volume" + else + echo " $n at $addr, OSD on the system disk" + fi NODE_ADDRS+=("$addr") done ok @@ -898,20 +1223,26 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $node_src" ok - # Handover, and this list is the whole of it — the same four variables the - # vmtest phase sets. If it grows, the test is drifting away from production. + # Before any node is configured. + step 0 "Closing tcp egress to $HCLOUD_PROXY_HOST (Hetzner firewall)..." + hcloud_apply_egress + ok + + # Handover: the variables the vmtest phase sets, and HCLOUDTEST. SITE=hcloud NODES=("${HCLOUD_NODES[@]}") WORKERS=() ALL_NODES=("${NODES[@]}") K8S_API_HOST="${NODE_ADDRS[0]}" + HCLOUDTEST=1 # Isolation from the production cluster. Both halves are required: the # _PATH variables are derived before any phase runs, and the talos phase # writes to those. - HCLOUD_DIR=$(mktemp -d) - export TALOSCONFIG="$HCLOUD_DIR/talosconfig" - export KUBECONFIG="$HCLOUD_DIR/kubeconfig" + mkdir -p "$HCLOUD_STATE_DIR" + chmod 700 "$HCLOUD_STATE_DIR" + export TALOSCONFIG="$HCLOUD_STATE_DIR/talosconfig" + export KUBECONFIG="$HCLOUD_STATE_DIR/kubeconfig" TALOSCONFIG_PATH="$TALOSCONFIG" KUBECONFIG_PATH="$KUBECONFIG" @@ -1578,9 +1909,13 @@ EOF ok # Stash configs for subsequent phases / export-access step. Skipped under - # vmtest: those are a throwaway cluster's credentials, and writing them here - # would leave the flux phase pointed at VMs that no longer exist. - if [[ "${VMTEST:-0}" == "1" ]]; then + # vmtest and hcloudtest: those are a throwaway cluster's credentials, and a + # later production `bootstrap.sh flux` copies access-staging/ over the + # operator's kubeconfig and talosconfig. + if [[ "${HCLOUDTEST:-0}" == "1" ]]; then + hcloud_egress_check + echo -e "${GREEN}Talos phase complete.${NC} Access files in $HCLOUD_STATE_DIR/ (not staged)" + elif [[ "${VMTEST:-0}" == "1" ]]; then # Nodes being Ready says nothing about storage. The CephCluster picks # OSD nodes by a label out of the machine config and their device by the # partition label RawVolumeConfig creates, so assert both here — running @@ -1829,7 +2164,9 @@ bring_up_openbao() { # entered by mistake. Wait for the container to be running first. echo -n " waiting for ${pods[0]} to answer" local status="" status_err="" - for i in $(seq 1 90); do + # Bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to + # provision its volume. + for i in $(seq 1 180); do if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then echo; break fi @@ -1847,7 +2184,7 @@ bring_up_openbao() { if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi printf '.' sleep 10 - if [[ $i -eq 90 ]]; then + if [[ $i -eq 180 ]]; then echo echo " last error: $status_err" >&2 fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised" @@ -1956,9 +2293,33 @@ if phase_runs flux; then case "$FLUX_SOURCE" in gitea) GITEA_TOKEN=$(read_secret git-token); export GITEA_TOKEN ;; github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;; - *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github)" ;; + restore-test) + GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN + command -v gh >/dev/null || fail "gh not found (registers the read-only deploy key)" + command -v jq >/dev/null || fail "jq not found (reads the DoH answer for the egress check)" + # The throwaway cluster carries production secrets, and this phase + # copies access files over KUBECONFIG_PATH and TALOSCONFIG_PATH: + # neither may be the operator's production file. + [[ "$KUBECONFIG_PATH" != "$HOME/.kube/config" && "$TALOSCONFIG_PATH" != "$HOME/.talos/config" ]] \ + || fail "FLUX_SOURCE=restore-test needs KUBECONFIG and TALOSCONFIG pointed at the test cluster ($HCLOUD_STATE_DIR)" + ;; + *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github|restore-test)" ;; esac - [[ "$PHASE" == "flux" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG, which + # hcloudtest writes; access-staging/ may hold an older run's files. + [[ "$PHASE" == "flux" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # Whatever the paths say, the cluster itself must be the test cluster, + # before anything below writes a secret to it. + node_names=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') \ + || fail "cannot list the nodes of the target cluster" + [[ -n "$node_names" ]] || fail "the target cluster has no nodes" + for n in $node_names; do + [[ "$n" == "$HCLOUD_NAME_PREFIX"-* ]] \ + || fail "node '$n' is not a $HCLOUD_NAME_PREFIX-* test node — refusing the restore test against this cluster" + done + hcloud_egress_check + fi echo -n "Creating SOPS age secret... " kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f - @@ -2000,9 +2361,43 @@ if phase_runs flux; then # Before Flux: cnpg-object-stores fails without it rather than render empty # paths, and the paths must exist before any Cluster bootstraps. - write_cnpg_paths "$CNPG_BUCKET" + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + create_cnpg_test_bucket + write_cnpg_paths "$CNPG_TEST_BUCKET" + else + write_cnpg_paths "$CNPG_BUCKET" + fi - if [[ "$FLUX_SOURCE" == "github" ]]; then + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # The production tree through kubernetes/restore-test/flux-root.yaml, + # whose patches are the test's whole deviation from production. Nothing + # is committed anywhere: the source is the GitHub mirror, read through a + # deploy key GitHub itself holds to read-only, so the test cluster + # cannot push (image automation) even with production's manifests. + step 0 "Bootstrapping FluxCD for the restore test (restore-test/flux-root.yaml)..." + flux install --components-extra=image-reflector-controller,image-automation-controller \ + || fail "flux install failed" + restore_test_repo + deploy_key="$OPENBAO_SECRETS_DIR/restore-test-git" + rm -f "$deploy_key" "$deploy_key.pub" + ssh-keygen -q -t ed25519 -N '' -C "restore-test" -f "$deploy_key" + gh api -X POST "repos/$restore_repo/keys" \ + -f title="restore-test $(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -f key="$(cat "$deploy_key.pub")" -F read_only=true >/dev/null \ + || fail "could not register the read-only deploy key on $restore_repo" + flux create secret git restore-test-git \ + --url="$restore_url" \ + --private-key-file="$deploy_key" >/dev/null \ + || fail "could not create the restore-test-git Secret" + # On FLUX_BRANCH, so a branch can be tried here before it is merged. + # shellcheck disable=SC2016 # $b is a jq variable + yq -y --arg b "$FLUX_BRANCH" \ + 'if .kind == "GitRepository" and .metadata.name == "restore-test" then .spec.ref.branch = $b else . end' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml" | kubectl apply -f - >/dev/null \ + || fail "could not apply restore-test/flux-root.yaml" + echo " source: $restore_url, branch $FLUX_BRANCH" + ok + elif [[ "$FLUX_SOURCE" == "github" ]]; then step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..." echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}" echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}" @@ -2106,7 +2501,8 @@ if phase_runs wait-apps; then echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}" command -v kubectl >/dev/null || fail "kubectl not found" command -v flux >/dev/null || fail "flux not found" - [[ "$PHASE" == "wait-apps" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG. + [[ "$PHASE" == "wait-apps" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access # Wait for a Flux Kustomization to become Ready=True. Prints a status line # whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a diff --git a/kubernetes/restore-test/egress-check.yaml b/kubernetes/restore-test/egress-check.yaml new file mode 100644 index 00000000..d49be521 --- /dev/null +++ b/kubernetes/restore-test/egress-check.yaml @@ -0,0 +1,36 @@ +# Asserts the restore test's containment from a pod, before Flux restores +# production's secrets. Applied by bootstrap.sh (hcloudtest, and +# FLUX_SOURCE=restore-test flux), which substitutes PROXY_ADDRS; exits non-zero +# if any expectation does not hold. +apiVersion: v1 +kind: Pod +metadata: + name: egress-check + namespace: default +spec: + restartPolicy: Never + containers: + - name: check + image: docker.io/nicolaka/netshoot:v0.13 + command: + - /bin/bash + - -c + - | + rc=0 + expect() { # + if [[ "$3" == $2 ]]; then echo "PASS $1: $3"; else echo "FAIL $1: got '$3', want '$2'"; rc=1; fi + } + code() { curl -sS -m 8 -o /dev/null -w '%{http_code} via %{remote_ip}' "$1" 2>&1 || true; } + port() { local out; if out=$(nc -z -w 5 "$1" "$2" 2>&1); then echo open; else echo "closed ${out}"; fi; } + expect "public DNS resolves" "[0-9]*.[0-9]*.[0-9]*.[0-9]*" "$(dig +short example.com A 2>&1 | head -1)" + expect "github.com https reachable" "[23]?? via *" "$(code https://github.com)" + expect "github.com ssh reachable" "open" "$(port github.com 22)" + expect "ghcr.io reachable" "401 via *" "$(code https://ghcr.io/v2/)" + expect "object storage reachable" "[1-4]?? via *" "$(code https://fsn1.your-objectstorage.com)" + for a in $PROXY_ADDRS; do + expect "proxy $a:443 blocked" "closed*" "$(port "$a" 443)" + done + exit $rc + env: + - name: PROXY_ADDRS + value: PROXY_ADDRS_PLACEHOLDER diff --git a/kubernetes/restore-test/flux-root.yaml b/kubernetes/restore-test/flux-root.yaml new file mode 100644 index 00000000..7aacc935 --- /dev/null +++ b/kubernetes/restore-test/flux-root.yaml @@ -0,0 +1,144 @@ +# Flux entry point of the off-site restore test (bootstrap.sh hcloudtest, then +# FLUX_SOURCE=restore-test bootstrap.sh flux). Not reconciled by production. +# +# Applies the production tree unchanged. spec.patches below is the complete +# list of differences between the test cluster and production; each is marked +# source, hardware or safety. +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + url: ssh://git@github.com/konairius/Homelab.git + ref: + branch: master + # Read-only deploy key, created per run by bootstrap.sh. + secretRef: + name: restore-test-git +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + path: ./kubernetes/clusters/homelab + prune: true + sourceRef: + kind: GitRepository + name: restore-test + patches: + # 1. Source: the GitHub mirror, read-only. In-cluster Gitea is restored by + # this test, not available to it. + - target: + kind: GitRepository + name: flux-system + namespace: flux-system + patch: | + - op: replace + path: /spec/url + value: ssh://git@github.com/konairius/Homelab.git + - op: replace + path: /spec/secretRef/name + value: restore-test-git + # 2. Source: the tree's own root sync is suspended; this Kustomization + # replaces it. + - target: + kind: Kustomization + name: flux-system + namespace: flux-system + patch: | + - op: add + path: /spec/suspend + value: true + # 3. Source: every Kustomization of the tree reads restore-test (the branch + # bootstrap.sh sets) and carries the patches below; a target a + # Kustomization does not contain matches nothing. + - target: + kind: Kustomization + patch: | + - op: replace + path: /spec/sourceRef/name + value: restore-test + - op: add + path: /spec/patches + value: + # 3a. Hardware: no Ceph cluster network. Hetzner networks are + # layer 3 and a macvlan child does not cross servers; Ceph + # replicates over the public (pod) network. + - target: + kind: CephCluster + name: rook-ceph + patch: | + - op: remove + path: /spec/network + # 3b. Safety: Velero ReadOnly blocks backups, deletion, GC and + # repository maintenance; restores and backup sync still work. + - target: + kind: HelmRelease + name: velero + patch: | + - op: add + path: /spec/values/configuration/backupStorageLocation/0/accessMode + value: ReadOnly + # CNPG needs no patch: bootstrap.sh points every cluster's archive + # at the test bucket (cnpg-paths), and the source stores carry no + # retentionPolicy, so WAL, base backups and retention all act on + # the test bucket only. + # 3f. Safety: no MariaDB logical dumps. + - target: + group: k8s.mariadb.com + kind: Backup + patch: | + - op: replace + path: /spec/schedule/suspend + value: true + # 3g. Safety: credential rotators would roll production's tokens + # at their issuers. + - target: + group: batch + kind: CronJob + labelSelector: app.kubernetes.io/component=secret-rotator + patch: | + - op: add + path: /spec/suspend + value: true + # 3h. Safety: Gitea gets no egress beyond the cluster. Its restored + # push mirror force-pushes (--mirror) over the GitHub mirror. + - target: + kind: CiliumNetworkPolicy + name: default-deny + namespace: gitea + patch: | + - op: test + path: /spec/egress/5/toEntities + value: [host, remote-node, world] + - op: replace + path: /spec/egress/5/toEntities + value: [host, remote-node] + # 3i. Safety: no writes to production NetBird. netbird-service-sync + # rewrites the public proxy services at netbird.renner.uno to + # this cluster's Traefik, fighting production's own controller. + - target: + kind: Deployment + name: netbird-service-sync + namespace: netbird-service-sync + patch: | + - op: replace + path: /spec/replicas + value: 0 + # 3j. Safety: no routing peers in the production mesh. They would + # announce this cluster's service and LoadBalancer ranges, the + # same ranges production announces. + - target: + kind: DaemonSet + name: netbird-ingress + namespace: ingress + patch: | + - op: add + path: /spec/template/spec/nodeSelector + value: + restore-test.renner.uno/never: "true" diff --git a/kubernetes/talos/patches/hcloud-1.yaml b/kubernetes/talos/patches/hcloud-1.yaml index 298a0251..13a7f6d6 100644 --- a/kubernetes/talos/patches/hcloud-1.yaml +++ b/kubernetes/talos/patches/hcloud-1.yaml @@ -55,15 +55,21 @@ kind: LinkConfig name: ceph-san up: true --- +# Capped so the OSD partition fits beside it on cpx52's 480 GB local disk. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- +# The OSD is a partition on the local system disk; bootstrap.sh creates no +# Hetzner volume for a raw volume matching system_disk. apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - # Two disks: the system disk and the attached volume. Matching on transport - # as the qemu patches do selects nothing here — a Hetzner volume arrives - # over virtio-scsi, and a selector matching zero disks is skipped silently. - match: '!system_disk' - # bootstrap.sh reads this size and creates the volume to match. - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-2.yaml b/kubernetes/talos/patches/hcloud-2.yaml index a32341e7..32d3b3d0 100644 --- a/kubernetes/talos/patches/hcloud-2.yaml +++ b/kubernetes/talos/patches/hcloud-2.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-3.yaml b/kubernetes/talos/patches/hcloud-3.yaml index 05a424b3..cdcceb03 100644 --- a/kubernetes/talos/patches/hcloud-3.yaml +++ b/kubernetes/talos/patches/hcloud-3.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index e2ce3856..fad5a8f0 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -19,7 +19,64 @@ machine: validSubnets: - 10.90.0.0/16 cluster: + # etcd peers on the node segment. Hetzner firewalls do not filter private + # networks; the public addresses admit only this host and the nodes. + etcd: + advertisedSubnets: + - 10.90.0.0/16 # Three control planes that also carry workloads, which is the production # shape. The qemu site cannot mirror it: that provisioner gives a second disk # to workers only, so a Ceph OSD forces 1 control plane and 3 workers there. allowSchedulingOnControlPlanes: true + # The site's part of Cilium's values (see site-homelab.yaml). Hetzner + # networks route only server addresses and only IPv4, so pod traffic between + # nodes is tunnelled (VXLAN) over the node segment. Their gateway drops any + # source address not assigned to the sending server (uRPF), so only the pod + # CIDRs are exempt from masquerading. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: tunnel + tunnelProtocol: vxlan + autoDirectNodeRoutes: false + ipv4NativeRoutingCIDR: 10.244.0.0/16 + ipv6NativeRoutingCIDR: fd10:244::/48 + # The site's upstream for k8s-gateway (its pod resolv.conf): the + # resolvers Hetzner's DHCP hands out. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: None + dnsConfig: + nameservers: + - 185.12.64.1 + - 185.12.64.2 +--- +# Host DNS asks k8s-gateway first, so containerd resolves in-cluster names +# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right +# after the CNI. Hetzner's resolvers after it. This list replaces the +# nameservers from DHCP. +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: 10.96.0.11 + - address: 185.12.64.1 + - address: 185.12.64.2 -- 2.54.0