From 174e08624330bbaec4cc25feba9ba8cb44cd1de0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 4 Sep 2026 11:42:05 +0200 Subject: [PATCH] fix(bootstrap): isolate the Hetzner phase from $HOME, and retry etcd bootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two faults the first real runs exposed, one of them damaging. ISOLATION. The phase exported TALOSCONFIG and KUBECONFIG to a temp directory and stopped there. But TALOSCONFIG_PATH and KUBECONFIG_PATH are derived at the top of this script, from those variables, before any phase runs — so exporting them afterwards moves nothing, and the talos phase writes to the _PATH ones. The throwaway cluster's credentials were written over ~/.kube/config and ~/.talos/config, and another session working on the real cluster lost its view of it mid-task. The vmtest phase has always set both halves; this one now does too. BOOTSTRAP RETRY. `talosctl bootstrap` was called once, and anything that was not "already exists" was reported and then ignored. "bootstrap is not available yet" is a moment, not a fault: the preceding wait only establishes that the node answers with its hostname, while etcd accepting the call comes later and how much later depends on the machine. On qemu the gap is short enough to miss. On a Hetzner cloud server it is not — the single attempt failed with FailedPrecondition, the run carried on, and then sat out its full timeout waiting for an apiserver that could never start because etcd had never been bootstrapped. It now retries for ten minutes, treats "already exists" as success, and fails loudly if the call never lands. Also adds a check that the generated registry patch is a readable file, after a bug where it was not: require_secrets prints "All secrets present." on stdout, and calling it inside registry_auth_patch made that line part of the captured path. talosctl reported `open All secrets present. | /tmp/…: no such file` sixty times while the node sat in maintenance. The call moved out of the substitution; a function whose value is its stdout must be the only thing writing there. --- kubernetes/bootstrap.sh | 95 ++++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 0d2a0aed..931b294e 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -468,17 +468,32 @@ if [[ "$PHASE" == "hcloudtest" ]]; then # apply-config reaches a node that has no config yet. On a public address # that has to be closed to everyone but this host for the minutes it takes. step 0 "Creating a firewall for this host only..." - MY_IP=$(curl -fsS https://ifconfig.me) || fail "could not determine this host's public address" - [[ -n "$MY_IP" ]] || fail "public address came back empty" + # Both families, and each with its own prefix length. The first version + # asked ifconfig.me once and appended /32: this host answered over IPv6, and + # Hetzner rejected `2003:d6:…:86d2/32` because /32 of that address is not a + # network boundary. A single-address rule is /32 for v4 and /128 for v6. + # + # Both are added rather than whichever happened to answer, because the rule + # has to admit the address talosctl actually leaves from, and that is chosen + # per connection by the resolver — not by whatever this lookup returned. + # No `local` here: this phase runs in the script body, not in a function, + # and `local` is a runtime error there — one `bash -n` does not catch. + fw_sources=() + v4=$(curl -4 -fsS --max-time 10 https://ifconfig.me 2>/dev/null) && [[ -n "$v4" ]] && fw_sources+=("$v4/32") + v6=$(curl -6 -fsS --max-time 10 https://ifconfig.me 2>/dev/null) && [[ -n "$v6" ]] && fw_sources+=("$v6/128") + [[ ${#fw_sources[@]} -gt 0 ]] \ + || fail "could not determine this host's public address over either IPv4 or IPv6" + fw_src=$(IFS=,; echo "${fw_sources[*]}") + hcloud firewall create --name "hcloudtest-$$" --label "$HCLOUD_LABEL" >/dev/null \ || fail "could not create the firewall" hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 50000 \ - --source-ips "$MY_IP/32" --description "Talos API, this host only" >/dev/null \ + --source-ips "$fw_src" --description "Talos API, this host only" >/dev/null \ || fail "could not add the Talos API rule" hcloud firewall add-rule "hcloudtest-$$" --direction in --protocol tcp --port 6443 \ - --source-ips "$MY_IP/32" --description "Kubernetes API, this host only" >/dev/null \ + --source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \ || fail "could not add the Kubernetes API rule" - echo " open to $MY_IP only" + echo " open to $fw_src only" ok # cx23, not the cx22 the Talos guide names: that type no longer exists, and @@ -508,9 +523,18 @@ if [[ "$PHASE" == "hcloudtest" ]]; then ALL_NODES=("${NODES[@]}") K8S_API_HOST="${NODE_ADDRS[0]}" + # Hard isolation from the production cluster. BOTH halves are required and + # the first version had only the exported ones: TALOSCONFIG_PATH and + # KUBECONFIG_PATH are derived at the top of this script, before any phase + # runs, so exporting the variables afterwards does not move them. The talos + # phase writes to the _PATH ones — and wrote this throwaway cluster's + # credentials over ~/.kube/config, which took another session's view of the + # real cluster with it. HCLOUD_DIR=$(mktemp -d) export TALOSCONFIG="$HCLOUD_DIR/talosconfig" export KUBECONFIG="$HCLOUD_DIR/kubeconfig" + TALOSCONFIG_PATH="$TALOSCONFIG" + KUBECONFIG_PATH="$KUBECONFIG" PHASE=talos fi @@ -791,12 +815,6 @@ if phase_runs talos; then # it carries. registry_auth_patch() { local out="$OPENBAO_SECRETS_DIR/registry-auth.yaml" token - # This phase can run on its own (`bootstrap.sh talos`), in which case - # nothing has demanded the secrets yet — the full require_secrets is - # gated on the flux phase. Ask for the one this needs, and only that - # one: pulling in the whole list would make an unattended talos run - # depend on the OpenBao unseal key, which never reaches CI. - require_secrets ghcr-pull token=$(read_secret ghcr-pull) || fail "could not read the GHCR pull token" ( umask 077; cat > "$out" <&1); then - grep -qi 'already\|AlreadyExists' <<<"$bootstrap_err" \ - || echo -e "\n${YELLOW}bootstrap reported:${NC} ${bootstrap_err//$'\n'/ | }" - fi + # Re-running against an already-bootstrapped cluster is not an error — that + # is why this does not simply fail on a non-zero exit. Anything that is not + # "already exists" is shown rather than swallowed, because a silent failure + # here surfaces much later as an API that never answers. + # + # And it RETRIES, because "bootstrap is not available yet" is a moment, not + # a fault. The wait above only establishes that the node answers with its + # hostname; etcd accepting the bootstrap call comes later, and how much + # later depends on the machine. On qemu the gap is short enough to miss. On + # a Hetzner cloud server it is not: the single attempt failed with + # FailedPrecondition, the run carried on regardless, and then sat out its + # full timeout waiting for an apiserver that could never start because etcd + # had never been bootstrapped. + bootstrap_err="" + for i in $(seq 1 60); do + if bootstrap_err=$(talosctl bootstrap -n "$bootstrap_target" -e "$bootstrap_target" 2>&1); then + break + fi + if grep -qi 'already\|AlreadyExists' <<<"$bootstrap_err"; then + bootstrap_err="" + break + fi + # Not yet ready is the expected answer for the first attempts; anything + # else is reported once and then retried anyway, since the alternative + # is failing on a condition that may still clear. + if ! grep -qi 'not available yet\|connection refused\|FailedPrecondition' <<<"$bootstrap_err"; then + [[ $i -eq 1 ]] && echo -e "\n${YELLOW}bootstrap reported:${NC} ${bootstrap_err//$'\n'/ | }" + fi + sleep 10 + [[ $i -eq 60 ]] && { echo; echo "${bootstrap_err//$'\n'/ | }" >&2; fail "etcd never accepted the bootstrap call"; } + done ok echo -n "Getting kubeconfig... "