diff --git a/kubernetes/RUNBOOK-baremetal-conversion.md b/kubernetes/RUNBOOK-baremetal-conversion.md index 38db173b..3be319bc 100644 --- a/kubernetes/RUNBOOK-baremetal-conversion.md +++ b/kubernetes/RUNBOOK-baremetal-conversion.md @@ -198,7 +198,7 @@ talosctl gen secrets --from-controlplane-config cp.yaml -o secrets.yaml talosctl gen config homelab https://k8s.renner.uno:6443 \ --with-secrets secrets.yaml --output-types controlplane \ --config-patch @talos/patches/cluster.yaml \ - --config-patch @talos/patches/registry-mirror.yaml \ + --config-patch @talos/patches/containerd-layer-retention.yaml \ --config-patch @talos/patches/k8s-N.yaml -o install.yaml ``` diff --git a/kubernetes/RUNBOOK-cold-rebuild.md b/kubernetes/RUNBOOK-cold-rebuild.md index 1c32f6f5..e92f020e 100644 --- a/kubernetes/RUNBOOK-cold-rebuild.md +++ b/kubernetes/RUNBOOK-cold-rebuild.md @@ -136,11 +136,18 @@ in `talos/patches/k8s-N.yaml`. ## Step 2 — talos: form the cluster + get kubeconfig -**Do not use `kubernetes/bootstrap.sh talos`.** That phase is dead: it fails on -a `cilium-cni.yaml` that no longer exists, and before that it failed because it -passed the file whole (#581). Nobody has fixed it because nothing needs it — -the procedure below is what is actually used. Its other phases — `flux`, -`wait-apps` — are still used, in Steps 3, 5 and 6. +`kubernetes/bootstrap.sh talos` does this, including the CNI install that used +to be a hand-typed step. It was unusable for months because it passed +`cilium-cni.yaml` whole and the file's `$patch: delete` document failed against +a fresh config (#581); that file no longer exists, so the failure is gone with +it. + +Verified as far as it can be without wiping a cluster: `talosctl gen config` +with exactly the patches the script passes produces a config carrying +`cni: none` and `proxy: disabled` that passes `talosctl validate --mode metal`. +The apply-and-bootstrap half has not been exercised since the rewrite. Read the +manual procedure below before running the script, so you know what it is doing +and can take over at any step if it stops. The procedure below is what actually works, and is the same one `RUNBOOK-baremetal-conversion.md` uses. @@ -160,7 +167,7 @@ for N in 1 2 3; do talosctl gen config homelab https://k8s.renner.uno:6443 \ --with-secrets secrets.yaml --output-types controlplane \ --config-patch @talos/patches/cluster.yaml \ - --config-patch @talos/patches/registry-mirror.yaml \ + --config-patch @talos/patches/containerd-layer-retention.yaml \ --config-patch @talos/patches/k8s-$N.yaml -o install-k8s-$N.yaml done ``` @@ -224,6 +231,9 @@ place it is configured, and this installs from those same values so the cluster comes up on what git describes rather than on a second, drifting copy. See PROPOSAL-cilium-single-owner.md for why it works this way. +`bootstrap.sh talos` already did this. What follows is the same thing by hand, +for when the script stopped partway or you are not using it. + Every input is read out of `helmrelease.yaml`. Nothing here is a version or a URL you have to keep in step with anything — copy the block as it stands: diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 726300b7..605f5c5d 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -21,6 +21,13 @@ set -euo pipefail TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)" REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# Cilium's single source of truth. Talos ships `cni: none` (talos/patches/ +# cluster.yaml) and nothing else, so the CNI has to be installed here, between +# `talosctl bootstrap` and Flux — Flux's own controllers are pods and cannot +# reconcile the network they need in order to start. Everything the install +# needs is read out of this file at run time so there is no second copy of a +# version or a value to keep in step. See PROPOSAL-cilium-single-owner.md. +CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml" ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging" NODES=(k8s-1 k8s-2 k8s-3) # No dedicated workers since the k8s-edge VM was retired (2026-07-02) — @@ -82,9 +89,11 @@ if phase_runs talos; then echo -e "${YELLOW}=== Phase: Talos bootstrap ===${NC}" command -v talosctl >/dev/null || fail "talosctl not found" command -v kubectl >/dev/null || fail "kubectl not found" + command -v helm >/dev/null || fail "helm not found (needed to install the CNI)" + command -v yq >/dev/null || fail "yq not found (needed to read the Cilium HelmRelease)" [[ -f "$TALOS_DIR/patches/cluster.yaml" ]] || fail "cluster.yaml patch not found" - [[ -f "$TALOS_DIR/patches/cilium-cni.yaml" ]] || fail "cilium-cni.yaml patch not found" - [[ -f "$TALOS_DIR/patches/registry-mirror.yaml" ]] || fail "registry-mirror.yaml patch not found" + [[ -f "$TALOS_DIR/patches/containerd-layer-retention.yaml" ]] || fail "containerd-layer-retention.yaml patch not found" + [[ -f "$CILIUM_HR" ]] || fail "cilium helmrelease not found at $CILIUM_HR" step 0 "Generating Talos configs..." TALOS_GEN_DIR=$(mktemp -d) @@ -123,8 +132,7 @@ if phase_runs talos; then if err=$(talosctl apply-config --insecure -n "$node_ip" \ --file "$base_config" \ --config-patch "@$TALOS_DIR/patches/cluster.yaml" \ - --config-patch "@$TALOS_DIR/patches/cilium-cni.yaml" \ - --config-patch "@$TALOS_DIR/patches/registry-mirror.yaml" \ + --config-patch "@$TALOS_DIR/patches/containerd-layer-retention.yaml" \ --config-patch "@$TALOS_DIR/patches/${node}.yaml" 2>&1); then echo " applied via $node_ip ${GREEN}(OK)${NC}" return 0 @@ -174,6 +182,35 @@ if phase_runs talos; then done ok + step 0 "Installing Cilium (CNI)..." + # Must run before the node-Ready wait below: with `cni: none` the kubelet + # reports NotReady until a CNI is present, so that loop would time out. + # + # Chart repo, version constraint and values all come out of the HelmRelease + # so this cannot drift from what Flux will reconcile minutes later. helm + # accepts the semver range verbatim and resolves it the same way Flux does. + # + # serviceMonitor is the one override, and it is a bootstrap constraint + # rather than a value: its CRD arrives with the monitoring stack much + # later, and the object would fail to apply. Nothing else is overridden, + # which is what lets helm-controller adopt this release instead of + # replacing it. + CILIUM_VALUES=$(mktemp) + yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES" + grep -q 'policyMapMax' "$CILIUM_VALUES" \ + || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR" + + helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null + helm repo update cilium >/dev/null + helm upgrade --install cilium cilium/cilium \ + --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \ + --namespace kube-system \ + -f "$CILIUM_VALUES" \ + --set prometheus.serviceMonitor.enabled=false \ + --wait --timeout 10m + rm -f "$CILIUM_VALUES" + ok + step 0 "Waiting for K8s nodes..." expected=${#ALL_NODES[@]} for i in $(seq 1 60); do diff --git a/kubernetes/talos/patches/registry-mirror.yaml b/kubernetes/talos/patches/containerd-layer-retention.yaml similarity index 100% rename from kubernetes/talos/patches/registry-mirror.yaml rename to kubernetes/talos/patches/containerd-layer-retention.yaml