k.rennerandClaude Opus 4.7
da76201f1c
Reverts the bootstrap.sh kubectl-taint hack and fixes it properly in the Talos machine config. Talos machine.nodeTaints is silently ineffective on workers — NodeRestriction admission blocks post-join taint writes by worker kubelets. Kubelets CAN register with taints via the --register-with-taints flag, so pass it through kubelet.extraArgs and the taint lands at node registration instead of post-hoc reconcile. Also drops the workload-class nodeLabel (same NodeRestriction issue: arbitrary-namespace labels aren't kubelet-settable); only the kubernetes.io zone label stays since that one is on the allow-list. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
299 lines
10 KiB
Bash
Executable File
299 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# K8s Cluster Bootstrap — split into phases so CI can export access as early as possible.
|
||
#
|
||
# Usage:
|
||
# ./bootstrap.sh # run all phases locally
|
||
# ./bootstrap.sh talos # Talos apply + K8s bootstrap + kubeconfig (writes access-staging/)
|
||
# ./bootstrap.sh flux # reads access-staging/, installs flux, waits for infrastructure
|
||
# ./bootstrap.sh wait-apps # waits for apps/authentik/paperless to reconcile
|
||
#
|
||
# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt
|
||
|
||
TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
|
||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||
ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging"
|
||
NODES=(k8s-1 k8s-2 k8s-3)
|
||
WORKERS=(k8s-edge)
|
||
ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
|
||
DOMAIN="srv.renner.uno"
|
||
DNS_SERVER="10.2.40.53"
|
||
GITEA_HOST="gitea.renner.uno"
|
||
GITEA_OWNER="gitea_admin"
|
||
GITEA_REPO="homelab"
|
||
FLUX_PATH="kubernetes/clusters/homelab"
|
||
SOPS_KEY="$HOME/.config/sops/age/keys.txt"
|
||
|
||
PHASE="${1:-all}"
|
||
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m'
|
||
|
||
is_tty() { [[ -t 1 ]]; }
|
||
|
||
step() {
|
||
local _ignored=$1
|
||
local label="$2"
|
||
if is_tty; then
|
||
printf "${BLUE}==>${NC} %s\n" "$label"
|
||
else
|
||
printf "==> %s\n" "$label"
|
||
fi
|
||
}
|
||
|
||
fail() { echo -e "\n${RED}FAILED:${NC} $1"; exit 1; }
|
||
ok() { echo -e "${GREEN}OK${NC}"; }
|
||
|
||
resolve_node_ip() {
|
||
local fqdn=$1
|
||
dig +short "$fqdn" A @"$DNS_SERVER" | head -1
|
||
}
|
||
|
||
phase_runs() {
|
||
[[ "$PHASE" == "all" || "$PHASE" == "$1" ]]
|
||
}
|
||
|
||
if phase_runs talos; then
|
||
echo -e "${YELLOW}=== Phase: Talos bootstrap ===${NC}"
|
||
command -v talosctl >/dev/null || fail "talosctl not found"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
[[ -f "$TALOS_DIR/patches/cluster.yaml" ]] || fail "cluster.yaml patch not found"
|
||
[[ -f "$TALOS_DIR/patches/cilium-cni.yaml" ]] || fail "cilium-cni.yaml patch not found"
|
||
|
||
step 0 "Generating Talos configs..."
|
||
TALOS_GEN_DIR=$(mktemp -d)
|
||
talosctl gen config homelab "https://${NODES[0]}.${DOMAIN}:6443" \
|
||
--output "$TALOS_GEN_DIR" \
|
||
--force
|
||
# Strip the auto-generated HostnameConfig docs (conflicts with machine.network.hostname in per-node patches)
|
||
for f in "$TALOS_GEN_DIR"/controlplane.yaml "$TALOS_GEN_DIR"/worker.yaml; do
|
||
awk '
|
||
BEGIN { doc="" }
|
||
/^---[ \t]*$/ {
|
||
if (doc !~ /\nkind: HostnameConfig\n/ && doc !~ /^kind: HostnameConfig\n/) printf "%s", doc
|
||
doc=$0 "\n"; next
|
||
}
|
||
{ doc = doc $0 "\n" }
|
||
END {
|
||
if (doc !~ /\nkind: HostnameConfig\n/ && doc !~ /^kind: HostnameConfig\n/) printf "%s", doc
|
||
}
|
||
' "$f" > "$f.tmp" && mv "$f.tmp" "$f"
|
||
done
|
||
mkdir -p ~/.talos
|
||
cp "$TALOS_GEN_DIR/talosconfig" ~/.talos/config
|
||
|
||
step 0 "Applying Talos configs to nodes..."
|
||
apply_node_config() {
|
||
local node=$1 base_config=$2
|
||
local fqdn="${node}.${DOMAIN}"
|
||
echo " $fqdn:"
|
||
for i in $(seq 1 60); do
|
||
node_ip=$(resolve_node_ip "$fqdn")
|
||
if [[ -z "$node_ip" ]]; then
|
||
echo " attempt $i/60: DNS returned no IP"
|
||
sleep 10
|
||
continue
|
||
fi
|
||
if err=$(talosctl apply-config --insecure -n "$node_ip" \
|
||
--file "$base_config" \
|
||
--config-patch "@$TALOS_DIR/patches/cluster.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/cilium-cni.yaml" \
|
||
--config-patch "@$TALOS_DIR/patches/${node}.yaml" 2>&1); then
|
||
echo " applied via $node_ip ${GREEN}(OK)${NC}"
|
||
return 0
|
||
fi
|
||
echo " attempt $i/60: $node_ip - ${err//$'\n'/ | }"
|
||
sleep 10
|
||
done
|
||
fail "Timeout waiting for $fqdn"
|
||
}
|
||
|
||
for node in "${NODES[@]}"; do
|
||
apply_node_config "$node" "$TALOS_GEN_DIR/controlplane.yaml"
|
||
done
|
||
for node in "${WORKERS[@]}"; do
|
||
apply_node_config "$node" "$TALOS_GEN_DIR/worker.yaml"
|
||
done
|
||
|
||
echo -n "Configuring talosctl... "
|
||
talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
|
||
talosctl config node "${NODES[0]}.${DOMAIN}"
|
||
ok
|
||
|
||
step 0 "Waiting for Talos cluster formation..."
|
||
for i in $(seq 1 30); do
|
||
members=$(talosctl get members 2>/dev/null || true)
|
||
missing=0
|
||
for node in "${ALL_NODES[@]}"; do
|
||
grep -q "$node" <<<"$members" || missing=1
|
||
done
|
||
[[ $missing -eq 0 ]] && break
|
||
sleep 5
|
||
[[ $i -eq 30 ]] && fail "Talos cluster didn't form"
|
||
done
|
||
|
||
echo -n "Bootstrapping Kubernetes... "
|
||
talosctl bootstrap 2>/dev/null || true
|
||
ok
|
||
|
||
echo -n "Getting kubeconfig... "
|
||
for i in $(seq 1 30); do
|
||
if talosctl kubeconfig ~/.kube/config --force 2>/dev/null; then
|
||
sed -i "s|server:.*|server: https://${NODES[0]}.${DOMAIN}:6443|" ~/.kube/config
|
||
break
|
||
fi
|
||
sleep 5
|
||
[[ $i -eq 30 ]] && fail "Couldn't get kubeconfig"
|
||
done
|
||
ok
|
||
|
||
step 0 "Waiting for K8s nodes..."
|
||
expected=${#ALL_NODES[@]}
|
||
for i in $(seq 1 60); do
|
||
ready=$(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready" || true)
|
||
if [[ "$ready" -eq "$expected" ]]; then
|
||
break
|
||
fi
|
||
sleep 5
|
||
[[ $i -eq 60 ]] && fail "Not all nodes ready (got $ready/$expected)"
|
||
done
|
||
|
||
# Stash configs for subsequent phases / export-access step
|
||
mkdir -p "$ACCESS_STAGING"
|
||
cp ~/.talos/config "$ACCESS_STAGING/talosconfig"
|
||
cp ~/.kube/config "$ACCESS_STAGING/kubeconfig"
|
||
echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
|
||
fi
|
||
|
||
# Helper: load staged access files into home dir (for later CI steps)
|
||
load_staged_access() {
|
||
[[ -f "$ACCESS_STAGING/talosconfig" && -f "$ACCESS_STAGING/kubeconfig" ]] \
|
||
|| fail "access-staging/ missing talosconfig or kubeconfig (run talos phase first)"
|
||
mkdir -p ~/.talos ~/.kube
|
||
cp "$ACCESS_STAGING/talosconfig" ~/.talos/config
|
||
cp "$ACCESS_STAGING/kubeconfig" ~/.kube/config
|
||
}
|
||
|
||
if phase_runs flux; then
|
||
echo -e "${YELLOW}=== Phase: FluxCD bootstrap ===${NC}"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v flux >/dev/null || fail "flux not found"
|
||
[[ -f "$SOPS_KEY" ]] || fail "SOPS age key not found at $SOPS_KEY"
|
||
[[ -n "${GITEA_TOKEN:-}" ]] || fail "GITEA_TOKEN env var not set"
|
||
[[ "$PHASE" == "flux" ]] && load_staged_access
|
||
|
||
echo -n "Creating SOPS age secret... "
|
||
kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f - 2>/dev/null
|
||
kubectl create secret generic sops-age -n flux-system \
|
||
--from-file=age.agekey="$SOPS_KEY" \
|
||
--dry-run=client -o yaml | kubectl apply -f - 2>/dev/null
|
||
ok
|
||
|
||
step 0 "Bootstrapping FluxCD..."
|
||
flux bootstrap gitea \
|
||
--hostname="$GITEA_HOST" \
|
||
--owner="$GITEA_OWNER" \
|
||
--repository="$GITEA_REPO" \
|
||
--branch=master \
|
||
--path="$FLUX_PATH" \
|
||
--personal \
|
||
--token-auth 2>&1 | tail -5
|
||
|
||
step 0 "Waiting for infrastructure..."
|
||
for i in $(seq 1 60); do
|
||
ready=$(flux get kustomization infrastructure --no-header 2>/dev/null | grep -c "True" || true)
|
||
if [[ "$ready" -ge 1 ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
[[ $i -eq 60 ]] && echo -e "${YELLOW}Warning: infrastructure not ready yet, continuing...${NC}"
|
||
done
|
||
|
||
step 0 "Waiting for Ceph cluster..."
|
||
for i in $(seq 1 60); do
|
||
health=$(kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)
|
||
if [[ "$health" == "Ready" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
[[ $i -eq 60 ]] && echo -e "${YELLOW}Warning: Ceph not ready yet, continuing...${NC}"
|
||
done
|
||
|
||
step 0 "Waiting for Ceph Object Store..."
|
||
for i in $(seq 1 60); do
|
||
phase=$(kubectl get cephobjectstore -n rook-ceph migration -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||
if [[ "$phase" == "Ready" ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
[[ $i -eq 60 ]] && echo -e "${YELLOW}Warning: Object Store not ready yet, continuing...${NC}"
|
||
done
|
||
fi
|
||
|
||
if phase_runs wait-apps; then
|
||
echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}"
|
||
command -v kubectl >/dev/null || fail "kubectl not found"
|
||
command -v flux >/dev/null || fail "flux not found"
|
||
[[ "$PHASE" == "wait-apps" ]] && load_staged_access
|
||
|
||
step 0 "Waiting for apps..."
|
||
for i in $(seq 1 30); do
|
||
ready=$(flux get kustomization apps --no-header 2>/dev/null | grep -c "True" || true)
|
||
if [[ "$ready" -ge 1 ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
done
|
||
|
||
step 0 "Waiting for Authentik..."
|
||
for i in $(seq 1 30); do
|
||
ready=$(flux get kustomization authentik --no-header 2>/dev/null | grep -c "True" || true)
|
||
if [[ "$ready" -ge 1 ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
done
|
||
|
||
step 0 "Waiting for Paperless (DB + ~3.5GB media restore)..."
|
||
for i in $(seq 1 90); do
|
||
ready=$(flux get kustomization paperless --no-header 2>/dev/null | grep -c "True" || true)
|
||
if [[ "$ready" -ge 1 ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
done
|
||
|
||
step 0 "Waiting for Audiobookshelf (library tar + S3 round-trip)..."
|
||
# Measured ~70 min on a 70 GB library (17 min upload + 52 min extract).
|
||
# 480 × 10s = 80 min gives headroom for growth.
|
||
for i in $(seq 1 480); do
|
||
ready=$(flux get kustomization audiobookshelf --no-header 2>/dev/null | grep -c "True" || true)
|
||
if [[ "$ready" -ge 1 ]]; then
|
||
break
|
||
fi
|
||
sleep 10
|
||
done
|
||
|
||
echo ""
|
||
echo -e "${YELLOW}=== Verification ===${NC}"
|
||
echo -n "Nodes: "
|
||
kubectl get nodes --no-headers 2>/dev/null | grep -c Ready | tr -d '\n'
|
||
echo "/${#ALL_NODES[@]}"
|
||
|
||
echo -n "HelmReleases: "
|
||
flux get helmrelease --all-namespaces --no-header 2>/dev/null | grep -c "True" | tr -d '\n'
|
||
echo " ready"
|
||
|
||
echo -n "Ceph: "
|
||
kubectl get cephcluster -n rook-ceph -o jsonpath='{.items[0].status.ceph.health}' 2>/dev/null || echo "pending"
|
||
echo ""
|
||
|
||
echo -n "Ingress (local): "
|
||
curl -sk --connect-timeout 5 --resolve status.renner.uno:443:10.2.40.80 https://status.renner.uno -o /dev/null -w "%{http_code}" 2>/dev/null || echo "N/A"
|
||
echo ""
|
||
fi
|