fix: [alert] NetbirdVpsBackupStale (netbird-backup) (#714)

ansible/roles/netbird-server/files/netbird-backup-dump

@@ -14,5 +14,10 @@
 set -eu
 tmp=$(mktemp -d)
 trap 'rm -rf "$tmp"' EXIT
-sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'"
+# sqlite3's busy timeout defaults to 0 ms, so the online-backup API gives
+# up the instant it cannot take its read lock on store.db — which is what
+# a live netbird-management write does. Wait the writer out instead of
+# failing the nightly job; 60 s fits well inside the puller's
+# activeDeadlineSeconds: 600 against a normal runtime of ~15-19 s.
+sqlite3 -cmd '.timeout 60000' /opt/netbird/management/store.db ".backup '$tmp/store.db'"
 tar -C "$tmp" -czf - store.db

kubernetes/apps/trmnl/valkey.yaml

@@ -27,7 +27,7 @@ spec:
         # Valkey major version tracks upstream's compose.yml (9.x). Sidekiq
         # is the only consumer; a major bump is a separate migration.
         - name: valkey
-          image: valkey/valkey:9.1.2-alpine # {"$imagepolicy": "flux-system:valkey-trmnl"}
+          image: valkey/valkey:9.1.1-alpine # {"$imagepolicy": "flux-system:valkey-trmnl"}
           args:
             - valkey-server
             - --save

kubernetes/bootstrap.sh

@@ -8,9 +8,6 @@ set -euo pipefail
 #   ./bootstrap.sh talos        # Talos apply + K8s bootstrap + kubeconfig (writes access-staging/)
 #   ./bootstrap.sh flux         # reads access-staging/, installs flux, waits for infrastructure
 #   ./bootstrap.sh wait-apps    # waits for apps/authentik/paperless to reconcile
-#   ./bootstrap.sh vmtest       # throwaway QEMU VMs, then the talos phase
-#                               # against them unchanged. Needs root for the
-#                               # provisioner and leaves nothing behind.
 #
 # Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls
 # from is itself Flux-managed, so a cold rebuild cannot pull from it until the
@@ -43,24 +40,6 @@ DNS_SERVER="10.2.40.1"  # UDM (old PVE DNS VIP 10.2.40.53 decommissioned)
 # Cluster API endpoint — Talos VIP (10.2.40.60) elected via etcd, fronted by
 # UniFi DNS A record k8s.renner.uno. Defined in talos/patches/k8s-{1,2,3}.yaml.
 K8S_API_HOST="k8s.renner.uno"
-# Which site patch the nodes get: talos/patches/site-$SITE.yaml. The three
-# variables above and this one are everything the vmtest phase has to change in
-# order to run this same code against throwaway VMs — if that list ever grows,
-# the test is drifting away from the thing it claims to exercise.
-SITE="${SITE:-homelab}"
-# Where this script writes the cluster access files. These are the paths
-# talosctl and kubectl read from anyway, so unset they resolve to exactly the
-# old behaviour. Set, the whole run stays out of $HOME — which is what lets the
-# vmtest phase drive a throwaway cluster without replacing the operator's
-# production access with a VM's.
-TALOSCONFIG_PATH="${TALOSCONFIG:-$HOME/.talos/config}"
-KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}"
-case "$KUBECONFIG_PATH" in
-    *:*) echo "KUBECONFIG holds multiple paths; set it to a single file" >&2; exit 1 ;;
-esac
-# Addresses to apply to, in the order of NODES. Empty means resolve the node
-# FQDNs, which is right for real hardware on a network that has DNS for it.
-NODE_ADDRS=()
 GITEA_HOST="gitea.renner.uno"
 GITEA_OWNER="gitea_admin"
 GITEA_REPO="homelab"
@@ -72,19 +51,6 @@ FLUX_SOURCE="${FLUX_SOURCE:-gitea}"
 GITHUB_OWNER="${GITHUB_OWNER:-konairius}"
 GITHUB_REPO="${GITHUB_REPO:-Homelab}"
 FLUX_PATH="kubernetes/clusters/homelab"
-# Branch Flux reconciles from, and that `flux bootstrap` commits its own sync
-# manifests into. master for a real rebuild.
-#
-# Overridable so the phase can be rehearsed without writing to master — point it
-# at a throwaway branch, which has to exist already: `flux bootstrap` clones the
-# branch and fails with `couldn't find remote ref` rather than creating it.
-#
-#     git push origin origin/master:refs/heads/rehearsal-1
-#     FLUX_BRANCH=rehearsal-1 ./bootstrap.sh flux
-#
-# Nothing else about the phase changes, which is the point. Delete the branch
-# afterwards.
-FLUX_BRANCH="${FLUX_BRANCH:-master}"
 SOPS_KEY="$HOME/.config/sops/age/keys.txt"
 
 PHASE="${1:-all}"
@@ -119,114 +85,6 @@ phase_runs() {
     [[ "$PHASE" == "all" || "$PHASE" == "$1" ]]
 }
 
-# Provision throwaway QEMU VMs, then run the Talos phase below against
-# them unchanged. Deliberately not a copy of that phase: it sets the site
-# variables and falls through, because a rehearsal that runs different code
-# proves less than one that does not. Everything it changes is listed here — if
-# that list grows, the test is drifting away from what it claims to exercise.
-#
-# What it cannot cover, and what therefore still has to be true on hardware:
-# Ceph (the provisioner gives extra disks to workers only), the mgmt and v30iot
-# VLANs and the Layer2 VIP (one flat NIC per VM), and Flux (its own phase).
-#
-# Needs root for the QEMU provisioner (CNI and KVM), and only for that step.
-if [[ "$PHASE" == "vmtest" ]]; then
-    echo -e "${YELLOW}=== Phase: VM rebuild test ===${NC}"
-    command -v talosctl >/dev/null || fail "talosctl not found"
-    command -v yq >/dev/null       || fail "yq not found"
-    # The node patches are the list; the VM count follows from them rather than
-    # being stated twice. vm-1 is the control plane, the rest are the workers
-    # that carry an OSD device — see the comment on NODES below for why.
-    VM_CP_NODES=(vm-1)
-    VM_WORKER_NODES=(vm-2 vm-3 vm-4)
-    VM_CPS=${#VM_CP_NODES[@]}
-    VM_WORKER_COUNT=${#VM_WORKER_NODES[@]}
-    for n in "${VM_CP_NODES[@]}" "${VM_WORKER_NODES[@]}"; do
-        [[ -f "$TALOS_DIR/patches/$n.yaml" ]] || fail "$TALOS_DIR/patches/$n.yaml not found"
-    done
-    [[ -f "$TALOS_DIR/patches/site-qemu.yaml" ]] || fail "site-qemu.yaml not found"
-
-    # Talos version and Image Factory schematic are read out of the installer
-    # image in cluster.yaml. Naming them here would be a second copy to keep in
-    # step, and the day they diverged the test would quietly stop booting the
-    # image a rebuild installs.
-    VM_IMAGE=$(yq -r '.machine.install.image' "$TALOS_DIR/patches/cluster.yaml")
-    VM_SCHEMATIC=${VM_IMAGE#*metal-installer/}
-    VM_SCHEMATIC=${VM_SCHEMATIC%%:*}
-    VM_VERSION=${VM_IMAGE##*:}
-    [[ ${#VM_SCHEMATIC} -eq 64 && "$VM_VERSION" == v* ]] \
-        || fail "could not read a schematic and version out of '$VM_IMAGE'"
-
-    # sun_path caps a unix socket path at 108 bytes and the provisioner puts a
-    # qemu monitor socket under the state directory, so this has to stay short.
-    # A path under the repo or a long mktemp dir silently fails to boot.
-    VM_STATE=/tmp/bootstrap-vmtest
-    VM_CLUSTER=bootstrap-vmtest
-    VM_DIR=$(mktemp -d)
-    TALOSCTL_BIN=$(command -v talosctl)
-
-    # Hard isolation from the production cluster: talosctl and kubectl both
-    # honour these, and the phase below writes to them rather than to $HOME.
-    export TALOSCONFIG="$VM_DIR/talosconfig"
-    export KUBECONFIG="$VM_DIR/kubeconfig"
-    TALOSCONFIG_PATH="$TALOSCONFIG"
-    KUBECONFIG_PATH="$KUBECONFIG"
-
-    cleanup_vmtest() {
-        echo -n "Destroying throwaway VMs... "
-        sudo -E "$TALOSCTL_BIN" cluster destroy --name "$VM_CLUSTER" --state "$VM_STATE" >/dev/null || true
-        # destroy removes the cluster's own directory but leaves the state root.
-        sudo rm -rf "$VM_STATE"
-        rm -rf "$VM_DIR"
-        echo "done"
-    }
-    trap cleanup_vmtest EXIT
-
-    step 0 "Creating $VM_CPS control plane and $VM_WORKER_COUNT workers in maintenance mode..."
-    # maintenance means the provisioner applies no configuration of its own, so
-    # every byte of machine config below comes from talos/patches.
-    sudo -E "$TALOSCTL_BIN" cluster create qemu \
-        --name "$VM_CLUSTER" \
-        --state "$VM_STATE" \
-        --presets iso,maintenance \
-        --controlplanes "$VM_CPS" \
-        --workers "$VM_WORKER_COUNT" \
-        --talos-version "$VM_VERSION" \
-        --schematic-id "$VM_SCHEMATIC" \
-        --memory-controlplanes 2560MiB \
-        --memory-workers 4GiB \
-        --cpus-controlplanes 2 \
-        --cpus-workers 2 \
-        --disks virtio:10GiB,virtio:8GiB \
-        --talosconfig-destination "$TALOSCONFIG" \
-        || fail "could not create the throwaway VMs"
-    sudo chown -R "$(id -u):$(id -g)" "$VM_STATE" "$TALOSCONFIG"
-    ok
-
-    # The provisioner hands out its CIDR sequentially from .2; .1 is the
-    # gateway and the load balancer it puts in front of the API. If that ever
-    # changes, the apply loop below reports which address stopped answering
-    # rather than failing silently.
-    # One control plane and three workers, not three control planes: the
-    # provisioner gives the second disk only to workers ("disks after the first
-    # one are added only to worker machines"), and an OSD needs it. Three of
-    # them because the CephCluster runs three mons with allowMultiplePerNode
-    # false over pools that are replica 3 across failureDomain host.
-    NODES=("${VM_CP_NODES[@]}")
-    WORKERS=("${VM_WORKER_NODES[@]}")
-    ALL_NODES=("${NODES[@]}" "${WORKERS[@]}")
-    NODE_ADDRS=()
-    for i in $(seq 0 $(( ${#ALL_NODES[@]} - 1 ))); do
-        NODE_ADDRS+=("10.5.0.$(( i + 2 ))")
-    done
-    SITE=qemu
-    K8S_API_HOST=10.5.0.1
-    VMTEST=1
-
-    # Run the real phase, not a reimplementation of it.
-    PHASE=talos
-fi
-
 if phase_runs talos; then
     echo -e "${YELLOW}=== Phase: Talos bootstrap ===${NC}"
     command -v talosctl >/dev/null || fail "talosctl not found"
@@ -242,22 +100,30 @@ if phase_runs talos; then
     talosctl gen config homelab "https://${K8S_API_HOST}:6443" \
         --output "$TALOS_GEN_DIR" \
         --force
-    mkdir -p "$(dirname "$TALOSCONFIG_PATH")"
-    cp "$TALOS_GEN_DIR/talosconfig" "$TALOSCONFIG_PATH"
+    # Strip the auto-generated HostnameConfig docs (conflicts with machine.network.hostname in per-node patches)
+    for f in "$TALOS_GEN_DIR"/controlplane.yaml "$TALOS_GEN_DIR"/worker.yaml; do
+        awk '
+          BEGIN { doc="" }
+          /^---[ \t]*$/ {
+            if (doc !~ /\nkind: HostnameConfig\n/ && doc !~ /^kind: HostnameConfig\n/) printf "%s", doc
+            doc=$0 "\n"; next
+          }
+          { doc = doc $0 "\n" }
+          END {
+            if (doc !~ /\nkind: HostnameConfig\n/ && doc !~ /^kind: HostnameConfig\n/) printf "%s", doc
+          }
+        ' "$f" > "$f.tmp" && mv "$f.tmp" "$f"
+    done
+    mkdir -p ~/.talos
+    cp "$TALOS_GEN_DIR/talosconfig" ~/.talos/config
 
     step 0 "Applying Talos configs to nodes..."
     apply_node_config() {
-        local node=$1 base_config=$2 fixed_addr=${3:-}
+        local node=$1 base_config=$2
         local fqdn="${node}.${DOMAIN}"
-        echo "  ${fixed_addr:-$fqdn}:"
+        echo "  $fqdn:"
         for i in $(seq 1 60); do
-            # A fixed address skips DNS. Real hardware has a name to resolve;
-            # a throwaway VM has an address and nothing else.
-            if [[ -n "$fixed_addr" ]]; then
-                node_ip="$fixed_addr"
-            else
-                node_ip=$(resolve_node_ip "$fqdn")
-            fi
+            node_ip=$(resolve_node_ip "$fqdn")
             if [[ -z "$node_ip" ]]; then
                 echo "    attempt $i/60: DNS returned no IP"
                 sleep 10
@@ -266,130 +132,53 @@ if phase_runs talos; then
             if err=$(talosctl apply-config --insecure -n "$node_ip" \
                 --file "$base_config" \
                 --config-patch "@$TALOS_DIR/patches/cluster.yaml" \
-                --config-patch "@$TALOS_DIR/patches/site-${SITE}.yaml" \
                 --config-patch "@$TALOS_DIR/patches/containerd-layer-retention.yaml" \
                 --config-patch "@$TALOS_DIR/patches/${node}.yaml" 2>&1); then
-                echo -e "    applied via $node_ip ${GREEN}(OK)${NC}"
+                echo "    applied via $node_ip ${GREEN}(OK)${NC}"
                 return 0
             fi
             echo "    attempt $i/60: $node_ip - ${err//$'\n'/ | }"
             sleep 10
         done
-        fail "Timeout waiting for ${fixed_addr:-$fqdn}"
+        fail "Timeout waiting for $fqdn"
     }
 
-    # One loop over ALL_NODES rather than one per role, so NODE_ADDRS can be a
-    # single list in the same order. Control planes come first in ALL_NODES,
-    # which is what decides the base config each node gets.
-    for idx in "${!ALL_NODES[@]}"; do
-        if [[ $idx -lt ${#NODES[@]} ]]; then
-            base="$TALOS_GEN_DIR/controlplane.yaml"
-        else
-            base="$TALOS_GEN_DIR/worker.yaml"
-        fi
-        apply_node_config "${ALL_NODES[$idx]}" "$base" "${NODE_ADDRS[$idx]:-}"
+    for node in "${NODES[@]}"; do
+        apply_node_config "$node" "$TALOS_GEN_DIR/controlplane.yaml"
+    done
+    for node in "${WORKERS[@]}"; do
+        apply_node_config "$node" "$TALOS_GEN_DIR/worker.yaml"
     done
 
     echo -n "Configuring talosctl... "
-    # `gen config` leaves talosconfig endpoints empty; they are the Talos API
-    # addresses and unrelated to cluster.controlPlane.endpoint.
-    if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
-        talosctl config endpoint "${NODE_ADDRS[@]}"
-        talosctl config node "${NODE_ADDRS[0]}"
-    else
-        talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
-        talosctl config node "${NODES[0]}.${DOMAIN}"
-    fi
+    talosctl config endpoint "${NODES[@]/%/.${DOMAIN}}"
+    talosctl config node "${NODES[0]}.${DOMAIN}"
     ok
 
-    step 0 "Waiting for every node to come back configured..."
-    # Asks each machine directly, by address, and expects it to answer with the
-    # hostname its patch assigns. That is what has to be true before etcd is
-    # bootstrapped: the config took, the node rebooted into it, and the machine
-    # at that address is the one we think it is.
-    #
-    # This used to wait on `talosctl get members`, which was wrong twice over.
-    # It queried through the configured endpoint list rather than each node, so
-    # apid refused it with `PermissionDenied: no request forwarding` — the same
-    # thing that broke the bootstrap call. And membership comes from the
-    # discovery service, because the Kubernetes registry is disabled: a cold
-    # rebuild would have been waiting on discovery.talos.dev being reachable,
-    # before the cluster it is trying to rebuild even exists.
-    #
-    # Each node installs to disk and reboots first, so the window is generous.
-    for idx in "${!ALL_NODES[@]}"; do
-        want="${ALL_NODES[$idx]}"
-        addr="${NODE_ADDRS[$idx]:-${want}.${DOMAIN}}"
-        echo -n "  $want ($addr)... "
-        node_err=""
-        for i in $(seq 1 60); do
-            got=$(talosctl -n "$addr" -e "$addr" get hostname \
-                    -o jsonpath='{.spec.hostname}' 2>"$TALOS_GEN_DIR/hostname.err") || true
-            node_err=$(cat "$TALOS_GEN_DIR/hostname.err")
-            [[ "$got" == "$want" ]] && break
-            sleep 10
-            if [[ $i -eq 60 ]]; then
-                echo
-                [[ -n "$node_err" ]] && echo "  last talosctl error: $node_err" >&2
-                fail "$addr never came back as $want (last answer: '${got:-none}')"
-            fi
+    step 0 "Waiting for Talos cluster formation..."
+    for i in $(seq 1 30); do
+        members=$(talosctl get members 2>/dev/null || true)
+        missing=0
+        for node in "${ALL_NODES[@]}"; do
+            grep -q "$node" <<<"$members" || missing=1
         done
-        ok
+        [[ $missing -eq 0 ]] && break
+        sleep 5
+        [[ $i -eq 30 ]] && fail "Talos cluster didn't form"
     done
 
     echo -n "Bootstrapping Kubernetes... "
-    # -n AND -e on one control plane, deliberately. etcd bootstrap is the one
-    # call apid refuses to forward: reached through any other node it fails with
-    # `PermissionDenied: no request forwarding`. Without them talosctl picks an
-    # endpoint from the configured list — with more than one control plane, or a
-    # list that includes workers, that is a coin flip.
-    if [[ ${#NODE_ADDRS[@]} -gt 0 ]]; then
-        bootstrap_target="${NODE_ADDRS[0]}"
-    else
-        bootstrap_target="${NODES[0]}.${DOMAIN}"
-    fi
-    # Re-running against an already-bootstrapped cluster is not an error and is
-    # the reason for `|| true` — but that used to swallow every other reason as
-    # well, and the run carried on to fail somewhere further downstream with no
-    # trace of the actual cause. Anything that is not "already exists" is shown.
-    if ! bootstrap_err=$(talosctl bootstrap -n "$bootstrap_target" -e "$bootstrap_target" 2>&1); then
-        grep -qi 'already\|AlreadyExists' <<<"$bootstrap_err" \
-            || echo -e "\n${YELLOW}bootstrap reported:${NC} ${bootstrap_err//$'\n'/ | }"
-    fi
+    talosctl bootstrap 2>/dev/null || true
     ok
 
     echo -n "Getting kubeconfig... "
-    kubeconfig_err=""
     for i in $(seq 1 30); do
-        mkdir -p "$(dirname "$KUBECONFIG_PATH")"
-        if kubeconfig_err=$(talosctl kubeconfig "$KUBECONFIG_PATH" --force 2>&1); then
-            sed -i "s|server:.*|server: https://${K8S_API_HOST}:6443|" "$KUBECONFIG_PATH"
+        if talosctl kubeconfig ~/.kube/config --force 2>/dev/null; then
+            sed -i "s|server:.*|server: https://${K8S_API_HOST}:6443|" ~/.kube/config
             break
         fi
         sleep 5
-        # Held back while retrying — the first attempts fail by design — but
-        # printed if it never works, because that message is the diagnosis.
-        [[ $i -eq 30 ]] && { echo; echo "$kubeconfig_err" >&2; fail "Couldn't get kubeconfig"; }
-    done
-    ok
-
-    echo -n "Waiting for the Kubernetes API to answer... "
-    # Having a kubeconfig is not the same as having an apiserver. `talosctl
-    # kubeconfig` succeeds as soon as Talos can hand out credentials, which is
-    # before kube-apiserver is serving on the endpoint the file points at — so
-    # the very next command could fail with an EOF against a port nothing is
-    # listening on yet. On hardware the gap usually closes on its own; on a
-    # scratch cluster it does not, which is how this was found.
-    # `get --raw /version` needs the server; `kubectl version` would answer from
-    # the client alone and pass against nothing at all.
-    api_err=""
-    for i in $(seq 1 60); do
-        api_err=$(kubectl get --raw /version 2>&1) && break
-        sleep 5
-        if [[ $i -eq 60 ]]; then
-            echo "$api_err" >&2
-            fail "Kubernetes API at https://${K8S_API_HOST}:6443 never answered"
-        fi
+        [[ $i -eq 30 ]] && fail "Couldn't get kubeconfig"
     done
     ok
 
@@ -424,58 +213,20 @@ if phase_runs talos; then
 
     step 0 "Waiting for K8s nodes..."
     expected=${#ALL_NODES[@]}
-    nodes_err=""
     for i in $(seq 1 60); do
-        # The STATUS column, matched whole. `grep -c Ready` counted NotReady
-        # too — the substring is in it — so this waited for nodes to register
-        # rather than to become ready, and passed while every one of them was
-        # still NotReady.
-        nodes_out=$(kubectl get nodes --no-headers 2>"$TALOS_GEN_DIR/nodes.err") || true
-        nodes_err=$(cat "$TALOS_GEN_DIR/nodes.err")
-        ready=$(awk '$2 == "Ready"' <<<"$nodes_out" | grep -c . || true)
+        ready=$(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready" || true)
         if [[ "$ready" -eq "$expected" ]]; then
             break
         fi
         sleep 5
-        if [[ $i -eq 60 ]]; then
-            echo
-            [[ -n "$nodes_err" ]] && echo "last kubectl error: $nodes_err" >&2
-            echo "${nodes_out:-  (no nodes listed)}" >&2
-            fail "Not all nodes ready (got $ready/$expected)"
-        fi
+        [[ $i -eq 60 ]] && fail "Not all nodes ready (got $ready/$expected)"
     done
 
-    # Stash configs for subsequent phases / export-access step. Skipped under
-    # vmtest: those are a throwaway cluster's credentials, and writing them here
-    # would leave the flux phase pointed at VMs that no longer exist.
-    if [[ "${VMTEST:-0}" == "1" ]]; then
-        # Nodes being Ready says nothing about storage. The CephCluster picks
-        # OSD nodes by a label out of the machine config and their device by the
-        # partition label RawVolumeConfig creates, so assert both here — running
-        # it by hand afterwards races the teardown, and a test that does not
-        # check its own premise is worth little.
-        step 0 "Checking the OSD prerequisites Ceph selects on..."
-        for idx in $(seq "${#NODES[@]}" $(( ${#ALL_NODES[@]} - 1 ))); do
-            vm_node="${ALL_NODES[$idx]}"
-            vm_addr="${NODE_ADDRS[$idx]}"
-            vols=$(talosctl -n "$vm_addr" -e "$vm_addr" get discoveredvolumes 2>&1) \
-                || fail "could not read volumes from $vm_node ($vm_addr): ${vols//$'\n'/ | }"
-            awk '$NF == "r-ceph-osd" {found=1} END {exit !found}' <<<"$vols" \
-                || fail "$vm_node has no r-ceph-osd partition — RawVolumeConfig did not take"
-            kubectl get node "$vm_node" \
-                -o jsonpath='{.metadata.labels.storage\.renner\.uno/ceph-osd}' \
-                | grep -q true \
-                || fail "$vm_node is missing the storage.renner.uno/ceph-osd label"
-            echo -e "  $vm_node: partition and label ${GREEN}(OK)${NC}"
-        done
-
-        echo -e "${GREEN}VM rebuild test passed.${NC} ${#ALL_NODES[@]} nodes Ready on the repo's own config."
-    else
-        mkdir -p "$ACCESS_STAGING"
-        cp "$TALOSCONFIG_PATH" "$ACCESS_STAGING/talosconfig"
-        cp "$KUBECONFIG_PATH"  "$ACCESS_STAGING/kubeconfig"
-        echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
-    fi
+    # Stash configs for subsequent phases / export-access step
+    mkdir -p "$ACCESS_STAGING"
+    cp ~/.talos/config "$ACCESS_STAGING/talosconfig"
+    cp ~/.kube/config  "$ACCESS_STAGING/kubeconfig"
+    echo -e "${GREEN}Talos phase complete.${NC} Access files in $ACCESS_STAGING/"
 fi
 
 # Helper: load staged access files into home dir (for later CI steps)
@@ -514,7 +265,7 @@ if phase_runs flux; then
         flux bootstrap github \
             --owner="$GITHUB_OWNER" \
             --repository="$GITHUB_REPO" \
-            --branch="$FLUX_BRANCH" \
+            --branch=master \
             --path="$FLUX_PATH" \
             --components-extra=image-reflector-controller,image-automation-controller \
             --personal \
@@ -525,7 +276,7 @@ if phase_runs flux; then
             --hostname="$GITEA_HOST" \
             --owner="$GITEA_OWNER" \
             --repository="$GITEA_REPO" \
-            --branch="$FLUX_BRANCH" \
+            --branch=master \
             --path="$FLUX_PATH" \
             --components-extra=image-reflector-controller,image-automation-controller \
             --personal \

kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml

@@ -1,118 +0,0 @@
----
-# Advertises the on-board Intel iGPU as the schedulable resource
-# `gpu.intel.com/i915`, so pods can request it without a /dev/dri hostPath or
-# any privilege escalation — the plugin hands the render node to the container
-# through the kubelet device-plugin API. The same resource covers both GPU
-# compute (LLM inference) and QuickSync video transcoding; it is not
-# compute-specific.
-#
-# All three nodes are MS-01 with an i9-12900H (Alder Lake-P, PCI 8086:46a6,
-# Iris Xe 96 EU) at 0000:00:02.0. Talos carries the driver in the `i915` system
-# extension, already present in the schematic, so no machineconfig change is
-# needed.
-#
-# Vendored from upstream rather than installed via the
-# `intel-device-plugins-gpu` Helm chart: that chart renders a `GpuDevicePlugin`
-# custom resource, which only works alongside the Intel Device Plugin Operator
-# and its CRDs, cert-manager webhook and controller. With one device type on
-# three identical nodes, the operator manages nothing this DaemonSet does not.
-#
-# Source: intel/intel-device-plugins-for-kubernetes, v0.36.0,
-# deployments/gpu_plugin/base/intel-gpu-plugin.yaml
-# Upgrading means re-syncing this file against that path at the new tag and
-# bumping the image tag below.
-#
-# Namespace is kube-system because the plugin mounts hostPaths (/dev/dri,
-# /sys/class/drm, the kubelet device-plugin socket and the CDI path). The
-# cluster enforces PodSecurity `baseline`, which forbids hostPath volumes, and
-# kube-system is the only namespace in the API server's exemption list.
-#
-# Node Feature Discovery is not deployed, so upstream's `nfd_labeled_nodes`
-# overlay is omitted along with its `intel.feature.node.kubernetes.io/gpu`
-# nodeSelector. Every node carries the same iGPU, leaving nothing to discover;
-# the plugin exits cleanly on a node without /dev/dri.
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: intel-gpu-plugin
-  namespace: kube-system
-  labels:
-    app: intel-gpu-plugin
-spec:
-  selector:
-    matchLabels:
-      app: intel-gpu-plugin
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxSurge: 0
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        app: intel-gpu-plugin
-    spec:
-      containers:
-        - name: intel-gpu-plugin
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: HOST_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.hostIP
-          image: intel/intel-gpu-plugin:0.36.0
-          imagePullPolicy: IfNotPresent
-          args:
-            # Slots the single iGPU is handed out as, letting that many
-            # containers share it concurrently. A slot count, not fractional
-            # accounting — nothing partitions the device between sharers, so
-            # they contend for it. Sized for an inference server plus room for
-            # a transcoder and a benchmark pod alongside it.
-            - -shared-dev-num=4
-          securityContext:
-            seLinuxOptions:
-              type: "container_device_plugin_t"
-            readOnlyRootFilesystem: true
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-            seccompProfile:
-              type: RuntimeDefault
-          resources:
-            requests:
-              memory: "45Mi"
-              cpu: 40m
-            limits:
-              memory: "90Mi"
-              cpu: 100m
-          volumeMounts:
-            - name: devfs
-              mountPath: /dev/dri
-              readOnly: true
-            - name: sysfsdrm
-              mountPath: /sys/class/drm
-              readOnly: true
-            - name: kubeletsockets
-              mountPath: /var/lib/kubelet/device-plugins
-            - name: cdipath
-              mountPath: /var/run/cdi
-      volumes:
-        - name: devfs
-          hostPath:
-            path: /dev/dri
-        - name: sysfsdrm
-          hostPath:
-            path: /sys/class/drm
-        - name: kubeletsockets
-          hostPath:
-            path: /var/lib/kubelet/device-plugins
-        - name: cdipath
-          hostPath:
-            path: /var/run/cdi
-            type: DirectoryOrCreate
-      nodeSelector:
-        kubernetes.io/arch: amd64

kubernetes/infrastructure/intel-gpu-plugin/kustomization.yaml

@@ -1,4 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - daemonset.yaml

kubernetes/infrastructure/kustomization.yaml

@@ -10,7 +10,6 @@ resources:
   - cert-manager/
   - snapshot-controller/
   - metrics-server/
-  - intel-gpu-plugin/
   - crowdsec/
   - traefik/
   - rook-ceph/

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -118,76 +118,43 @@ spec:
     # simultaneously — every PG inactive until they return. Binds any change
     # that rewrites the OSD deployment spec, not just version upgrades.
     osdMaxUpdatesInParallel: 1
-    # Which machines carry storage is declared by the machines, as the node
-    # label storage.renner.uno/ceph-osd (talos/patches/k8s-*.yaml), and read
-    # here through placement.osd.nodeAffinity below. Listing hostnames here
-    # instead said the same thing in a second place and tied this file to three
-    # specific boxes: a cluster built anywhere else got no OSDs at all, however
-    # its machines were equipped.
-    #
-    # useAllNodes only means "do not keep a node list here"; nodeAffinity is
-    # what restricts OSDs, and without it this would be every node.
-    useAllNodes: true
+    useAllNodes: false
     useAllDevices: false
-    # Two forms while the production drives are migrated:
-    #
-    #   by-partlabel/r-ceph-osd  the target. Talos names the drive in the
-    #                            machine config (RawVolumeConfig ceph-osd), so
-    #                            this file needs to know nothing about models,
-    #                            serials or hostnames.
-    #   by-id/nvme-SAMSUNG…      the three MS-01 drives as they are today:
-    #                            bluestore written straight onto the raw disk,
-    #                            with no partition table and therefore no
-    #                            partition label to match. Matching by model
-    #                            rather than by the three individual serials it
-    #                            replaced — a drive swap is no longer a repo
-    #                            edit. It cannot match the Kingston system disk
-    #                            or the Crucial that carries EPHEMERAL.
-    #
-    # The second half goes when those three are repartitioned, one node at a
-    # time. Until then it is what keeps the running OSDs claimed.
-    devicePathFilter: "^/dev/disk/by-(partlabel/r-ceph-osd|id/nvme-SAMSUNG_MZQLB960HAJR)"
+    nodes:
+      - name: k8s-1
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M400426
+      - name: k8s-2
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M403161
+      - name: k8s-3
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NF0M501176
     config:
       osdsPerDevice: "1"
   # Highest built-in class: kubelet evicts these last and the kernel gives
   # them the lowest oom_score_adj.
   priorityClassNames:
     all: system-node-critical
-  # Requests are what a daemon actually uses; limits stay where they were, as
-  # burst headroom. The two were equal, for Guaranteed QoS, and the gap was
-  # large enough to matter on machines where memory is the scarce resource:
-  # measured against the running cluster, mon sat at ~420Mi against a 2Gi
-  # request, mgr at ~650Mi against 4Gi, and the OSDs at ~2.2Gi against 4Gi —
-  # about 17GiB reserved cluster-wide and never used. Only the requests moved.
-  #
-  # Guaranteed QoS is given up deliberately. What it bought was eviction order,
-  # and priorityClassNames below already pins these to system-node-critical,
-  # which the kubelet weighs first.
-  #
-  # CPU limits are deliberately NOT set — Ceph daemons burst during recovery and
-  # a limit would throttle them.
+  # requests == limits gives Guaranteed QoS. CPU limits are deliberately NOT
+  # set — Ceph daemons burst during recovery and a limit would throttle them.
   resources:
     mon:
       requests:
         cpu: 500m
-        memory: 512Mi
+        memory: 2Gi
       limits:
         memory: 2Gi
     osd:
-      # The OSD request is not simply "what it uses": an OSD grows its cache
-      # towards Ceph's own osd_memory_target, which defaults to 4Gi, so the
-      # measured 2.2Gi is a floor and the 4Gi limit is what that tunable implies.
-      # Lower the limit without lowering osd_memory_target and the OSD gets
-      # OOM-killed once the cache fills.
       requests:
         cpu: 500m
-        memory: 2560Mi
+        memory: 4Gi
       limits:
         memory: 4Gi
     mgr:
       requests:
         cpu: 250m
-        memory: 1Gi
+        memory: 4Gi
       limits:
         memory: 4Gi
   placement:
@@ -195,16 +162,3 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/control-plane
-    # The other half of selecting OSD nodes by label: with useAllNodes: true
-    # this is the only thing keeping OSDs off machines that carry no OSD drive.
-    # Removing it does not fall back to the old node list — it puts OSDs
-    # everywhere.
-    osd:
-      nodeAffinity:
-        requiredDuringSchedulingIgnoredDuringExecution:
-          nodeSelectorTerms:
-            - matchExpressions:
-                - key: storage.renner.uno/ceph-osd
-                  operator: In
-                  values:
-                    - "true"

kubernetes/talos/patches/cluster.yaml

@@ -27,9 +27,7 @@ machine:
     image: factory.talos.dev/metal-installer/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9:v1.13.9
   features:
     rbac: true
-    # stableHostname is NOT here: it is the v1alpha1 spelling of what the
-    # generated HostnameConfig document already says with `auto: stable`.
-    # Asking for the same thing in both schemas is what Talos rejects.
+    stableHostname: true
     apidCheckExtKeyUsage: true
     diskQuotaSupport: true
     kubePrism:
@@ -41,12 +39,13 @@ machine:
   nodeLabels:
     node.kubernetes.io/exclude-from-external-load-balancers: ""
   kubelet:
-    # nodeIP.validSubnets is NOT here: it names a network this cluster happens
-    # to sit on, not something true of every cluster built from this file, so it
-    # lives in the site patch (site-homelab.yaml). Here it forced every machine
-    # onto 10.2.40.0/24 whether or not it has an interface there, which is what
-    # made this file unusable anywhere but on the three production boxes.
-    #
+    # Constrain kubelet's auto-picked node IP to the mgmt LAN. Without
+    # this, kubelet picks the first non-loopback IP, which could land on
+    # the v30iot NIC (ens19, 10.2.30.0/24). Control-plane traffic +
+    # hostNetwork pod IPs must stay on mgmt LAN.
+    nodeIP:
+      validSubnets:
+        - 10.2.40.0/24
     # Lower than the kubelet defaults (85/80) to bound containerd's image
     # cache, the largest single consumer of /var.
     #
@@ -58,11 +57,8 @@ machine:
       imageGCHighThresholdPercent: 70
       imageGCLowThresholdPercent: 60
 cluster:
-  # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its
-  # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443
-  # there. Setting it again in this patch restated the same value in a second
-  # place and overrode whatever endpoint a caller had generated for, which is the
-  # only reason a config generated for a different cluster could not be used.
+  controlPlane:
+    endpoint: https://k8s.renner.uno:6443
   apiServer:
     certSANs:
       - k8s.renner.uno

kubernetes/talos/patches/k8s-1.yaml

@@ -2,6 +2,8 @@
 # machine.network.interfaces. Keep every link in one schema: the docs do not
 # define what happens when both configure the same link.
 machine:
+  network:
+    hostname: k8s-1
   sysctls:
     # ens19 (VLAN 30, v30iot) is LINK ONLY — it is the macvlan parent for the
     # v30iot pod attachments and must never get host addressing. A connected
@@ -24,28 +26,9 @@ machine:
     - k8s-1.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: jupiter
-    # This machine holds a Ceph OSD drive. The CephCluster selects OSD nodes by
-    # this label instead of listing hostnames, so which boxes carry storage is
-    # stated once, here, by the machine that knows.
-    storage.renner.uno/ceph-osd: "true"
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
-# The hostname, in the schema talosctl generates rather than in
-# machine.network.hostname. `talosctl gen config` always emits a HostnameConfig
-# document, and naming the host in both schemas is what Talos rejects — at apply
-# time, with `static hostname is already set in v1alpha1 config`, not merely in
-# validate. bootstrap.sh used to cut the generated document back out with awk to
-# keep the v1alpha1 spelling working; setting the name here instead means there
-# is nothing to cut.
-#
-# `auto: off` switches off the generated `auto: stable`, which is otherwise
-# mutually exclusive with a static name.
-apiVersion: v1alpha1
-kind: HostnameConfig
-auto: off
-hostname: k8s-1
----
 # EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
 # contents are reconstructible, so a worn disk is consumable here in a way it
 # is not as an OSD. Note /var/lib/etcd lives here too, which is the one thing

kubernetes/talos/patches/k8s-2.yaml

@@ -2,6 +2,8 @@
 # to k8s-1.yaml — see that file for the reasoning behind each choice; only the
 # serials, MACs and addresses differ.
 machine:
+  network:
+    hostname: k8s-2
   sysctls:
     # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It is the
     # macvlan parent for the v30iot pod attachments; host addresses on that L2
@@ -21,18 +23,9 @@ machine:
     - k8s-2.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: saturn
-    # Holds a Ceph OSD drive — see k8s-1.yaml.
-    storage.renner.uno/ceph-osd: "true"
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
-# Hostname in the generated schema, not machine.network.hostname — see
-# k8s-1.yaml for why `auto: off` is set alongside it.
-apiVersion: v1alpha1
-kind: HostnameConfig
-auto: off
-hostname: k8s-2
----
 # EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
 # contents are reconstructible, so a worn disk is consumable in this role.
 #

kubernetes/talos/patches/k8s-3.yaml

@@ -1,6 +1,8 @@
 # Structurally identical to k8s-1.yaml — see that file for the reasoning
 # behind each choice; only the serials and addresses differ.
 machine:
+  network:
+    hostname: k8s-3
   sysctls:
     # v30iot NIC carries no host addressing — see k8s-1.yaml.
     net.ipv6.conf.ens19.disable_ipv6: "1"
@@ -17,18 +19,9 @@ machine:
     - k8s-3.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: neptune
-    # Holds a Ceph OSD drive — see k8s-1.yaml.
-    storage.renner.uno/ceph-osd: "true"
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
-# Hostname in the generated schema, not machine.network.hostname — see
-# k8s-1.yaml for why `auto: off` is set alongside it.
-apiVersion: v1alpha1
-kind: HostnameConfig
-auto: off
-hostname: k8s-3
----
 # EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
 # contents are reconstructible, so a worn disk is consumable in this role.
 #

kubernetes/talos/patches/site-homelab.yaml

@@ -1,16 +0,0 @@
-# Facts about the network this cluster is wired into, as opposed to facts about
-# the cluster (cluster.yaml) or about one machine (k8s-1.yaml, vm-1.yaml).
-#
-# Applied to all three production nodes and to nothing else. A cluster brought up
-# somewhere else — a test VM on the provisioner's network — takes its own site
-# patch instead, which is the whole point of separating this out: it is the only
-# part of the config that could not be true anywhere but here.
-machine:
-  kubelet:
-    # Constrain kubelet's auto-picked node IP to the mgmt LAN. Without this,
-    # kubelet picks the first non-loopback IP, which could land on the v30iot
-    # NIC (ens19, 10.2.30.0/24). Control-plane traffic and hostNetwork pod IPs
-    # must stay on mgmt LAN.
-    nodeIP:
-      validSubnets:
-        - 10.2.40.0/24

kubernetes/talos/patches/site-qemu.yaml

@@ -1,15 +0,0 @@
-# The site patch for a cluster on the QEMU provisioner's network, used in place
-# of site-homelab.yaml when a rebuild is tried on a VM rather than on the three
-# MS-01 boxes.
-#
-# This file is the complete list of what a test run assumes about its network.
-# Everything else it uses — cluster.yaml and a node patch — is the same file a
-# real rebuild reads. If this file ever grows beyond network facts, the test has
-# started proving less than it appears to.
-machine:
-  kubelet:
-    # The provisioner's default CIDR (talosctl cluster create --cidr). One NIC,
-    # no VLANs, so there is nothing to keep kubelet away from.
-    nodeIP:
-      validSubnets:
-        - 10.5.0.0/24

kubernetes/talos/patches/vm-1.yaml

@@ -1,45 +0,0 @@
-# Node patch for a virtual test node, alongside k8s-1/2/3 and in the same shape.
-#
-# It exists so a rebuild can be tried against the real cluster.yaml instead of a
-# hand-picked extract of it. An extract tests the extract: an earlier attempt
-# dropped allowSchedulingOnControlPlanes and every Flux controller went
-# unschedulable, which says nothing about the config a recovery would use.
-#
-# No chassis-serial marker, unlike the k8s-* patches. Virtual machines report no
-# serial at all — `talosctl get systeminformation` has no serialnumber field —
-# so bootstrap.sh's patch_for_machine cannot identify one. This file is named
-# explicitly by the test harness.
-#
-# Everything k8s-1.yaml carries that is missing here is missing because the
-# hardware is:
-#
-#   no LinkConfig / VLANConfig  the VM has one NIC. There is no X710 trunk to
-#                               tag, no v30iot macvlan parent to keep
-#                               addressless, and no Ceph cluster network.
-#   no Layer2VIPConfig          the shared control-plane VIP is elected between
-#                               three machines; one node has nothing to share it
-#                               with.
-#   no VolumeConfig EPHEMERAL   that moves /var onto a retired OSD drive. One
-#                               disk here, so /var stays on the system disk.
-machine:
-  install:
-    # Named directly rather than selected by serial. The serial selector on the
-    # real nodes guards against the installer taking the Ceph OSD drive; with a
-    # single disk there is nothing to pick wrongly.
-    disk: /dev/vda
-  certSANs:
-    - vm-1
-cluster:
-  allowSchedulingOnControlPlanes: true
----
-# The hostname, in the schema talosctl generates rather than in
-# machine.network.hostname. `talosctl gen config` always emits a HostnameConfig
-# document, and naming the host in both schemas is what Talos rejects at
-# apply time with `static hostname is already set in v1alpha1 config`.
-#
-# `auto: off` switches off the generated `auto: stable`, which is otherwise
-# mutually exclusive with a static name.
-apiVersion: v1alpha1
-kind: HostnameConfig
-auto: off
-hostname: vm-1

kubernetes/talos/patches/vm-2.yaml

@@ -1,37 +0,0 @@
-# Virtual test worker carrying a Ceph OSD device.
-#
-# A worker rather than a control plane for one reason: the QEMU provisioner
-# hands out only the first disk to control planes ("disks after the first one
-# are added only to worker machines"), and an OSD needs a second one. Ceph does
-# not care which role a node has — mons and OSDs schedule on workers fine.
-machine:
-  install:
-    disk: /dev/vda
-  certSANs:
-    - vm-2
-  nodeLabels:
-    # This machine carries a Ceph OSD device. The CephCluster selects OSD nodes
-    # by this label rather than by hostname, so which machines hold storage is
-    # stated once, by the machine that has it.
-    storage.renner.uno/ceph-osd: "true"
----
-apiVersion: v1alpha1
-kind: HostnameConfig
-auto: off
-hostname: vm-2
----
-# The OSD device, named by the machine that has it. Ceph then consumes
-# /dev/disk/by-partlabel/r-ceph-osd and needs to know nothing about models,
-# serials or hostnames — the same CephCluster works here and on the MS-01s.
-#
-# The selector is a CEL expression, so each machine says whatever identifies its
-# own drive. Here that is "the virtio disk that is not the system disk"; on the
-# real nodes it will name the Samsung.
-apiVersion: v1alpha1
-kind: RawVolumeConfig
-name: ceph-osd
-provisioning:
-  diskSelector:
-    match: disk.transport == "virtio" && !system_disk
-  minSize: 5GiB
-  grow: true

kubernetes/talos/patches/vm-3.yaml

@@ -1,23 +0,0 @@
-# Virtual test worker carrying a Ceph OSD device. Identical to vm-2.yaml — see
-# that file for the reasoning; only the hostname differs.
-machine:
-  install:
-    disk: /dev/vda
-  certSANs:
-    - vm-3
-  nodeLabels:
-    storage.renner.uno/ceph-osd: "true"
----
-apiVersion: v1alpha1
-kind: HostnameConfig
-auto: off
-hostname: vm-3
----
-apiVersion: v1alpha1
-kind: RawVolumeConfig
-name: ceph-osd
-provisioning:
-  diskSelector:
-    match: disk.transport == "virtio" && !system_disk
-  minSize: 5GiB
-  grow: true

kubernetes/talos/patches/vm-4.yaml

@@ -1,27 +0,0 @@
-# Virtual test worker carrying a Ceph OSD device. Identical to vm-2.yaml — see
-# that file for the reasoning; only the hostname differs.
-#
-# Three OSD workers because the CephCluster runs three mons with
-# allowMultiplePerNode: false and its pools are replica 3 over failureDomain
-# host — two nodes would never reach quorum or place a third replica.
-machine:
-  install:
-    disk: /dev/vda
-  certSANs:
-    - vm-4
-  nodeLabels:
-    storage.renner.uno/ceph-osd: "true"
----
-apiVersion: v1alpha1
-kind: HostnameConfig
-auto: off
-hostname: vm-4
----
-apiVersion: v1alpha1
-kind: RawVolumeConfig
-name: ceph-osd
-provisioning:
-  diskSelector:
-    match: disk.transport == "virtio" && !system_disk
-  minSize: 5GiB
-  grow: true