fix: [alert] TargetDown (pve-node) (#573)

Reviewed-on: k.renner/homelab#575

kubernetes/apps/pve/endpointslice.yaml

@@ -16,11 +16,29 @@ ports:
     port: 8006
     protocol: TCP
 endpoints:
+  # jupiter is OUT OF SERVICE as a PVE node: its MS-01 hardware is being
+  # converted to run Talos directly as k8s-1 (see
+  # kubernetes/talos/patches/k8s-1.yaml), and nothing answers the PVE API on
+  # .201 while that is in progress — verified 2026-08-25 from pve-exporter,
+  # which fails with "[Errno 113] Host is unreachable" for .201.
+  #
+  # This slice is manually managed, and Kubernetes health-checks nothing it did
+  # not create — a hardcoded `ready: true` keeps kube-proxy and Traefik load
+  # balancing onto a dead address forever. That silently defeated the whole
+  # point of pointing the pve scrape job at this Service rather than at a node
+  # IP (see the job_name: pve comment in
+  # kubernetes/apps/monitoring/helmrelease-prometheus.yaml): cluster metrics
+  # were meant to survive any one node being down, but instead
+  # up{job="pve"} flapped 0/1 as connections landed on .201, and pve.renner.uno
+  # served an error for the same fraction of requests.
+  #
+  # Marking it not-ready is the supported way to drain a manual endpoint. If
+  # jupiter is ever rebuilt as a PVE member, flip this back to true.
   - addresses:
       - 10.2.40.201
     hostname: jupiter
     conditions:
-      ready: true
+      ready: false
   - addresses:
       - 10.2.40.202
     hostname: neptune

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -11,43 +11,6 @@ spec:
   mon:
     count: 3
     allowMultiplePerNode: false
-  # TEMPORARY — SET BACK TO false (or drop this block) ONCE k8s-1 HAS REJOINED.
-  #
-  # Stops Rook's mon failover loop for the jupiter window. k8s-1 is no longer a
-  # Node object at all, so the health checker takes the "assigned node not
-  # found" path and fails over *immediately* instead of after the 600s
-  # out-of-quorum timeout:
-  #
-  #   op-mon: mon "c" NOT found in quorum and its assigned node "k8s-1" is not
-  #           found, failing over immediately
-  #
-  # The failover can never complete. Mons run hostNetwork on 3300/6789 and
-  # allowMultiplePerNode is false, so with k8s-1 gone the replacement's canary
-  # has nowhere to land ("2 node(s) didn't have free ports for the requested
-  # pod ports"). The operator gives up after ~2.5 min, deletes the canary, and
-  # the 45s health-check interval starts the next attempt. Each cycle churns a
-  # Deployment plus pod, rewrites the mon-endpoints ConfigMap and the ceph-csi
-  # connection/clientProfile CRs, and buries everything else in the operator log.
-  #
-  # This does NOT hide the degradation: daemonHealth.status is untouched, so
-  # MON_DOWN stays in CephCluster.status.ceph and CephHealthWarn still fires,
-  # and rook-ceph-mon-c stays 0/1 Pending so KubeDeploymentReplicasMismatch
-  # keeps firing. Nothing is given up, because there is no node the failover
-  # could have succeeded on anyway.
-  #
-  # It also keeps mon "c" intact for the end of the window: with the checker
-  # live, the moment k8s-1 rejoins the operator races to stand mon-d up there
-  # and drop "c" from the monmap. Off, mon-c reschedules onto k8s-1 and rejoins
-  # with its existing store under /var/lib/rook/mon-c. If that store did not
-  # survive the reinstall, re-enable the checker first — then failover is
-  # exactly what you want.
-  #
-  # Reverts together with the `RE-ADD IT` note under storage.nodes below and
-  # the `RESTORE TO 1` note in kubernetes/apps/immich/machine-learning.yaml.
-  healthCheck:
-    daemonHealth:
-      mon:
-        disabled: true
   mgr:
     count: 2
     allowMultiplePerNode: false

kubernetes/talos/patches/k8s-1.yaml

@@ -1,7 +1,7 @@
 # k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
 # still Proxmox VMs. Every machine-specific selector below differs from theirs
-# for that reason: there are no virtio NICs, no PCI-passthrough layer, and both
-# NVMe drives are visible to Talos at once.
+# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
+# four physical NICs and all three NVMe drives are visible to Talos at once.
 #
 # Networking uses the v1.12 multi-document config rather than the deprecated
 # machine.network.interfaces. That is deliberate: the docs do not define what
@@ -24,10 +24,12 @@ machine:
     # This disables v6 entirely on that link, including link-local.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    # Both NVMe drives are now ordinary block devices to Talos: the Kingston
-    # OM8PGP41024Q-A0 system disk and the Samsung PM983 that Rook claims as the
-    # Ceph OSD. Selecting the installer target by serial means it cannot land
-    # on the OSD drive regardless of which one the kernel enumerates first.
+    # Three NVMe drives are ordinary block devices to Talos here: the Kingston
+    # OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
+    # Ceph OSD, and the retired Crucial P3 that now carries EPHEMERAL (see the
+    # VolumeConfig below). Every one of them is selected by serial. Nothing in
+    # this file may use a broad selector: `disk.transport == 'nvme'` matches all
+    # three, and the failure mode is Talos taking the OSD drive.
     diskSelector:
       serial: 50026B7382B48B86
   certSANs:
@@ -37,9 +39,39 @@ machine:
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
+#
+# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
+# the PM983 replaced it. /var is exactly the right role for a disk in that
+# state: containerd's image and snapshot store, pod logs and the Ceph mon store
+# are all reconstructible, so the drive is consumable here in a way it is not as
+# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
+# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
+# they absorb another node's pods.
+#
+# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; a config applied after EPHEMERAL
+# has been created on the system disk is silently ignored, and relocating it
+# then requires wiping the disk and booting install media again. So this
+# document has to be present in the very first `apply-config --insecure`.
+#
+# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
+# would match the PM983 as well, and losing that coin flip means Talos formats
+# the OSD drive.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B749A'
+  minSize: 50GiB
+  grow: true
+---
 # Physical NICs are aliased by permanent MAC. Both X710 ports bind i40e, so the
 # driver selector k8s-2/k8s-3 use for their single passed-through port would
-# match two links here. MAC matching is unambiguous and survives udev renaming.
+# match two links here. MAC matching is unambiguous and survives udev renaming —
+# which is not hypothetical: adding the third NVMe renumbered the PCI buses and
+# moved the second I226 from enp89s0 to enp90s0.
 apiVersion: v1alpha1
 kind: LinkAliasConfig
 name: lan0
@@ -81,7 +113,7 @@ link: mgmt0
 # that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
 # serve both VM and bare-metal nodes for as long as the cluster is mixed.
 # Nothing else on this machine is called ens19: the physical NICs are
-# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp89s0.
+# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp90s0.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19