fix: [alert] TargetDown (pve-node) (#573)

Reviewed-on: k.renner/homelab#575

kubernetes/apps/pve/endpointslice.yaml

@@ -16,11 +16,29 @@ ports:
     port: 8006
     protocol: TCP
 endpoints:
+  # jupiter is OUT OF SERVICE as a PVE node: its MS-01 hardware is being
+  # converted to run Talos directly as k8s-1 (see
+  # kubernetes/talos/patches/k8s-1.yaml), and nothing answers the PVE API on
+  # .201 while that is in progress — verified 2026-08-25 from pve-exporter,
+  # which fails with "[Errno 113] Host is unreachable" for .201.
+  #
+  # This slice is manually managed, and Kubernetes health-checks nothing it did
+  # not create — a hardcoded `ready: true` keeps kube-proxy and Traefik load
+  # balancing onto a dead address forever. That silently defeated the whole
+  # point of pointing the pve scrape job at this Service rather than at a node
+  # IP (see the job_name: pve comment in
+  # kubernetes/apps/monitoring/helmrelease-prometheus.yaml): cluster metrics
+  # were meant to survive any one node being down, but instead
+  # up{job="pve"} flapped 0/1 as connections landed on .201, and pve.renner.uno
+  # served an error for the same fraction of requests.
+  #
+  # Marking it not-ready is the supported way to drain a manual endpoint. If
+  # jupiter is ever rebuilt as a PVE member, flip this back to true.
   - addresses:
       - 10.2.40.201
     hostname: jupiter
     conditions:
-      ready: true
+      ready: false
   - addresses:
       - 10.2.40.202
     hostname: neptune

kubernetes/talos/patches/k8s-1.yaml

@@ -1,7 +1,7 @@
 # k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
 # still Proxmox VMs. Every machine-specific selector below differs from theirs
-# for that reason: there are no virtio NICs, no PCI-passthrough layer, and both
-# NVMe drives are visible to Talos at once.
+# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
+# four physical NICs and all three NVMe drives are visible to Talos at once.
 #
 # Networking uses the v1.12 multi-document config rather than the deprecated
 # machine.network.interfaces. That is deliberate: the docs do not define what
@@ -24,10 +24,12 @@ machine:
     # This disables v6 entirely on that link, including link-local.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    # Both NVMe drives are now ordinary block devices to Talos: the Kingston
-    # OM8PGP41024Q-A0 system disk and the Samsung PM983 that Rook claims as the
-    # Ceph OSD. Selecting the installer target by serial means it cannot land
-    # on the OSD drive regardless of which one the kernel enumerates first.
+    # Three NVMe drives are ordinary block devices to Talos here: the Kingston
+    # OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
+    # Ceph OSD, and the retired Crucial P3 that now carries EPHEMERAL (see the
+    # VolumeConfig below). Every one of them is selected by serial. Nothing in
+    # this file may use a broad selector: `disk.transport == 'nvme'` matches all
+    # three, and the failure mode is Talos taking the OSD drive.
     diskSelector:
       serial: 50026B7382B48B86
   certSANs:
@@ -37,9 +39,39 @@ machine:
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
+#
+# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
+# the PM983 replaced it. /var is exactly the right role for a disk in that
+# state: containerd's image and snapshot store, pod logs and the Ceph mon store
+# are all reconstructible, so the drive is consumable here in a way it is not as
+# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
+# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
+# they absorb another node's pods.
+#
+# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; a config applied after EPHEMERAL
+# has been created on the system disk is silently ignored, and relocating it
+# then requires wiping the disk and booting install media again. So this
+# document has to be present in the very first `apply-config --insecure`.
+#
+# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
+# would match the PM983 as well, and losing that coin flip means Talos formats
+# the OSD drive.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B749A'
+  minSize: 50GiB
+  grow: true
+---
 # Physical NICs are aliased by permanent MAC. Both X710 ports bind i40e, so the
 # driver selector k8s-2/k8s-3 use for their single passed-through port would
-# match two links here. MAC matching is unambiguous and survives udev renaming.
+# match two links here. MAC matching is unambiguous and survives udev renaming —
+# which is not hypothetical: adding the third NVMe renumbered the PCI buses and
+# moved the second I226 from enp89s0 to enp90s0.
 apiVersion: v1alpha1
 kind: LinkAliasConfig
 name: lan0
@@ -81,7 +113,7 @@ link: mgmt0
 # that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
 # serve both VM and bare-metal nodes for as long as the cluster is mixed.
 # Nothing else on this machine is called ens19: the physical NICs are
-# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp89s0.
+# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp90s0.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19