fix: [alert] TargetDown (pve-node) (#573)

Reviewed-on: k.renner/homelab#575

kubernetes/apps/immich/machine-learning.yaml

@@ -6,7 +6,17 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-machine-learning
 spec:
-  replicas: 1
+  # TEMPORARY — RESTORE TO 1 ONCE k8s-1 HAS REJOINED.
+  #
+  # Down for the jupiter bare-metal conversion. With one of three nodes out,
+  # the remaining pair have no room: cluster CPU requests exceed two nodes'
+  # capacity, and k8s-3 runs ~5.8 GiB above the kubelet's eviction threshold,
+  # so every pod that does not need to be running is one less thing competing
+  # for both. This one is the best candidate: 500m reserved against a measured
+  # 2m idle draw, and inference bursts hard enough that lowering the request
+  # instead would be the wrong trade. Photo processing pauses; nothing else
+  # depends on it.
+  replicas: 0
   strategy:
     type: RollingUpdate
   selector:

kubernetes/apps/immich/server.yaml

@@ -36,7 +36,11 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-server
 spec:
-  replicas: 1
+  # TEMPORARY — RESTORE TO 1 ONCE k8s-1 HAS REJOINED. See the note on
+  # immich-machine-learning; immich is offline for the jupiter conversion.
+  # Its Postgres cluster is deliberately left running, so nothing needs
+  # restoring beyond these two replica counts.
+  replicas: 0
   strategy:
     type: Recreate
   selector:

kubernetes/apps/pve/endpointslice.yaml

@@ -16,11 +16,29 @@ ports:
     port: 8006
     protocol: TCP
 endpoints:
+  # jupiter is OUT OF SERVICE as a PVE node: its MS-01 hardware is being
+  # converted to run Talos directly as k8s-1 (see
+  # kubernetes/talos/patches/k8s-1.yaml), and nothing answers the PVE API on
+  # .201 while that is in progress — verified 2026-08-25 from pve-exporter,
+  # which fails with "[Errno 113] Host is unreachable" for .201.
+  #
+  # This slice is manually managed, and Kubernetes health-checks nothing it did
+  # not create — a hardcoded `ready: true` keeps kube-proxy and Traefik load
+  # balancing onto a dead address forever. That silently defeated the whole
+  # point of pointing the pve scrape job at this Service rather than at a node
+  # IP (see the job_name: pve comment in
+  # kubernetes/apps/monitoring/helmrelease-prometheus.yaml): cluster metrics
+  # were meant to survive any one node being down, but instead
+  # up{job="pve"} flapped 0/1 as connections landed on .201, and pve.renner.uno
+  # served an error for the same fraction of requests.
+  #
+  # Marking it not-ready is the supported way to drain a manual endpoint. If
+  # jupiter is ever rebuilt as a PVE member, flip this back to true.
   - addresses:
       - 10.2.40.201
     hostname: jupiter
     conditions:
-      ready: true
+      ready: false
   - addresses:
       - 10.2.40.202
     hostname: neptune

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -160,19 +160,34 @@ spec:
         - "10.2.40.0/24"
       cluster:
         - "10.99.0.0/29"
+  # Nodes are listed explicitly rather than useAllNodes, so that a node can be
+  # taken out of the storage fleet by removing it here. Under useAllNodes the
+  # operator re-provisions an OSD on any node whose device still carries a
+  # BlueStore signature, which makes purging an OSD ahead of pulling its drive
+  # impossible: the OSD is recreated within a reconcile and starts backfilling
+  # onto the drive being replaced.
+  #
+  # k8s-1 is absent while jupiter's OSD drive is swapped and the node is
+  # converted to bare metal. RE-ADD IT once the PM983 is in and the node has
+  # rejoined; the cluster runs on two OSDs until then.
+  #
+  # nvme0n1 on k8s-2/k8s-3 is the NVMe drive PCI-passed-through from saturn and
+  # neptune, and is the only block device those guests can see. Bare-metal k8s-1
+  # will see two — the OSD drive in MS-01 slot 1 (PCI 01:00.0) and the Talos
+  # system disk in slot 2 (58:00.0) — so when it is re-added, name its device
+  # deliberately rather than assuming slot 1 keeps enumerating first. The system
+  # disk is separately protected: talos/patches/k8s-1.yaml pins the installer to
+  # it by serial, and Rook skips a disk carrying partitions.
   storage:
-    useAllNodes: true
+    useAllNodes: false
     useAllDevices: false
-    devices:
-      # On k8s-2/k8s-3 this is the NVMe drive PCI-passed-through from saturn
-      # and neptune, and it is the only block device the guest can see. k8s-1
-      # is bare metal on jupiter and sees two: the OSD drive in MS-01 slot 1
-      # (PCI 01:00.0) and the Talos system disk in slot 2 (58:00.0). Slot 1
-      # enumerates first, so nvme0n1 is still the OSD drive there — but that
-      # is enumeration order, not a guarantee. The system disk is protected
-      # from the other direction instead: talos/patches/k8s-1.yaml pins the
-      # installer to it by serial, and Rook skips a disk carrying partitions.
-      - name: nvme0n1
+    nodes:
+      - name: k8s-2
+        devices:
+          - name: nvme0n1
+      - name: k8s-3
+        devices:
+          - name: nvme0n1
     config:
       osdsPerDevice: "1"
   # Storage layer must never be evicted by kubelet under node pressure

kubernetes/talos/patches/k8s-1.yaml

@@ -1,7 +1,7 @@
 # k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
 # still Proxmox VMs. Every machine-specific selector below differs from theirs
-# for that reason: there are no virtio NICs, no PCI-passthrough layer, and both
-# NVMe drives are visible to Talos at once.
+# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
+# four physical NICs and all three NVMe drives are visible to Talos at once.
 #
 # Networking uses the v1.12 multi-document config rather than the deprecated
 # machine.network.interfaces. That is deliberate: the docs do not define what
@@ -24,10 +24,12 @@ machine:
     # This disables v6 entirely on that link, including link-local.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    # Both NVMe drives are now ordinary block devices to Talos: the Kingston
-    # OM8PGP41024Q-A0 system disk and the Samsung PM983 that Rook claims as the
-    # Ceph OSD. Selecting the installer target by serial means it cannot land
-    # on the OSD drive regardless of which one the kernel enumerates first.
+    # Three NVMe drives are ordinary block devices to Talos here: the Kingston
+    # OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
+    # Ceph OSD, and the retired Crucial P3 that now carries EPHEMERAL (see the
+    # VolumeConfig below). Every one of them is selected by serial. Nothing in
+    # this file may use a broad selector: `disk.transport == 'nvme'` matches all
+    # three, and the failure mode is Talos taking the OSD drive.
     diskSelector:
       serial: 50026B7382B48B86
   certSANs:
@@ -37,9 +39,39 @@ machine:
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
+#
+# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
+# the PM983 replaced it. /var is exactly the right role for a disk in that
+# state: containerd's image and snapshot store, pod logs and the Ceph mon store
+# are all reconstructible, so the drive is consumable here in a way it is not as
+# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
+# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
+# they absorb another node's pods.
+#
+# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; a config applied after EPHEMERAL
+# has been created on the system disk is silently ignored, and relocating it
+# then requires wiping the disk and booting install media again. So this
+# document has to be present in the very first `apply-config --insecure`.
+#
+# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
+# would match the PM983 as well, and losing that coin flip means Talos formats
+# the OSD drive.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B749A'
+  minSize: 50GiB
+  grow: true
+---
 # Physical NICs are aliased by permanent MAC. Both X710 ports bind i40e, so the
 # driver selector k8s-2/k8s-3 use for their single passed-through port would
-# match two links here. MAC matching is unambiguous and survives udev renaming.
+# match two links here. MAC matching is unambiguous and survives udev renaming —
+# which is not hypothetical: adding the third NVMe renumbered the PCI buses and
+# moved the second I226 from enp89s0 to enp90s0.
 apiVersion: v1alpha1
 kind: LinkAliasConfig
 name: lan0
@@ -81,7 +113,7 @@ link: mgmt0
 # that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
 # serve both VM and bare-metal nodes for as long as the cluster is mixed.
 # Nothing else on this machine is called ens19: the physical NICs are
-# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp89s0.
+# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp90s0.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19