fix: [alert] TargetDown (pve) (#579)

Reviewed-on: k.renner/homelab#580

kubernetes/apps/monitoring/helmrelease-prometheus.yaml

@@ -108,12 +108,16 @@ spec:
           # identical `id`, differing `instance` — a real TSDB cardinality
           # cost that dashboards had to paper over with max by (id).
           #
-          # Target the pve Service (apps ns) rather than a hardcoded node IP:
-          # its EndpointSlice load-balances the PVE API across all three nodes,
-          # so cluster metrics keep flowing as long as ANY node is up, instead
-          # of pausing whenever .201 (jupiter) specifically is down. The
-          # exporter reaches the API on 8006 (verify_ssl:false, so the LB'd
+          # Target the pve Service (apps ns) rather than a hardcoded node IP,
+          # so this job does not have to be edited when a node is renumbered.
+          # The exporter reaches the API on 8006 (verify_ssl:false, so the LB'd
           # cert hostname mismatch is a non-issue).
+          #
+          # This is round robin, NOT failover: the EndpointSlice is hand-written
+          # with `ready: true` asserted and nothing health-checks it. One scrape
+          # makes several API calls, so a single down member fails 100% of this
+          # job's scrapes rather than 1/N of them (issue #579). Keeping the
+          # EndpointSlice free of decommissioned nodes is what keeps this job up.
           - job_name: pve
             static_configs:
               - targets:
@@ -135,12 +139,12 @@ spec:
           # scraped — this is the split recommended by prometheus-pve-exporter
           # upstream to avoid the cluster-collector duplication above.
           # Targets are the node FQDNs, not raw IPs: the instance label then
-          # reads jupiter/neptune/saturn.renner.uno, and a renumbered node is
-          # picked up from DNS without editing this file.
+          # reads neptune/saturn.renner.uno, and a renumbered node is picked up
+          # from DNS without editing this file. jupiter is absent on purpose --
+          # its hardware runs Talos as k8s-1 now and has no PVE API to scrape.
           - job_name: pve-node
             static_configs:
               - targets:
-                  - "jupiter.renner.uno"
                   - "neptune.renner.uno"
                   - "saturn.renner.uno"
             metrics_path: /pve

kubernetes/apps/pve/endpointslice.yaml

@@ -1,7 +1,17 @@
 # PVE node IPs are hardcoded here despite the general "IPs aren't static"
 # rule for the homelab: corosync requires stable cluster member identity,
-# so jupiter/neptune/saturn keep .201/.202/.203 across reboots and PVE
-# upgrades. If a node is permanently renumbered, edit this file.
+# so neptune/saturn keep .202/.203 across reboots and PVE upgrades. If a node
+# is permanently renumbered, edit this file.
+#
+# There are no health checks on these entries -- `ready: true` is asserted, not
+# observed -- so a member listed here that is down still takes its share of the
+# round robin. A decommissioned node must be REMOVED, not left in rotation:
+# the monitoring `pve` job makes several API calls per scrape, and any one of
+# them landing on a dead member fails the whole scrape (TargetDown, issue #579).
+#
+# jupiter (.201) was removed for exactly that reason: its MS-01 hardware now
+# runs Talos directly as k8s-1 and it is no longer a Proxmox node. See
+# kubernetes/talos/patches/k8s-1.yaml and .woodpecker/deploy-infrastructure.yml.
 apiVersion: discovery.k8s.io/v1
 kind: EndpointSlice
 metadata:
@@ -16,11 +26,6 @@ ports:
     port: 8006
     protocol: TCP
 endpoints:
-  - addresses:
-      - 10.2.40.201
-    hostname: jupiter
-    conditions:
-      ready: true
   - addresses:
       - 10.2.40.202
     hostname: neptune

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -167,17 +167,20 @@ spec:
   # impossible: the OSD is recreated within a reconcile and starts backfilling
   # onto the drive being replaced.
   #
-  # k8s-1 is absent while jupiter's OSD drive is swapped and the node is
-  # converted to bare metal. RE-ADD IT once the PM983 is in and the node has
-  # rejoined; the cluster runs on two OSDs until then.
-  #
   # nvme0n1 on k8s-2/k8s-3 is the NVMe drive PCI-passed-through from saturn and
-  # neptune, and is the only block device those guests can see. Bare-metal k8s-1
-  # will see two — the OSD drive in MS-01 slot 1 (PCI 01:00.0) and the Talos
-  # system disk in slot 2 (58:00.0) — so when it is re-added, name its device
-  # deliberately rather than assuming slot 1 keeps enumerating first. The system
-  # disk is separately protected: talos/patches/k8s-1.yaml pins the installer to
-  # it by serial, and Rook skips a disk carrying partitions.
+  # neptune, and is the only block device those guests can see, so a bare device
+  # name is unambiguous there.
+  #
+  # k8s-1 is bare metal and sees three NVMe drives, so it names its OSD by
+  # /dev/disk/by-id path instead. A bare name would be actively wrong: Talos
+  # enumerates this machine's drives in a different order than Proxmox did, and
+  # nvme0n1 here is the Kingston SYSTEM disk. Rook would probably decline it for
+  # carrying partitions, but that is luck rather than intent. The by-id path is
+  # derived from model and serial and does not depend on enumeration at all.
+  #
+  #   nvme0n1  50026B7382B48B86  Kingston  — Talos system disk
+  #   nvme1n1  2332E86B749A      Crucial   — Talos EPHEMERAL (/var)
+  #   nvme2n1  S437NX0M400426    PM983     — this OSD
   storage:
     useAllNodes: false
     useAllDevices: false
@@ -188,6 +191,9 @@ spec:
       - name: k8s-3
         devices:
           - name: nvme0n1
+      - name: k8s-1
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M400426
     config:
       osdsPerDevice: "1"
   # Storage layer must never be evicted by kubelet under node pressure

kubernetes/talos/patches/k8s-1.yaml

@@ -1,7 +1,7 @@
 # k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
 # still Proxmox VMs. Every machine-specific selector below differs from theirs
-# for that reason: there are no virtio NICs, no PCI-passthrough layer, and both
-# NVMe drives are visible to Talos at once.
+# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
+# four physical NICs and all three NVMe drives are visible to Talos at once.
 #
 # Networking uses the v1.12 multi-document config rather than the deprecated
 # machine.network.interfaces. That is deliberate: the docs do not define what
@@ -9,9 +9,6 @@
 # every link in one schema instead of straddling them.
 machine:
   network:
-    # Shared control-plane VIP 10.2.40.60 is configured as a Layer2VIPConfig
-    # below rather than machine.network.interfaces[].vip. Exposed via UniFi DNS
-    # as k8s.renner.uno.
     hostname: k8s-1
   sysctls:
     # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It exists as
@@ -24,10 +21,12 @@ machine:
     # This disables v6 entirely on that link, including link-local.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    # Both NVMe drives are now ordinary block devices to Talos: the Kingston
-    # OM8PGP41024Q-A0 system disk and the Samsung PM983 that Rook claims as the
-    # Ceph OSD. Selecting the installer target by serial means it cannot land
-    # on the OSD drive regardless of which one the kernel enumerates first.
+    # Three NVMe drives are ordinary block devices to Talos here: the Kingston
+    # OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
+    # Ceph OSD, and the retired Crucial P3 that carries EPHEMERAL (see the
+    # VolumeConfig below). Every one of them is selected by serial. Nothing in
+    # this file may use a broad selector: `disk.transport == 'nvme'` matches all
+    # three, and the failure mode is Talos taking the OSD drive.
     diskSelector:
       serial: 50026B7382B48B86
   certSANs:
@@ -37,40 +36,70 @@ machine:
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
-# Physical NICs are aliased by permanent MAC. Both X710 ports bind i40e, so the
-# driver selector k8s-2/k8s-3 use for their single passed-through port would
-# match two links here. MAC matching is unambiguous and survives udev renaming.
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
+#
+# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
+# the PM983 replaced it. /var is exactly the right role for a disk in that
+# state: containerd's image and snapshot store, pod logs and the Ceph mon store
+# are all reconstructible, so the drive is consumable here in a way it is not as
+# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
+# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
+# they absorb another node's pods.
+#
+# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; a config applied after EPHEMERAL
+# has been created elsewhere is silently ignored, and relocating it then
+# requires wiping the disk and booting install media again.
+#
+# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
+# would match the PM983 as well, and losing that coin flip means Talos formats
+# the OSD drive.
 apiVersion: v1alpha1
-kind: LinkAliasConfig
-name: lan0
-selector:
-  match: glob("58:47:ca:74:70:27", mac(link.permanent_addr))
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B749A'
+  minSize: 50GiB
+  grow: true
 ---
-apiVersion: v1alpha1
-kind: LinkAliasConfig
-name: ceph0
-selector:
-  match: glob("58:47:ca:74:70:28", mac(link.permanent_addr))
----
-# lan0 is the switch trunk (X710 port 0). It carries no addressing of its own;
-# VLAN 40 and VLAN 30 ride on top. Proxmox's vlan-aware vmbr0 did this tagging
-# before, which is why the VM saw two already-untagged virtio NICs instead.
+# X710 port 0 carries the switch trunk. It takes no addressing itself — the
+# port's untagged VLAN is the 192.168.8.0/24 network, which this node has no
+# business being on — so management and v30iot both ride it as tagged VLANs.
+#
+# Links are addressed by predictable kernel name. Those derive from PCI
+# topology, not probe order — though not immune: adding the third NVMe
+# renumbered the buses and moved the unused second I226 from enp89s0 to
+# enp90s0. Neither name used here was affected.
+#
+# LinkAliasConfig would be the better way to pin these, and is worth
+# reintroducing — but only once it is proven. Aliasing both X710 ports by
+# permanent MAC and using the aliases as VLAN parents produced a node with no
+# address on any interface: the aliases were never created, so every document
+# referencing them did nothing, including the static address on port 1. Verify
+# any alias-based revision with `talosctl apply-config --mode=try`, which rolls
+# back on its own, before making it permanent.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: lan0
+name: enp2s0f0np0
 up: true
 ---
+# Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
+# network. The k8s-1 VM reached 10.2.40.174 over this same physical port via
+# Proxmox's v40srv VNet (vmbr0.40), so the switch trunks VLAN 40 here.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: lan0
+parent: enp2s0f0np0
 up: true
 ---
 apiVersion: v1alpha1
 kind: DHCPv4Config
 name: mgmt0
 ---
+# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
+# k8s.renner.uno.
 apiVersion: v1alpha1
 kind: Layer2VIPConfig
 name: 10.2.40.60
@@ -80,21 +109,20 @@ link: mgmt0
 # `master: ens19`, and k8s-2/k8s-3 still present the VLAN as a virtio NIC of
 # that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
 # serve both VM and bare-metal nodes for as long as the cluster is mixed.
-# Nothing else on this machine is called ens19: the physical NICs are
-# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp89s0.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: lan0
+parent: enp2s0f0np0
 up: true
 ---
-# Dedicated Ceph cluster network, VLAN 49 / 10.99.0.0/29, untagged on the wire
-# on X710 port 1. Used by Rook for OSD<->OSD replication via
-# CephCluster.spec.network.addressRanges.cluster. No default route.
+# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
+# untagged on the wire. Used by Rook for OSD<->OSD replication via
+# CephCluster.spec.network.addressRanges.cluster. Not routed off its own
+# segment, and deliberately has no default route.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: ceph0
+name: enp2s0f1np1
 up: true
 mtu: 9000
 addresses: