fix: [alert] TargetDown (pve) (#579 )

2026-08-25 19:45:14 +00:00
2 changed files with 24 additions and 15 deletions
@@ -108,12 +108,16 @@ spec:
          # identical `id`, differing `instance` — a real TSDB cardinality
          # cost that dashboards had to paper over with max by (id).
          #
-          # Target the pve Service (apps ns) rather than a hardcoded node IP:
-          # its EndpointSlice load-balances the PVE API across all three nodes,
-          # so cluster metrics keep flowing as long as ANY node is up, instead
-          # of pausing whenever .201 (jupiter) specifically is down. The
-          # exporter reaches the API on 8006 (verify_ssl:false, so the LB'd
+          # Target the pve Service (apps ns) rather than a hardcoded node IP,
+          # so this job does not have to be edited when a node is renumbered.
+          # The exporter reaches the API on 8006 (verify_ssl:false, so the LB'd
          # cert hostname mismatch is a non-issue).
+          #
+          # This is round robin, NOT failover: the EndpointSlice is hand-written
+          # with `ready: true` asserted and nothing health-checks it. One scrape
+          # makes several API calls, so a single down member fails 100% of this
+          # job's scrapes rather than 1/N of them (issue #579). Keeping the
+          # EndpointSlice free of decommissioned nodes is what keeps this job up.
          - job_name: pve
            static_configs:
              - targets:
@@ -135,12 +139,12 @@ spec:
          # scraped — this is the split recommended by prometheus-pve-exporter
          # upstream to avoid the cluster-collector duplication above.
          # Targets are the node FQDNs, not raw IPs: the instance label then
-          # reads jupiter/neptune/saturn.renner.uno, and a renumbered node is
-          # picked up from DNS without editing this file.
+          # reads neptune/saturn.renner.uno, and a renumbered node is picked up
+          # from DNS without editing this file. jupiter is absent on purpose --
+          # its hardware runs Talos as k8s-1 now and has no PVE API to scrape.
          - job_name: pve-node
            static_configs:
              - targets:
-                  - "jupiter.renner.uno"
                  - "neptune.renner.uno"
                  - "saturn.renner.uno"
            metrics_path: /pve
@@ -1,7 +1,17 @@
 # PVE node IPs are hardcoded here despite the general "IPs aren't static"
 # rule for the homelab: corosync requires stable cluster member identity,
-# so jupiter/neptune/saturn keep .201/.202/.203 across reboots and PVE
-# upgrades. If a node is permanently renumbered, edit this file.
+# so neptune/saturn keep .202/.203 across reboots and PVE upgrades. If a node
+# is permanently renumbered, edit this file.
+#
+# There are no health checks on these entries -- `ready: true` is asserted, not
+# observed -- so a member listed here that is down still takes its share of the
+# round robin. A decommissioned node must be REMOVED, not left in rotation:
+# the monitoring `pve` job makes several API calls per scrape, and any one of
+# them landing on a dead member fails the whole scrape (TargetDown, issue #579).
+#
+# jupiter (.201) was removed for exactly that reason: its MS-01 hardware now
+# runs Talos directly as k8s-1 and it is no longer a Proxmox node. See
+# kubernetes/talos/patches/k8s-1.yaml and .woodpecker/deploy-infrastructure.yml.
 apiVersion: discovery.k8s.io/v1
 kind: EndpointSlice
 metadata:
@@ -16,11 +26,6 @@ ports:
    port: 8006
    protocol: TCP
 endpoints:
-  - addresses:
-      - 10.2.40.201
-    hostname: jupiter
-    conditions:
-      ready: true
  - addresses:
      - 10.2.40.202
    hostname: neptune