fix: [alert] TargetDown (pve-node) (#573 )

2026-08-25 18:48:48 +00:00
1 changed files with 19 additions and 1 deletions
@@ -16,11 +16,29 @@ ports:
    port: 8006
    protocol: TCP
 endpoints:
+  # jupiter is OUT OF SERVICE as a PVE node: its MS-01 hardware is being
+  # converted to run Talos directly as k8s-1 (see
+  # kubernetes/talos/patches/k8s-1.yaml), and nothing answers the PVE API on
+  # .201 while that is in progress — verified 2026-08-25 from pve-exporter,
+  # which fails with "[Errno 113] Host is unreachable" for .201.
+  #
+  # This slice is manually managed, and Kubernetes health-checks nothing it did
+  # not create — a hardcoded `ready: true` keeps kube-proxy and Traefik load
+  # balancing onto a dead address forever. That silently defeated the whole
+  # point of pointing the pve scrape job at this Service rather than at a node
+  # IP (see the job_name: pve comment in
+  # kubernetes/apps/monitoring/helmrelease-prometheus.yaml): cluster metrics
+  # were meant to survive any one node being down, but instead
+  # up{job="pve"} flapped 0/1 as connections landed on .201, and pve.renner.uno
+  # served an error for the same fraction of requests.
+  #
+  # Marking it not-ready is the supported way to drain a manual endpoint. If
+  # jupiter is ever rebuilt as a PVE member, flip this back to true.
  - addresses:
      - 10.2.40.201
    hostname: jupiter
    conditions:
-      ready: true
+      ready: false
  - addresses:
      - 10.2.40.202
    hostname: neptune