From a7ccbd7a76d765e5cc4ec62c16a245ccdedca960 Mon Sep 17 00:00:00 2001 From: Alert Fix Agent Date: Tue, 25 Aug 2026 18:48:48 +0000 Subject: [PATCH] fix: [alert] TargetDown (pve-node) (#573) --- kubernetes/apps/pve/endpointslice.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/kubernetes/apps/pve/endpointslice.yaml b/kubernetes/apps/pve/endpointslice.yaml index b179fd23..82637074 100644 --- a/kubernetes/apps/pve/endpointslice.yaml +++ b/kubernetes/apps/pve/endpointslice.yaml @@ -16,11 +16,29 @@ ports: port: 8006 protocol: TCP endpoints: + # jupiter is OUT OF SERVICE as a PVE node: its MS-01 hardware is being + # converted to run Talos directly as k8s-1 (see + # kubernetes/talos/patches/k8s-1.yaml), and nothing answers the PVE API on + # .201 while that is in progress — verified 2026-08-25 from pve-exporter, + # which fails with "[Errno 113] Host is unreachable" for .201. + # + # This slice is manually managed, and Kubernetes health-checks nothing it did + # not create — a hardcoded `ready: true` keeps kube-proxy and Traefik load + # balancing onto a dead address forever. That silently defeated the whole + # point of pointing the pve scrape job at this Service rather than at a node + # IP (see the job_name: pve comment in + # kubernetes/apps/monitoring/helmrelease-prometheus.yaml): cluster metrics + # were meant to survive any one node being down, but instead + # up{job="pve"} flapped 0/1 as connections landed on .201, and pve.renner.uno + # served an error for the same fraction of requests. + # + # Marking it not-ready is the supported way to drain a manual endpoint. If + # jupiter is ever rebuilt as a PVE member, flip this back to true. - addresses: - 10.2.40.201 hostname: jupiter conditions: - ready: true + ready: false - addresses: - 10.2.40.202 hostname: neptune -- 2.54.0