From 26aadd0629657378f9a50780d39cd4768b4b1173 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 17 Jul 2026 20:22:35 +0200 Subject: [PATCH 1/2] fix(monitoring): scrape pve cluster metrics via Service VIP, not a fixed node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pve cluster-collector job hardcoded 10.2.40.201, so cluster-wide metrics (pve_up, guest info, storage) paused whenever jupiter specifically was down, even with the cluster healthy. Point it at the pve Service DNS (pve.apps.svc.cluster.local) whose EndpointSlice load-balances the PVE API across all three nodes, so the scrape follows whichever node is up. Node-local job keeps the three explicit targets — it needs per-node identity. Verified: exporter returns pve_up for cluster + all nodes when targeted at the Service DNS. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_015K7gLUVVnutvDnzV2TrMRJ --- .../apps/monitoring/helmrelease-prometheus.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml index 55ed2a46..40db20a8 100644 --- a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml +++ b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml @@ -81,12 +81,17 @@ spec: # (the pre-2026-07 config) ingested every node/guest series 3x — # identical `id`, differing `instance` — a real TSDB cardinality # cost that dashboards had to paper over with max by (id). - # Trade-off: cluster metrics pause if .201 is down; node-local - # metrics below keep flowing from the surviving nodes. + # + # Target the pve Service (apps ns) rather than a hardcoded node IP: + # its EndpointSlice load-balances the PVE API across all three nodes, + # so cluster metrics keep flowing as long as ANY node is up, instead + # of pausing whenever .201 (jupiter) specifically is down. The + # exporter reaches the API on 8006 (verify_ssl:false, so the LB'd + # cert hostname mismatch is a non-issue). - job_name: pve static_configs: - targets: - - "10.2.40.201" + - "pve.apps.svc.cluster.local" metrics_path: /pve params: cluster: ["1"] -- 2.54.0 From 74f387f1852c58ae4061d9e0b0229438dba80bb1 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 17 Jul 2026 20:23:56 +0200 Subject: [PATCH 2/2] fix(monitoring): use node FQDNs for pve-node scrape, drop raw IPs No hardcoded IP remains in the pve scrape config: the node job now targets jupiter/neptune/saturn.renner.uno. instance label reads the hostname, and a renumbered node follows DNS without a config edit. Verified: exporter returns node-local metrics when targeted at jupiter.renner.uno from inside the cluster. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_015K7gLUVVnutvDnzV2TrMRJ --- kubernetes/apps/monitoring/helmrelease-prometheus.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml index 40db20a8..735f1f98 100644 --- a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml +++ b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml @@ -108,12 +108,15 @@ spec: # node-local and genuinely distinct per target, so every node is # scraped — this is the split recommended by prometheus-pve-exporter # upstream to avoid the cluster-collector duplication above. + # Targets are the node FQDNs, not raw IPs: the instance label then + # reads jupiter/neptune/saturn.renner.uno, and a renumbered node is + # picked up from DNS without editing this file. - job_name: pve-node static_configs: - targets: - - "10.2.40.201" - - "10.2.40.202" - - "10.2.40.203" + - "jupiter.renner.uno" + - "neptune.renner.uno" + - "saturn.renner.uno" metrics_path: /pve params: cluster: ["0"] -- 2.54.0