From 16d4996dff30de4d10b2245ddf6f9ca5910c15c1 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 17 Jul 2026 08:46:30 +0200 Subject: [PATCH] fix(monitoring): stop 3x-ingesting pve cluster metrics The single pve scrape job hit all three node IPs with cluster=1, and pve-exporter returns the full cluster-wide resource set regardless of which node answers - every node/guest series landed in the TSDB three times (same id, differing instance). Dashboards papered over it with max by (id); the cardinality cost stayed. Split per upstream prometheus-pve-exporter guidance: cluster collectors scrape one node only, node-local collectors keep scraping all three under a new pve-node job. No dashboard queries filter on job=, and all already aggregate by id, so this is query-compatible. Co-Authored-By: Claude Fable 5 --- .../monitoring/helmrelease-prometheus.yaml | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml index 3a2d2c46..55ed2a46 100644 --- a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml +++ b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml @@ -74,7 +74,36 @@ spec: # instance queries each PVE node's API on the user's behalf. # The targets list is the PVE node addresses; the actual # scrape goes to pve-exporter.monitoring.svc:9221. + # + # Cluster collectors (pve_up, pve_guest_info, …) return the SAME + # cluster-wide resource set no matter which node answers, so they + # are scraped from a single node only. Scraping them from all three + # (the pre-2026-07 config) ingested every node/guest series 3x — + # identical `id`, differing `instance` — a real TSDB cardinality + # cost that dashboards had to paper over with max by (id). + # Trade-off: cluster metrics pause if .201 is down; node-local + # metrics below keep flowing from the surviving nodes. - job_name: pve + static_configs: + - targets: + - "10.2.40.201" + metrics_path: /pve + params: + cluster: ["1"] + node: ["0"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: pve-exporter.monitoring.svc.cluster.local:9221 + + # Node collectors (pve_version_info, replication status) are + # node-local and genuinely distinct per target, so every node is + # scraped — this is the split recommended by prometheus-pve-exporter + # upstream to avoid the cluster-collector duplication above. + - job_name: pve-node static_configs: - targets: - "10.2.40.201" @@ -82,7 +111,7 @@ spec: - "10.2.40.203" metrics_path: /pve params: - cluster: ["1"] + cluster: ["0"] node: ["1"] relabel_configs: - source_labels: [__address__] -- 2.54.0