diff --git a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml index 3a2d2c46..55ed2a46 100644 --- a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml +++ b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml @@ -74,7 +74,36 @@ spec: # instance queries each PVE node's API on the user's behalf. # The targets list is the PVE node addresses; the actual # scrape goes to pve-exporter.monitoring.svc:9221. + # + # Cluster collectors (pve_up, pve_guest_info, …) return the SAME + # cluster-wide resource set no matter which node answers, so they + # are scraped from a single node only. Scraping them from all three + # (the pre-2026-07 config) ingested every node/guest series 3x — + # identical `id`, differing `instance` — a real TSDB cardinality + # cost that dashboards had to paper over with max by (id). + # Trade-off: cluster metrics pause if .201 is down; node-local + # metrics below keep flowing from the surviving nodes. - job_name: pve + static_configs: + - targets: + - "10.2.40.201" + metrics_path: /pve + params: + cluster: ["1"] + node: ["0"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: pve-exporter.monitoring.svc.cluster.local:9221 + + # Node collectors (pve_version_info, replication status) are + # node-local and genuinely distinct per target, so every node is + # scraped — this is the split recommended by prometheus-pve-exporter + # upstream to avoid the cluster-collector duplication above. + - job_name: pve-node static_configs: - targets: - "10.2.40.201" @@ -82,7 +111,7 @@ spec: - "10.2.40.203" metrics_path: /pve params: - cluster: ["1"] + cluster: ["0"] node: ["1"] relabel_configs: - source_labels: [__address__]