claude-botandClaude Opus 5.5
a35bc6b91d
Add a blackbox-http probe of kuma.renner.uno with a dedicated critical UptimeKumaUnreachable rule (excluded from BlackboxProbeFailing), scrape Kuma /metrics with the API key from OpenBao, and alert on sustained slow monitor response times. No egress policy exists in the monitoring namespace, so no CNP change is needed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
728 lines
32 KiB
YAML
728 lines
32 KiB
YAML
apiVersion: source.toolkit.fluxcd.io/v1
|
|
kind: HelmRepository
|
|
metadata:
|
|
name: prometheus-community
|
|
namespace: monitoring
|
|
spec:
|
|
interval: 24h
|
|
url: https://prometheus-community.github.io/helm-charts
|
|
---
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: kube-prometheus-stack
|
|
namespace: monitoring
|
|
spec:
|
|
interval: 30m
|
|
chart:
|
|
spec:
|
|
chart: kube-prometheus-stack
|
|
version: "92.3.0"
|
|
sourceRef:
|
|
kind: HelmRepository
|
|
name: prometheus-community
|
|
namespace: monitoring
|
|
install:
|
|
crds: CreateReplace
|
|
remediation:
|
|
retries: 3
|
|
upgrade:
|
|
crds: CreateReplace
|
|
remediation:
|
|
retries: 3
|
|
values:
|
|
# Prometheus
|
|
prometheus:
|
|
# prometheus.renner.uno is served by a standalone, Authentik-gated
|
|
# Ingress (prometheus-gate-ingress.yaml, class authentik-gate), NOT this Helm-generated one.
|
|
# Prometheus had no auth of its own — its full 30d metrics DB is an
|
|
# infra-recon oracle (hostnames, IPs, service inventory). Gated behind
|
|
# the Authentik gate, same as Alertmanager (#146).
|
|
# The chart Ingress is disabled here — two Ingresses for one host would
|
|
# conflict. tls (prometheus-tls) and externalUrl below are unchanged;
|
|
# the standalone Ingress reuses the same secret.
|
|
#
|
|
# Grafana is unaffected: its datasource targets the ClusterIP Service
|
|
# (kube-prometheus-stack-prometheus.monitoring:9090), not this ingress.
|
|
# Blackbox uptime probes that target https://prometheus.renner.uno
|
|
# follow the gate's 302 to the Authentik login (200), so they stay
|
|
# green — but now measure Authentik reachability, not Prometheus. Same
|
|
# already-true for alerts.renner.uno since #146. Repointing those probes
|
|
# to an internal endpoint is a separate follow-up.
|
|
ingress:
|
|
enabled: false
|
|
prometheusSpec:
|
|
# Scrape ServiceMonitors/PodMonitors regardless of their labels.
|
|
#
|
|
# By default kube-prometheus-stack sets these selectors to
|
|
# `matchLabels: {release: kube-prometheus-stack}`. ServiceMonitors
|
|
# created at RUNTIME by third-party operators — Rook, goauthentik.io —
|
|
# carry that operator's own labels instead and can never be made to
|
|
# match: no static manifest or Kustomize patch can relabel an object
|
|
# the operator creates itself. Left strict, those targets are dropped
|
|
# silently, with no failed resource and no gap visible anywhere except
|
|
# in the missing series.
|
|
#
|
|
# Because these two are empty, the `release` label on this repo's own
|
|
# ServiceMonitors and PodMonitors is decorative. It is still set for
|
|
# consistency, and the comments there say so.
|
|
#
|
|
# ruleSelector, probeSelector and scrapeConfigSelector are deliberately
|
|
# NOT loosened: nothing outside this repo creates those, so keeping the
|
|
# selectors strict stops a stray third-party rule from firing here. For
|
|
# PrometheusRules and Probes the label IS required.
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
podMonitorSelectorNilUsesHelmValues: false
|
|
# Rewrites the in-cluster URL embedded in alert generatorURLs and
|
|
# status pages so click-through from emails works.
|
|
externalUrl: https://prometheus.renner.uno
|
|
# Time retention is the one in force; the size cap is a backstop
|
|
# (REN-107, owner decision: 30 days on the existing disk). This was
|
|
# 90d/30GB, and at ~1 GB/day of ingest the 30GiB cap bound first: the
|
|
# real window was a 20.5-27 day saw-tooth while this line said 90d.
|
|
# 30d peaks near 38 GB (30d plus up to one 3-day block awaiting
|
|
# deletion, plus WAL), under 40GiB (42.9 GB), under the 50Gi PVC.
|
|
# Raising either number means re-doing that arithmetic against the
|
|
# PVC; PrometheusRetentionNotInForce (monitoring-rules) fires if the
|
|
# size cap starts binding again.
|
|
retention: 30d
|
|
retentionSize: 40GiB
|
|
scrapeInterval: 30s
|
|
evaluationInterval: 30s
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: ceph-block
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: 50Gi
|
|
# Mounted at /etc/prometheus/secrets/<name>/; see
|
|
# ./uptime-kuma-metrics-externalsecret.yaml.
|
|
secrets:
|
|
- uptime-kuma-metrics
|
|
# Additional scrape targets not covered by ServiceMonitors.
|
|
additionalScrapeConfigs:
|
|
# Gitea exposes /metrics anonymously on its main HTTP port. The
|
|
# `gitea_*` metrics power the gitea-overview dashboard. Scrapes the
|
|
# ClusterIP Service of the StatefulSet in the gitea namespace.
|
|
- job_name: gitea
|
|
static_configs:
|
|
- targets: ["gitea.gitea.svc.cluster.local:3000"]
|
|
metrics_path: /metrics
|
|
# Off-site Uptime Kuma. /metrics takes the API key as the basic-auth
|
|
# password with an empty username.
|
|
- job_name: uptime-kuma
|
|
scheme: https
|
|
metrics_path: /metrics
|
|
basic_auth:
|
|
username: ""
|
|
password_file: /etc/prometheus/secrets/uptime-kuma-metrics/api_key
|
|
static_configs:
|
|
- targets: ["kuma.renner.uno:443"]
|
|
|
|
# Grafana
|
|
grafana:
|
|
defaultDashboardsEnabled: true
|
|
defaultDashboardsTimezone: Europe/Berlin
|
|
# Single replica on an RWO (ceph-block) PVC — RollingUpdate deadlocks
|
|
# on Multi-Attach because both Pods would need the volume simultaneously.
|
|
deploymentStrategy:
|
|
type: Recreate
|
|
ingress:
|
|
enabled: true
|
|
ingressClassName: traefik
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
|
hosts:
|
|
- dash.renner.uno
|
|
tls:
|
|
- secretName: grafana-tls
|
|
hosts:
|
|
- dash.renner.uno
|
|
# Postgres backend (gf-pg CNPG cluster). SQLite-on-RBD made schema
|
|
# migrations on cold rebuilds blow past the liveness probe budget;
|
|
# Postgres batches WAL fsyncs server-side and migrations finish in
|
|
# seconds. The gf-pg-app secret is auto-created by CNPG.
|
|
env:
|
|
GF_DATABASE_TYPE: postgres
|
|
GF_DATABASE_HOST: gf-pg-rw.monitoring.svc:5432
|
|
GF_DATABASE_NAME: grafana
|
|
GF_DATABASE_SSL_MODE: require
|
|
envValueFrom:
|
|
GF_DATABASE_USER:
|
|
secretKeyRef:
|
|
name: gf-pg-app
|
|
key: username
|
|
GF_DATABASE_PASSWORD:
|
|
secretKeyRef:
|
|
name: gf-pg-app
|
|
key: password
|
|
# PVC no longer needed for the DB; keep it for the (small) Grafana
|
|
# plugin/data dir which Grafana still writes to.
|
|
persistence:
|
|
enabled: true
|
|
storageClassName: ceph-block
|
|
size: 5Gi
|
|
replicas: 1
|
|
additionalDataSources:
|
|
- name: Loki
|
|
uid: loki
|
|
type: loki
|
|
url: http://loki.monitoring.svc.cluster.local:3100
|
|
access: proxy
|
|
isDefault: false
|
|
# Legacy Prometheus (LXC 227) — bridges host-level dashboards
|
|
# (Alloy, docker, pve host node_exporter, etc.) that depend on
|
|
# metrics only the legacy Prom scrapes. Retires at global cut-over.
|
|
- name: Prometheus (legacy)
|
|
uid: prometheus-legacy
|
|
type: prometheus
|
|
url: http://prometheus.srv.renner.uno:9090
|
|
access: proxy
|
|
isDefault: false
|
|
jsonData:
|
|
httpMethod: POST
|
|
# In-cluster InfluxDB (chart `influxdb-influxdb2`). Both HA (write)
|
|
# and Grafana (read) hit this directly via Service DNS. The
|
|
# `grafana-influxdb-token` Secret is created by the Job
|
|
# `influxdb-grafana-token` (kubernetes/apps/influxdb/grafana-
|
|
# token-job.yaml), which mints a read-only auth on the
|
|
# suedstr28 bucket every fresh deploy.
|
|
- name: InfluxDB
|
|
uid: influxdb
|
|
type: influxdb
|
|
url: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086
|
|
access: proxy
|
|
isDefault: false
|
|
jsonData:
|
|
version: Flux
|
|
organization: renner
|
|
defaultBucket: suedstr28
|
|
tlsSkipVerify: true
|
|
secureJsonData:
|
|
token: ${INFLUXDB_TOKEN}
|
|
sidecar:
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 128Mi
|
|
limits:
|
|
memory: 256Mi
|
|
dashboards:
|
|
enabled: true
|
|
label: grafana_dashboard
|
|
labelValue: "1"
|
|
searchNamespace: monitoring
|
|
folder: /tmp/dashboards
|
|
provider:
|
|
foldersFromFilesStructure: true
|
|
dashboardProviders:
|
|
dashboardproviders.yaml:
|
|
apiVersion: 1
|
|
providers:
|
|
- name: default
|
|
orgId: 1
|
|
folder: ""
|
|
type: file
|
|
disableDeletion: false
|
|
editable: true
|
|
options:
|
|
path: /var/lib/grafana/dashboards/default
|
|
dashboards:
|
|
default:
|
|
velero:
|
|
gnetId: 11055
|
|
revision: 2
|
|
datasource: Prometheus
|
|
plugins:
|
|
- volkovlabs-echarts-panel
|
|
envFromSecrets:
|
|
- name: grafana-oauth
|
|
- name: grafana-influxdb-token
|
|
grafana.ini:
|
|
server:
|
|
root_url: https://dash.renner.uno
|
|
auth.generic_oauth:
|
|
enabled: true
|
|
name: Authentik
|
|
allow_sign_up: true
|
|
scopes: openid profile email
|
|
auth_url: https://auth.renner.uno/application/o/authorize/
|
|
token_url: https://auth.renner.uno/application/o/token/
|
|
api_url: https://auth.renner.uno/application/o/userinfo/
|
|
use_pkce: true
|
|
role_attribute_path: "contains(groups[*], 'Infrastructure Admins') && 'Admin' || 'Viewer'"
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 768Mi
|
|
limits:
|
|
memory: 1Gi
|
|
|
|
# Alertmanager
|
|
alertmanager:
|
|
# alerts.renner.uno is served by a standalone, Authentik-gated Ingress
|
|
# (alertmanager-gate-ingress.yaml) instead of this Helm-generated one.
|
|
# Alertmanager has no auth and its v2 API is writable (silence-all),
|
|
# so it must sit behind the Authentik gate (class authentik-gate). The
|
|
# chart Ingress is disabled here — two Ingresses for the same host
|
|
# would conflict. The tls secret (alertmanager-tls) and externalUrl
|
|
# below are unchanged; the standalone Ingress reuses the same secret.
|
|
ingress:
|
|
enabled: false
|
|
alertmanagerSpec:
|
|
# Embedded in the AlertmanagerURL footer of every email plus
|
|
# the "Source" link of every grouped alert, so the email link
|
|
# resolves externally instead of pointing at the in-cluster
|
|
# service hostname.
|
|
externalUrl: https://alerts.renner.uno
|
|
storage:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: ceph-block
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
# Mounts ./alertmanager-smtp Secret keys at
|
|
# /etc/alertmanager/secrets/alertmanager-smtp/<key>. The
|
|
# smtp_auth_password_file below references that path.
|
|
secrets:
|
|
- alertmanager-smtp
|
|
# SMTP via Proton submission (smtp.protonmail.ch:587 STARTTLS).
|
|
# The Proton SMTP token is a per-account credential generated in
|
|
# the Proton webapp (Settings -> IMAP/SMTP). It is stored in
|
|
# OpenBao at apps/monitoring/alertmanager-smtp and projected into
|
|
# the alertmanager-smtp Secret by
|
|
# ./alertmanager-smtp-externalsecret.yaml — rotating the token
|
|
# means updating OpenBao, not this repo. Outbound 587 from the
|
|
# monitoring ns is not blocked by any CNP today.
|
|
config:
|
|
global:
|
|
smtp_smarthost: 'smtp.protonmail.ch:587'
|
|
smtp_from: 'cluster@renner.uno'
|
|
smtp_auth_username: 'cluster@renner.uno'
|
|
smtp_auth_password_file: '/etc/alertmanager/secrets/alertmanager-smtp/smtp-password'
|
|
smtp_require_tls: true
|
|
resolve_timeout: 5m
|
|
route:
|
|
receiver: email
|
|
group_by: ['alertname', 'cluster', 'namespace']
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 12h
|
|
routes:
|
|
# The kube-prometheus-stack chart ships a `Watchdog` alert
|
|
# designed to fire continuously as a heartbeat for an
|
|
# external dead-man's-switch. We don't have a DMS endpoint,
|
|
# and a 12h-repeat email of an always-firing alert is
|
|
# noise — drop it.
|
|
- matchers:
|
|
- alertname = "Watchdog"
|
|
receiver: 'null'
|
|
# KubeCPUOvercommit fires whenever the cluster cannot
|
|
# tolerate the failure of one node — by design unsatisfiable
|
|
# on a small homelab cluster regardless of actual load.
|
|
- matchers:
|
|
- alertname = "KubeCPUOvercommit"
|
|
receiver: 'null'
|
|
# Helper alert that only drives the info inhibit rule below.
|
|
- matchers:
|
|
- alertname = "InfoInhibitor"
|
|
receiver: 'null'
|
|
# Flux notification-controller error events (flux-failures Alert).
|
|
- matchers:
|
|
- source = "flux"
|
|
receiver: email
|
|
group_wait: 1m
|
|
group_interval: 10m
|
|
repeat_interval: 24h
|
|
# From kube-prometheus manifests/alertmanager-secret.yaml.
|
|
inhibit_rules:
|
|
- source_matchers:
|
|
- severity = "critical"
|
|
target_matchers:
|
|
- severity =~ "warning|info"
|
|
equal: ['namespace', 'alertname']
|
|
- source_matchers:
|
|
- severity = "warning"
|
|
target_matchers:
|
|
- severity = "info"
|
|
equal: ['namespace', 'alertname']
|
|
- source_matchers:
|
|
- alertname = "InfoInhibitor"
|
|
target_matchers:
|
|
- severity = "info"
|
|
equal: ['namespace']
|
|
receivers:
|
|
- name: email
|
|
email_configs:
|
|
- to: 'admin@renner.uno'
|
|
send_resolved: true
|
|
- name: 'null'
|
|
|
|
# Node Exporter (DaemonSet on all K8s nodes)
|
|
nodeExporter:
|
|
enabled: true
|
|
|
|
# kube-state-metrics
|
|
kubeStateMetrics:
|
|
enabled: true
|
|
|
|
# Resources pass-through to the kube-state-metrics subchart.
|
|
# Top-level `kubeStateMetrics.resources` is not honored — must
|
|
# be set under the subchart name (with hyphens).
|
|
kube-state-metrics:
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 64Mi
|
|
limits:
|
|
memory: 256Mi
|
|
|
|
# Groups Velero maintenance Jobs per repository in prometheusrule-jobs.yaml.
|
|
metricLabelsAllowlist:
|
|
- jobs=[velero.io/repo-name]
|
|
|
|
# --- Flux per-object readiness (REN-102) ---------------------------------
|
|
#
|
|
# WHY THIS IS HERE AND NOT ON THE FLUX CONTROLLERS. `gotk_reconcile_condition`
|
|
# and `gotk_suspend_status` are still declared by fluxcd/pkg/runtime, but no
|
|
# controller populates them at the tags pinned in
|
|
# clusters/homelab/flux-system/gotk-components.yaml, and an unpopulated
|
|
# GaugeVec exports no series. Verified after the REN-97 scrape landed: the
|
|
# PodMonitor now yields 2,273 `gotk_*` series and
|
|
# `count({__name__=~"gotk_reconcile_condition|gotk_suspend_status"})` is still
|
|
# empty. Upstream moved per-object readiness to kube-state-metrics
|
|
# custom-resource-state, which is what this block turns on.
|
|
#
|
|
# DO NOT ADD `collectors: []` OR `--custom-resource-state-only=true`.
|
|
# Upstream's reference config (fluxcd/flux2-monitoring-example,
|
|
# monitoring/controllers/kube-prometheus-stack/kube-state-metrics-config.yaml)
|
|
# opens with exactly those two lines. Copying them replaces every built-in
|
|
# collector with the Flux ones: measured 2026-09-26, that would take
|
|
# `count({job="kube-state-metrics"})` from 27,792 series to ~183 and blind
|
|
# every `Kube*` alert in the chart. Both keys are deliberately absent below.
|
|
#
|
|
# ROLLS THE kube-state-metrics POD. Enabling customResourceState adds an arg
|
|
# and a volume to the Deployment, so the pod is replaced once on apply. That
|
|
# is the whole blast radius of this block — KSM only watches, it writes
|
|
# nothing, and no Flux object or workload is touched. During the roll (a few
|
|
# seconds) `kube_*` has a gap; `KubeStateMetricsDown` and `TargetDown` cover
|
|
# it if it does not come back.
|
|
#
|
|
# NOTE FOR LATER EDITS: the subchart puts no config checksum on the
|
|
# Deployment's pod template, so editing `config` below changes the ConfigMap
|
|
# WITHOUT rolling the pod, and KSM keeps serving the old config until it
|
|
# restarts for another reason. Any future change to this config needs a
|
|
# deliberate pod restart to take effect.
|
|
#
|
|
# RBAC: list + watch, nothing else. `get` is NOT needed — KSM drives
|
|
# reflectors, never point-reads. Also note the chart adds
|
|
# `customresourcedefinitions: [list, watch]` on its own via
|
|
# rbac.customResourceState.createClusterRoleRules (default true); KSM needs it
|
|
# to resolve the GVKs below. Both land in the existing
|
|
# kube-prometheus-stack-kube-state-metrics ClusterRole rather than a second
|
|
# binding to maintain. No write verb, no wildcard, no cluster-admin.
|
|
rbac:
|
|
extraRules:
|
|
- apiGroups:
|
|
- source.toolkit.fluxcd.io
|
|
- kustomize.toolkit.fluxcd.io
|
|
- helm.toolkit.fluxcd.io
|
|
- image.toolkit.fluxcd.io
|
|
# Trimmed to the kinds this cluster actually has. Omitted from
|
|
# upstream's list, each for a checked reason:
|
|
# buckets, receivers — zero objects exist in this repo.
|
|
# helmcharts — created by source-controller at
|
|
# runtime, one per HelmRelease; a not-Ready HelmChart is always
|
|
# mirrored by the HelmRelease that owns it, so alerting on both
|
|
# is one failure reported twice.
|
|
# alerts, providers — notification.toolkit.fluxcd.io/v1beta3
|
|
# Alert and Provider have NO status.conditions in the CRD at all
|
|
# (checked in gotk-components.yaml), so there is no readiness to
|
|
# report; upstream omits `ready` for them for the same reason. A
|
|
# series carrying only `suspended` is not alertable.
|
|
resources:
|
|
- gitrepositories
|
|
- helmrepositories
|
|
- ocirepositories
|
|
- kustomizations
|
|
- helmreleases
|
|
- imagerepositories
|
|
- imagepolicies
|
|
- imageupdateautomations
|
|
verbs: ["list", "watch"]
|
|
|
|
customResourceState:
|
|
enabled: true
|
|
config:
|
|
spec:
|
|
resources:
|
|
# Label set is upstream's, minus the churning ones. Dropped
|
|
# deliberately: `revision` (status.artifact.revision /
|
|
# lastAppliedRevision — a git SHA, so every commit that reaches a
|
|
# Kustomization mints a NEW SERIES), and `url` / `endpoint`.
|
|
# Prometheus is at 84.5% of its 30GB retentionSize cap (REN-107);
|
|
# a label that changes several times a day across 33 Kustomizations
|
|
# is exactly what must not be added. Kept: `ready` and `suspended`
|
|
# (what the alerts read), `name` / `exported_namespace` (identity),
|
|
# `source_name` (the join key, see prometheusrule-flux.yaml), plus
|
|
# `image` and `tag` on the two image kinds, which are stable enough
|
|
# to ride the existing series.
|
|
#
|
|
# `exported_namespace` is upstream's own label name for this, not a
|
|
# Prometheus collision rename: the chart's KSM ServiceMonitor runs
|
|
# honorLabels, so `namespace` on a KSM series is already the
|
|
# OBJECT's namespace and nothing gets renamed. The rules use the
|
|
# same string because the config declares it.
|
|
#
|
|
# KSM adds `customresource_group`, `customresource_version` and
|
|
# `customresource_kind` to every custom-resource-state series by
|
|
# itself. The rules key off `customresource_kind`.
|
|
- groupVersionKind:
|
|
group: kustomize.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: Kustomization
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux Kustomization resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
source_name: [spec, sourceRef, name]
|
|
- groupVersionKind:
|
|
group: helm.toolkit.fluxcd.io
|
|
version: v2
|
|
kind: HelmRelease
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux HelmRelease resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: GitRepository
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux GitRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: HelmRepository
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux HelmRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: OCIRepository
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux OCIRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
- groupVersionKind:
|
|
group: image.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: ImageRepository
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux ImageRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
image: [spec, image]
|
|
# The repository-side half of the Ready-but-frozen signal. A
|
|
# policy can hold Ready=True forever while its filter excludes
|
|
# every new release, so readiness cannot see it; what can is
|
|
# "the repository kept finding tags and the policy's answer
|
|
# never moved". .status.lastScanResult.tagCount is a required
|
|
# field on this CRD, so the series exists for every object.
|
|
- name: "image_repository_tag_count"
|
|
help: "Number of tags found by the ImageRepository's last scan."
|
|
each:
|
|
type: Gauge
|
|
gauge:
|
|
path: [status, lastScanResult]
|
|
valueFrom: [tagCount]
|
|
# `name` belongs HERE and not under `gauge`. KSM resolves
|
|
# `gauge.labelsFromPath` RELATIVE TO `gauge.path`, so
|
|
# [metadata, name] under there would be looked up inside
|
|
# status.lastScanResult and silently yield no label. The
|
|
# metric-level labelsFromPath resolves from the object root.
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
exported_namespace: [metadata, namespace]
|
|
- groupVersionKind:
|
|
group: image.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: ImagePolicy
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux ImagePolicy resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
# The resolved tag, as a label on the Info series that
|
|
# already exists — so the policy-side half of the frozen
|
|
# signal costs zero extra series at rest. It retires one
|
|
# series and starts another only when the tag actually
|
|
# changes: 191 automated bumps in 147 days across 34
|
|
# policies is ~1.3 new series/day, which ages out with
|
|
# retention.
|
|
tag: [status, latestRef, tag]
|
|
# The join key. `metadata.name` happens to equal
|
|
# `spec.imageRepositoryRef.name` for all 34 pairs in this
|
|
# repo today, but the REFERENCE is this field, and a policy
|
|
# named differently from its repository would silently drop
|
|
# out of a name-based join.
|
|
source_name: [spec, imageRepositoryRef, name]
|
|
- groupVersionKind:
|
|
group: image.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: ImageUpdateAutomation
|
|
metricNamePrefix: gotk
|
|
errorLogV: 10
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux ImageUpdateAutomation resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [metadata, name]
|
|
labelsFromPath:
|
|
exported_namespace: [metadata, namespace]
|
|
ready: [status, conditions, "[type=Ready]", status]
|
|
suspended: [spec, suspend]
|
|
source_name: [spec, sourceRef, name]
|
|
|
|
# Drop the apiserver's high-cardinality latency histograms.
|
|
#
|
|
# Measured 2026-08-20: Prometheus held 298,236 active series, of which
|
|
# ~108,000 — over a third — were apiserver/etcd `_bucket` histograms. Each
|
|
# histogram fans out to one series per bucket x verb x resource x scope, so
|
|
# a single metric like apiserver_request_duration_seconds_bucket was 23,810
|
|
# series on its own. For comparison, every Home Assistant metric combined is
|
|
# ~4,100.
|
|
#
|
|
# That cardinality is the dominant driver of Prometheus write volume
|
|
# (~11 GB/day of TSDB + WAL, doubled again by Ceph replication), and none of
|
|
# it is observable value here: per-bucket apiserver latency quantiles are for
|
|
# clusters with SLAs and dozens of nodes, not three.
|
|
#
|
|
# ONLY metrics with ZERO rule dependencies are dropped. Verified against the
|
|
# live rule set before writing this — each metric checked individually for
|
|
# references in alert and recording rules:
|
|
#
|
|
# apiserver_request_duration_seconds_bucket rules=0 23,810 series
|
|
# apiserver_request_body_size_bytes_bucket rules=0 23,200
|
|
# etcd_request_duration_seconds_bucket rules=0 15,840
|
|
# apiserver_watch_list_duration_seconds_bucket rules=0 11,538
|
|
# apiserver_watch_cache_read_wait_seconds_bucket rules=0 8,232
|
|
# apiserver_response_sizes_bucket rules=0 7,592
|
|
# apiserver_watch_events_sizes_bucket rules=0 5,220
|
|
#
|
|
# apiserver_request_sli_duration_seconds_bucket is DELIBERATELY KEPT: 23
|
|
# recording rules depend on it, including the apiserver availability SLO and
|
|
# the KubeAPIErrorBudgetBurn burn-rate series. Dropping it would silently
|
|
# break that alerting, which is the failure mode this comment exists to
|
|
# prevent if someone later "tidies up" this regex.
|
|
#
|
|
# Only the `_bucket` series are dropped. The matching `_count` and `_sum`
|
|
# remain, so request rates and mean latencies still work — what is lost is
|
|
# per-bucket quantiles.
|
|
#
|
|
# etcd_* appears here even though kubeEtcd is disabled below: these are the
|
|
# apiserver's own etcd-client metrics, exposed on the apiserver endpoint, not
|
|
# a separate etcd scrape.
|
|
kubeApiServer:
|
|
serviceMonitor:
|
|
metricRelabelings:
|
|
- action: drop
|
|
sourceLabels: [__name__]
|
|
regex: (apiserver_request_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|etcd_request_duration_seconds_bucket|apiserver_watch_list_duration_seconds_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket)
|
|
|
|
# Disable etcd/scheduler/controller-manager scraping (Talos doesn't expose them by default)
|
|
kubeEtcd:
|
|
enabled: false
|
|
kubeScheduler:
|
|
enabled: false
|
|
kubeControllerManager:
|
|
enabled: false
|
|
kubeProxy:
|
|
enabled: false
|
|
|
|
# Replaced by monitoring-rules/prometheusrule-jobs.yaml.
|
|
defaultRules:
|
|
disabled:
|
|
KubeJobFailed: true
|
|
KubeContainerWaiting: true
|