diff --git a/kubernetes/apps/monitoring-rules/cilium-servicemonitor.yaml b/kubernetes/apps/monitoring-rules/cilium-servicemonitor.yaml index ed40a611..6716aabf 100644 --- a/kubernetes/apps/monitoring-rules/cilium-servicemonitor.yaml +++ b/kubernetes/apps/monitoring-rules/cilium-servicemonitor.yaml @@ -1,8 +1,75 @@ +--- +# The agent metrics Service, declared here because the chart does not render +# one under our values (REN-147). +# +# cilium's templates/cilium-agent/service.yaml is gated twice: +# +# {{- if and .Values.agent (not .Values.preflight.enabled) .Values.prometheus.enabled }} +# {{- if (or .Values.prometheus.serviceMonitor.enabled .Values.prometheus.metricsService) }} +# +# infrastructure/cilium/helmrelease.yaml sets `prometheus.enabled: true` and +# `prometheus.serviceMonitor.enabled: false`, and leaves `metricsService` at its +# default of false. First gate passes, second does not. The chart's fallback +# branch is envoy-only and needs envoy in the agent pod, but envoy runs as its +# own DaemonSet here — so no `cilium-agent` Service was rendered at all. +# +# The consequence was silent: agents exported metrics on :9962 all along, the +# ServiceMonitor below selected a Service that did not exist, and the scrape +# pool sat at 0 active / 46 dropped targets. `cilium_*` had zero series, so +# CiliumBPFMapPressureHigh and CiliumBPFMapFull in prometheusrule-cilium.yaml +# evaluated an empty vector forever and could never fire. +# +# Why a Service here and not `prometheus.metricsService: true` in the +# HelmRelease: that flag is the chart-native fix and its rendered diff is +# exactly this one object (it appears in no other template — not the +# ConfigMap, not the DaemonSet). But setting it changes the HelmRelease values, +# which makes helm-controller run `helm upgrade` on the CNI release, and that +# three-way-merges against live objects. PROPOSAL-cilium-single-owner.md +# records that `cilium-config` has carried a `kubectl-patch` field manager; if +# any of that drift is still live, the upgrade reconciles it, and the +# `configmap.reloader.stakater.com/reload: cilium-config` annotation on the +# HelmRelease then rolls every agent node by node. A Service applied by the +# apps Kustomization cannot do that. If the drift is ever proven gone, delete +# this object and set `prometheus.metricsService: true` instead. +# +# Name and selector follow authentik-outpost-servicemonitors.yaml: a distinct +# name plus `app.kubernetes.io/managed-by: homelab`, so flipping either chart +# flag later renders the chart's own `cilium-agent` Service without an +# ownership conflict and without double-scraping the same endpoints. +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent-metrics + namespace: kube-system + labels: + app.kubernetes.io/managed-by: homelab + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + # Copied onto every series by the ServiceMonitor's targetLabels below. + # The dashboard the chart ships queries + # cilium_bpf_map_pressure{k8s_app="cilium"}, so this label is load-bearing. + k8s-app: cilium +spec: + type: ClusterIP + # Headless on purpose: agent metrics are per-node and every endpoint has to + # be scraped. A ClusterIP would hand Prometheus one random agent per scrape. + clusterIP: None + # Agent pods only. cilium-envoy carries k8s-app: cilium-envoy and exposes + # 9964, so it cannot match. + selector: + k8s-app: cilium + ports: + # Named and numbered exactly as the chart's own Service, so the + # ServiceMonitor below stays a field-for-field copy of the chart's. + - name: metrics + port: 9962 + targetPort: prometheus + protocol: TCP +--- # Cilium's agent ServiceMonitor, declared here rather than by the chart. # -# Reproduces what cilium 1.20.1 renders with prometheus.serviceMonitor.enabled, -# taken field for field off the live object so scraping is unchanged: same name, -# same namespace, same selector, same 10s interval and node relabeling. +# Reproduces what cilium renders with prometheus.serviceMonitor.enabled: same +# name, same namespace, same 10s interval and node relabeling. # # It lives here because the chart's template is gated on monitoring.coreos.com/v1 # existing at install time, and on an empty cluster it does not — those CRDs @@ -22,13 +89,14 @@ metadata: app.kubernetes.io/part-of: cilium spec: selector: + # Pinned to the Service above. Without managed-by this would also match a + # chart-rendered `cilium-agent` Service and scrape the same endpoints twice. matchLabels: + app.kubernetes.io/managed-by: homelab app.kubernetes.io/name: cilium-agent namespaceSelector: matchNames: - kube-system - # The chart's Service keeps rendering (prometheus.enabled), so the target and - # its `metrics` port are there regardless of who declares the ServiceMonitor. targetLabels: - k8s-app endpoints: diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index f7e01483..3e9f4c39 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -30,6 +30,9 @@ resources: - prometheusrule-dns-resolution.yaml - probes-dns.yaml - alloy-servicemonitor.yaml + # Agent metrics. Ships the metrics Service as well as the ServiceMonitor: + # the chart renders neither under our values, so this scraped nothing and + # the two CiliumBPFMapPressure alerts could never fire. See the file. - cilium-servicemonitor.yaml - cnpg-podmonitors.yaml - crowdsec-servicemonitor.yaml diff --git a/kubernetes/infrastructure/cilium/helmrelease.yaml b/kubernetes/infrastructure/cilium/helmrelease.yaml index 8377090c..b1f8145e 100644 --- a/kubernetes/infrastructure/cilium/helmrelease.yaml +++ b/kubernetes/infrastructure/cilium/helmrelease.yaml @@ -72,10 +72,29 @@ spec: # alertable beforehand. # # That reasoning is about the metrics, which this line provides. The - # ServiceMonitor is only the scrape wiring and is declared standalone in - # apps/monitoring-rules instead — the same treatment velero/helmrelease - # already gives its own, and for the same reason: the chart's template is - # gated on monitoring.coreos.com/v1 existing at install time. + # scrape wiring — both the ServiceMonitor and the metrics Service — is + # declared standalone in apps/monitoring-rules/cilium-servicemonitor.yaml + # instead. For the ServiceMonitor that is the same treatment + # velero/helmrelease already gives its own, and for the same reason: the + # chart's template is gated on monitoring.coreos.com/v1 existing at + # install time. + # + # The Service is there for a different reason, and it is the one that + # bit us (REN-147). templates/cilium-agent/service.yaml renders only when + # `prometheus.enabled` AND (`serviceMonitor.enabled` OR `metricsService`). + # With serviceMonitor.enabled false and metricsService at its default, + # no Service was rendered, so the standalone ServiceMonitor selected + # nothing and cilium_* had zero series for as long as this file has + # looked like this. Turning either flag on here is the chart-native fix + # and would render exactly that one object — but it is a values change, + # so helm-controller would upgrade the CNI release and three-way-merge + # against live objects, and the reloader annotation above turns any + # resulting cilium-config change into a rolling restart of every agent. + # Hence the Service is applied by the apps Kustomization, which cannot. + # + # If you ever do set `serviceMonitor.enabled` or `metricsService` here, + # delete the matching object from cilium-servicemonitor.yaml in the same + # change. # # Here it is worse than for velero. Cilium's chart does not skip the # template, it aborts rendering — "Service Monitor requires