diff --git a/kubernetes/apps/monitoring-rules/external-secrets-podmonitor.yaml b/kubernetes/apps/monitoring-rules/external-secrets-podmonitor.yaml new file mode 100644 index 00000000..d1c14eaf --- /dev/null +++ b/kubernetes/apps/monitoring-rules/external-secrets-podmonitor.yaml @@ -0,0 +1,71 @@ +# Scrape the external-secrets (ESO) controller's own metrics. Until this +# existed, nothing in this cluster could see a failed ExternalSecret sync: +# `externalsecret_status_condition` held 0 series, so no rule could be written +# about a key that silently stops syncing (REN-100). +# +# This is the scrape only. The alert rule is deliberately NOT in this change — +# its `for:` window has to be tuned against real data, and that data does not +# exist until this has been live for a while. See REN-103. +# +# A PodMonitor, NOT the chart's `serviceMonitor.enabled`, and NOT a hand-written +# ServiceMonitor. Both alternatives were checked against chart 2.11.0: +# +# * A hand-written ServiceMonitor has nothing to select. `metrics.service.enabled` +# defaults to false and infrastructure/external-secrets/helmrelease.yaml +# overrides nothing, so templates/service.yaml renders no Service for the +# controller. The container port exists (`name: metrics`, containerPort 8080) +# but nothing fronts it — the same trap that forced a PodMonitor for the Flux +# controllers. +# * Setting `serviceMonitor.enabled: true` would work, but it triggers a Helm +# UPGRADE of ESO, which re-renders the Deployment and re-applies its CRDs +# (`crds: CreateReplace` in the HelmRelease). ESO is what all 54 ExternalSecret +# manifests in this repo depend on. A PodMonitor gets the same series while +# touching the release not at all, which is the trade this repo already made +# for Velero's node-agent (velero-podmonitor.yaml) and for Flux. +# +# THIS NEEDS A NETWORK POLICY CHANGE, unlike the Flux PodMonitor: the cluster is +# default-deny and external-secrets-ns.yaml admitted only intra-namespace traffic, +# host/remote-node and ICMP — no `monitoring` grant, so this scrape would have +# been dropped and the metric would have stayed absent. The matching ingress rule +# is added to infrastructure/network-policies/external-secrets-ns.yaml in the +# same pull request. Without it this file is decoration. +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: external-secrets + namespace: monitoring + labels: + app.kubernetes.io/name: external-secrets + # Not required for discovery — podMonitorSelector is empty, see + # apps/monitoring/helmrelease-prometheus.yaml — but set for consistency with + # the other monitors in this directory. + release: kube-prometheus-stack +spec: + # Lives in `monitoring` rather than in `external-secrets`, matching + # external-dns-servicemonitor.yaml and the Flux PodMonitor. Every object in + # the external-secrets namespace is owned by the ESO HelmRelease; keeping this + # out of that namespace means this change cannot interact with the release. + namespaceSelector: + matchNames: + - external-secrets + selector: + matchLabels: + # The CONTROLLER only. The webhook and cert-controller carry + # `app.kubernetes.io/name: external-secrets-webhook` and + # `-cert-controller` respectively (see the chart's + # external-secrets-webhook.selectorLabels / -cert-controller.selectorLabels + # helpers), so an exact match on the unsuffixed name excludes both. That is + # intended: the `externalsecret_*` series are emitted by the controller, and + # the other two would only add idle targets. + app.kubernetes.io/name: external-secrets + app.kubernetes.io/instance: external-secrets + podMetricsEndpoints: + # Named container port, not a hardcoded 8080, so this follows + # `metrics.listen.port` if it is ever overridden in the HelmRelease. + - port: metrics + # 60s, not the chart's 30s default: `externalsecret_status_condition` is a + # slow-moving gauge over ~54 ExternalSecrets, and the alert built on it in + # REN-103 will use a multi-minute `for:` window. 30s would double the + # samples for no gain in detection time. + interval: 60s + scrapeTimeout: 10s diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index 00dbebf8..1b3afcf9 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -48,4 +48,8 @@ resources: - loki-servicemonitor.yaml - smartctl-exporter-servicemonitor.yaml - intel-gpu-exporter-servicemonitor.yaml + # ESO's controller metrics. Needs the `monitoring` ingress grant added to + # infrastructure/network-policies/external-secrets-ns.yaml in the same change, + # or it scrapes nothing. See the file. + - external-secrets-podmonitor.yaml - probes-blackbox.yaml diff --git a/kubernetes/infrastructure/network-policies/external-secrets-ns.yaml b/kubernetes/infrastructure/network-policies/external-secrets-ns.yaml index 6edd77e1..5f8d6b23 100644 --- a/kubernetes/infrastructure/network-policies/external-secrets-ns.yaml +++ b/kubernetes/infrastructure/network-policies/external-secrets-ns.yaml @@ -10,6 +10,29 @@ spec: - fromEndpoints: - matchLabels: "k8s:io.kubernetes.pod.namespace": external-secrets + # Prometheus scrape of the ESO controller's metrics (REN-100). Until this + # existed a failed ExternalSecret sync was invisible to Prometheus: this + # namespace admitted only the rules either side of this one, so the scrape was + # dropped and `externalsecret_status_condition` held 0 series. + # + # Port-restricted, unlike the equivalent rule in velero-ns.yaml, which admits + # `monitoring` to the whole namespace. 8080 is the metrics port and the only + # thing `monitoring` has any business reaching here; the webhook (10250) and + # the health port stay closed to it. 8080 is the chart default + # (`metrics.listen.port`) and helmrelease.yaml does not override it — if that + # is ever overridden, this port and the PodMonitor's named `metrics` endpoint + # both have to follow. + # + # The consumer is apps/monitoring-rules/external-secrets-podmonitor.yaml. + # Additive: this rule cannot deny anything previously allowed, so ESO's + # existing reachability to OpenBao and the apiserver is unchanged. + - fromEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": monitoring + toPorts: + - ports: + - port: "8080" + protocol: TCP # kube-apiserver (host identity in Cilium) drives the ExternalSecret/ # SecretStore validating webhook and the controller's watch/list/create # calls. Without this, every ExternalSecret apply hangs the admission