2026-09-26 00:16:00 +00:00
3 changed files with 98 additions and 0 deletions
@@ -0,0 +1,71 @@
+# Scrape the external-secrets (ESO) controller's own metrics. Until this
+# existed, nothing in this cluster could see a failed ExternalSecret sync:
+# `externalsecret_status_condition` held 0 series, so no rule could be written
+# about a key that silently stops syncing (REN-100).
+#
+# This is the scrape only. The alert rule is deliberately NOT in this change —
+# its `for:` window has to be tuned against real data, and that data does not
+# exist until this has been live for a while. See REN-103.
+#
+# A PodMonitor, NOT the chart's `serviceMonitor.enabled`, and NOT a hand-written
+# ServiceMonitor. Both alternatives were checked against chart 2.11.0:
+#
+#   * A hand-written ServiceMonitor has nothing to select. `metrics.service.enabled`
+#     defaults to false and infrastructure/external-secrets/helmrelease.yaml
+#     overrides nothing, so templates/service.yaml renders no Service for the
+#     controller. The container port exists (`name: metrics`, containerPort 8080)
+#     but nothing fronts it — the same trap that forced a PodMonitor for the Flux
+#     controllers.
+#   * Setting `serviceMonitor.enabled: true` would work, but it triggers a Helm
+#     UPGRADE of ESO, which re-renders the Deployment and re-applies its CRDs
+#     (`crds: CreateReplace` in the HelmRelease). ESO is what all 54 ExternalSecret
+#     manifests in this repo depend on. A PodMonitor gets the same series while
+#     touching the release not at all, which is the trade this repo already made
+#     for Velero's node-agent (velero-podmonitor.yaml) and for Flux.
+#
+# THIS NEEDS A NETWORK POLICY CHANGE, unlike the Flux PodMonitor: the cluster is
+# default-deny and external-secrets-ns.yaml admitted only intra-namespace traffic,
+# host/remote-node and ICMP — no `monitoring` grant, so this scrape would have
+# been dropped and the metric would have stayed absent. The matching ingress rule
+# is added to infrastructure/network-policies/external-secrets-ns.yaml in the
+# same pull request. Without it this file is decoration.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: external-secrets
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: external-secrets
+    # Not required for discovery — podMonitorSelector is empty, see
+    # apps/monitoring/helmrelease-prometheus.yaml — but set for consistency with
+    # the other monitors in this directory.
+    release: kube-prometheus-stack
+spec:
+  # Lives in `monitoring` rather than in `external-secrets`, matching
+  # external-dns-servicemonitor.yaml and the Flux PodMonitor. Every object in
+  # the external-secrets namespace is owned by the ESO HelmRelease; keeping this
+  # out of that namespace means this change cannot interact with the release.
+  namespaceSelector:
+    matchNames:
+      - external-secrets
+  selector:
+    matchLabels:
+      # The CONTROLLER only. The webhook and cert-controller carry
+      # `app.kubernetes.io/name: external-secrets-webhook` and
+      # `-cert-controller` respectively (see the chart's
+      # external-secrets-webhook.selectorLabels / -cert-controller.selectorLabels
+      # helpers), so an exact match on the unsuffixed name excludes both. That is
+      # intended: the `externalsecret_*` series are emitted by the controller, and
+      # the other two would only add idle targets.
+      app.kubernetes.io/name: external-secrets
+      app.kubernetes.io/instance: external-secrets
+  podMetricsEndpoints:
+    # Named container port, not a hardcoded 8080, so this follows
+    # `metrics.listen.port` if it is ever overridden in the HelmRelease.
+    - port: metrics
+      # 60s, not the chart's 30s default: `externalsecret_status_condition` is a
+      # slow-moving gauge over ~54 ExternalSecrets, and the alert built on it in
+      # REN-103 will use a multi-minute `for:` window. 30s would double the
+      # samples for no gain in detection time.
+      interval: 60s
+      scrapeTimeout: 10s
@@ -48,4 +48,8 @@ resources:
  - loki-servicemonitor.yaml
  - smartctl-exporter-servicemonitor.yaml
  - intel-gpu-exporter-servicemonitor.yaml
+  # ESO's controller metrics. Needs the `monitoring` ingress grant added to
+  # infrastructure/network-policies/external-secrets-ns.yaml in the same change,
+  # or it scrapes nothing. See the file.
+  - external-secrets-podmonitor.yaml
  - probes-blackbox.yaml
@@ -10,6 +10,29 @@ spec:
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": external-secrets
+    # Prometheus scrape of the ESO controller's metrics (REN-100). Until this
+    # existed a failed ExternalSecret sync was invisible to Prometheus: this
+    # namespace admitted only the rules either side of this one, so the scrape was
+    # dropped and `externalsecret_status_condition` held 0 series.
+    #
+    # Port-restricted, unlike the equivalent rule in velero-ns.yaml, which admits
+    # `monitoring` to the whole namespace. 8080 is the metrics port and the only
+    # thing `monitoring` has any business reaching here; the webhook (10250) and
+    # the health port stay closed to it. 8080 is the chart default
+    # (`metrics.listen.port`) and helmrelease.yaml does not override it — if that
+    # is ever overridden, this port and the PodMonitor's named `metrics` endpoint
+    # both have to follow.
+    #
+    # The consumer is apps/monitoring-rules/external-secrets-podmonitor.yaml.
+    # Additive: this rule cannot deny anything previously allowed, so ESO's
+    # existing reachability to OpenBao and the apiserver is unchanged.
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": monitoring
+      toPorts:
+        - ports:
+            - port: "8080"
+              protocol: TCP
    # kube-apiserver (host identity in Cilium) drives the ExternalSecret/
    # SecretStore validating webhook and the controller's watch/list/create
    # calls. Without this, every ExternalSecret apply hangs the admission