From 6ec37253710e4d5fb8244421a1f923c9925ab9f4 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Fri, 10 Jul 2026 17:27:45 +0200 Subject: [PATCH] Gate Alertmanager behind Authentik MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alertmanager (alerts.renner.uno, monitoring ns, :9093) had no auth of its own, and its v2 API is writable: an unauthenticated LAN client could POST /api/v2/silences and suppress ALL alerting — a tamper capability, not just disclosure (2026-07-10 audit, highest-teeth of the deferred unauth findings). Gate it with the same Authentik proxy-outpost pattern as paperless/vscode/shelly-manager: - New proxy provider + application + dedicated outpost (ak-outpost-alertmanager-gate), mode proxy, forwarding to the ClusterIP Service (kube-prometheus-stack-alertmanager.monitoring.svc:9093). - New git-owned "Monitoring Access" group (k.renner) bound to the app. Named generically so it can also gate the still-deferred Prometheus UI. - The Helm-chart-generated Alertmanager Ingress is disabled (alertmanager.ingress.enabled=false); a standalone gate Ingress (alertmanager-gate-ingress.yaml) backends the outpost ExternalName — two Ingresses for the same host would conflict. Mirrors loki-ingress. - Scoped CNPs: monitoring ingress rule allowing only the outpost -> :9093; matching egress rule in the authentik namespace. implicit-consent flow, NOT passkey step-up: Alertmanager is a frequently-checked monitoring UI, and requiring an authenticated Authentik session already closes the unauthenticated-LAN exposure. Passkey re-auth on every visit is disproportionate friction for a dashboard (paperless-gate uses implicit-consent for the same reason). Bump to passkey-stepup-authorization if the write API warrants it. No skip_path_regex: the Alertmanager UI is a plain Elm SPA with no service worker and no manifest (verified 2026-07-10 — index.html has no serviceWorker/manifest/webmanifest), so none of the credentials:'omit' PWA fetches that forced skip carve-outs elsewhere. Prometheus's alerting path is UNAFFECTED: it posts to the ClusterIP Service (verified: spec.alerting.alertmanagers targets monitoring/kube-prometheus-stack-alertmanager:http-web), not this ingress. Prometheus's own ingress is untouched (still enabled) — its unauth exposure is a separate deferred finding, and it is read-only. Verified: kubectl kustomize builds for authentik, monitoring and network-policies; the embedded outpost survived the blueprint insert; the rendered ingress backends ak-outpost-alertmanager-gate:9000, egress is scoped to alertmanager:9093, and alertmanager.ingress.enabled=false while prometheus.ingress.enabled=true. Post-merge checks to run: only the gate Ingress exists for alerts.renner.uno (the Helm one is gone); unauthenticated GET -> 302 to auth.renner.uno; POST /api/v2/silences unauthenticated is refused; the outpost forwards to :9093 after login; Prometheus is still delivering alerts to Alertmanager. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../authentik/alertmanager-gate-egress.yaml | 25 ++++ .../apps/authentik/blueprints-configmap.yaml | 112 ++++++++++++++++++ kubernetes/apps/authentik/kustomization.yaml | 1 + .../monitoring/alertmanager-gate-ingress.yaml | 39 ++++++ .../monitoring/alertmanager-gate-service.yaml | 22 ++++ .../monitoring/helmrelease-prometheus.yaml | 20 ++-- kubernetes/apps/monitoring/kustomization.yaml | 2 + .../network-policies/monitoring-ns.yaml | 15 +++ 8 files changed, 224 insertions(+), 12 deletions(-) create mode 100644 kubernetes/apps/authentik/alertmanager-gate-egress.yaml create mode 100644 kubernetes/apps/monitoring/alertmanager-gate-ingress.yaml create mode 100644 kubernetes/apps/monitoring/alertmanager-gate-service.yaml diff --git a/kubernetes/apps/authentik/alertmanager-gate-egress.yaml b/kubernetes/apps/authentik/alertmanager-gate-egress.yaml new file mode 100644 index 00000000..8e9eb18e --- /dev/null +++ b/kubernetes/apps/authentik/alertmanager-gate-egress.yaml @@ -0,0 +1,25 @@ +--- +# Allow the dedicated alertmanager-gate outpost to egress into the +# monitoring namespace and reach Alertmanager (:9093). Without this, the +# authentik-namespace baseline CNP has no egress rule to monitoring, so the +# gate outpost's forwarded requests are dropped. Scoped to the gate +# outpost's pod label so no other authentik component can reach it. +# Mirrors shelly-manager-gate-egress.yaml. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: alertmanager-gate-egress + namespace: authentik +spec: + endpointSelector: + matchLabels: + goauthentik.io/outpost-name: alertmanager-gate + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": monitoring + app.kubernetes.io/name: alertmanager + toPorts: + - ports: + - port: "9093" + protocol: TCP diff --git a/kubernetes/apps/authentik/blueprints-configmap.yaml b/kubernetes/apps/authentik/blueprints-configmap.yaml index 08fb73a3..0d68e315 100644 --- a/kubernetes/apps/authentik/blueprints-configmap.yaml +++ b/kubernetes/apps/authentik/blueprints-configmap.yaml @@ -433,6 +433,118 @@ data: kubernetes_ingress_secret_name: "authentik-outpost-tls" kubernetes_json_patches: null + # ------------------------------------------------------------------ + # alertmanager gate + # + # Alertmanager (alerts.renner.uno, monitoring ns, :9093) had no auth + # of its own: its v2 API is writable, so any unauthenticated LAN + # client could POST /api/v2/silences and suppress ALL alerting — + # a tamper capability, not just disclosure (2026-07-10 audit). + # + # Same proxy-outpost pattern as the gates above. Traefik routes + # alerts.renner.uno only to this outpost (the Helm-generated ingress + # is disabled; a standalone gate ingress replaces it), and the + # monitoring CNP allows ONLY this outpost to reach :9093 from the + # authentik namespace (see monitoring-ns.yaml + + # alertmanager-gate-egress.yaml). Prometheus's own alerting path is + # untouched — it posts to the ClusterIP Service + # (kube-prometheus-stack-alertmanager:9093), not through the ingress. + # + # implicit-consent flow, NOT passkey step-up: Alertmanager is a + # frequently-checked monitoring UI, and requiring an authenticated + # Authentik session already closes the unauthenticated-LAN exposure. + # Passkey re-auth on every visit would be disproportionate friction + # for a dashboard (paperless-gate uses implicit-consent for the same + # reason). Bump to passkey-stepup-authorization if the write API + # warrants it. + - id: monitoring-access-group + model: authentik_core.group + identifiers: + name: Monitoring Access + state: present + attrs: + is_superuser: false + # Git owns membership, as with the other gate groups. Named + # generically ("Monitoring") so the same allowlist can gate the + # still-deferred Prometheus UI when that is done. Add/remove + # members HERE, not in the UI — blueprint apply is authoritative. + users: + - !Find [authentik_core.user, [username, k.renner]] + + - id: alertmanager-gate-provider + model: authentik_providers_proxy.proxyprovider + identifiers: + name: alertmanager-gate + state: present + attrs: + authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]] + invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]] + mode: proxy + internal_host: http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093 + internal_host_ssl_validation: false + external_host: https://alerts.renner.uno + cookie_domain: renner.uno + access_token_validity: hours=24 + refresh_token_validity: days=30 + intercept_header_auth: true + # No skip_path_regex: the Alertmanager UI is a plain Elm SPA with + # no service worker and no manifest (verified 2026-07-10 — the + # index.html has no serviceWorker/manifest/webmanifest refs), so + # it has none of the credentials:'omit' PWA fetches that forced + # skip carve-outs on paperless/vscode and killed the + # music-assistant gate. Everything is a same-origin credentialed + # fetch and stays gated. + + - model: authentik_core.application + identifiers: + slug: alertmanager-gate + state: present + attrs: + name: Alertmanager (Gate) + slug: alertmanager-gate + provider: !KeyOf alertmanager-gate-provider + meta_launch_url: https://alerts.renner.uno + meta_description: "Authentik-gated Alertmanager" + + # Only the Monitoring Access group passes the gate. + - model: authentik_policies.policybinding + identifiers: + target: !Find [authentik_core.application, [slug, alertmanager-gate]] + group: !KeyOf monitoring-access-group + order: 0 + state: present + attrs: + enabled: true + negate: false + failure_result: false + + - model: authentik_outposts.outpost + identifiers: + name: alertmanager-gate + state: present + attrs: + type: proxy + service_connection: !KeyOf k8s-conn + providers: + - !KeyOf alertmanager-gate-provider + config: + log_level: info + authentik_host: "http://authentik-server.authentik.svc.cluster.local" + authentik_host_insecure: false + authentik_host_browser: "https://auth.renner.uno/" + refresh_interval: "minutes=5" + container_image: null + object_naming_template: "ak-outpost-%(name)s" + kubernetes_namespace: authentik + kubernetes_replicas: 1 + kubernetes_service_type: ClusterIP + kubernetes_disabled_components: ["ingress"] + kubernetes_image_pull_secrets: [] + kubernetes_ingress_annotations: {} + kubernetes_ingress_class_name: null + kubernetes_ingress_secret_name: "authentik-outpost-tls" + kubernetes_json_patches: null + - model: authentik_outposts.outpost identifiers: name: authentik Embedded Outpost diff --git a/kubernetes/apps/authentik/kustomization.yaml b/kubernetes/apps/authentik/kustomization.yaml index c124cc73..5c1c9c08 100644 --- a/kubernetes/apps/authentik/kustomization.yaml +++ b/kubernetes/apps/authentik/kustomization.yaml @@ -9,6 +9,7 @@ resources: - paperless-gate-egress.yaml - vscode-gate-egress.yaml - shelly-manager-gate-egress.yaml + - alertmanager-gate-egress.yaml - outpost-servicemonitors.yaml - ldap-outpost-cert.yaml - ldap-outpost-lb.yaml diff --git a/kubernetes/apps/monitoring/alertmanager-gate-ingress.yaml b/kubernetes/apps/monitoring/alertmanager-gate-ingress.yaml new file mode 100644 index 00000000..05b39c48 --- /dev/null +++ b/kubernetes/apps/monitoring/alertmanager-gate-ingress.yaml @@ -0,0 +1,39 @@ +--- +# alerts.renner.uno — Alertmanager, gated behind Authentik. +# +# Alertmanager has no auth of its own and its v2 API is writable +# (POST /api/v2/silences can suppress all alerting), so Traefik must not +# route to it directly. This Ingress backends the Authentik +# alertmanager-gate proxy outpost (ExternalName → authentik ns); the +# monitoring CNP allows only that outpost to reach :9093. Replaces the +# Helm-chart-generated Alertmanager Ingress (disabled via +# alertmanager.ingress.enabled=false in helmrelease-prometheus.yaml) — +# two Ingresses for the same host would conflict. Standalone-manifest +# pattern mirrors loki-ingress.yaml. +# +# Prometheus's alerting path is unaffected: it posts to the ClusterIP +# Service (kube-prometheus-stack-alertmanager:9093), not this hostname. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alertmanager-gate + namespace: monitoring + annotations: + cert-manager.io/cluster-issuer: letsencrypt-cluster +spec: + ingressClassName: traefik + rules: + - host: alerts.renner.uno + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ak-outpost-alertmanager-gate + port: + number: 9000 + tls: + - secretName: alertmanager-tls + hosts: + - alerts.renner.uno diff --git a/kubernetes/apps/monitoring/alertmanager-gate-service.yaml b/kubernetes/apps/monitoring/alertmanager-gate-service.yaml new file mode 100644 index 00000000..126ce9cc --- /dev/null +++ b/kubernetes/apps/monitoring/alertmanager-gate-service.yaml @@ -0,0 +1,22 @@ +--- +# ExternalName mirror so the monitoring-namespace alertmanager gate Ingress +# can backend to the alertmanager-gate outpost that Authentik deploys into +# the authentik namespace (cross-namespace Service refs aren't supported in +# a standard Ingress). Mirrors ak-outpost-shelly-manager-gate. +# +# Traffic path: +# browser → traefik (alerts.renner.uno) +# → ak-outpost-alertmanager-gate (ExternalName → authentik ns) +# → outpost validates the Authentik session +# → http://kube-prometheus-stack-alertmanager.monitoring.svc:9093 +apiVersion: v1 +kind: Service +metadata: + name: ak-outpost-alertmanager-gate + namespace: monitoring +spec: + type: ExternalName + externalName: ak-outpost-alertmanager-gate.authentik.svc.cluster.local + ports: + - port: 9000 + targetPort: 9000 diff --git a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml index 2236b115..0a8ff976 100644 --- a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml +++ b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml @@ -246,19 +246,15 @@ spec: # Alertmanager alertmanager: - # LAN-only ingress at alerts.renner.uno. Same pattern as - # Prometheus above: unifi-DNS only, not Cloudflare-published. + # alerts.renner.uno is served by a standalone, Authentik-gated Ingress + # (alertmanager-gate-ingress.yaml) instead of this Helm-generated one. + # Alertmanager has no auth and its v2 API is writable (silence-all), + # so it must sit behind the alertmanager-gate proxy outpost. The + # chart Ingress is disabled here — two Ingresses for the same host + # would conflict. The tls secret (alertmanager-tls) and externalUrl + # below are unchanged; the standalone Ingress reuses the same secret. ingress: - enabled: true - ingressClassName: traefik - annotations: - cert-manager.io/cluster-issuer: letsencrypt-cluster - hosts: - - alerts.renner.uno - tls: - - secretName: alertmanager-tls - hosts: - - alerts.renner.uno + enabled: false alertmanagerSpec: # Embedded in the AlertmanagerURL footer of every email plus # the "Source" link of every grouped alert, so the email link diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml index cc241916..ce0f57bb 100644 --- a/kubernetes/apps/monitoring/kustomization.yaml +++ b/kubernetes/apps/monitoring/kustomization.yaml @@ -12,6 +12,8 @@ resources: - loki-basic-auth.sops.yaml - loki-basic-auth-middleware.yaml - loki-ingress.yaml + - alertmanager-gate-service.yaml + - alertmanager-gate-ingress.yaml - dashboards/ # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a # separate Flux Kustomization that dependsOn `monitoring` — otherwise the diff --git a/kubernetes/infrastructure/network-policies/monitoring-ns.yaml b/kubernetes/infrastructure/network-policies/monitoring-ns.yaml index 9483ac89..40c2c604 100644 --- a/kubernetes/infrastructure/network-policies/monitoring-ns.yaml +++ b/kubernetes/infrastructure/network-policies/monitoring-ns.yaml @@ -24,6 +24,21 @@ spec: - fromEndpoints: - matchLabels: "k8s:io.kubernetes.pod.namespace": flux-system + # The Authentik alertmanager-gate outpost forwards gated browser traffic + # to Alertmanager :9093. Scoped to the outpost's pod label + port, so no + # other authentik pod gains monitoring-namespace reach. Matching egress + # rule: authentik/alertmanager-gate-egress.yaml. NOTE: this CNP's + # endpointSelector is {} (namespace-wide), so this :9093 grant reaches + # every monitoring pod listening on 9093 — today only Alertmanager; the + # egress side is what actually confines the outpost to Alertmanager. + - fromEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": authentik + goauthentik.io/outpost-name: alertmanager-gate + toPorts: + - ports: + - port: "9093" + protocol: TCP # Kubelet probes + control-plane (PrometheusRule/ServiceMonitor admission # webhooks via kube-prometheus-stack-operator are called by kube-apiserver). # Also covers external alloy-syslog traffic — ETP=Cluster on the LB SNATs -- 2.54.0