From fe2e163b6a5895ba5a6b921130d7b0726bc003c9 Mon Sep 17 00:00:00 2001 From: atlas-bot Date: Sun, 4 Oct 2026 12:20:27 +0000 Subject: [PATCH] monitoring: GET-only Loki read gateway for the paperclip read SA (REN-621) Loki runs SingleBinary with auth disabled and answers GET on /ingester/shutdown and /flush, so an RBAC `get` on services/proxy for loki:3100 would not be read-only. Put an nginx allow-list in front of it (query, query_range, labels, label//values, series; GET only; upstream URI rebuilt per location) and grant the paperclip read SA `get` on that Service only. Owner decision: REN-622 (option A2). Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/monitoring/kustomization.yaml | 8 ++ .../apps/monitoring/loki-read-gateway.conf | 60 ++++++++++++ .../apps/monitoring/loki-read-gateway.yaml | 97 +++++++++++++++++++ .../apps/paperclip/prometheus-read-rbac.yaml | 11 ++- 4 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 kubernetes/apps/monitoring/loki-read-gateway.conf create mode 100644 kubernetes/apps/monitoring/loki-read-gateway.yaml diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml index d570ee49..5df35af4 100644 --- a/kubernetes/apps/monitoring/kustomization.yaml +++ b/kubernetes/apps/monitoring/kustomization.yaml @@ -10,6 +10,7 @@ resources: - loki-basic-auth-externalsecret.yaml - loki-basic-auth-middleware.yaml - loki-ingress.yaml + - loki-read-gateway.yaml - alertmanager-gate-service.yaml - alertmanager-gate-ingress.yaml - prometheus-gate-service.yaml @@ -18,6 +19,13 @@ resources: - smartctl-exporter.yaml - intel-gpu-exporter.yaml - dashboards/ +configMapGenerator: + # Hash suffix kept on purpose: an edit to the .conf rolls the gateway pod, + # which is stateless and off the ingest path. + - name: loki-read-gateway + namespace: monitoring + files: + - loki-read-gateway.conf # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a # separate Flux Kustomization that dependsOn `monitoring` — otherwise the # server-side dry-run of the PrometheusRule fails on a fresh cluster diff --git a/kubernetes/apps/monitoring/loki-read-gateway.conf b/kubernetes/apps/monitoring/loki-read-gateway.conf new file mode 100644 index 00000000..10d58fda --- /dev/null +++ b/kubernetes/apps/monitoring/loki-read-gateway.conf @@ -0,0 +1,60 @@ +# GET-only allow-list in front of Loki (REN-621, owner decision on REN-622). +# +# Loki runs SingleBinary with auth_enabled: false, and its HTTP port mixes +# reads with control endpoints that also answer GET (/ingester/shutdown, +# /flush). An RBAC `get` on services/proxy for loki:3100 would therefore be a +# kill switch for log ingest. This server is the read-only boundary instead: +# the paperclip read ServiceAccount may proxy to this Service only. +# +# Each allowed location forwards a URI built here, never the client's raw +# request URI, so dot-segments or encoded slashes cannot reach another Loki +# path. Anything not listed gets 403. /loki/api/v1/tail (websocket) is left +# out on purpose. Adding a path is a new card and a new review. + +upstream loki { + server loki.monitoring.svc.cluster.local:3100; +} + +server { + listen 8080; + server_tokens off; + + proxy_http_version 1.1; + proxy_set_header Host loki; + proxy_read_timeout 60s; + + location / { + return 403; + } + + location = /healthz { + access_log off; + default_type text/plain; + return 200 "ok\n"; + } + + location = /loki/api/v1/query { + limit_except GET { deny all; } + proxy_pass http://loki/loki/api/v1/query$is_args$args; + } + + location = /loki/api/v1/query_range { + limit_except GET { deny all; } + proxy_pass http://loki/loki/api/v1/query_range$is_args$args; + } + + location = /loki/api/v1/labels { + limit_except GET { deny all; } + proxy_pass http://loki/loki/api/v1/labels$is_args$args; + } + + location = /loki/api/v1/series { + limit_except GET { deny all; } + proxy_pass http://loki/loki/api/v1/series$is_args$args; + } + + location ~ ^/loki/api/v1/label/(?[A-Za-z_][A-Za-z0-9_]*)/values$ { + limit_except GET { deny all; } + proxy_pass http://loki/loki/api/v1/label/$label_name/values$is_args$args; + } +} diff --git a/kubernetes/apps/monitoring/loki-read-gateway.yaml b/kubernetes/apps/monitoring/loki-read-gateway.yaml new file mode 100644 index 00000000..c8b3d651 --- /dev/null +++ b/kubernetes/apps/monitoring/loki-read-gateway.yaml @@ -0,0 +1,97 @@ +--- +# Read-only Loki gateway for the agents in the paperclip pod (REN-621, +# owner decision on REN-622). The allow-list and the reason it exists are in +# loki-read-gateway.conf. Callers reach it through the apiserver proxy only: +# /api/v1/namespaces/monitoring/services/loki-read-gateway:8080/proxy/loki/api/v1/query_range?... +# granted by ../paperclip/prometheus-read-rbac.yaml. +# +# Stateless and off the ingest path: Alloy pushes straight to the `loki` +# Service, so restarting or deleting this pod never touches log ingest. +# Revert: delete this file, the .conf and their kustomization entries, and the +# resourceName in the paperclip Role. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki-read-gateway + namespace: monitoring + labels: + app.kubernetes.io/name: loki-read-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: loki-read-gateway + template: + metadata: + labels: + app.kubernetes.io/name: loki-read-gateway + spec: + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: nginx + image: nginxinc/nginx-unprivileged:1.27-alpine + ports: + - name: http + containerPort: 8080 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + readinessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 30 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + volumeMounts: + # Replaces the image's default.conf; the entrypoint's ipv6 and + # envsubst steps skip a read-only conf.d and the server starts. + - name: config + mountPath: /etc/nginx/conf.d + readOnly: true + # pid file and proxy temp paths of the unprivileged image. + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: loki-read-gateway + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 32Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: loki-read-gateway + namespace: monitoring + labels: + app.kubernetes.io/name: loki-read-gateway +spec: + selector: + app.kubernetes.io/name: loki-read-gateway + ports: + - name: http + port: 8080 + targetPort: http + protocol: TCP diff --git a/kubernetes/apps/paperclip/prometheus-read-rbac.yaml b/kubernetes/apps/paperclip/prometheus-read-rbac.yaml index f7766a57..97113628 100644 --- a/kubernetes/apps/paperclip/prometheus-read-rbac.yaml +++ b/kubernetes/apps/paperclip/prometheus-read-rbac.yaml @@ -24,10 +24,11 @@ # # Every process in the paperclip pod can read the mounted ServiceAccount token, # so the token is worth the sum of everything bound to this ServiceAccount. This -# Role's own share of that is `get` on the two Services named below and nothing +# Role's own share of that is `get` on the Services named below and nothing # else. Widening it is a new card and a new review, not an edit here — the # Alertmanager name was added that way (REN-95, for read-only alert-delivery -# checks; parent REN-94). +# checks; parent REN-94). The Loki read gateway was added that way too (REN-621, +# owner decision REN-622). # # THIS IS NO LONGER THE ONLY GRANT ON THIS ServiceAccount # @@ -43,6 +44,7 @@ # port-qualified forms are granted, so the only URLs that work are # /api/v1/namespaces/monitoring/services/kube-prometheus-stack-prometheus:9090/proxy/... # /api/v1/namespaces/monitoring/services/kube-prometheus-stack-alertmanager:9093/proxy/... +# /api/v1/namespaces/monitoring/services/loki-read-gateway:8080/proxy/loki/api/v1/... # A 403 on some other form is the grant working. Do not fix it by adding a # string. apiVersion: v1 @@ -79,6 +81,11 @@ rules: resourceNames: - "kube-prometheus-stack-prometheus:9090" - "kube-prometheus-stack-alertmanager:9093" + # REN-621 (owner decision REN-622): the GET-only Loki gateway, never + # `loki:3100` itself. Loki answers GET /ingester/shutdown and /flush, + # so a `get` on it would not be read-only. See + # ../monitoring/loki-read-gateway.conf. + - "loki-read-gateway:8080" --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding -- 2.54.0