kubernetes/apps/monitoring/kustomization.yaml

@@ -10,6 +10,7 @@ resources:
   - loki-basic-auth-externalsecret.yaml
   - loki-basic-auth-middleware.yaml
   - loki-ingress.yaml
+  - loki-read-gateway.yaml
   - alertmanager-gate-service.yaml
   - alertmanager-gate-ingress.yaml
   - prometheus-gate-service.yaml
@@ -18,6 +19,13 @@ resources:
   - smartctl-exporter.yaml
   - intel-gpu-exporter.yaml
   - dashboards/
+configMapGenerator:
+  # Hash suffix kept on purpose: an edit to the .conf rolls the gateway pod,
+  # which is stateless and off the ingest path.
+  - name: loki-read-gateway
+    namespace: monitoring
+    files:
+      - loki-read-gateway.conf
 # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
 # separate Flux Kustomization that dependsOn `monitoring` — otherwise the
 # server-side dry-run of the PrometheusRule fails on a fresh cluster

kubernetes/apps/monitoring/loki-read-gateway.conf

@@ -0,0 +1,60 @@
+# GET-only allow-list in front of Loki (REN-621, owner decision on REN-622).
+#
+# Loki runs SingleBinary with auth_enabled: false, and its HTTP port mixes
+# reads with control endpoints that also answer GET (/ingester/shutdown,
+# /flush). An RBAC `get` on services/proxy for loki:3100 would therefore be a
+# kill switch for log ingest. This server is the read-only boundary instead:
+# the paperclip read ServiceAccount may proxy to this Service only.
+#
+# Each allowed location forwards a URI built here, never the client's raw
+# request URI, so dot-segments or encoded slashes cannot reach another Loki
+# path. Anything not listed gets 403. /loki/api/v1/tail (websocket) is left
+# out on purpose. Adding a path is a new card and a new review.
+
+upstream loki {
+    server loki.monitoring.svc.cluster.local:3100;
+}
+
+server {
+    listen 8080;
+    server_tokens off;
+
+    proxy_http_version 1.1;
+    proxy_set_header Host loki;
+    proxy_read_timeout 60s;
+
+    location / {
+        return 403;
+    }
+
+    location = /healthz {
+        access_log off;
+        default_type text/plain;
+        return 200 "ok\n";
+    }
+
+    location = /loki/api/v1/query {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/query$is_args$args;
+    }
+
+    location = /loki/api/v1/query_range {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/query_range$is_args$args;
+    }
+
+    location = /loki/api/v1/labels {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/labels$is_args$args;
+    }
+
+    location = /loki/api/v1/series {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/series$is_args$args;
+    }
+
+    location ~ ^/loki/api/v1/label/(?<label_name>[A-Za-z_][A-Za-z0-9_]*)/values$ {
+        limit_except GET { deny all; }
+        proxy_pass http://loki/loki/api/v1/label/$label_name/values$is_args$args;
+    }
+}

kubernetes/apps/monitoring/loki-read-gateway.yaml

@@ -0,0 +1,97 @@
+---
+# Read-only Loki gateway for the agents in the paperclip pod (REN-621,
+# owner decision on REN-622). The allow-list and the reason it exists are in
+# loki-read-gateway.conf. Callers reach it through the apiserver proxy only:
+#   /api/v1/namespaces/monitoring/services/loki-read-gateway:8080/proxy/loki/api/v1/query_range?...
+# granted by ../paperclip/prometheus-read-rbac.yaml.
+#
+# Stateless and off the ingest path: Alloy pushes straight to the `loki`
+# Service, so restarting or deleting this pod never touches log ingest.
+# Revert: delete this file, the .conf and their kustomization entries, and the
+# resourceName in the paperclip Role.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki-read-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: loki-read-gateway
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: loki-read-gateway
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: loki-read-gateway
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 101
+        runAsGroup: 101
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: nginx
+          image: nginxinc/nginx-unprivileged:1.27-alpine
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            periodSeconds: 30
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+          volumeMounts:
+            # Replaces the image's default.conf; the entrypoint's ipv6 and
+            # envsubst steps skip a read-only conf.d and the server starts.
+            - name: config
+              mountPath: /etc/nginx/conf.d
+              readOnly: true
+            # pid file and proxy temp paths of the unprivileged image.
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: loki-read-gateway
+        - name: tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: 32Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki-read-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: loki-read-gateway
+spec:
+  selector:
+    app.kubernetes.io/name: loki-read-gateway
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+      protocol: TCP

kubernetes/apps/paperclip/prometheus-read-rbac.yaml

@@ -24,10 +24,11 @@
 #
 # Every process in the paperclip pod can read the mounted ServiceAccount token,
 # so the token is worth the sum of everything bound to this ServiceAccount. This
-# Role's own share of that is `get` on the two Services named below and nothing
+# Role's own share of that is `get` on the Services named below and nothing
 # else. Widening it is a new card and a new review, not an edit here — the
 # Alertmanager name was added that way (REN-95, for read-only alert-delivery
-# checks; parent REN-94).
+# checks; parent REN-94). The Loki read gateway was added that way too (REN-621,
+# owner decision REN-622).
 #
 # THIS IS NO LONGER THE ONLY GRANT ON THIS ServiceAccount
 #
@@ -43,6 +44,7 @@
 # port-qualified forms are granted, so the only URLs that work are
 #   /api/v1/namespaces/monitoring/services/kube-prometheus-stack-prometheus:9090/proxy/...
 #   /api/v1/namespaces/monitoring/services/kube-prometheus-stack-alertmanager:9093/proxy/...
+#   /api/v1/namespaces/monitoring/services/loki-read-gateway:8080/proxy/loki/api/v1/...
 # A 403 on some other form is the grant working. Do not fix it by adding a
 # string.
 apiVersion: v1
@@ -79,6 +81,11 @@ rules:
     resourceNames:
       - "kube-prometheus-stack-prometheus:9090"
       - "kube-prometheus-stack-alertmanager:9093"
+      # REN-621 (owner decision REN-622): the GET-only Loki gateway, never
+      # `loki:3100` itself. Loki answers GET /ingester/shutdown and /flush,
+      # so a `get` on it would not be read-only. See
+      # ../monitoring/loki-read-gateway.conf.
+      - "loki-read-gateway:8080"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding