k.renner
98dbf2dc4a
Loki had no Prometheus scrape config — no loki_* metrics, no way to alert on the log stack itself going sideways. ServiceMonitor covers the single-binary + canary services, alerts cover Down / 5xx / panics / canary-detected log loss.
57 lines
2.4 KiB
YAML
57 lines
2.4 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: loki
|
|
namespace: monitoring
|
|
labels:
|
|
# kube-prometheus-stack's Prometheus resource selects on this label;
|
|
# without it the rule is created but never loaded into Prometheus.
|
|
release: kube-prometheus-stack
|
|
spec:
|
|
groups:
|
|
- name: loki.rules
|
|
rules:
|
|
- alert: LokiDown
|
|
expr: up{job=~"loki|loki-canary"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Loki target {{ $labels.job }} is down"
|
|
description: "Prometheus has been unable to scrape {{ $labels.job }} ({{ $labels.instance }}) for 5 minutes. Log ingest and queries are likely affected."
|
|
|
|
- alert: LokiRequestErrors
|
|
# 5xx ratio on Loki's HTTP routes. Excludes health/debug routes
|
|
# which can be noisy without indicating a real problem.
|
|
expr: |
|
|
sum by (route) (rate(loki_request_duration_seconds_count{job="loki",status_code=~"5..",route!~"debug.*|.*ready.*|.*health.*"}[5m]))
|
|
/
|
|
sum by (route) (rate(loki_request_duration_seconds_count{job="loki",route!~"debug.*|.*ready.*|.*health.*"}[5m]))
|
|
> 0.10
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Loki {{ $labels.route }} >10% 5xx errors"
|
|
description: "Loki route {{ $labels.route }} has returned 5xx responses for more than 10% of requests over the last 15m."
|
|
|
|
- alert: LokiPanics
|
|
expr: sum(increase(loki_panic_total{job="loki"}[10m])) > 0
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Loki panicked"
|
|
description: "loki_panic_total increased in the last 10m — Loki encountered an internal panic. Check pod logs."
|
|
|
|
- alert: LokiCanaryMissingEntries
|
|
# loki-canary writes test logs and reads them back. Missing
|
|
# entries mean the ingest → store → query path lost data.
|
|
expr: sum(increase(loki_canary_missing_entries_total[10m])) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Loki canary detected missing log entries"
|
|
description: "loki-canary reported entries it wrote were not returned by the read path in the last 10m — log data loss."
|