Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  claude-botandClaude Opus 5
|
24b2a0178d | ||
 k.renner
|
b8061ea05d | ||
|
claude-botandClaude Opus 5
|
d0bea0dc15 | ||
|
k.renner
|
8ccf5ac8e5 | ||
|
claude-bot
|
08b2fe7ef6 | ||
|
k.renner
|
04df5da041 | ||
|
claude-botandClaude Opus 5
|
e8b217e8a1 | ||
|
k.renner
|
8c03a6e310 |
@@ -6,6 +6,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- prometheusrule-ceph.yaml
|
||||
- prometheusrule-cilium.yaml
|
||||
- prometheusrule-disk-wear.yaml
|
||||
- prometheusrule-hetzner-backup.yaml
|
||||
- prometheusrule-loki.yaml
|
||||
- prometheusrule-mariadb-backup.yaml
|
||||
@@ -24,4 +25,5 @@ resources:
|
||||
- home-assistant-servicemonitor.yaml
|
||||
- immich-servicemonitor.yaml
|
||||
- loki-servicemonitor.yaml
|
||||
- smartctl-exporter-servicemonitor.yaml
|
||||
- probes-blackbox.yaml
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: disk-wear
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus resource selects on this label;
|
||||
# without it the rule is created but never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: disk-wear.rules
|
||||
rules:
|
||||
# THE GAP THESE RULES CLOSE
|
||||
#
|
||||
# Drive wear is invisible in every other signal this cluster has.
|
||||
# `ceph -s` reads active+clean at any wear level, no Ceph health check
|
||||
# fires until a device actually fails, and the block-layer metrics show
|
||||
# throughput and latency but never endurance. That blind spot is how a
|
||||
# set of OSD drives reached 81-87% before anyone looked (#549, #434).
|
||||
#
|
||||
# The numbers below are wear percentages reported by the drive itself:
|
||||
# 100 means the manufacturer's rated endurance is used up. Drives do
|
||||
# not stop at 100 — they typically keep working and start consuming
|
||||
# spare blocks — so treat these as planning thresholds, not failure
|
||||
# predictions.
|
||||
- alert: NvmeDriveWearHigh
|
||||
# 80% with months of runway left is the point where ordering a
|
||||
# replacement is still a calm decision rather than an outage.
|
||||
# `for` is long because wear is monotonic and slow: it cannot
|
||||
# flap, and a short window would only make the alert noisier
|
||||
# after a restart.
|
||||
expr: smartctl_device_percentage_used >= 80
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear"
|
||||
description: >-
|
||||
The drive has used {{ $value }}% of its rated write endurance.
|
||||
Nothing is failing and no data is at risk yet. Check what the
|
||||
drive is carrying before deciding urgency: an OSD is replaceable
|
||||
with a rebalance, whereas the EPHEMERAL volume has no such
|
||||
escape — a node simply stops when it goes. The Disk Health
|
||||
dashboard resolves which drive holds which role.
|
||||
- alert: NvmeDriveWearCritical
|
||||
expr: smartctl_device_percentage_used >= 95
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear"
|
||||
description: >-
|
||||
The drive is at or past the end of its rated endurance. It will
|
||||
likely keep working for a while, but it is now operating outside
|
||||
what the manufacturer stands behind, and replacement should be
|
||||
scheduled rather than considered.
|
||||
# A drive that has consumed its spare area is no longer masking media
|
||||
# defects, which is a materially different state from being worn:
|
||||
# wear is a budget, spare exhaustion is damage.
|
||||
- alert: NvmeDriveSpareExhausted
|
||||
expr: smartctl_device_available_spare <= smartctl_device_available_spare_threshold
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} has exhausted its spare area"
|
||||
description: >-
|
||||
Available spare has fallen to the manufacturer's threshold. The
|
||||
drive has run out of reserve blocks to remap failures into.
|
||||
Replace it.
|
||||
- alert: NvmeDriveCriticalWarning
|
||||
# The NVMe critical warning byte is a bitfield: spare below
|
||||
# threshold, temperature out of range, degraded reliability,
|
||||
# read-only mode, volatile-memory backup failed. Any non-zero value
|
||||
# is the drive itself asking for attention.
|
||||
expr: smartctl_device_critical_warning > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} reports critical warning {{ $value }}"
|
||||
description: >-
|
||||
The drive has raised its own critical warning flag. Read the
|
||||
detail with `smartctl -a` on the node; the value is a bitfield,
|
||||
not a count.
|
||||
- alert: NvmeDriveMediaErrors
|
||||
# Unlike wear, this counter should never move. Alert on the increase
|
||||
# rather than the absolute value, so a drive that recorded errors
|
||||
# long ago does not alert forever while a drive that starts failing
|
||||
# today does.
|
||||
expr: increase(smartctl_device_media_errors[6h]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} recorded {{ $value }} new media errors"
|
||||
description: >-
|
||||
The drive reported unrecoverable data-integrity errors in the
|
||||
last six hours. A healthy drive never increments this. Check
|
||||
whether Ceph has logged scrub errors on the same node.
|
||||
# Without this the whole group is silently useless: no series means no
|
||||
# alert fires, and a dashboard of empty panels reads much like a
|
||||
# healthy one. This is the check that the checks are running.
|
||||
- alert: SmartctlExporterIncomplete
|
||||
# Compared against node-exporter's own NVMe inventory rather than a
|
||||
# written-down fleet size. Adding or removing a drive then moves both
|
||||
# sides together, so this cannot rot into a permanent false alarm the
|
||||
# way a hardcoded count would.
|
||||
expr: count(smartctl_device_percentage_used) < count(node_nvme_info) or absent(smartctl_device_percentage_used)
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fewer NVMe drives report SMART than node-exporter can see"
|
||||
description: >-
|
||||
Some drives visible to node-exporter are not reporting SMART, or
|
||||
none are. Either a DaemonSet pod is down or a drive stopped
|
||||
answering — both mean the wear alerts above have stopped watching
|
||||
part of the fleet, which looks identical to everything being
|
||||
fine.
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: smartctl-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
# kube-prometheus-stack's Prometheus resource selects ServiceMonitors
|
||||
# by this label; without it the resource exists but is never
|
||||
# discovered.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
endpoints:
|
||||
# smartctl itself polls the drives every 60s, so scraping faster than that
|
||||
# returns the same numbers with a new timestamp. Wear moves on a scale of
|
||||
# weeks; 60s is already far finer than the signal.
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 60s
|
||||
relabelings:
|
||||
# Without this every series is labelled with the exporter pod's IP and
|
||||
# nothing else. A DaemonSet pod's IP says nothing about which machine
|
||||
# the drive is in, and it changes on every reschedule — so the wear
|
||||
# figure would be unattributable, which is most of its value gone.
|
||||
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
targetLabel: node
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance/wear is absent; see the note at the bottom and issue #549.",
|
||||
"description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance comes from smartctl_exporter, whose series are labelled by node rather than by the exporter pod's IP — see the note at the bottom for what it still cannot tell you.",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
@@ -19,7 +19,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 0, "y": 0},
|
||||
"id": 10,
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_health_status", "legendFormat": "", "refId": "A"}],
|
||||
@@ -38,7 +38,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 5, "y": 0},
|
||||
"id": 11,
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [
|
||||
@@ -61,7 +61,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 10, "y": 0},
|
||||
"id": 12,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_cluster_total_used_bytes / ceph_cluster_total_bytes", "legendFormat": "", "refId": "A"}],
|
||||
@@ -82,7 +82,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 15, "y": 0},
|
||||
"id": 13,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(1 - node_filesystem_avail_bytes{mountpoint=\"/var\"} / node_filesystem_size_bytes{mountpoint=\"/var\"})", "legendFormat": "", "refId": "A"}],
|
||||
@@ -114,7 +114,7 @@
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Drive inventory",
|
||||
"title": "Drive inventory (identity and role)",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
@@ -127,6 +127,100 @@
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Rated endurance consumed, worst drive in the fleet. This is the drive's own accounting against what the manufacturer warrants, not a failure prediction — a drive past 100 usually keeps working and starts eating its spare area. Read it for planning; read the SMART table for whether anything is failing now.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"mappings": [],
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
|
||||
"id": 14,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(smartctl_device_percentage_used)", "legendFormat": "", "refId": "A"}],
|
||||
"title": "Worst NVMe wear",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Rated endurance consumed per drive. The line is nearly flat by nature — wear moves on a scale of weeks — so the useful reading is the spread between drives and the slope, not today's value. Joined to smartctl_device for the model name, because the device label alone is not comparable across nodes.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "dashed"}},
|
||||
"mappings": [],
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
|
||||
"id": 15,
|
||||
"options": {"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device",
|
||||
"legendFormat": "{{node}} {{model_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "NVMe wear",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "The failure-now signals, as opposed to the endurance budget above. Spare below its threshold means the drive has run out of blocks to remap defects into; media errors are unrecoverable data-integrity failures and should never increment; the critical warning is a bitfield the drive raises itself. Written bytes is the counter behind the wear percentage, useful for projecting when a drive will reach it.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"custom": {"align": "auto", "cellOptions": {"type": "auto"}, "inspect": false},
|
||||
"mappings": [],
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}
|
||||
},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "wear"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "spare"}, "properties": [{"id": "unit", "value": "percent"}]},
|
||||
{"matcher": {"id": "byName", "options": "media errors"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "critical warning"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "powered on"}, "properties": [{"id": "unit", "value": "s"}]},
|
||||
{"matcher": {"id": "byName", "options": "written"}, "properties": [{"id": "unit", "value": "bytes"}]}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
|
||||
"id": 16,
|
||||
"options": {"cellHeight": "sm", "footer": {"countRows": false, "fields": "", "reducer": ["sum"], "show": false}, "showHeader": true, "sortBy": [{"desc": true, "displayName": "wear"}]},
|
||||
"targets": [
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "A"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_available_spare * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "B"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_media_errors * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "C"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_critical_warning * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "D"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_power_on_seconds * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "E"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_bytes_written * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "F"}
|
||||
],
|
||||
"title": "SMART health",
|
||||
"transformations": [
|
||||
{"id": "merge", "options": {}},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {"Time": true, "__name__": true, "container": true, "endpoint": true, "instance": true, "job": true, "namespace": true, "pod": true, "service": true},
|
||||
"indexByName": {},
|
||||
"renameByName": {"node": "node", "device": "device", "model_name": "model", "Value #A": "wear", "Value #B": "spare", "Value #C": "media errors", "Value #D": "critical warning", "Value #E": "powered on", "Value #F": "written"}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Read and write throughput per drive.",
|
||||
@@ -140,7 +234,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
|
||||
"id": 2,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -173,7 +267,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
|
||||
"id": 3,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -206,7 +300,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
|
||||
"id": 4,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -244,7 +338,7 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
|
||||
"id": 5,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -277,7 +371,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 40},
|
||||
"id": 6,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -304,7 +398,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 40},
|
||||
"id": 7,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -339,7 +433,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 40},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 49},
|
||||
"id": 9,
|
||||
"options": {"legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -355,11 +449,11 @@
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 11, "w": 24, "x": 0, "y": 48},
|
||||
"gridPos": {"h": 11, "w": 24, "x": 0, "y": 57},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
|
||||
"content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Endurance is missing.** No wear, no written-bytes total, no power-on hours, no media-error count — no exporter here produces them. NVMe reports those over an admin ioctl that needs elevated privileges; temperature is the only health value the kernel publishes through sysfs. Tracked as **#549**. Do not read a green top row as a healthy drive.\n\nCeph does record wear, but only inside its own devicehealth module and not in Prometheus: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. It covers OSDs and nothing else, and a drive removed from Ceph is purged from that registry — its history goes with it, so the wear figure of a retired drive is not recoverable after the fact.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, so the drive with the least endurance left can be the one carrying container state.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
|
||||
"content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Wear is a budget, not a prediction.** The percentage is what the drive reports against its own rated endurance. Drives do not stop at 100 — they usually keep working and start consuming spare blocks — so read it for planning, and read available spare and media errors for whether something is failing now. Those are different questions and the alert rules treat them as such.\n\n**A green top row is not a healthy drive.** Endurance says nothing about the drive that dies of a controller fault tomorrow, and this dashboard cannot see one coming.\n\nCeph keeps its own wear figures for OSDs in the devicehealth module, which is a useful cross-check but covers OSDs only: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. A drive removed from Ceph is purged from that registry and its history goes with it.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, so the drive with the least endurance left can be the one carrying container state.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Gaps and caveats",
|
||||
|
||||
@@ -15,6 +15,7 @@ resources:
|
||||
- prometheus-gate-service.yaml
|
||||
- prometheus-gate-ingress.yaml
|
||||
- ci-diagnostics-rbac.yaml
|
||||
- smartctl-exporter.yaml
|
||||
- dashboards/
|
||||
# prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
|
||||
# separate Flux Kustomization that dependsOn `monitoring` — otherwise the
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# SMART/endurance metrics for every NVMe drive in the fleet (#549).
|
||||
#
|
||||
# WHY THIS HOLDS ELEVATED CAPABILITIES, and why that is a deliberate exception
|
||||
# rather than an oversight:
|
||||
#
|
||||
# NVMe reports health over an admin ioctl on the character device
|
||||
# (NVME_IOCTL_ADMIN_CMD), which the kernel gates on CAP_SYS_ADMIN. There is no
|
||||
# sysfs path for wear, media errors or written-bytes totals — hwmon publishes
|
||||
# the controller temperature and nothing else. Upstream smartctl_exporter
|
||||
# documents `privileged: true` and `user: root` for exactly this reason.
|
||||
#
|
||||
# The alternative that needs no privileges — reading Ceph's devicehealth
|
||||
# archive, which does hold full smartctl output — covers OSD drives only. In
|
||||
# this cluster that is three of nine, and the six it misses include the drives
|
||||
# that were retired from Ceph for wear and now carry EPHEMERAL. Covering the
|
||||
# healthy drives and not the worn ones was judged not worth a new service; the
|
||||
# owner chose fleet-wide coverage with the privilege exception instead.
|
||||
#
|
||||
# The exception is `privileged: true`, and narrower settings were tried and
|
||||
# measured to fail — see the securityContext below for what was ruled out and
|
||||
# how. It is still bounded: no host namespaces, no hostPID, no hostNetwork,
|
||||
# read-only root filesystem, and /dev as the only host mount.
|
||||
#
|
||||
# If this ever needs revisiting, there are exactly two things worth checking,
|
||||
# and neither is "does another exporter ask for less":
|
||||
# - whether the kernel has grown a sysfs interface for NVMe health, which
|
||||
# would remove the need for device access altogether
|
||||
# - whether enough workloads need raw device access to justify a device
|
||||
# plugin, which is the only way Kubernetes grants individual devices
|
||||
# The requirement comes from the device cgroup and the ioctl gate, not from
|
||||
# the choice of tool.
|
||||
#
|
||||
# Upstream had no release between 2025-04-22 (v0.14.0) and this deployment.
|
||||
# The repository is not archived; the pin is simply its newest tag.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: smartctl-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
spec:
|
||||
# Control planes are also the storage nodes here, so the exporter has to
|
||||
# tolerate the control-plane taint or it would cover nothing at all.
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: smartctl-exporter
|
||||
image: quay.io/prometheuscommunity/smartctl-exporter:v0.14.0
|
||||
# No --smartctl.device on purpose: the exporter scans for devices
|
||||
# itself and rescans every 10m by default. Device names are not
|
||||
# stable across these machines, so a hardcoded list would quietly
|
||||
# stop covering a drive the next time one is replaced.
|
||||
args:
|
||||
- --web.listen-address=:9633
|
||||
ports:
|
||||
- name: http-metrics
|
||||
containerPort: 9633
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
# PRIVILEGED, and there is no narrower setting that works. This
|
||||
# was established by measurement after two narrower attempts
|
||||
# failed, so the reasoning is recorded here rather than rediscovered:
|
||||
#
|
||||
# Attempt 1 — CAP_SYS_ADMIN + CAP_SYS_RAWIO, everything else
|
||||
# dropped. The exporter found all three drives per node and
|
||||
# could open none: "Smartctl open device: /dev/nvme0 failed:
|
||||
# Operation not permitted".
|
||||
#
|
||||
# Attempt 2 — the same plus seccompProfile: Unconfined, on the
|
||||
# theory that EPERM is the default seccomp action for an
|
||||
# unlisted syscall and the kubelet runs seccompDefault: true.
|
||||
# /proc/1/status showed CapEff 0x220000 and Seccomp 2, so both
|
||||
# capabilities were effective and a filter was loaded. Removing
|
||||
# the filter changed nothing.
|
||||
#
|
||||
# What actually settled it: `head -c 512 /dev/nvme0n1` inside
|
||||
# the container fails with EPERM too. A plain read of a block
|
||||
# device involves no ioctl, no capability and no syscall a
|
||||
# seccomp profile would filter. The block is the device cgroup —
|
||||
# containerd denies non-privileged containers access to host
|
||||
# device nodes whether or not /dev is mounted.
|
||||
#
|
||||
# Kubernetes exposes no per-device allowlist in a Pod spec; there
|
||||
# is no equivalent of `docker --device`. The alternatives are
|
||||
# `privileged: true` or a device plugin advertising the NVMe
|
||||
# drives as a resource. A device plugin is the genuinely narrower
|
||||
# answer and is worth building if more workloads ever need raw
|
||||
# device access; for a single read-only metrics exporter it is a
|
||||
# component to write, deploy and maintain in exchange for the same
|
||||
# access.
|
||||
runAsUser: 0
|
||||
runAsNonRoot: false
|
||||
privileged: true
|
||||
# Stated rather than omitted: the API rejects `privileged: true`
|
||||
# together with `allowPrivilegeEscalation: false`, so leaving it
|
||||
# out would only hide that privileged already implies it.
|
||||
allowPrivilegeEscalation: true
|
||||
readOnlyRootFilesystem: true
|
||||
volumeMounts:
|
||||
# Not read-only. smartmontools opens NVMe devices read-write on
|
||||
# some paths, and once the container is privileged a read-only
|
||||
# flag on this mount is not a boundary anyone should rely on —
|
||||
# keeping it would suggest a protection that is not there.
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
# smartctl is not known to need scratch space, but the root
|
||||
# filesystem is read-only and a tool that turns out to want /tmp
|
||||
# would crashloop instead of degrading. Cheaper to provide it than
|
||||
# to find out from a restart counter.
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: http-metrics
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
type: Directory
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: smartctl-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
ports:
|
||||
- name: http-metrics
|
||||
port: 9633
|
||||
targetPort: http-metrics
|
||||
protocol: TCP
|
||||
Reference in New Issue
Block a user