Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 claude-bot
|
da1482e33f | ||
 k.renner
|
c6b1aa109b | ||
|
claude-bot
|
2f3d952656 | ||
|
k.renner
|
0a87f91116 | ||
|
claude-bot
|
c59f0050b3 | ||
|
k.renner
|
48c46610a6 | ||
 claude-botandClaude Opus 5
|
53887dd88f | ||
|
k.renner
|
c9a10ab015 | ||
|
claude-botandClaude Opus 5
|
24b2a0178d | ||
|
k.renner
|
b8061ea05d | ||
|
claude-botandClaude Opus 5
|
d0bea0dc15 | ||
|
k.renner
|
8ccf5ac8e5 | ||
|
claude-bot
|
08b2fe7ef6 | ||
|
k.renner
|
04df5da041 | ||
|
claude-botandClaude Opus 5
|
e8b217e8a1 | ||
|
k.renner
|
8c03a6e310 |
@@ -1,18 +0,0 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: barman-cloud-creds
|
||||
namespace: authentik
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->authentik-KS handoff (barman creds).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/authentik/barman-creds
|
||||
@@ -15,4 +15,4 @@ spec:
|
||||
name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/authentik/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -27,5 +27,4 @@ resources:
|
||||
- cluster.yaml
|
||||
- objectstore.yaml
|
||||
- scheduled-backup.yaml
|
||||
- barman-creds-externalsecret.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
|
||||
@@ -1,21 +1,8 @@
|
||||
---
|
||||
# Barman S3 credentials for ha-pg.
|
||||
#
|
||||
# FOLLOW-UP (#397): sourced from seed/paperless/hetzner-creds, which is not where
|
||||
# they belong. woodpecker/hetzner-creds-externalsecret.yaml has the identical
|
||||
# problem and the same note. The values are the same today, so this is correct
|
||||
# but coupled: rotating paperless's Hetzner credentials would silently change
|
||||
# the backup credentials for Home Assistant and Woodpecker too.
|
||||
#
|
||||
# The fix is one dedicated path per consumer (seed/home-assistant/hetzner-creds)
|
||||
# and a one-line change to the `key:` below. Creating it needs an OpenBao write:
|
||||
#
|
||||
# bao kv put homelab/seed/home-assistant/hetzner-creds \
|
||||
# @<(bao kv get -format=json homelab/seed/paperless/hetzner-creds | jq '.data.data')
|
||||
#
|
||||
# Note the mount is `homelab`, so the path is homelab/seed/... — #397 records a
|
||||
# 403 on `bao kv list seed`, which was a wrong-path error rather than a
|
||||
# permissions problem.
|
||||
# One shared path for one shared credential — see the note in
|
||||
# ../paperless/hetzner-creds-externalsecret.yaml.
|
||||
#
|
||||
# The namespace allowlist on the openbao ClusterSecretStore
|
||||
# (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
|
||||
@@ -35,4 +22,4 @@ spec:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: barman-cloud-creds
|
||||
namespace: immich
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->immich-KS handoff (barman creds).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/immich/barman-creds
|
||||
@@ -15,4 +15,4 @@ spec:
|
||||
name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/immich/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -13,6 +13,5 @@ resources:
|
||||
- cluster.yaml
|
||||
- objectstore.yaml
|
||||
- scheduled-backup.yaml
|
||||
- barman-creds-externalsecret.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
- pvcs.yaml
|
||||
|
||||
@@ -6,6 +6,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- prometheusrule-ceph.yaml
|
||||
- prometheusrule-cilium.yaml
|
||||
- prometheusrule-disk-wear.yaml
|
||||
- prometheusrule-hetzner-backup.yaml
|
||||
- prometheusrule-loki.yaml
|
||||
- prometheusrule-mariadb-backup.yaml
|
||||
@@ -24,4 +25,5 @@ resources:
|
||||
- home-assistant-servicemonitor.yaml
|
||||
- immich-servicemonitor.yaml
|
||||
- loki-servicemonitor.yaml
|
||||
- smartctl-exporter-servicemonitor.yaml
|
||||
- probes-blackbox.yaml
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: disk-wear
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus resource selects on this label;
|
||||
# without it the rule is created but never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: disk-wear.rules
|
||||
rules:
|
||||
# Wear percentages are self-reported: 100 means the rated endurance is
|
||||
# spent, not that the drive stops. Planning thresholds, not predictions.
|
||||
- alert: NvmeDriveWearHigh
|
||||
# Long `for` because wear is monotonic and cannot flap.
|
||||
expr: smartctl_device_percentage_used >= 80
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear"
|
||||
description: >-
|
||||
{{ $value }}% of rated write endurance used. Nothing is failing
|
||||
yet. Urgency depends on what the drive carries — the Disk Health
|
||||
dashboard shows the role. Losing an OSD costs a rebalance;
|
||||
EPHEMERAL is /var and rebuilds, apart from the node's etcd member
|
||||
data, which has to be removed and re-added.
|
||||
- alert: NvmeDriveWearCritical
|
||||
expr: smartctl_device_percentage_used >= 95
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear"
|
||||
description: >-
|
||||
The drive is at or past the end of its rated endurance. It will
|
||||
likely keep working for a while, but it is now operating outside
|
||||
what the manufacturer stands behind, and replacement should be
|
||||
scheduled rather than considered.
|
||||
# Wear is a budget; spare exhaustion is damage.
|
||||
- alert: NvmeDriveSpareExhausted
|
||||
expr: smartctl_device_available_spare <= smartctl_device_available_spare_threshold
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} has exhausted its spare area"
|
||||
description: >-
|
||||
Available spare has fallen to the manufacturer's threshold. The
|
||||
drive has run out of reserve blocks to remap failures into.
|
||||
Replace it.
|
||||
- alert: NvmeDriveCriticalWarning
|
||||
expr: smartctl_device_critical_warning > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} reports critical warning {{ $value }}"
|
||||
description: >-
|
||||
The drive has raised its own critical warning flag. Read the
|
||||
detail with `smartctl -a` on the node; the value is a bitfield,
|
||||
not a count.
|
||||
- alert: NvmeDriveMediaErrors
|
||||
# On the increase, not the total: pre-existing errors would otherwise
|
||||
# alert forever. Standing counts are visible on the dashboard only.
|
||||
expr: increase(smartctl_device_media_errors[6h]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} recorded {{ $value }} new media errors"
|
||||
description: >-
|
||||
The drive reported unrecoverable data-integrity errors in the
|
||||
last six hours. A healthy drive never increments this. Check
|
||||
whether Ceph has logged scrub errors on the same node.
|
||||
# The check that the checks are running: no series fires no alert.
|
||||
# Counted against node-exporter's inventory, not a fixed fleet size.
|
||||
- alert: SmartctlExporterIncomplete
|
||||
expr: count(smartctl_device_percentage_used) < count(node_nvme_info) or absent(smartctl_device_percentage_used)
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fewer NVMe drives report SMART than node-exporter can see"
|
||||
description: >-
|
||||
Some drives visible to node-exporter are not reporting SMART, or
|
||||
none are. Either a DaemonSet pod is down or a drive stopped
|
||||
answering — both mean the wear alerts above have stopped watching
|
||||
part of the fleet, which looks identical to everything being
|
||||
fine.
|
||||
@@ -0,0 +1,25 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: smartctl-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
# kube-prometheus-stack's Prometheus resource selects ServiceMonitors
|
||||
# by this label; without it the resource exists but is never
|
||||
# discovered.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
endpoints:
|
||||
# smartctl polls the drives every 60s; scraping faster gains nothing.
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 60s
|
||||
relabelings:
|
||||
# Without this a series carries only the exporter pod's IP, which does
|
||||
# not identify the machine the drive is in.
|
||||
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
targetLabel: node
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance/wear is absent; see the note at the bottom and issue #549.",
|
||||
"description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance comes from smartctl_exporter, whose series are labelled by node rather than by the exporter pod's IP — see the note at the bottom for what it still cannot tell you.",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
@@ -19,7 +19,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 0, "y": 0},
|
||||
"id": 10,
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_health_status", "legendFormat": "", "refId": "A"}],
|
||||
@@ -28,7 +28,7 @@
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "OSDs the cluster knows about but which are not up, and not in. Both should read zero. Counted as a difference against ceph_osd_metadata rather than compared to a fixed number, so adding or removing an OSD does not turn this panel into a false alarm.",
|
||||
"description": "OSDs the cluster knows about but which are not up. Should read zero. Counted as a difference against ceph_osd_metadata rather than compared to a fixed number, so adding or removing an OSD does not turn this panel into a false alarm. An OSD that is up but out shows as a Ceph health warning instead — it is a state, not a fault, and it does not belong in a red-or-green box.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
@@ -38,14 +38,13 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 5, "y": 0},
|
||||
"id": 11,
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(ceph_osd_metadata) - sum(ceph_osd_up)", "legendFormat": "down", "refId": "A"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(ceph_osd_metadata) - sum(ceph_osd_in)", "legendFormat": "out", "refId": "B"}
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(ceph_osd_metadata) - sum(ceph_osd_up)", "legendFormat": "", "refId": "A"}
|
||||
],
|
||||
"title": "OSDs down / out",
|
||||
"title": "OSDs down",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
@@ -62,7 +61,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 10, "y": 0},
|
||||
"id": 12,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_cluster_total_used_bytes / ceph_cluster_total_bytes", "legendFormat": "", "refId": "A"}],
|
||||
@@ -71,7 +70,7 @@
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Fullest EPHEMERAL volume across the fleet — Talos's /var, holding container images, pod scratch space and kubelet state. A full /var takes the node down, and unlike Ceph nothing rebalances away from it.",
|
||||
"description": "Fullest EPHEMERAL volume across the fleet — Talos's /var. Its contents are meant to be disposable: container images, kubelet state and pod scratch all rebuild. Filling it is still a problem, because a node with no room to write stops being useful long before anything is lost, and nothing rebalances /var the way Ceph rebalances a full OSD.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
@@ -83,7 +82,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 15, "y": 0},
|
||||
"id": 13,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(1 - node_filesystem_avail_bytes{mountpoint=\"/var\"} / node_filesystem_size_bytes{mountpoint=\"/var\"})", "legendFormat": "", "refId": "A"}],
|
||||
@@ -115,7 +114,7 @@
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Drive inventory",
|
||||
"title": "Drive inventory (identity and role)",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
@@ -128,6 +127,100 @@
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Rated endurance consumed, worst drive in the fleet. This is the drive's own accounting against what the manufacturer warrants, not a failure prediction — a drive past 100 usually keeps working and starts eating its spare area. Read it for planning; read the SMART table for whether anything is failing now.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"mappings": [],
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
|
||||
"id": 14,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(smartctl_device_percentage_used)", "legendFormat": "", "refId": "A"}],
|
||||
"title": "Worst NVMe wear",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Rated endurance consumed per drive. The line is nearly flat by nature — wear moves on a scale of weeks — so the useful reading is the spread between drives and the slope, not today's value. Joined to smartctl_device for the model name, because the device label alone is not comparable across nodes.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "dashed"}},
|
||||
"mappings": [],
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
|
||||
"id": 15,
|
||||
"options": {"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device",
|
||||
"legendFormat": "{{node}} {{model_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "NVMe wear",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "The failure-now signals, as opposed to the endurance budget above. Spare below its threshold means the drive has run out of blocks to remap defects into; media errors are unrecoverable data-integrity failures and should never increment; the critical warning is a bitfield the drive raises itself. Written bytes is the counter behind the wear percentage, useful for projecting when a drive will reach it.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"custom": {"align": "auto", "cellOptions": {"type": "auto"}, "inspect": false},
|
||||
"mappings": [],
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}
|
||||
},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "wear"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "spare"}, "properties": [{"id": "unit", "value": "percent"}]},
|
||||
{"matcher": {"id": "byName", "options": "media errors"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "critical warning"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "powered on"}, "properties": [{"id": "unit", "value": "s"}]},
|
||||
{"matcher": {"id": "byName", "options": "written"}, "properties": [{"id": "unit", "value": "bytes"}]}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
|
||||
"id": 16,
|
||||
"options": {"cellHeight": "sm", "footer": {"countRows": false, "fields": "", "reducer": ["sum"], "show": false}, "showHeader": true, "sortBy": [{"desc": true, "displayName": "wear"}]},
|
||||
"targets": [
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "A"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_available_spare * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "B"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_media_errors * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "C"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_critical_warning * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "D"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_power_on_seconds * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "E"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_bytes_written * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "F"}
|
||||
],
|
||||
"title": "SMART health",
|
||||
"transformations": [
|
||||
{"id": "merge", "options": {}},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {"Time": true, "__name__": true, "container": true, "endpoint": true, "instance": true, "job": true, "namespace": true, "pod": true, "service": true},
|
||||
"indexByName": {},
|
||||
"renameByName": {"node": "node", "device": "device", "model_name": "model", "Value #A": "wear", "Value #B": "spare", "Value #C": "media errors", "Value #D": "critical warning", "Value #E": "powered on", "Value #F": "written"}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Read and write throughput per drive.",
|
||||
@@ -141,7 +234,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
|
||||
"id": 2,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -174,7 +267,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
|
||||
"id": 3,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -207,7 +300,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
|
||||
"id": 4,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -245,7 +338,7 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
|
||||
"id": 5,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -278,7 +371,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 40},
|
||||
"id": 6,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -305,7 +398,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 40},
|
||||
"id": 7,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -340,7 +433,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 40},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 49},
|
||||
"id": 9,
|
||||
"options": {"legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -356,11 +449,11 @@
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 11, "w": 24, "x": 0, "y": 48},
|
||||
"gridPos": {"h": 11, "w": 24, "x": 0, "y": 57},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
|
||||
"content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Endurance is missing.** No wear, no written-bytes total, no power-on hours, no media-error count — no exporter here produces them. NVMe reports those over an admin ioctl that needs elevated privileges; temperature is the only health value the kernel publishes through sysfs. Tracked as **#549**. Do not read a green top row as a healthy drive.\n\nCeph does record wear, but only inside its own devicehealth module and not in Prometheus: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. It covers OSDs and nothing else, and a drive removed from Ceph is purged from that registry — its history goes with it, so the wear figure of a retired drive is not recoverable after the fact.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, so the drive with the least endurance left can be the one carrying container state.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
|
||||
"content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Wear is a budget, not a prediction.** The percentage is what the drive reports against its own rated endurance. Drives do not stop at 100 — they usually keep working and start consuming spare blocks — so read it for planning, and read available spare and media errors for whether something is failing now. Those are different questions and the alert rules treat them as such.\n\n**A green top row is not a healthy drive.** Endurance says nothing about the drive that dies of a controller fault tomorrow, and this dashboard cannot see one coming.\n\nCeph keeps its own wear figures for OSDs in the devicehealth module, which is a useful cross-check but covers OSDs only: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. A drive removed from Ceph is purged from that registry and its history goes with it.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, and putting a worn one on EPHEMERAL is a sensible destination rather than a hazard — that volume is built to be rebuilt. Check the inventory table for what a given drive holds before reading its wear as urgent.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Gaps and caveats",
|
||||
|
||||
@@ -15,6 +15,7 @@ resources:
|
||||
- prometheus-gate-service.yaml
|
||||
- prometheus-gate-ingress.yaml
|
||||
- ci-diagnostics-rbac.yaml
|
||||
- smartctl-exporter.yaml
|
||||
- dashboards/
|
||||
# prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
|
||||
# separate Flux Kustomization that dependsOn `monitoring` — otherwise the
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# SMART/endurance metrics for every NVMe drive in the fleet (#549).
|
||||
#
|
||||
# Runs privileged: host device access is gated by the container device cgroup,
|
||||
# and Kubernetes offers no per-device allowlist. Narrower settings do not work;
|
||||
# the alternatives that were tried are in the commit history.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: smartctl-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
spec:
|
||||
# Control planes are also the storage nodes here, so the exporter has to
|
||||
# tolerate the control-plane taint or it would cover nothing at all.
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: smartctl-exporter
|
||||
image: quay.io/prometheuscommunity/smartctl-exporter:v0.14.0
|
||||
# No --smartctl.device: the exporter scans and rescans on its own.
|
||||
# Device names are not stable across these machines.
|
||||
args:
|
||||
- --web.listen-address=:9633
|
||||
ports:
|
||||
- name: http-metrics
|
||||
containerPort: 9633
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
runAsNonRoot: false
|
||||
privileged: true
|
||||
# The API rejects privileged together with
|
||||
# allowPrivilegeEscalation: false.
|
||||
allowPrivilegeEscalation: true
|
||||
readOnlyRootFilesystem: true
|
||||
volumeMounts:
|
||||
# Not read-only: smartmontools opens NVMe devices read-write.
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
# Scratch space, because the root filesystem is read-only.
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: http-metrics
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
type: Directory
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: smartctl-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
ports:
|
||||
- name: http-metrics
|
||||
port: 9633
|
||||
targetPort: http-metrics
|
||||
protocol: TCP
|
||||
@@ -1,18 +0,0 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: barman-cloud-creds
|
||||
namespace: paperless
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->paperless-KS handoff (barman creds).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/barman-creds
|
||||
@@ -1,3 +1,9 @@
|
||||
# One Hetzner S3 key, one path, shared by every barman ObjectStore. Separate
|
||||
# per-app paths would not make rotation independent — that needs one Hetzner
|
||||
# key per consumer (#397).
|
||||
#
|
||||
# velero reads the same key from infrastructure/velero-hetzner, which holds it
|
||||
# as a credentials file rather than these three fields.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
@@ -15,4 +21,4 @@ spec:
|
||||
name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -20,7 +20,6 @@ resources:
|
||||
- cluster.yaml
|
||||
- objectstore.yaml
|
||||
- scheduled-backup.yaml
|
||||
- barman-creds-externalsecret.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
- pvcs.yaml
|
||||
configMapGenerator:
|
||||
|
||||
@@ -12,4 +12,4 @@ spec:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/trmnl/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
---
|
||||
# Barman S3 credentials for wp-pg.
|
||||
#
|
||||
# Sourced from seed/paperless/hetzner-creds, which is not where they belong.
|
||||
# Issue #397 wants woodpecker to own seed/woodpecker/hetzner-creds; creating
|
||||
# that path needs OpenBao write access (`bao kv list seed` → 403 for the
|
||||
# identities available here), so it stays open. The values are identical, so it
|
||||
# is a one-line swap of the `key:` below once the path exists.
|
||||
# One shared path for one shared credential — see the note in
|
||||
# ../paperless/hetzner-creds-externalsecret.yaml.
|
||||
#
|
||||
# The namespace allowlist on the openbao ClusterSecretStore
|
||||
# (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
|
||||
@@ -24,4 +21,4 @@ spec:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -36,11 +36,9 @@ spec:
|
||||
target:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
# Reuses the paperless copy of the shared Hetzner S3 credentials (identical
|
||||
# ACCESS_KEY_ID / ACCESS_SECRET_KEY / REGION) to avoid seeding a new OpenBao
|
||||
# path for this migration — same approach woodpecker took. FOLLOW-UP: give
|
||||
# wordpress its own seed/wordpress/hetzner-creds path once OpenBao can be
|
||||
# written, and repoint this here (the value is the same, so it's a drop-in
|
||||
# swap).
|
||||
# One shared path for one shared credential. There is a single Hetzner S3
|
||||
# key behind all of these — see the note in
|
||||
# ../paperless/hetzner-creds-externalsecret.yaml for why per-app paths were
|
||||
# the wrong shape and what would have to change for them to be right.
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
Reference in New Issue
Block a user