Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 claude-bot
|
ed86592978 | ||
 k.renner
|
a8c2f79f84 | ||
|
k.renner
|
9f10ea793d | ||
 issue-fix-bot
|
bdffe1804b | ||
|
claude-bot
|
da1482e33f | ||
|
k.renner
|
c6b1aa109b | ||
|
claude-bot
|
2f3d952656 | ||
|
k.renner
|
0a87f91116 | ||
|
claude-bot
|
c59f0050b3 | ||
|
k.renner
|
48c46610a6 | ||
 claude-botandClaude Opus 5
|
53887dd88f | ||
|
k.renner
|
c9a10ab015 | ||
|
claude-botandClaude Opus 5
|
24b2a0178d | ||
|
k.renner
|
b8061ea05d |
@@ -1,18 +0,0 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: barman-cloud-creds
|
||||
namespace: authentik
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->authentik-KS handoff (barman creds).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/authentik/barman-creds
|
||||
@@ -15,4 +15,4 @@ spec:
|
||||
name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/authentik/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -27,5 +27,4 @@ resources:
|
||||
- cluster.yaml
|
||||
- objectstore.yaml
|
||||
- scheduled-backup.yaml
|
||||
- barman-creds-externalsecret.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
|
||||
@@ -1,21 +1,8 @@
|
||||
---
|
||||
# Barman S3 credentials for ha-pg.
|
||||
#
|
||||
# FOLLOW-UP (#397): sourced from seed/paperless/hetzner-creds, which is not where
|
||||
# they belong. woodpecker/hetzner-creds-externalsecret.yaml has the identical
|
||||
# problem and the same note. The values are the same today, so this is correct
|
||||
# but coupled: rotating paperless's Hetzner credentials would silently change
|
||||
# the backup credentials for Home Assistant and Woodpecker too.
|
||||
#
|
||||
# The fix is one dedicated path per consumer (seed/home-assistant/hetzner-creds)
|
||||
# and a one-line change to the `key:` below. Creating it needs an OpenBao write:
|
||||
#
|
||||
# bao kv put homelab/seed/home-assistant/hetzner-creds \
|
||||
# @<(bao kv get -format=json homelab/seed/paperless/hetzner-creds | jq '.data.data')
|
||||
#
|
||||
# Note the mount is `homelab`, so the path is homelab/seed/... — #397 records a
|
||||
# 403 on `bao kv list seed`, which was a wrong-path error rather than a
|
||||
# permissions problem.
|
||||
# One shared path for one shared credential — see the note in
|
||||
# ../paperless/hetzner-creds-externalsecret.yaml.
|
||||
#
|
||||
# The namespace allowlist on the openbao ClusterSecretStore
|
||||
# (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
|
||||
@@ -35,4 +22,4 @@ spec:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: barman-cloud-creds
|
||||
namespace: immich
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->immich-KS handoff (barman creds).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/immich/barman-creds
|
||||
@@ -15,4 +15,4 @@ spec:
|
||||
name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/immich/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -13,6 +13,5 @@ resources:
|
||||
- cluster.yaml
|
||||
- objectstore.yaml
|
||||
- scheduled-backup.yaml
|
||||
- barman-creds-externalsecret.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
- pvcs.yaml
|
||||
|
||||
@@ -11,25 +11,10 @@ spec:
|
||||
groups:
|
||||
- name: disk-wear.rules
|
||||
rules:
|
||||
# THE GAP THESE RULES CLOSE
|
||||
#
|
||||
# Drive wear is invisible in every other signal this cluster has.
|
||||
# `ceph -s` reads active+clean at any wear level, no Ceph health check
|
||||
# fires until a device actually fails, and the block-layer metrics show
|
||||
# throughput and latency but never endurance. That blind spot is how a
|
||||
# set of OSD drives reached 81-87% before anyone looked (#549, #434).
|
||||
#
|
||||
# The numbers below are wear percentages reported by the drive itself:
|
||||
# 100 means the manufacturer's rated endurance is used up. Drives do
|
||||
# not stop at 100 — they typically keep working and start consuming
|
||||
# spare blocks — so treat these as planning thresholds, not failure
|
||||
# predictions.
|
||||
# Wear percentages are self-reported: 100 means the rated endurance is
|
||||
# spent, not that the drive stops. Planning thresholds, not predictions.
|
||||
- alert: NvmeDriveWearHigh
|
||||
# 80% with months of runway left is the point where ordering a
|
||||
# replacement is still a calm decision rather than an outage.
|
||||
# `for` is long because wear is monotonic and slow: it cannot
|
||||
# flap, and a short window would only make the alert noisier
|
||||
# after a restart.
|
||||
# Long `for` because wear is monotonic and cannot flap.
|
||||
expr: smartctl_device_percentage_used >= 80
|
||||
for: 1h
|
||||
labels:
|
||||
@@ -37,12 +22,11 @@ spec:
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear"
|
||||
description: >-
|
||||
The drive has used {{ $value }}% of its rated write endurance.
|
||||
Nothing is failing and no data is at risk yet. Check what the
|
||||
drive is carrying before deciding urgency: an OSD is replaceable
|
||||
with a rebalance, whereas the EPHEMERAL volume has no such
|
||||
escape — a node simply stops when it goes. The Disk Health
|
||||
dashboard resolves which drive holds which role.
|
||||
{{ $value }}% of rated write endurance used. Nothing is failing
|
||||
yet. Urgency depends on what the drive carries — the Disk Health
|
||||
dashboard shows the role. Losing an OSD costs a rebalance;
|
||||
EPHEMERAL is /var and rebuilds, apart from the node's etcd member
|
||||
data, which has to be removed and re-added.
|
||||
- alert: NvmeDriveWearCritical
|
||||
expr: smartctl_device_percentage_used >= 95
|
||||
for: 1h
|
||||
@@ -55,9 +39,7 @@ spec:
|
||||
likely keep working for a while, but it is now operating outside
|
||||
what the manufacturer stands behind, and replacement should be
|
||||
scheduled rather than considered.
|
||||
# A drive that has consumed its spare area is no longer masking media
|
||||
# defects, which is a materially different state from being worn:
|
||||
# wear is a budget, spare exhaustion is damage.
|
||||
# Wear is a budget; spare exhaustion is damage.
|
||||
- alert: NvmeDriveSpareExhausted
|
||||
expr: smartctl_device_available_spare <= smartctl_device_available_spare_threshold
|
||||
for: 15m
|
||||
@@ -70,10 +52,6 @@ spec:
|
||||
drive has run out of reserve blocks to remap failures into.
|
||||
Replace it.
|
||||
- alert: NvmeDriveCriticalWarning
|
||||
# The NVMe critical warning byte is a bitfield: spare below
|
||||
# threshold, temperature out of range, degraded reliability,
|
||||
# read-only mode, volatile-memory backup failed. Any non-zero value
|
||||
# is the drive itself asking for attention.
|
||||
expr: smartctl_device_critical_warning > 0
|
||||
for: 5m
|
||||
labels:
|
||||
@@ -85,10 +63,8 @@ spec:
|
||||
detail with `smartctl -a` on the node; the value is a bitfield,
|
||||
not a count.
|
||||
- alert: NvmeDriveMediaErrors
|
||||
# Unlike wear, this counter should never move. Alert on the increase
|
||||
# rather than the absolute value, so a drive that recorded errors
|
||||
# long ago does not alert forever while a drive that starts failing
|
||||
# today does.
|
||||
# On the increase, not the total: pre-existing errors would otherwise
|
||||
# alert forever. Standing counts are visible on the dashboard only.
|
||||
expr: increase(smartctl_device_media_errors[6h]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
@@ -99,14 +75,9 @@ spec:
|
||||
The drive reported unrecoverable data-integrity errors in the
|
||||
last six hours. A healthy drive never increments this. Check
|
||||
whether Ceph has logged scrub errors on the same node.
|
||||
# Without this the whole group is silently useless: no series means no
|
||||
# alert fires, and a dashboard of empty panels reads much like a
|
||||
# healthy one. This is the check that the checks are running.
|
||||
# The check that the checks are running: no series fires no alert.
|
||||
# Counted against node-exporter's inventory, not a fixed fleet size.
|
||||
- alert: SmartctlExporterIncomplete
|
||||
# Compared against node-exporter's own NVMe inventory rather than a
|
||||
# written-down fleet size. Adding or removing a drive then moves both
|
||||
# sides together, so this cannot rot into a permanent false alarm the
|
||||
# way a hardcoded count would.
|
||||
expr: count(smartctl_device_percentage_used) < count(node_nvme_info) or absent(smartctl_device_percentage_used)
|
||||
for: 30m
|
||||
labels:
|
||||
|
||||
@@ -14,16 +14,12 @@ spec:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: smartctl-exporter
|
||||
endpoints:
|
||||
# smartctl itself polls the drives every 60s, so scraping faster than that
|
||||
# returns the same numbers with a new timestamp. Wear moves on a scale of
|
||||
# weeks; 60s is already far finer than the signal.
|
||||
# smartctl polls the drives every 60s; scraping faster gains nothing.
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 60s
|
||||
relabelings:
|
||||
# Without this every series is labelled with the exporter pod's IP and
|
||||
# nothing else. A DaemonSet pod's IP says nothing about which machine
|
||||
# the drive is in, and it changes on every reschedule — so the wear
|
||||
# figure would be unattributable, which is most of its value gone.
|
||||
# Without this a series carries only the exporter pod's IP, which does
|
||||
# not identify the machine the drive is in.
|
||||
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
targetLabel: node
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance/wear is absent; see the note at the bottom and issue #549.",
|
||||
"description": "NVMe health and performance per node. The top row answers 'is anything wrong'; the graphs below answer 'why'. Panels resolve each drive through node_nvme_info and node_uname_info rather than plotting the Linux device name, because NVMe enumeration order is not guaranteed to agree across machines — the same name can be a different drive on the next node. Which drive currently holds which role is in the inventory table, deliberately not in this text. Endurance comes from smartctl_exporter, whose series are labelled by node rather than by the exporter pod's IP — see the note at the bottom for what it still cannot tell you.",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
@@ -19,7 +19,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 0, "y": 0},
|
||||
"id": 10,
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_health_status", "legendFormat": "", "refId": "A"}],
|
||||
@@ -28,7 +28,7 @@
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "OSDs the cluster knows about but which are not up, and not in. Both should read zero. Counted as a difference against ceph_osd_metadata rather than compared to a fixed number, so adding or removing an OSD does not turn this panel into a false alarm.",
|
||||
"description": "OSDs the cluster knows about but which are not up. Should read zero. Counted as a difference against ceph_osd_metadata rather than compared to a fixed number, so adding or removing an OSD does not turn this panel into a false alarm. An OSD that is up but out shows as a Ceph health warning instead — it is a state, not a fault, and it does not belong in a red-or-green box.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
@@ -38,14 +38,13 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 5, "y": 0},
|
||||
"id": 11,
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(ceph_osd_metadata) - sum(ceph_osd_up)", "legendFormat": "down", "refId": "A"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(ceph_osd_metadata) - sum(ceph_osd_in)", "legendFormat": "out", "refId": "B"}
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(ceph_osd_metadata) - sum(ceph_osd_up)", "legendFormat": "", "refId": "A"}
|
||||
],
|
||||
"title": "OSDs down / out",
|
||||
"title": "OSDs down",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
@@ -62,7 +61,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 10, "y": 0},
|
||||
"id": 12,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "ceph_cluster_total_used_bytes / ceph_cluster_total_bytes", "legendFormat": "", "refId": "A"}],
|
||||
@@ -71,7 +70,7 @@
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Fullest EPHEMERAL volume across the fleet — Talos's /var, holding container images, pod scratch space and kubelet state. A full /var takes the node down, and unlike Ceph nothing rebalances away from it.",
|
||||
"description": "Fullest EPHEMERAL volume across the fleet — Talos's /var. Its contents are meant to be disposable: container images, kubelet state and pod scratch all rebuild. Filling it is still a problem, because a node with no room to write stops being useful long before anything is lost, and nothing rebalances /var the way Ceph rebalances a full OSD.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
@@ -83,7 +82,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
|
||||
"gridPos": {"h": 4, "w": 5, "x": 15, "y": 0},
|
||||
"id": 13,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(1 - node_filesystem_avail_bytes{mountpoint=\"/var\"} / node_filesystem_size_bytes{mountpoint=\"/var\"})", "legendFormat": "", "refId": "A"}],
|
||||
@@ -115,7 +114,7 @@
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Drive inventory",
|
||||
"title": "Drive inventory (identity and role)",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
@@ -128,6 +127,100 @@
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Rated endurance consumed, worst drive in the fleet. This is the drive's own accounting against what the manufacturer warrants, not a failure prediction — a drive past 100 usually keeps working and starts eating its spare area. Read it for planning; read the SMART table for whether anything is failing now.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"mappings": [],
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
|
||||
"id": 14,
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(smartctl_device_percentage_used)", "legendFormat": "", "refId": "A"}],
|
||||
"title": "Worst NVMe wear",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Rated endurance consumed per drive. The line is nearly flat by nature — wear moves on a scale of weeks — so the useful reading is the spread between drives and the slope, not today's value. Joined to smartctl_device for the model name, because the device label alone is not comparable across nodes.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "dashed"}},
|
||||
"mappings": [],
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
|
||||
"id": 15,
|
||||
"options": {"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device",
|
||||
"legendFormat": "{{node}} {{model_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "NVMe wear",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "The failure-now signals, as opposed to the endurance budget above. Spare below its threshold means the drive has run out of blocks to remap defects into; media errors are unrecoverable data-integrity failures and should never increment; the critical warning is a bitfield the drive raises itself. Written bytes is the counter behind the wear percentage, useful for projecting when a drive will reach it.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"custom": {"align": "auto", "cellOptions": {"type": "auto"}, "inspect": false},
|
||||
"mappings": [],
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}
|
||||
},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "wear"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 80}, {"color": "red", "value": 95}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "spare"}, "properties": [{"id": "unit", "value": "percent"}]},
|
||||
{"matcher": {"id": "byName", "options": "media errors"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "critical warning"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-text"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "powered on"}, "properties": [{"id": "unit", "value": "s"}]},
|
||||
{"matcher": {"id": "byName", "options": "written"}, "properties": [{"id": "unit", "value": "bytes"}]}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
|
||||
"id": 16,
|
||||
"options": {"cellHeight": "sm", "footer": {"countRows": false, "fields": "", "reducer": ["sum"], "show": false}, "showHeader": true, "sortBy": [{"desc": true, "displayName": "wear"}]},
|
||||
"targets": [
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_percentage_used * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "A"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_available_spare * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "B"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_media_errors * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "C"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_critical_warning * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "D"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_power_on_seconds * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "E"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "smartctl_device_bytes_written * on (node, device) group_left(model_name) smartctl_device", "format": "table", "instant": true, "refId": "F"}
|
||||
],
|
||||
"title": "SMART health",
|
||||
"transformations": [
|
||||
{"id": "merge", "options": {}},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {"Time": true, "__name__": true, "container": true, "endpoint": true, "instance": true, "job": true, "namespace": true, "pod": true, "service": true},
|
||||
"indexByName": {},
|
||||
"renameByName": {"node": "node", "device": "device", "model_name": "model", "Value #A": "wear", "Value #B": "spare", "Value #C": "media errors", "Value #D": "critical warning", "Value #E": "powered on", "Value #F": "written"}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Read and write throughput per drive.",
|
||||
@@ -141,7 +234,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 13},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
|
||||
"id": 2,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -174,7 +267,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 13},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
|
||||
"id": 3,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -207,7 +300,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 22},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
|
||||
"id": 4,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -245,7 +338,7 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 22},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
|
||||
"id": 5,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -278,7 +371,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 31},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 40},
|
||||
"id": 6,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -305,7 +398,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 31},
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 40},
|
||||
"id": 7,
|
||||
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -340,7 +433,7 @@
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 40},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 49},
|
||||
"id": 9,
|
||||
"options": {"legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
@@ -356,11 +449,11 @@
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 11, "w": 24, "x": 0, "y": 48},
|
||||
"gridPos": {"h": 11, "w": 24, "x": 0, "y": 57},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
|
||||
"content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Endurance is missing.** No wear, no written-bytes total, no power-on hours, no media-error count — no exporter here produces them. NVMe reports those over an admin ioctl that needs elevated privileges; temperature is the only health value the kernel publishes through sysfs. Tracked as **#549**. Do not read a green top row as a healthy drive.\n\nCeph does record wear, but only inside its own devicehealth module and not in Prometheus: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. It covers OSDs and nothing else, and a drive removed from Ceph is purged from that registry — its history goes with it, so the wear figure of a retired drive is not recoverable after the fact.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, so the drive with the least endurance left can be the one carrying container state.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
|
||||
"content": "### What this dashboard cannot tell you\n\nThese notes are about mechanisms, not about the drives currently fitted. The inventory table above is the live answer to what is in the machines; nothing here restates it, because prose does not get re-read when hardware is swapped.\n\n**Wear is a budget, not a prediction.** The percentage is what the drive reports against its own rated endurance. Drives do not stop at 100 — they usually keep working and start consuming spare blocks — so read it for planning, and read available spare and media errors for whether something is failing now. Those are different questions and the alert rules treat them as such.\n\n**A green top row is not a healthy drive.** Endurance says nothing about the drive that dies of a controller fault tomorrow, and this dashboard cannot see one coming.\n\nCeph keeps its own wear figures for OSDs in the devicehealth module, which is a useful cross-check but covers OSDs only: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph device ls`. A drive removed from Ceph is purged from that registry and its history goes with it.\n\n**Roles are assigned in configuration, not by device name.** Talos selects the system disk and the EPHEMERAL volume by serial, Rook selects the OSD by `/dev/disk/by-id` path. Two consequences outlive any particular set of drives: a broad selector such as `disk.transport == 'nvme'` matches every drive in the box, with the installer taking the OSD as the failure mode — and a drive retired from Ceph is not necessarily out of service, and putting a worn one on EPHEMERAL is a sensible destination rather than a hazard — that volume is built to be rebuilt. Check the inventory table for what a given drive holds before reading its wear as urgent.\n\n**A dying NVMe here hangs rather than errors.** Talos boots with `nvme_core.io_timeout=4294967295`, so NVMe commands never time out. Expect a stall, not an I/O error, and expect the latency panel to go flat rather than spike.\n\n**Kernel messages are not retained.** Talos kernel logs do not reach Loki — the only log jobs are Kubernetes pods and a systemd-journal from elsewhere. `talosctl -n <node> dmesg` is the only source, it is a ring buffer, and a reboot empties it. An NVMe error from before the last reboot left no trace anywhere.",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Gaps and caveats",
|
||||
|
||||
@@ -1,37 +1,8 @@
|
||||
# SMART/endurance metrics for every NVMe drive in the fleet (#549).
|
||||
#
|
||||
# WHY THIS HOLDS ELEVATED CAPABILITIES, and why that is a deliberate exception
|
||||
# rather than an oversight:
|
||||
#
|
||||
# NVMe reports health over an admin ioctl on the character device
|
||||
# (NVME_IOCTL_ADMIN_CMD), which the kernel gates on CAP_SYS_ADMIN. There is no
|
||||
# sysfs path for wear, media errors or written-bytes totals — hwmon publishes
|
||||
# the controller temperature and nothing else. Upstream smartctl_exporter
|
||||
# documents `privileged: true` and `user: root` for exactly this reason.
|
||||
#
|
||||
# The alternative that needs no privileges — reading Ceph's devicehealth
|
||||
# archive, which does hold full smartctl output — covers OSD drives only. In
|
||||
# this cluster that is three of nine, and the six it misses include the drives
|
||||
# that were retired from Ceph for wear and now carry EPHEMERAL. Covering the
|
||||
# healthy drives and not the worn ones was judged not worth a new service; the
|
||||
# owner chose fleet-wide coverage with the privilege exception instead.
|
||||
#
|
||||
# The exception is `privileged: true`, and narrower settings were tried and
|
||||
# measured to fail — see the securityContext below for what was ruled out and
|
||||
# how. It is still bounded: no host namespaces, no hostPID, no hostNetwork,
|
||||
# read-only root filesystem, and /dev as the only host mount.
|
||||
#
|
||||
# If this ever needs revisiting, there are exactly two things worth checking,
|
||||
# and neither is "does another exporter ask for less":
|
||||
# - whether the kernel has grown a sysfs interface for NVMe health, which
|
||||
# would remove the need for device access altogether
|
||||
# - whether enough workloads need raw device access to justify a device
|
||||
# plugin, which is the only way Kubernetes grants individual devices
|
||||
# The requirement comes from the device cgroup and the ioctl gate, not from
|
||||
# the choice of tool.
|
||||
#
|
||||
# Upstream had no release between 2025-04-22 (v0.14.0) and this deployment.
|
||||
# The repository is not archived; the pin is simply its newest tag.
|
||||
# Runs privileged: host device access is gated by the container device cgroup,
|
||||
# and Kubernetes offers no per-device allowlist. Narrower settings do not work;
|
||||
# the alternatives that were tried are in the commit history.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
@@ -57,10 +28,8 @@ spec:
|
||||
containers:
|
||||
- name: smartctl-exporter
|
||||
image: quay.io/prometheuscommunity/smartctl-exporter:v0.14.0
|
||||
# No --smartctl.device on purpose: the exporter scans for devices
|
||||
# itself and rescans every 10m by default. Device names are not
|
||||
# stable across these machines, so a hardcoded list would quietly
|
||||
# stop covering a drive the next time one is replaced.
|
||||
# No --smartctl.device: the exporter scans and rescans on its own.
|
||||
# Device names are not stable across these machines.
|
||||
args:
|
||||
- --web.listen-address=:9633
|
||||
ports:
|
||||
@@ -68,56 +37,18 @@ spec:
|
||||
containerPort: 9633
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
# PRIVILEGED, and there is no narrower setting that works. This
|
||||
# was established by measurement after two narrower attempts
|
||||
# failed, so the reasoning is recorded here rather than rediscovered:
|
||||
#
|
||||
# Attempt 1 — CAP_SYS_ADMIN + CAP_SYS_RAWIO, everything else
|
||||
# dropped. The exporter found all three drives per node and
|
||||
# could open none: "Smartctl open device: /dev/nvme0 failed:
|
||||
# Operation not permitted".
|
||||
#
|
||||
# Attempt 2 — the same plus seccompProfile: Unconfined, on the
|
||||
# theory that EPERM is the default seccomp action for an
|
||||
# unlisted syscall and the kubelet runs seccompDefault: true.
|
||||
# /proc/1/status showed CapEff 0x220000 and Seccomp 2, so both
|
||||
# capabilities were effective and a filter was loaded. Removing
|
||||
# the filter changed nothing.
|
||||
#
|
||||
# What actually settled it: `head -c 512 /dev/nvme0n1` inside
|
||||
# the container fails with EPERM too. A plain read of a block
|
||||
# device involves no ioctl, no capability and no syscall a
|
||||
# seccomp profile would filter. The block is the device cgroup —
|
||||
# containerd denies non-privileged containers access to host
|
||||
# device nodes whether or not /dev is mounted.
|
||||
#
|
||||
# Kubernetes exposes no per-device allowlist in a Pod spec; there
|
||||
# is no equivalent of `docker --device`. The alternatives are
|
||||
# `privileged: true` or a device plugin advertising the NVMe
|
||||
# drives as a resource. A device plugin is the genuinely narrower
|
||||
# answer and is worth building if more workloads ever need raw
|
||||
# device access; for a single read-only metrics exporter it is a
|
||||
# component to write, deploy and maintain in exchange for the same
|
||||
# access.
|
||||
runAsUser: 0
|
||||
runAsNonRoot: false
|
||||
privileged: true
|
||||
# Stated rather than omitted: the API rejects `privileged: true`
|
||||
# together with `allowPrivilegeEscalation: false`, so leaving it
|
||||
# out would only hide that privileged already implies it.
|
||||
# The API rejects privileged together with
|
||||
# allowPrivilegeEscalation: false.
|
||||
allowPrivilegeEscalation: true
|
||||
readOnlyRootFilesystem: true
|
||||
volumeMounts:
|
||||
# Not read-only. smartmontools opens NVMe devices read-write on
|
||||
# some paths, and once the container is privileged a read-only
|
||||
# flag on this mount is not a boundary anyone should rely on —
|
||||
# keeping it would suggest a protection that is not there.
|
||||
# Not read-only: smartmontools opens NVMe devices read-write.
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
# smartctl is not known to need scratch space, but the root
|
||||
# filesystem is read-only and a tool that turns out to want /tmp
|
||||
# would crashloop instead of degrading. Cheaper to provide it than
|
||||
# to find out from a restart counter.
|
||||
# Scratch space, because the root filesystem is read-only.
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
resources:
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: barman-cloud-creds
|
||||
namespace: paperless
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->paperless-KS handoff (barman creds).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: barman-cloud-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/barman-creds
|
||||
@@ -1,3 +1,9 @@
|
||||
# One Hetzner S3 key, one path, shared by every barman ObjectStore. Separate
|
||||
# per-app paths would not make rotation independent — that needs one Hetzner
|
||||
# key per consumer (#397).
|
||||
#
|
||||
# velero reads the same key from infrastructure/velero-hetzner, which holds it
|
||||
# as a credentials file rather than these three fields.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
@@ -15,4 +21,4 @@ spec:
|
||||
name: hetzner-s3-creds # keeps the old Secret's name — consumers unchanged
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -20,7 +20,6 @@ resources:
|
||||
- cluster.yaml
|
||||
- objectstore.yaml
|
||||
- scheduled-backup.yaml
|
||||
- barman-creds-externalsecret.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
- pvcs.yaml
|
||||
configMapGenerator:
|
||||
|
||||
@@ -12,4 +12,4 @@ spec:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/trmnl/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
---
|
||||
# Barman S3 credentials for wp-pg.
|
||||
#
|
||||
# Sourced from seed/paperless/hetzner-creds, which is not where they belong.
|
||||
# Issue #397 wants woodpecker to own seed/woodpecker/hetzner-creds; creating
|
||||
# that path needs OpenBao write access (`bao kv list seed` → 403 for the
|
||||
# identities available here), so it stays open. The values are identical, so it
|
||||
# is a one-line swap of the `key:` below once the path exists.
|
||||
# One shared path for one shared credential — see the note in
|
||||
# ../paperless/hetzner-creds-externalsecret.yaml.
|
||||
#
|
||||
# The namespace allowlist on the openbao ClusterSecretStore
|
||||
# (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
|
||||
@@ -24,4 +21,4 @@ spec:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -36,11 +36,9 @@ spec:
|
||||
target:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
# Reuses the paperless copy of the shared Hetzner S3 credentials (identical
|
||||
# ACCESS_KEY_ID / ACCESS_SECRET_KEY / REGION) to avoid seeding a new OpenBao
|
||||
# path for this migration — same approach woodpecker took. FOLLOW-UP: give
|
||||
# wordpress its own seed/wordpress/hetzner-creds path once OpenBao can be
|
||||
# written, and repoint this here (the value is the same, so it's a drop-in
|
||||
# swap).
|
||||
# One shared path for one shared credential. There is a single Hetzner S3
|
||||
# key behind all of these — see the note in
|
||||
# ../paperless/hetzner-creds-externalsecret.yaml for why per-app paths were
|
||||
# the wrong shape and what would have to change for them to be right.
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
key: infrastructure/hetzner-s3
|
||||
|
||||
@@ -5,52 +5,24 @@ machine:
|
||||
- k8s-3.srv.renner.uno
|
||||
- k8s.renner.uno
|
||||
- 10.2.40.60
|
||||
# Install from the Image Factory metal installer, not the stock
|
||||
# ghcr.io/siderolabs/installer that `talosctl gen config` defaults to.
|
||||
# Extensions live in the installer image, so with the stock one the schematic
|
||||
# is only ever a property of the boot media and nothing lands on disk —
|
||||
# `talosctl get extensions` returns empty on a node installed that way.
|
||||
# Image Factory metal installer, NOT the stock ghcr.io/siderolabs/installer
|
||||
# that `talosctl gen config` defaults to. Extensions live in the installer
|
||||
# image; install with the stock one and nothing lands on disk —
|
||||
# `talosctl get extensions` comes back empty.
|
||||
#
|
||||
# What the schematic carries, and why each matters here:
|
||||
# The schematic carries intel-ucode, thunderbolt and i915, and sets
|
||||
# net.ifnames=1, which is what the per-node VLAN parents match against.
|
||||
# Changing which extensions are present means a new schematic hash and a
|
||||
# reinstall or upgrade, not an edit here.
|
||||
#
|
||||
# siderolabs/intel-ucode CPU microcode. On a VM the hypervisor supplied
|
||||
# it; on bare metal nothing does except the BIOS.
|
||||
# siderolabs/thunderbolt Inert until machine.kernel.modules asks for
|
||||
# thunderbolt/thunderbolt_net. Present so enabling
|
||||
# the MS-01 mesh later needs neither a new image
|
||||
# nor a reinstall — which is exactly the property
|
||||
# lost by installing without it.
|
||||
# siderolabs/i915 Intel GPU microcode and kernel modules for the
|
||||
# Alder Lake-P Iris Xe in the MS-01. No workload
|
||||
# requests /dev/dri today; it is here for the same
|
||||
# reason as thunderbolt, so that wanting hardware
|
||||
# transcoding later is a config change rather than
|
||||
# another rolling upgrade of every node.
|
||||
# Read only at install and upgrade time — editing this does nothing to a
|
||||
# running node. The tag must match the version being upgraded to, or the
|
||||
# upgrade is a downgrade. Talos only tests migration between adjacent minor
|
||||
# releases: take the latest patch of each intermediate minor rather than
|
||||
# skipping one.
|
||||
#
|
||||
# Not included, and why, so the absences are not mistaken for oversights:
|
||||
# the X710 (i40e) and I226 (igc) NICs need no firmware extension —
|
||||
# intel-ice-firmware covers E810/ice only. mei is a prerequisite for Intel
|
||||
# Arc discrete GPUs, of which there are none. iscsi-tools, nvme-cli and
|
||||
# util-linux-tools have no consumer: storage is Ceph RBD, and device health
|
||||
# is reported by Ceph itself rather than a node-level exporter.
|
||||
#
|
||||
# 0f6fb8bd… is the bare-metal schematic: same net.ifnames=1 as the Proxmox
|
||||
# one (the predictable interface names the per-node VLAN parents match
|
||||
# against), minus qemu-guest-agent. It supersedes a21cec8f…, which was
|
||||
# identical but without i915 — see RUNBOOK-cold-rebuild.md.
|
||||
#
|
||||
# Only read at install and upgrade time. Changing it here does nothing to a
|
||||
# running node until that node is upgraded. Bump the tag with the Talos
|
||||
# version — it must match the running version, or the upgrade is a downgrade.
|
||||
#
|
||||
# Upgrades follow the documented path: migration is only tested between
|
||||
# adjacent minor releases, so go to the latest patch of each intermediate
|
||||
# minor rather than skipping. 1.12.11 -> 1.13.9 satisfies that — one minor
|
||||
# step from the last 1.12 patch, no intermediate needed.
|
||||
#
|
||||
# Kubernetes is a separate decision and is not forced by this. Talos 1.13
|
||||
# supports Kubernetes 1.36, 1.35, 1.34, 1.33, 1.32 and 1.31, so the cluster
|
||||
# stays on 1.35.2 until `talosctl upgrade-k8s` moves it deliberately.
|
||||
# The Kubernetes version is a separate decision, moved by
|
||||
# `talosctl upgrade-k8s`.
|
||||
install:
|
||||
image: factory.talos.dev/metal-installer/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9:v1.13.9
|
||||
features:
|
||||
@@ -74,20 +46,13 @@ machine:
|
||||
nodeIP:
|
||||
validSubnets:
|
||||
- 10.2.40.0/24
|
||||
# Lower the image GC thresholds from the kubelet defaults (85/80) to
|
||||
# bound containerd's image cache, which is the largest single consumer
|
||||
# of /var. Spegel P2P pull-through softens the re-pull cost when GC'd
|
||||
# images are needed back.
|
||||
# Lower than the kubelet defaults (85/80) to bound containerd's image
|
||||
# cache, the largest single consumer of /var.
|
||||
#
|
||||
# These thresholds do NOT control whether Ceph's mon reports
|
||||
# MON_DISK_LOW, and must not be tuned for that. Measured on k8s-2:
|
||||
# image GC frees only the content store (~8.6 GB), while the bulk of
|
||||
# /var/lib/containerd is ~20 GB of overlayfs snapshots — unpacked
|
||||
# layers shared with running containers, which GC cannot reclaim.
|
||||
# Dropping high to 60 deleted 28 images and moved free space by 0.02
|
||||
# percentage points. The mon's own data is ~106 MB; its 30%-free
|
||||
# warning is measuring a filesystem it merely shares, so the lever for
|
||||
# it is elsewhere (mon volume, /var size, or mon_data_avail_warn).
|
||||
# Do NOT tune these to clear Ceph's MON_DISK_LOW. Image GC reclaims only
|
||||
# the content store; the bulk of /var/lib/containerd is overlayfs
|
||||
# snapshots shared with running containers, which GC cannot touch. The
|
||||
# lever for the mon warning is elsewhere.
|
||||
extraConfig:
|
||||
imageGCHighThresholdPercent: 70
|
||||
imageGCLowThresholdPercent: 60
|
||||
|
||||
@@ -1,55 +1,25 @@
|
||||
# k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
|
||||
# still Proxmox VMs. Every machine-specific selector below differs from theirs
|
||||
# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
|
||||
# four physical NICs and all three NVMe drives are visible to Talos at once.
|
||||
#
|
||||
# Networking uses the v1.12 multi-document config rather than the deprecated
|
||||
# machine.network.interfaces. That is deliberate: the docs do not define what
|
||||
# happens when the same link is configured by both schemas, so this file keeps
|
||||
# every link in one schema instead of straddling them.
|
||||
# Networking uses the multi-document config, not the deprecated
|
||||
# machine.network.interfaces. Keep every link in one schema: the docs do not
|
||||
# define what happens when both configure the same link.
|
||||
machine:
|
||||
network:
|
||||
hostname: k8s-1
|
||||
sysctls:
|
||||
# ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It exists as
|
||||
# the macvlan parent for the v30iot pod attachments
|
||||
# (matter/otbr/shelly-manager/music-assistant), whose children do their own
|
||||
# DHCP on that L2. Host addresses were never needed and are actively
|
||||
# harmful: a connected 10.2.30.0/24 route makes node replies to IoT-subnet
|
||||
# clients bypass the UDM, which breaks every UDM-DNATed flow (hairpin NAT:
|
||||
# un-translated SYN-ACK -> client kernel RST, verified live 2026-07-23).
|
||||
# This disables v6 entirely on that link, including link-local.
|
||||
# ens19 (VLAN 30, v30iot) is LINK ONLY — it is the macvlan parent for the
|
||||
# v30iot pod attachments and must never get host addressing. A connected
|
||||
# 10.2.30.0/24 route makes node replies bypass the UDM and breaks every
|
||||
# DNATed flow.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
# Same argument, applied to the physical ports rather than a VLAN child.
|
||||
#
|
||||
# enp2s0f0np0 is nothing but the trunk parent for mgmt0 and ens19, and
|
||||
# enp87s0 is unused. Both sit on the untagged fallback network, where by
|
||||
# design nothing is supposed to run — and both were autoconfiguring
|
||||
# themselves from its router advertisements: an fd00::/64 ULA each, plus
|
||||
# an IPv6 default route each.
|
||||
#
|
||||
# The result was four IPv6 default routes per node at identical priority
|
||||
# 1024 — mgmt0, enp2s0f0np0, enp87s0 and lo — of which only mgmt0 has a
|
||||
# globally routable source address. With nothing to break the tie,
|
||||
# outbound IPv6 mostly left through an interface whose only usable source
|
||||
# was a ULA, so the cluster had no IPv6 egress at all while IPv4 worked
|
||||
# normally. It surfaced as two NetBird probes reporting the proxies down
|
||||
# for 30 hours; the proxies were fine and answered IPv6 in 13 ms from
|
||||
# outside.
|
||||
#
|
||||
# Verified on k8s-1 before rolling to the others: disabling v6 on a VLAN
|
||||
# parent leaves the VLAN child alone. mgmt0 kept its global address and
|
||||
# its fd40:: address, Ceph did not notice, and every failing probe
|
||||
# recovered on the next scrape.
|
||||
# These two sit on the untagged fallback network, where nothing may run.
|
||||
# Left enabled they autoconfigure a ULA and an IPv6 default route from its
|
||||
# RAs, which ties with mgmt0's and costs the cluster IPv6 egress.
|
||||
# Disabling v6 on a VLAN parent does not affect the VLAN child.
|
||||
net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
|
||||
net.ipv6.conf.enp87s0.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are ordinary block devices to Talos here: the Kingston
|
||||
# OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
|
||||
# Ceph OSD, and the retired Crucial P3 that carries EPHEMERAL (see the
|
||||
# VolumeConfig below). Every one of them is selected by serial. Nothing in
|
||||
# this file may use a broad selector: `disk.transport == 'nvme'` matches all
|
||||
# three, and the failure mode is Talos taking the OSD drive.
|
||||
# Select disks by serial only. A broad selector such as
|
||||
# `disk.transport == 'nvme'` matches every drive in the machine, and the
|
||||
# failure mode is the installer taking the Ceph OSD drive.
|
||||
diskSelector:
|
||||
serial: 50026B7382B48B86
|
||||
certSANs:
|
||||
@@ -59,15 +29,10 @@ machine:
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
|
||||
#
|
||||
# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
|
||||
# the PM983 replaced it. /var is exactly the right role for a disk in that
|
||||
# state: containerd's image and snapshot store, pod logs and the Ceph mon store
|
||||
# are all reconstructible, so the drive is consumable here in a way it is not as
|
||||
# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
|
||||
# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
|
||||
# they absorb another node's pods.
|
||||
# EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
|
||||
# contents are reconstructible, so a worn disk is consumable here in a way it
|
||||
# is not as an OSD. Note /var/lib/etcd lives here too, which is the one thing
|
||||
# on the volume that is not free to lose.
|
||||
#
|
||||
# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; a config applied after EPHEMERAL
|
||||
@@ -75,8 +40,8 @@ cluster:
|
||||
# requires wiping the disk and booting install media again.
|
||||
#
|
||||
# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
|
||||
# would match the PM983 as well, and losing that coin flip means Talos formats
|
||||
# the OSD drive.
|
||||
# still matches the OSD drive, and losing that coin flip means Talos formats
|
||||
# it.
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
@@ -86,30 +51,25 @@ provisioning:
|
||||
minSize: 50GiB
|
||||
grow: true
|
||||
---
|
||||
# X710 port 0 carries the switch trunk. It takes no addressing itself — the
|
||||
# port's untagged VLAN is the 192.168.8.0/24 network, which this node has no
|
||||
# business being on — so management and v30iot both ride it as tagged VLANs.
|
||||
# X710 port 0 carries the switch trunk and takes no addressing itself — its
|
||||
# untagged VLAN is a network this node has no business being on. Management
|
||||
# and v30iot ride it as tagged VLANs.
|
||||
#
|
||||
# Links are addressed by predictable kernel name. Those derive from PCI
|
||||
# topology, not probe order — though not immune: adding the third NVMe
|
||||
# renumbered the buses and moved the unused second I226 from enp89s0 to
|
||||
# enp90s0. Neither name used here was affected.
|
||||
# Links are named by predictable kernel name, which derives from PCI topology
|
||||
# and can still shift when hardware is added.
|
||||
#
|
||||
# LinkAliasConfig would be the better way to pin these, and is worth
|
||||
# reintroducing — but only once it is proven. Aliasing both X710 ports by
|
||||
# permanent MAC and using the aliases as VLAN parents produced a node with no
|
||||
# address on any interface: the aliases were never created, so every document
|
||||
# referencing them did nothing, including the static address on port 1. Verify
|
||||
# any alias-based revision with `talosctl apply-config --mode=try`, which rolls
|
||||
# back on its own, before making it permanent.
|
||||
# LinkAliasConfig would pin these better, but aliasing the X710 ports by MAC
|
||||
# and using the aliases as VLAN parents left the node with no address on any
|
||||
# interface — the aliases were never created and every document referencing
|
||||
# them silently did nothing. Prove any alias-based revision with
|
||||
# `talosctl apply-config --mode=try` first; it rolls back on its own.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
|
||||
# network. The k8s-1 VM reached 10.2.40.174 over this same physical port via
|
||||
# Proxmox's v40srv VNet (vmbr0.40), so the switch trunks VLAN 40 here.
|
||||
# network.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: mgmt0
|
||||
@@ -117,30 +77,15 @@ vlanID: 40
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Management addressing is DHCP and the lease is NOT reserved anywhere in this
|
||||
# repo — the k8s-{1,2,3}.srv.renner.uno reservations live on the UDM out of band
|
||||
# and opentofu/unifi.tf declares no live resources. The addresses do move under
|
||||
# a running node: k8s-3 went 10.2.40.168 -> .138 mid-flight on 2026-08-26
|
||||
# without its Node object ever re-registering (issue #635).
|
||||
# DHCP, and the lease is not reserved in this repo — reservations live on the
|
||||
# UDM out of band. Addresses do move under a running node.
|
||||
#
|
||||
# kube-apiserver does not follow when that happens. Talos starts it with
|
||||
# `--advertise-address=$(POD_IP)` and no `--endpoint-reconciler-type` override,
|
||||
# so the advertised address is fixed at pod start and the default lease
|
||||
# reconciler goes on renewing the masterlease under the OLD address. Nothing
|
||||
# expires it while the process lives, so `default/kubernetes` keeps handing out
|
||||
# a dead endpoint and the node's live address is never registered at all — 19h
|
||||
# of TargetDown on job="apiserver", a monitoring blind spot on one of three
|
||||
# apiservers, and it cleared only when the static pod restarted.
|
||||
#
|
||||
# So after any node re-IPs, check that
|
||||
# kube-apiserver does not follow a re-IP: it advertises $(POD_IP) fixed at pod
|
||||
# start and the lease reconciler keeps renewing the masterlease under the old
|
||||
# address, so `default/kubernetes` hands out a dead endpoint until the static
|
||||
# pod restarts (#635). After any node re-IPs, check that
|
||||
# `kubectl get endpoints kubernetes -n default` agrees with
|
||||
# `kubectl get nodes -o wide`, and restart the offending node
|
||||
# (`talosctl -n <node> reboot`) if it does not.
|
||||
#
|
||||
# Pinning mgmt0 with a static AddressConfig alongside this document would remove
|
||||
# the trigger, but it needs an address outside the UDM's DHCP pool — which is
|
||||
# not declared in this repo — and §6 of RUNBOOK-baremetal-conversion.md applies:
|
||||
# prove it with `talosctl apply-config --mode=try` before making it permanent.
|
||||
# `kubectl get nodes -o wide` and reboot the node if it does not.
|
||||
apiVersion: v1alpha1
|
||||
kind: DHCPv4Config
|
||||
name: mgmt0
|
||||
@@ -152,10 +97,8 @@ kind: Layer2VIPConfig
|
||||
name: 10.2.40.60
|
||||
link: mgmt0
|
||||
---
|
||||
# Named ens19 on purpose. The four v30iot macvlan NADs hard-code
|
||||
# `master: ens19`, and k8s-2/k8s-3 still present the VLAN as a virtio NIC of
|
||||
# that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
|
||||
# serve both VM and bare-metal nodes for as long as the cluster is mixed.
|
||||
# Named ens19 on purpose: the v30iot macvlan NADs hard-code `master: ens19`.
|
||||
# VLANConfig.name is free-form, so renaming this link means editing them too.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ens19
|
||||
@@ -163,19 +106,13 @@ vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
|
||||
# untagged on the wire. Carries OSD<->OSD replication and nothing else.
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
|
||||
# wire. OSD-to-OSD replication and nothing else.
|
||||
#
|
||||
# The node deliberately holds NO address here. Rook attaches this link to the
|
||||
# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
|
||||
# san-network-attachment.yaml), and a macvlan parent only has to be up — it
|
||||
# does not need an address of its own for its children to work.
|
||||
#
|
||||
# It used to carry 10.99.0.1/29, because reaching this link required
|
||||
# CephCluster.spec.network.provider: host, and that put every Ceph daemon on
|
||||
# host networking, mons included. Nothing needs it now, and by the rule this
|
||||
# segment is built on — Ceph-internal traffic and nothing else — the node has
|
||||
# no business on it either.
|
||||
# The node holds NO address here on purpose. Rook attaches the link to the OSD
|
||||
# pods through a Multus macvlan, and a macvlan parent only needs to be up.
|
||||
# Giving the node an address would put it on a segment reserved for
|
||||
# Ceph-internal traffic.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
|
||||
@@ -9,20 +9,14 @@ machine:
|
||||
# macvlan parent for the v30iot pod attachments; host addresses on that L2
|
||||
# make node replies bypass the UDM and break every UDM-DNATed flow.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
# Same argument for the physical ports: enp2s0f0np0 is only the trunk
|
||||
# parent for mgmt0 and ens19, enp87s0 is unused, and both sit on the
|
||||
# untagged fallback network where nothing is supposed to run. Both were
|
||||
# taking an fd00::/64 and an IPv6 default route from its router
|
||||
# advertisements, leaving four equal-priority default routes per node of
|
||||
# which only mgmt0 has a globally routable source. See k8s-1.yaml for the
|
||||
# full reasoning and how it was verified.
|
||||
# Same for the physical ports on the untagged fallback network: left
|
||||
# enabled they take an IPv6 default route from its RAs and cost the
|
||||
# cluster IPv6 egress.
|
||||
net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
|
||||
net.ipv6.conf.enp87s0.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are visible on bare metal. Select the installer target
|
||||
# by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
|
||||
# order the kernel enumerates them in — Talos and Proxmox do not agree on
|
||||
# that order.
|
||||
# Select the installer target by serial so it cannot land on the OSD drive
|
||||
# or on EPHEMERAL, whatever order the kernel enumerates them in.
|
||||
diskSelector:
|
||||
serial: 50026B7382DA5F6E
|
||||
certSANs:
|
||||
@@ -32,17 +26,15 @@ machine:
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
|
||||
# drive is at 81% of its rated endurance and is finished as an OSD, but
|
||||
# everything EPHEMERAL holds is reconstructible, so it is consumable in this
|
||||
# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had.
|
||||
# EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
|
||||
# contents are reconstructible, so a worn disk is consumable in this role.
|
||||
#
|
||||
# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; applied later it is silently
|
||||
# ignored and relocating needs a disk wipe and a boot from install media.
|
||||
#
|
||||
# Matched by serial deliberately: a broad selector such as
|
||||
# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
|
||||
# `disk.transport == 'nvme' && !system_disk` still matches the OSD drive.
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
@@ -85,9 +77,8 @@ kind: Layer2VIPConfig
|
||||
name: 10.2.40.60
|
||||
link: mgmt0
|
||||
---
|
||||
# Named ens19 on purpose: the four v30iot macvlan NADs hard-code
|
||||
# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that
|
||||
# name. VLANConfig.name is free-form, so one set of NADs serves both.
|
||||
# Named ens19 on purpose: the v30iot macvlan NADs hard-code `master: ens19`.
|
||||
# VLANConfig.name is free-form, so renaming this link means editing them too.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ens19
|
||||
@@ -95,18 +86,11 @@ vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
|
||||
# untagged on the wire. OSD<->OSD replication only. Not routed off its own
|
||||
# segment and deliberately has no default route.
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
|
||||
# wire. OSD-to-OSD replication only, not routed off its own segment.
|
||||
#
|
||||
# The node deliberately holds NO address here. Rook attaches this link to the
|
||||
# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
|
||||
# san-network-attachment.yaml), and a macvlan parent only has to be up — it
|
||||
# does not need an address of its own for its children to work.
|
||||
#
|
||||
# It used to carry 10.99.0.2/29, because reaching this link required
|
||||
# CephCluster.spec.network.provider: host, which put every Ceph daemon on
|
||||
# host networking, mons included. Nothing needs it now.
|
||||
# The node holds NO address here on purpose. Rook attaches the link to the OSD
|
||||
# pods through a Multus macvlan, and a macvlan parent only needs to be up.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
|
||||
@@ -1,27 +1,18 @@
|
||||
# k8s-3 runs Talos directly on neptune's MS-01 hardware. Structurally identical
|
||||
# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
|
||||
# serials and addresses differ. neptune was the last node converted, so all
|
||||
# three control-plane nodes now share this layout and no Proxmox remains.
|
||||
# Structurally identical to k8s-1.yaml — see that file for the reasoning
|
||||
# behind each choice; only the serials and addresses differ.
|
||||
machine:
|
||||
network:
|
||||
hostname: k8s-3
|
||||
sysctls:
|
||||
# v30iot NIC carries no host addressing — see k8s-1.yaml.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
# Same argument for the physical ports: enp2s0f0np0 is only the trunk
|
||||
# parent for mgmt0 and ens19, enp87s0 is unused, and both sit on the
|
||||
# untagged fallback network where nothing is supposed to run. Both were
|
||||
# taking an fd00::/64 and an IPv6 default route from its router
|
||||
# advertisements, leaving four equal-priority default routes per node of
|
||||
# which only mgmt0 has a globally routable source. See k8s-1.yaml for the
|
||||
# full reasoning and how it was verified.
|
||||
# Same for the physical ports on the untagged fallback network — see
|
||||
# k8s-1.yaml.
|
||||
net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
|
||||
net.ipv6.conf.enp87s0.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are visible on bare metal. Select the installer target
|
||||
# by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
|
||||
# order the kernel enumerates them in — Talos and Proxmox do not agree on
|
||||
# that order.
|
||||
# Select the installer target by serial so it cannot land on the OSD drive
|
||||
# or on EPHEMERAL, whatever order the kernel enumerates them in.
|
||||
diskSelector:
|
||||
serial: 50026B7382B48B90
|
||||
certSANs:
|
||||
@@ -31,10 +22,8 @@ machine:
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
|
||||
# drive is at 83% of its rated endurance and is finished as an OSD, but
|
||||
# everything EPHEMERAL holds is reconstructible, so it is consumable in this
|
||||
# role. Available spare is still 100% and it reports no media errors.
|
||||
# EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
|
||||
# contents are reconstructible, so a worn disk is consumable in this role.
|
||||
#
|
||||
# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; applied later it is silently
|
||||
@@ -81,15 +70,8 @@ vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network — see k8s-1.yaml.
|
||||
# The node deliberately holds NO address here. Rook attaches this link to the
|
||||
# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
|
||||
# san-network-attachment.yaml), and a macvlan parent only has to be up — it
|
||||
# does not need an address of its own for its children to work.
|
||||
#
|
||||
# It used to carry 10.99.0.3/29, because reaching this link required
|
||||
# CephCluster.spec.network.provider: host, which put every Ceph daemon on
|
||||
# host networking, mons included. Nothing needs it now.
|
||||
# Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
|
||||
# here on purpose; the link is only a macvlan parent for the OSD pods.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
|
||||
Reference in New Issue
Block a user