Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6b0617b4a7 | ||
|
|
6b7736ea58 | ||
|
|
7ff20e5b68 | ||
|
|
296accd1cb | ||
|
|
2665e5ebf3 | ||
|
|
0fe512746b | ||
|
|
4e2303f455 | ||
|
|
483093a104 | ||
|
|
8d348e7f88 | ||
|
|
6f9ff9d624 | ||
|
|
7bf65783be | ||
|
|
dd9fb01359 | ||
|
|
da8a725b32 | ||
|
|
6008d59c9f | ||
|
|
5aa6c0609d | ||
|
|
0ebe113abb | ||
|
|
1723f73b5d | ||
|
|
8d8df3b635 |
@@ -158,7 +158,7 @@ steps:
|
||||
# whenever it trails the deployed server. If you bump the server, bump here.
|
||||
- name: woodpecker-lint
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: woodpeckerci/woodpecker-cli:v3.17.0-alpine
|
||||
image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
|
||||
|
||||
ARG TALOS_VERSION=v1.12.6
|
||||
ARG TALOS_VERSION=v1.13.9
|
||||
ARG VELERO_VERSION=v1.16.0
|
||||
ARG KUBECTL_VERSION=v1.36.2
|
||||
ARG FLUX_VERSION=2.9.2
|
||||
ARG KUBECTL_VERSION=v1.36.3
|
||||
ARG FLUX_VERSION=2.9.5
|
||||
|
||||
RUN apk add --no-cache \
|
||||
aws-cli \
|
||||
@@ -18,7 +18,7 @@ RUN apk add --no-cache \
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/talosctl \
|
||||
"https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
|
||||
&& echo "4c41a3b10b075292e64b182283d558e9d8e935f43239de1ed4e4b14359593efb /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/talosctl
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
@@ -31,7 +31,7 @@ RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
|
||||
RUN curl -sfL -o /tmp/flux.tar.gz \
|
||||
"https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \
|
||||
&& echo "4092b367fd060097976fb7261601f79420ecf7266328417dfd8dfee27de0e6a3 /tmp/flux.tar.gz" | sha256sum -c - \
|
||||
&& echo "b853df82adfd7736f580692f9f734473d571606307139f8fd20c2a80dd1ff473 /tmp/flux.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/flux.tar.gz -C /usr/local/bin flux \
|
||||
&& chmod +x /usr/local/bin/flux \
|
||||
&& rm -f /tmp/flux.tar.gz
|
||||
|
||||
@@ -21,6 +21,9 @@ metadata:
|
||||
namespace: audiobookshelf
|
||||
labels:
|
||||
app.kubernetes.io/name: audiobookshelf
|
||||
annotations:
|
||||
# envFrom does not restart the pod when the ConfigMap changes; Reloader does.
|
||||
configmap.reloader.stakater.com/reload: audiobookshelf-config
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
|
||||
@@ -35,6 +35,9 @@ metadata:
|
||||
namespace: immich
|
||||
labels:
|
||||
app.kubernetes.io/name: immich-server
|
||||
annotations:
|
||||
# envFrom does not restart the pod when the ConfigMap changes; Reloader does.
|
||||
configmap.reloader.stakater.com/reload: immich-config
|
||||
spec:
|
||||
# Scaled to 0 alongside immich-machine-learning during single-node
|
||||
# maintenance; see the note there. The CNPG Postgres cluster stays running,
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
endpoints:
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
relabelings:
|
||||
# The series would otherwise carry only the exporter pod's IP.
|
||||
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
targetLabel: node
|
||||
@@ -39,4 +39,5 @@ resources:
|
||||
- immich-servicemonitor.yaml
|
||||
- loki-servicemonitor.yaml
|
||||
- smartctl-exporter-servicemonitor.yaml
|
||||
- intel-gpu-exporter-servicemonitor.yaml
|
||||
- probes-blackbox.yaml
|
||||
|
||||
@@ -34,7 +34,9 @@ spec:
|
||||
# query parameter, and an exact match would break silently if the
|
||||
# instance label ever encodes it differently.
|
||||
expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
|
||||
for: 5m
|
||||
# Two consecutive failed probes: enough to skip a single lost probe
|
||||
# through Traefik, not enough to sit on a sealed store.
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
|
||||
@@ -31,7 +31,9 @@ spec:
|
||||
# A replica that is sealed never recovers on its own, so the
|
||||
# duration is only there to ignore ordinary rescheduling: a healthy
|
||||
# pod moving between nodes is Ready again in well under a minute.
|
||||
expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 3
|
||||
# max by: kube-state-metrics stamps instance/pod on the series, so a
|
||||
# rescheduled exporter starts a new series and resets the timer.
|
||||
expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
@@ -53,11 +55,11 @@ spec:
|
||||
# the ClusterSecretStore goes InvalidProviderConfig and no
|
||||
# ExternalSecret can refresh.
|
||||
#
|
||||
# Shorter window than the warning above because this is an outage,
|
||||
# not a degradation — but not zero, so a rolling restart that
|
||||
# briefly dips does not page.
|
||||
expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 2
|
||||
for: 5m
|
||||
# No window: below quorum the secret store is down and every
|
||||
# ExternalSecret stalls with it. A sealed replica never recovers on
|
||||
# its own, so waiting only delays the page.
|
||||
expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
{
|
||||
"title": "Intel iGPU",
|
||||
"uid": "intel-gpu",
|
||||
"tags": ["gpu", "intel", "i915"],
|
||||
"schemaVersion": 39,
|
||||
"editable": true,
|
||||
"timezone": "",
|
||||
"refresh": "1m",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "node",
|
||||
"label": "Node",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": {
|
||||
"query": "label_values(gpumon_power, node)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"definition": "label_values(gpumon_power, node)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"current": { "selected": true, "text": ["All"], "value": ["$__all"] }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Render/3D busy now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "GPU power now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"decimals": 1,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "GT frequency now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "suffix: MHz",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Render/3D busy",
|
||||
"description": "Compute and 3D. llama.cpp's Vulkan backend runs here; 0 % during an inference means the model is on the CPU.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Video busy",
|
||||
"description": "Fixed-function decode/encode (QuickSync).",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Video\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "VideoEnhance busy",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"VideoEnhance\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Blitter busy",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Blitter\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GT frequency (actual)",
|
||||
"description": "0 while the GT sits in RC6.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "suffix: MHz",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GPU power",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Package power",
|
||||
"description": "Whole SoC (CPU + GPU + uncore), from the same PMU.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"pkg\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory controller bandwidth",
|
||||
"description": "Reads solid, writes dashed. The iGPU shares DDR5 with the CPU, so this is the whole package's traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "MiBs",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/write/" }, "properties": [{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_imc_read{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} read",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "gpumon_imc_write{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} write",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -77,6 +77,9 @@ configMapGenerator:
|
||||
- name: dashboard-infrastructure-overview
|
||||
files:
|
||||
- infrastructure-overview.json
|
||||
- name: dashboard-intel-gpu
|
||||
files:
|
||||
- intel-gpu.json
|
||||
- name: dashboard-netbird-overview
|
||||
files:
|
||||
- netbird-overview.json
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# Intel iGPU utilisation, frequency and power for every node (#752).
|
||||
#
|
||||
# Not privileged: /dev/dri comes from the device plugin's monitoring resource
|
||||
# (every GPU on the node, no hostPath), and the i915 PMU that intel_gpu_top
|
||||
# reads needs CAP_PERFMON — the one capability granted. No hostPID, no
|
||||
# hostNetwork.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
# Every node is a control plane; without this the exporter covers nothing.
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: intel-gpu-exporter
|
||||
image: ghcr.io/clambin/intel-gpu-exporter:0.7.4@sha256:d78761fef018834c759db154fe9dcf3dd9db5974703fa50ca21f6abc35221be1
|
||||
args:
|
||||
# No -device: intel_gpu_top picks the only GPU the plugin injects.
|
||||
- -interval=5s
|
||||
ports:
|
||||
- name: http-metrics
|
||||
containerPort: 9100
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
# Root so the PERFMON capability is effective without file
|
||||
# capabilities on the binary.
|
||||
runAsUser: 0
|
||||
runAsNonRoot: false
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["PERFMON"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
gpu.intel.com/monitoring: 1
|
||||
limits:
|
||||
memory: 128Mi
|
||||
gpu.intel.com/monitoring: 1
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: http-metrics
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
ports:
|
||||
- name: http-metrics
|
||||
port: 9100
|
||||
targetPort: http-metrics
|
||||
protocol: TCP
|
||||
@@ -16,6 +16,7 @@ resources:
|
||||
- prometheus-gate-ingress.yaml
|
||||
- ci-diagnostics-rbac.yaml
|
||||
- smartctl-exporter.yaml
|
||||
- intel-gpu-exporter.yaml
|
||||
- dashboards/
|
||||
# prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
|
||||
# separate Flux Kustomization that dependsOn `monitoring` — otherwise the
|
||||
|
||||
@@ -72,6 +72,9 @@ spec:
|
||||
# they contend for it. Sized for an inference server plus room for
|
||||
# a transcoder and a benchmark pod alongside it.
|
||||
- -shared-dev-num=4
|
||||
# Advertises gpu.intel.com/monitoring: one resource that grants a
|
||||
# pod every GPU on the node instead of a slot. The exporter needs it.
|
||||
- -enable-monitoring
|
||||
securityContext:
|
||||
seLinuxOptions:
|
||||
type: "container_device_plugin_t"
|
||||
|
||||
@@ -77,8 +77,10 @@ spec:
|
||||
containers:
|
||||
- name: controller
|
||||
# Pinned by digest: the date tag is rebuilt on every push of the
|
||||
# image directory, so the tag alone cannot express a new build.
|
||||
image: gitea.renner.uno/gitea_admin/netbird-proxy-services:2026-09-07@sha256:0d266b0288aec692c698ff5c05f76c826e247bfef365e0306a0b9a6541ebf180
|
||||
# image directory (branch and master alike) and the registry keeps
|
||||
# only the newest manifest per tag, so the digest must come from
|
||||
# the MASTER build of the code change.
|
||||
image: gitea.renner.uno/gitea_admin/netbird-proxy-services:2026-09-07@sha256:caffa308d802c6fcc6aea919ac94657c08b993651baa6b8d18de395bcc40523e
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
|
||||
@@ -25,10 +25,10 @@ spec:
|
||||
name: openbao
|
||||
namespace: openbao
|
||||
install:
|
||||
# A fresh OpenBao pod is 0/1 Ready until `bao operator init` + unseal
|
||||
# (readiness probe reports sealed). helm --wait would time out and
|
||||
# remediation would uninstall the release, so don't wait here; the
|
||||
# init/unseal procedure is manual (see kubernetes/MIGRATION-sops-openbao.md).
|
||||
# A fresh OpenBao pod is 0/1 Ready until `bao operator init` (readiness
|
||||
# reports sealed). helm --wait would time out and remediation would
|
||||
# uninstall the release, so don't wait here; init is manual (see
|
||||
# kubernetes/MIGRATION-sops-openbao.md). Unseal is automatic (static seal).
|
||||
disableWait: true
|
||||
remediation:
|
||||
retries: 3
|
||||
@@ -42,6 +42,11 @@ spec:
|
||||
# the agent-injector mutating webhook is unused surface.
|
||||
enabled: false
|
||||
server:
|
||||
# The static seal key, from the openbao-seal-key Secret (bootstrap.sh).
|
||||
extraSecretEnvironmentVars:
|
||||
- envName: BAO_SEAL_KEY
|
||||
secretName: openbao-seal-key
|
||||
secretKey: key
|
||||
# Raft (not standalone/file) for two reasons: `bao operator raft
|
||||
# snapshot` as a consistent backup primitive, and real HA across three
|
||||
# nodes. TLS terminates at the Traefik ingress; in-cluster traffic is
|
||||
@@ -53,16 +58,8 @@ spec:
|
||||
# the moment one pod is sealed or rescheduled, which is worse than
|
||||
# the single replica it replaced. Three tolerates one.
|
||||
#
|
||||
# What this buys, and what it does not: a node drain seals the replica
|
||||
# on it, and raft elects a new leader from the remaining standbys
|
||||
# within seconds, so the service does not go down. It does NOT remove
|
||||
# the manual unseal — a sealed server can neither serve requests nor
|
||||
# act as standby, so that replica stays out until someone unseals it.
|
||||
#
|
||||
# The consequence for rolling node maintenance is worth stating: a
|
||||
# Talos upgrade drains all three nodes in turn, sealing each replica
|
||||
# as it goes. Unseal each one as it comes back, or the third drain
|
||||
# takes the last unsealed replica with it and quorum is lost anyway.
|
||||
# A restarted replica unseals itself from the static seal and rejoins
|
||||
# raft, so a rolling reboot only ever costs one replica at a time.
|
||||
replicas: 3
|
||||
raft:
|
||||
enabled: true
|
||||
@@ -111,6 +108,14 @@ spec:
|
||||
|
||||
service_registration "kubernetes" {}
|
||||
|
||||
# Auto-unseal. The key is a 32-byte AES key in the openbao-seal-key
|
||||
# Secret, created from pass at bootstrap before this release. The
|
||||
# former shamir key is the recovery key (#805).
|
||||
seal "static" {
|
||||
current_key_id = "1"
|
||||
current_key = "env://BAO_SEAL_KEY"
|
||||
}
|
||||
|
||||
audit "file" "stdout" {
|
||||
description = "audit trail to pod stdout -> Loki"
|
||||
options {
|
||||
|
||||
Reference in New Issue
Block a user