From 69ecab3a064446fad6e131324411f7c97ad875b2 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 19 Jul 2026 00:11:22 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(proxmox):=20automated=20pve-exporter?= =?UTF-8?q?=20token=20rotation=20(Class-B,=20openbao-ns)=20=E2=80=94=20SUS?= =?UTF-8?q?PENDED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rotates pve-exporter@pve!monitoring via the Proxmox API. Proxmox has no in-place secret roll, so DELETE+re-create (privsep=0), regenerate pve.yml with the new token_value, patch OpenBao. openbao-ns pattern; per-secret rotator-proxmox role reads a bootstrap privileged Proxmox token + patches pve-exporter-config. refreshInterval 1h->10m; Reloader restarts pve-exporter. SHIPPED SUSPENDED + UNTESTED (riskiest Class-B): needs a bootstrap privileged Proxmox token at homelab/infrastructure/secret-rotators-bootstrap/proxmox, VERIFICATION that the token uses privsep=0 (else re-grant its ACL after re-create), and awareness of the brief monitoring gap during DELETE+create. Test before spec.suspend=false. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_015K7gLUVVnutvDnzV2TrMRJ --- .../pve-exporter-config-externalsecret.yaml | 4 +- .../monitoring/pve-exporter-deployment.yaml | 5 + .../openbao/policies/rotator-proxmox.hcl | 34 ++++++ .../openbao/rotation/kustomization.yaml | 1 + .../openbao/rotation/pve-exporter.yaml | 114 ++++++++++++++++++ 5 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl create mode 100644 kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml diff --git a/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml b/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml index e38a9fbf..2a159a9d 100644 --- a/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml +++ b/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml @@ -4,7 +4,9 @@ metadata: name: pve-exporter-config namespace: monitoring spec: - refreshInterval: 1h + # 10m (was 1h) so an automated token rotation propagates within ~10m; Reloader + # then restarts pve-exporter. See kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml. + refreshInterval: 10m secretStoreRef: kind: ClusterSecretStore name: openbao diff --git a/kubernetes/apps/monitoring/pve-exporter-deployment.yaml b/kubernetes/apps/monitoring/pve-exporter-deployment.yaml index 8f807a86..7738c39d 100644 --- a/kubernetes/apps/monitoring/pve-exporter-deployment.yaml +++ b/kubernetes/apps/monitoring/pve-exporter-deployment.yaml @@ -3,6 +3,11 @@ kind: Deployment metadata: name: pve-exporter namespace: monitoring + annotations: + # Reloader restarts pve-exporter when pve-exporter-config changes (token + # rotation) so it reloads pve.yml. See + # kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml. + secret.reloader.stakater.com/reload: pve-exporter-config spec: replicas: 1 selector: diff --git a/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl b/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl new file mode 100644 index 00000000..d153747e --- /dev/null +++ b/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl @@ -0,0 +1,34 @@ +# OpenBao policy: rotator-proxmox +# +# Class-B rotation for the pve-exporter Proxmox API token. Grants the +# proxmox-secret-rotator CronJob READ on its bootstrap credential (a PRIVILEGED +# Proxmox token allowed to delete/create tokens) and PATCH on the target +# pve-exporter-config path. Per-secret isolation (#213/#202). +# +# NOT applied by Flux. Apply ONCE with an admin token: +# +# bao policy write rotator-proxmox kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl +# bao write auth/kubernetes/role/rotator-proxmox \ +# bound_service_account_names=proxmox-secret-rotator \ +# bound_service_account_namespaces=openbao \ +# policies=rotator-proxmox ttl=15m +# +# BOOTSTRAP CREDENTIAL you must provision first (one-time, by hand): +# A SEPARATE privileged Proxmox API token (NOT the pve-exporter@pve!monitoring +# token this Job rotates) with permission to manage tokens on pve-exporter@pve +# (e.g. root@pam!, or a dedicated rotator token with the right ACL): +# bao kv put homelab/infrastructure/secret-rotators-bootstrap/proxmox \ +# api-url=https://jupiter.renner.uno:8006 \ +# auth='pve-rotator@pve!rotate=' +# +# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy). + +# Bootstrap: the privileged Proxmox token + API URL. +path "homelab/data/infrastructure/secret-rotators-bootstrap/proxmox" { + capabilities = ["read"] +} + +# Target: the pve-exporter config (holds the pve.yml with the token_value). +path "homelab/data/apps/monitoring/pve-exporter-config" { + capabilities = ["read", "patch"] +} diff --git a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml index cb05c1cc..28af1cbc 100644 --- a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml +++ b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml @@ -10,3 +10,4 @@ resources: - shelly-manager.yaml - authentik.yaml - crowdsec-bouncer-key.yaml + - pve-exporter.yaml diff --git a/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml b/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml new file mode 100644 index 00000000..65ae9bf7 --- /dev/null +++ b/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml @@ -0,0 +1,114 @@ +--- +# Class-B automated rotation of the pve-exporter Proxmox API token +# (pve-exporter@pve!monitoring). Proxmox has NO in-place secret roll, so this +# DELETEs and RE-CREATEs the token (brief monitoring gap), regenerates the +# pve.yml config with the new token_value, and patches OpenBao. openbao-ns +# pattern (Proxmox LAN API reachable via the openbao ns `world` egress). ESO +# syncs the pve-exporter-config Secret (ns monitoring, 10m refresh); Reloader +# restarts the pve-exporter Deployment to pick up the new config file. +# +# ┌─ SUSPENDED (spec.suspend: true) — the riskiest Class-B job ──────────────┐ +# │ UNTESTED. Before enabling: │ +# │ 1. Provision the bootstrap privileged Proxmox token (see rotator-proxmox.hcl). │ +# │ 2. VERIFY the token's privsep setting. This script re-creates with │ +# │ privsep=0 (token inherits pve-exporter@pve's own perms → no separate │ +# │ token-ACL to re-apply). If the current monitoring token uses privsep=1 │ +# │ with its OWN ACL, you MUST re-grant that ACL after re-creation, or the │ +# │ exporter loses read access. Check: GET /access/users/pve-exporter@pve/token/monitoring │ +# │ 3. DELETE+create causes a brief window with NO valid token — pve-exporter │ +# │ metrics gap until ESO(≤10m)+Reloader restart. Quarterly/off-hours ok. │ +# │ 4. Test on-demand, confirm metrics resume, THEN set suspend: false. │ +# └───────────────────────────────────────────────────────────────────────────┘ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-secret-rotator + namespace: openbao + labels: + app.kubernetes.io/name: pve-exporter + app.kubernetes.io/component: secret-rotator + # OpenBao: auth/kubernetes/role/rotator-proxmox binds this SA to rotator-proxmox + # (read bootstrap + patch the pve-exporter-config path). +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: proxmox-secret-rotate + namespace: openbao + labels: + app.kubernetes.io/name: pve-exporter + app.kubernetes.io/component: secret-rotator +spec: + suspend: true # UNTESTED + monitoring gap — see header; enable only after test + schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin + timeZone: Europe/Berlin + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 5 + startingDeadlineSeconds: 3600 + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 259200 # 3 d + template: + metadata: + labels: + app.kubernetes.io/name: pve-exporter + app.kubernetes.io/component: secret-rotator + spec: + serviceAccountName: proxmox-secret-rotator + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: rotate + image: alpine:3.21 # + runtime apk add curl jq (Class-B HTTP client) + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + - name: BAO_ADDR + value: http://openbao.openbao.svc.cluster.local:8200 + - name: PVE_USER + value: "pve-exporter@pve" + - name: PVE_TOKENID + value: "monitoring" + command: ["/bin/sh", "-ec"] + args: + - | + apk add --no-cache curl jq ca-certificates >/dev/null + JWT="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + T=$(curl -sf --request POST "$BAO_ADDR/v1/auth/kubernetes/login" \ + -d "{\"role\":\"rotator-proxmox\",\"jwt\":\"$JWT\"}" | jq -r .auth.client_token) + [ -n "$T" ] && [ "$T" != null ] || { echo "openbao login failed"; exit 1; } + BOOT=$(curl -sf -H "X-Vault-Token: $T" \ + "$BAO_ADDR/v1/homelab/data/infrastructure/secret-rotators-bootstrap/proxmox") + URL=$(echo "$BOOT" | jq -r .data.data."api-url") + AUTH=$(echo "$BOOT" | jq -r .data.data.auth) + [ -n "$URL" ] && [ "$URL" != null ] && [ -n "$AUTH" ] && [ "$AUTH" != null ] \ + || { echo "missing bootstrap api-url/auth"; exit 1; } + H="Authorization: PVEAPIToken=$AUTH" + # DELETE the old token, then re-create it (privsep=0) → new secret. + curl -sk -H "$H" -X DELETE "$URL/api2/json/access/users/$PVE_USER/token/$PVE_TOKENID" >/dev/null || true + NEW=$(curl -sk -H "$H" -X POST "$URL/api2/json/access/users/$PVE_USER/token/$PVE_TOKENID" \ + --data-urlencode "privsep=0" | jq -r '.data.value') + [ -n "$NEW" ] && [ "$NEW" != null ] || { echo "proxmox token re-create failed"; exit 1; } + # Regenerate pve.yml with the new token_value and patch OpenBao. + YML=$(printf 'default:\n user: %s\n token_name: %s\n token_value: %s\n verify_ssl: false\n' \ + "$PVE_USER" "$PVE_TOKENID" "$NEW") + jq -n --arg y "$YML" '{data:{"pve.yml":$y}}' \ + | curl -sf -H "X-Vault-Token: $T" -H "Content-Type: application/merge-patch+json" \ + -X PATCH "$BAO_ADDR/v1/homelab/data/apps/monitoring/pve-exporter-config" -d @- >/dev/null + echo "rotated Proxmox token $PVE_USER!$PVE_TOKENID + patched pve-exporter-config (value not logged)" + resources: + requests: + cpu: 25m + memory: 48Mi + limits: + memory: 96Mi -- 2.54.0 From fd85dbb90150ac0a7942602b5e9ebdaad972bff8 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 19 Jul 2026 09:13:55 +0200 Subject: [PATCH 2/2] fix(proxmox-rotation): python:alpine (apk fails non-root; lesson from netbird test) Same fix proven on netbird (#307/#308): alpine+apk -> python:3.13-alpine stdlib so it runs under the non-root restricted PSA. Uses a dedicated no-verify SSL context for the self-signed Proxmox API only (OpenBao stays plain http). Still suspended + untested; the privsep=0 caveat + brief monitoring gap on DELETE+recreate remain documented in the header. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_015K7gLUVVnutvDnzV2TrMRJ -- 2.54.0