kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml

@@ -4,7 +4,9 @@ metadata:
   name: pve-exporter-config
   namespace: monitoring
 spec:
-  refreshInterval: 1h
+  # 10m (was 1h) so an automated token rotation propagates within ~10m; Reloader
+  # then restarts pve-exporter. See kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml.
+  refreshInterval: 10m
   secretStoreRef:
     kind: ClusterSecretStore
     name: openbao

kubernetes/apps/monitoring/pve-exporter-deployment.yaml

@@ -3,6 +3,11 @@ kind: Deployment
 metadata:
   name: pve-exporter
   namespace: monitoring
+  annotations:
+    # Reloader restarts pve-exporter when pve-exporter-config changes (token
+    # rotation) so it reloads pve.yml. See
+    # kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml.
+    secret.reloader.stakater.com/reload: pve-exporter-config
 spec:
   replicas: 1
   selector:

kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl

@@ -0,0 +1,34 @@
+# OpenBao policy: rotator-proxmox
+#
+# Class-B rotation for the pve-exporter Proxmox API token. Grants the
+# proxmox-secret-rotator CronJob READ on its bootstrap credential (a PRIVILEGED
+# Proxmox token allowed to delete/create tokens) and PATCH on the target
+# pve-exporter-config path. Per-secret isolation (#213/#202).
+#
+# NOT applied by Flux. Apply ONCE with an admin token:
+#
+#   bao policy write rotator-proxmox kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl
+#   bao write auth/kubernetes/role/rotator-proxmox \
+#     bound_service_account_names=proxmox-secret-rotator \
+#     bound_service_account_namespaces=openbao \
+#     policies=rotator-proxmox ttl=15m
+#
+# BOOTSTRAP CREDENTIAL you must provision first (one-time, by hand):
+#   A SEPARATE privileged Proxmox API token (NOT the pve-exporter@pve!monitoring
+#   token this Job rotates) with permission to manage tokens on pve-exporter@pve
+#   (e.g. root@pam!<id>, or a dedicated rotator token with the right ACL):
+#   bao kv put homelab/infrastructure/secret-rotators-bootstrap/proxmox \
+#     api-url=https://jupiter.renner.uno:8006 \
+#     auth='pve-rotator@pve!rotate=<secret>'
+#
+# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
+
+# Bootstrap: the privileged Proxmox token + API URL.
+path "homelab/data/infrastructure/secret-rotators-bootstrap/proxmox" {
+  capabilities = ["read"]
+}
+
+# Target: the pve-exporter config (holds the pve.yml with the token_value).
+path "homelab/data/apps/monitoring/pve-exporter-config" {
+  capabilities = ["read", "patch"]
+}

kubernetes/infrastructure/openbao/rotation/kustomization.yaml

@@ -12,3 +12,4 @@ resources:
   - crowdsec-bouncer-key.yaml
   - netbird-ingress.yaml
   - cloudflare.yaml
+  - pve-exporter.yaml

kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml

@@ -0,0 +1,128 @@
+---
+# Class-B automated rotation of the pve-exporter Proxmox API token
+# (pve-exporter@pve!monitoring). Proxmox has NO in-place secret roll, so this
+# DELETEs and RE-CREATEs the token (brief monitoring gap), regenerates the
+# pve.yml config with the new token_value, and patches OpenBao. openbao-ns
+# pattern (Proxmox LAN API reachable via the openbao ns `world` egress). ESO
+# syncs the pve-exporter-config Secret (ns monitoring, 10m refresh); Reloader
+# restarts the pve-exporter Deployment to pick up the new config file.
+#
+# ┌─ SUSPENDED (spec.suspend: true) — the riskiest Class-B job ──────────────┐
+# │ UNTESTED. Before enabling:                                                │
+# │  1. Provision the bootstrap privileged Proxmox token (see rotator-proxmox.hcl). │
+# │  2. VERIFY the token's privsep setting. This script re-creates with       │
+# │     privsep=0 (token inherits pve-exporter@pve's own perms → no separate  │
+# │     token-ACL to re-apply). If the current monitoring token uses privsep=1 │
+# │     with its OWN ACL, you MUST re-grant that ACL after re-creation, or the │
+# │     exporter loses read access. Check: GET /access/users/pve-exporter@pve/token/monitoring │
+# │  3. DELETE+create causes a brief window with NO valid token — pve-exporter │
+# │     metrics gap until ESO(≤10m)+Reloader restart. Quarterly/off-hours ok.  │
+# │  4. Test on-demand, confirm metrics resume, THEN set suspend: false.       │
+# └───────────────────────────────────────────────────────────────────────────┘
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: proxmox-secret-rotator
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: pve-exporter
+    app.kubernetes.io/component: secret-rotator
+  # OpenBao: auth/kubernetes/role/rotator-proxmox binds this SA to rotator-proxmox
+  # (read bootstrap + patch the pve-exporter-config path).
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: proxmox-secret-rotate
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: pve-exporter
+    app.kubernetes.io/component: secret-rotator
+spec:
+  suspend: true # UNTESTED + monitoring gap — see header; enable only after test
+  schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin
+  timeZone: Europe/Berlin
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 2
+  failedJobsHistoryLimit: 5
+  startingDeadlineSeconds: 3600
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      activeDeadlineSeconds: 300
+      ttlSecondsAfterFinished: 259200 # 3 d
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: pve-exporter
+            app.kubernetes.io/component: secret-rotator
+        spec:
+          serviceAccountName: proxmox-secret-rotator
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            runAsGroup: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: rotate
+              # python:alpine ships an HTTP client + JSON via stdlib (urllib/json/ssl)
+              # — NO runtime package install, runs under the non-root restricted PSA
+              # (apk/apt need root and fail here).
+              image: python:3.13-alpine
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              env:
+                - name: BAO_ADDR
+                  value: http://openbao.openbao.svc.cluster.local:8200
+                - name: PVE_USER
+                  value: "pve-exporter@pve"
+                - name: PVE_TOKENID
+                  value: "monitoring"
+              command: ["python3", "-c"]
+              args:
+                - |
+                  import json, os, ssl, urllib.parse, urllib.request as u
+                  bao = os.environ["BAO_ADDR"]; user = os.environ["PVE_USER"]; tid = os.environ["PVE_TOKENID"]
+                  jwt = open("/var/run/secrets/kubernetes.io/serviceaccount/token").read()
+                  # Proxmox uses a self-signed cert → dedicated no-verify context (used
+                  # ONLY for the PVE calls; OpenBao is plain http in-cluster).
+                  novrfy = ssl.create_default_context(); novrfy.check_hostname = False; novrfy.verify_mode = ssl.CERT_NONE
+                  def call(method, url, headers=None, body=None, form=None, ct="application/json", ctx=None):
+                      data = None
+                      if form is not None:
+                          data = urllib.parse.urlencode(form).encode(); ct = "application/x-www-form-urlencoded"
+                      elif body is not None:
+                          data = json.dumps(body).encode()
+                      req = u.Request(url, data=data, method=method, headers=headers or {})
+                      if data is not None:
+                          req.add_header("Content-Type", ct)
+                      with u.urlopen(req, timeout=30, context=ctx) as r:
+                          return json.loads(r.read() or b"{}")
+                  tok = call("POST", bao + "/v1/auth/kubernetes/login",
+                             body={"role": "rotator-proxmox", "jwt": jwt})["auth"]["client_token"]
+                  boot = call("GET", bao + "/v1/homelab/data/infrastructure/secret-rotators-bootstrap/proxmox",
+                              {"X-Vault-Token": tok})["data"]["data"]
+                  url, auth = boot["api-url"], boot["auth"]
+                  H = {"Authorization": "PVEAPIToken=" + auth}
+                  base = url + "/api2/json/access/users/" + user + "/token/" + tid
+                  # Proxmox has no in-place roll: DELETE then re-create (privsep=0) → new secret.
+                  try:
+                      call("DELETE", base, H, ctx=novrfy)
+                  except Exception:
+                      pass
+                  new = call("POST", base, H, form={"privsep": "0"}, ctx=novrfy)["data"]["value"]
+                  yml = ("default:\n  user: %s\n  token_name: %s\n  token_value: %s\n  verify_ssl: false\n"
+                         % (user, tid, new))
+                  call("PATCH", bao + "/v1/homelab/data/apps/monitoring/pve-exporter-config",
+                       {"X-Vault-Token": tok}, {"data": {"pve.yml": yml}}, ct="application/merge-patch+json")
+                  print("rotated Proxmox token %s!%s + patched pve-exporter-config (value not logged)" % (user, tid))
+              resources:
+                requests:
+                  cpu: 25m
+                  memory: 48Mi
+                limits:
+                  memory: 96Mi