diff --git a/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml b/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml index e38a9fbf..2a159a9d 100644 --- a/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml +++ b/kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml @@ -4,7 +4,9 @@ metadata: name: pve-exporter-config namespace: monitoring spec: - refreshInterval: 1h + # 10m (was 1h) so an automated token rotation propagates within ~10m; Reloader + # then restarts pve-exporter. See kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml. + refreshInterval: 10m secretStoreRef: kind: ClusterSecretStore name: openbao diff --git a/kubernetes/apps/monitoring/pve-exporter-deployment.yaml b/kubernetes/apps/monitoring/pve-exporter-deployment.yaml index 8f807a86..7738c39d 100644 --- a/kubernetes/apps/monitoring/pve-exporter-deployment.yaml +++ b/kubernetes/apps/monitoring/pve-exporter-deployment.yaml @@ -3,6 +3,11 @@ kind: Deployment metadata: name: pve-exporter namespace: monitoring + annotations: + # Reloader restarts pve-exporter when pve-exporter-config changes (token + # rotation) so it reloads pve.yml. See + # kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml. + secret.reloader.stakater.com/reload: pve-exporter-config spec: replicas: 1 selector: diff --git a/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl b/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl new file mode 100644 index 00000000..d153747e --- /dev/null +++ b/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl @@ -0,0 +1,34 @@ +# OpenBao policy: rotator-proxmox +# +# Class-B rotation for the pve-exporter Proxmox API token. Grants the +# proxmox-secret-rotator CronJob READ on its bootstrap credential (a PRIVILEGED +# Proxmox token allowed to delete/create tokens) and PATCH on the target +# pve-exporter-config path. Per-secret isolation (#213/#202). +# +# NOT applied by Flux. Apply ONCE with an admin token: +# +# bao policy write rotator-proxmox kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl +# bao write auth/kubernetes/role/rotator-proxmox \ +# bound_service_account_names=proxmox-secret-rotator \ +# bound_service_account_namespaces=openbao \ +# policies=rotator-proxmox ttl=15m +# +# BOOTSTRAP CREDENTIAL you must provision first (one-time, by hand): +# A SEPARATE privileged Proxmox API token (NOT the pve-exporter@pve!monitoring +# token this Job rotates) with permission to manage tokens on pve-exporter@pve +# (e.g. root@pam!, or a dedicated rotator token with the right ACL): +# bao kv put homelab/infrastructure/secret-rotators-bootstrap/proxmox \ +# api-url=https://jupiter.renner.uno:8006 \ +# auth='pve-rotator@pve!rotate=' +# +# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy). + +# Bootstrap: the privileged Proxmox token + API URL. +path "homelab/data/infrastructure/secret-rotators-bootstrap/proxmox" { + capabilities = ["read"] +} + +# Target: the pve-exporter config (holds the pve.yml with the token_value). +path "homelab/data/apps/monitoring/pve-exporter-config" { + capabilities = ["read", "patch"] +} diff --git a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml index 94dc38f5..d674cf81 100644 --- a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml +++ b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml @@ -12,3 +12,4 @@ resources: - crowdsec-bouncer-key.yaml - netbird-ingress.yaml - cloudflare.yaml + - pve-exporter.yaml diff --git a/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml b/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml new file mode 100644 index 00000000..84318611 --- /dev/null +++ b/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml @@ -0,0 +1,128 @@ +--- +# Class-B automated rotation of the pve-exporter Proxmox API token +# (pve-exporter@pve!monitoring). Proxmox has NO in-place secret roll, so this +# DELETEs and RE-CREATEs the token (brief monitoring gap), regenerates the +# pve.yml config with the new token_value, and patches OpenBao. openbao-ns +# pattern (Proxmox LAN API reachable via the openbao ns `world` egress). ESO +# syncs the pve-exporter-config Secret (ns monitoring, 10m refresh); Reloader +# restarts the pve-exporter Deployment to pick up the new config file. +# +# ┌─ SUSPENDED (spec.suspend: true) — the riskiest Class-B job ──────────────┐ +# │ UNTESTED. Before enabling: │ +# │ 1. Provision the bootstrap privileged Proxmox token (see rotator-proxmox.hcl). │ +# │ 2. VERIFY the token's privsep setting. This script re-creates with │ +# │ privsep=0 (token inherits pve-exporter@pve's own perms → no separate │ +# │ token-ACL to re-apply). If the current monitoring token uses privsep=1 │ +# │ with its OWN ACL, you MUST re-grant that ACL after re-creation, or the │ +# │ exporter loses read access. Check: GET /access/users/pve-exporter@pve/token/monitoring │ +# │ 3. DELETE+create causes a brief window with NO valid token — pve-exporter │ +# │ metrics gap until ESO(≤10m)+Reloader restart. Quarterly/off-hours ok. │ +# │ 4. Test on-demand, confirm metrics resume, THEN set suspend: false. │ +# └───────────────────────────────────────────────────────────────────────────┘ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-secret-rotator + namespace: openbao + labels: + app.kubernetes.io/name: pve-exporter + app.kubernetes.io/component: secret-rotator + # OpenBao: auth/kubernetes/role/rotator-proxmox binds this SA to rotator-proxmox + # (read bootstrap + patch the pve-exporter-config path). +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: proxmox-secret-rotate + namespace: openbao + labels: + app.kubernetes.io/name: pve-exporter + app.kubernetes.io/component: secret-rotator +spec: + suspend: true # UNTESTED + monitoring gap — see header; enable only after test + schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin + timeZone: Europe/Berlin + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 5 + startingDeadlineSeconds: 3600 + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 259200 # 3 d + template: + metadata: + labels: + app.kubernetes.io/name: pve-exporter + app.kubernetes.io/component: secret-rotator + spec: + serviceAccountName: proxmox-secret-rotator + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: rotate + # python:alpine ships an HTTP client + JSON via stdlib (urllib/json/ssl) + # — NO runtime package install, runs under the non-root restricted PSA + # (apk/apt need root and fail here). + image: python:3.13-alpine + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + - name: BAO_ADDR + value: http://openbao.openbao.svc.cluster.local:8200 + - name: PVE_USER + value: "pve-exporter@pve" + - name: PVE_TOKENID + value: "monitoring" + command: ["python3", "-c"] + args: + - | + import json, os, ssl, urllib.parse, urllib.request as u + bao = os.environ["BAO_ADDR"]; user = os.environ["PVE_USER"]; tid = os.environ["PVE_TOKENID"] + jwt = open("/var/run/secrets/kubernetes.io/serviceaccount/token").read() + # Proxmox uses a self-signed cert → dedicated no-verify context (used + # ONLY for the PVE calls; OpenBao is plain http in-cluster). + novrfy = ssl.create_default_context(); novrfy.check_hostname = False; novrfy.verify_mode = ssl.CERT_NONE + def call(method, url, headers=None, body=None, form=None, ct="application/json", ctx=None): + data = None + if form is not None: + data = urllib.parse.urlencode(form).encode(); ct = "application/x-www-form-urlencoded" + elif body is not None: + data = json.dumps(body).encode() + req = u.Request(url, data=data, method=method, headers=headers or {}) + if data is not None: + req.add_header("Content-Type", ct) + with u.urlopen(req, timeout=30, context=ctx) as r: + return json.loads(r.read() or b"{}") + tok = call("POST", bao + "/v1/auth/kubernetes/login", + body={"role": "rotator-proxmox", "jwt": jwt})["auth"]["client_token"] + boot = call("GET", bao + "/v1/homelab/data/infrastructure/secret-rotators-bootstrap/proxmox", + {"X-Vault-Token": tok})["data"]["data"] + url, auth = boot["api-url"], boot["auth"] + H = {"Authorization": "PVEAPIToken=" + auth} + base = url + "/api2/json/access/users/" + user + "/token/" + tid + # Proxmox has no in-place roll: DELETE then re-create (privsep=0) → new secret. + try: + call("DELETE", base, H, ctx=novrfy) + except Exception: + pass + new = call("POST", base, H, form={"privsep": "0"}, ctx=novrfy)["data"]["value"] + yml = ("default:\n user: %s\n token_name: %s\n token_value: %s\n verify_ssl: false\n" + % (user, tid, new)) + call("PATCH", bao + "/v1/homelab/data/apps/monitoring/pve-exporter-config", + {"X-Vault-Token": tok}, {"data": {"pve.yml": yml}}, ct="application/merge-patch+json") + print("rotated Proxmox token %s!%s + patched pve-exporter-config (value not logged)" % (user, tid)) + resources: + requests: + cpu: 25m + memory: 48Mi + limits: + memory: 96Mi