kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl

@@ -13,13 +13,32 @@
 #     bound_service_account_namespaces=openbao \
 #     policies=rotator-proxmox ttl=15m
 #
-# BOOTSTRAP CREDENTIAL you must provision first (one-time, by hand):
-#   A SEPARATE privileged Proxmox API token (NOT the pve-exporter@pve!monitoring
-#   token this Job rotates) with permission to manage tokens on pve-exporter@pve
-#   (e.g. root@pam!<id>, or a dedicated rotator token with the right ACL):
+# BOOTSTRAP CREDENTIAL (PROVISIONED — recorded here for rebuild/DR).
+#
+# A SEPARATE least-privilege Proxmox identity, NOT root@pam. PVE has no
+# per-user ACL path (/access/users/<id> is rejected — see AccessControl.pm
+# check_path), and the token endpoints authorise via a `userid-group` check.
+# So the tightest possible scope is a group containing only the target user:
+#
+#   pveum role add SecretRotator -privs "User.Modify"
+#   pveum user add secret-rotator@pve
+#   pveum group add secret-rotated
+#   pveum user modify pve-exporter@pve --groups secret-rotated
+#   pveum acl modify /access/groups/secret-rotated \
+#     --users secret-rotator@pve --roles SecretRotator
+#   pveum user token add secret-rotator@pve rotate --privsep 0
+#
+# This grants exactly one thing: managing API tokens of users in the
+# secret-rotated group. Verified: no VM/storage visibility, and minting a
+# root@pam token returns 403.
+#
 #   bao kv put homelab/infrastructure/secret-rotators-bootstrap/proxmox \
-#     api-url=https://jupiter.renner.uno:8006 \
-#     auth='pve-rotator@pve!rotate=<secret>'
+#     api-url=https://pve.apps.svc.cluster.local:8006 \
+#     auth='secret-rotator@pve!rotate=<secret>'
+#
+# api-url MUST be the in-cluster Service (backed by all three PVE nodes).
+# The external name pve.renner.uno does NOT work from a pod — pods cannot
+# hairpin to their own cluster's ingress LB and the Job hangs.
 #
 # Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
 

kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml

@@ -7,17 +7,28 @@
 # syncs the pve-exporter-config Secret (ns monitoring, 10m refresh); Reloader
 # restarts the pve-exporter Deployment to pick up the new config file.
 #
-# ┌─ SUSPENDED (spec.suspend: true) — the riskiest Class-B job ──────────────┐
-# │ UNTESTED. Before enabling:                                                │
-# │  1. Provision the bootstrap privileged Proxmox token (see rotator-proxmox.hcl). │
-# │  2. VERIFY the token's privsep setting. This script re-creates with       │
-# │     privsep=0 (token inherits pve-exporter@pve's own perms → no separate  │
-# │     token-ACL to re-apply). If the current monitoring token uses privsep=1 │
-# │     with its OWN ACL, you MUST re-grant that ACL after re-creation, or the │
-# │     exporter loses read access. Check: GET /access/users/pve-exporter@pve/token/monitoring │
-# │  3. DELETE+create causes a brief window with NO valid token — pve-exporter │
-# │     metrics gap until ESO(≤10m)+Reloader restart. Quarterly/off-hours ok.  │
-# │  4. Test on-demand, confirm metrics resume, THEN set suspend: false.       │
+# ┌─ TESTED end-to-end and ENABLED ──────────────────────────────────────────┐
+# │ The privsep caveat from #306 is RESOLVED: pve-exporter@pve!monitoring is  │
+# │ privsep=0 and its ACL (/ → PVEAuditor) lives on the USER, not the token,  │
+# │ so DELETE+re-create with privsep=0 loses no permission — nothing to       │
+# │ re-grant. Verified by two full rotations; metrics resumed both times.     │
+# │                                                                           │
+# │ Bootstrap credential is a dedicated least-privilege PVE identity, NOT     │
+# │ root (see rotator-proxmox.hcl): secret-rotator@pve!rotate, holding role   │
+# │ SecretRotator (User.Modify) scoped to /access/groups/secret-rotated —     │
+# │ a group whose only member is pve-exporter@pve. Verified it can mint and   │
+# │ delete that user's tokens, sees no VMs/storage, and is 403 on minting a   │
+# │ root@pam token.                                                           │
+# │                                                                           │
+# │ api-url is the in-cluster Service https://pve.apps.svc.cluster.local:8006 │
+# │ (all three PVE nodes back it, so a single node being down is tolerated).  │
+# │ Do NOT use the external name pve.renner.uno — pods cannot hairpin to      │
+# │ their own cluster's ingress LB and the job hangs. Do NOT pin a node IP    │
+# │ or a single node FQDN (jupiter/…): both are a needless SPOF.              │
+# │                                                                           │
+# │ DELETE+create still leaves a brief window with no valid token —           │
+# │ pve-exporter metrics gap until ESO(≤10m)+Reloader restart. Quarterly      │
+# │ off-hours schedule keeps that acceptable.                                 │
 # └───────────────────────────────────────────────────────────────────────────┘
 apiVersion: v1
 kind: ServiceAccount
@@ -39,7 +50,7 @@ metadata:
     app.kubernetes.io/name: pve-exporter
     app.kubernetes.io/component: secret-rotator
 spec:
-  suspend: true # UNTESTED + monitoring gap — see header; enable only after test
+  suspend: false # tested end-to-end (two full rotations, metrics resumed) — see header
   schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin
   timeZone: Europe/Berlin
   concurrencyPolicy: Forbid