velero: restore-test purges its own PV residue

Ceph-block and ceph-filesystem ship with reclaimPolicy: Retain so that
production PVC deletions can be undone manually. The restore-test has
the opposite need: every PVC it creates is throwaway and must not
survive the iteration. Without explicit cleanup each weekly run leaked
O(100 GiB) of CephFS subvolumes, filling the cluster to 95% (OSD_FULL,
HEALTH_ERR, restore writes rejected) on 2026-05-17.

cleanup_service now sweeps every Released PV whose claimRef belongs to
this iteration — destination ns PVCs, velero/ data-mover backup PVCs
(*-rtest-<RUN_ID>-*), and source-side snapshot probe PVCs (rt-src-*-<RUN_ID>) —
patching reclaimPolicy to Delete before removing so CSI purges the
backing storage. RBAC adds patch on persistentvolumes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

kubernetes/infrastructure/velero/restore-test-rbac.yaml

@@ -24,8 +24,13 @@ rules:
     resources: ["namespaces"]
     verbs: ["get", "list", "create", "delete"]
   - apiGroups: [""]
-    resources: ["persistentvolumeclaims", "persistentvolumes"]
+    resources: ["persistentvolumeclaims"]
     verbs: ["get", "list", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    # patch needed to flip reclaimPolicy from Retain → Delete on test residue
+    # before deleting, so CSI purges the backing CephFS subvolume / RBD image.
+    verbs: ["get", "list", "delete", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
     verbs: ["get", "list", "create", "delete"]

kubernetes/infrastructure/velero/restore-test.sh

@@ -47,6 +47,30 @@ check() {
     fi
 }
 
+# Force-delete Released PVs whose claimRef matches a pattern. The ceph-block
+# and ceph-filesystem StorageClasses ship with reclaimPolicy: Retain so that
+# production PVC deletions can be undone manually. The restore-test has the
+# opposite need: every PVC it creates (data-mover backup PVCs in velero ns,
+# destination PVCs in *-restore-test namespaces, source-side snapshot probe
+# PVCs in source namespaces) is throwaway and MUST not survive the iteration.
+# Without this, each weekly run leaks O(100 GiB) of CephFS subvolumes and the
+# cluster eventually goes OSD_FULL (incident 2026-05-17).
+purge_released_pvs() {
+    local pattern="$1"   # regex applied to .spec.claimRef.namespace + "/" + .spec.claimRef.name
+    local pvs pv
+    pvs=$(kubectl get pv -o json 2>/dev/null \
+        | jq -r --arg p "$pattern" '
+            .items[]
+            | select(.status.phase == "Released")
+            | select(((.spec.claimRef.namespace // "") + "/" + (.spec.claimRef.name // "")) | test($p))
+            | .metadata.name')
+    for pv in $pvs; do
+        kubectl patch pv "$pv" --type=merge \
+            -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null 2>&1 || true
+        kubectl delete pv "$pv" --wait=false --ignore-not-found >/dev/null 2>&1 || true
+    done
+}
+
 cleanup_service() {
     local dst_ns="$1" restore="$2"
     [[ -n "$restore" ]] && kubectl delete restore.velero.io "$restore" \
@@ -54,11 +78,19 @@ cleanup_service() {
     kubectl delete ns "$dst_ns" --ignore-not-found --wait=false >/dev/null 2>&1 || true
     # Block until namespace is actually gone, otherwise the next service's
     # pre-cleanup races with leftover terminating pods.
+    local ns_gone=1
     for _ in $(seq 1 60); do
-        kubectl get ns "$dst_ns" >/dev/null 2>&1 || return 0
+        if ! kubectl get ns "$dst_ns" >/dev/null 2>&1; then
+            ns_gone=0; break
+        fi
         sleep 5
     done
-    return 1
+    # Reclaim PVs left behind in Released state for this iteration:
+    #   - destination PVCs in dst_ns
+    #   - Velero data-mover backup PVCs in velero/ named *-rtest-${RUN_ID}-*
+    #   - source-side snapshot-probe PVCs in any ns named rt-src-*-${RUN_ID}
+    purge_released_pvs "^(${dst_ns}/.*|velero/.*-rtest-${RUN_ID}-.*|.*/rt-src-.*-${RUN_ID})$"
+    return $ns_gone
 }
 
 # Mount the named PVC in a transient probe pod and emit the count of