fix: Scope OBC health check to each app's own bucket

Previously the OBC CEL health check lived on infrastructure-ceph-cluster,
which holds OBCs for every app. A slow bind in one namespace stalled
every other app that depends on infrastructure-ceph-cluster. Move each
OBC into its own app Kustomization (paperless-db, authentik-db) and
apply the health check there so each app only waits on its own bucket.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

kubernetes/apps/authentik-db/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - woodpecker-api.sops.yaml
+  - migration-bucket.yaml
   - postgres.yaml
   - trigger-backup-job.yaml
   - restore-job.yaml

kubernetes/apps/authentik-db/migration-bucket.yaml

@@ -0,0 +1,8 @@
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: migration-bucket
+  namespace: authentik
+spec:
+  generateBucketName: migration
+  storageClassName: ceph-bucket

kubernetes/apps/paperless-db/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - woodpecker-api.sops.yaml
+  - migration-bucket.yaml
   - postgres.yaml
   - trigger-backup-job.yaml
   - restore-db-job.yaml

kubernetes/apps/paperless-db/migration-bucket.yaml

@@ -0,0 +1,8 @@
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: migration-bucket
+  namespace: paperless
+spec:
+  generateBucketName: migration-paperless
+  storageClassName: ceph-bucket

kubernetes/clusters/homelab/authentik.yaml

@@ -13,6 +13,12 @@ spec:
   path: ./kubernetes/apps/authentik-db
   prune: true
   wait: true
+  healthCheckExprs:
+    - apiVersion: objectbucket.io/v1alpha1
+      kind: ObjectBucketClaim
+      current: 'status.phase == "Bound"'
+      inProgress: 'has(status.phase) == false || status.phase != "Bound"'
+      failed: 'status.phase == "Failed"'
   decryption:
     provider: sops
     secretRef:

kubernetes/clusters/homelab/infrastructure.yaml

@@ -57,11 +57,5 @@ spec:
   path: ./kubernetes/infrastructure/rook-ceph-cluster
   prune: true
   wait: true
-  healthCheckExprs:
-    - apiVersion: objectbucket.io/v1alpha1
-      kind: ObjectBucketClaim
-      current: 'status.phase == "Bound"'
-      inProgress: 'has(status.phase) == false || status.phase != "Bound"'
-      failed: 'status.phase == "Failed"'
   dependsOn:
     - name: infrastructure

kubernetes/clusters/homelab/paperless.yaml

@@ -13,6 +13,12 @@ spec:
   path: ./kubernetes/apps/paperless-db
   prune: true
   wait: true
+  healthCheckExprs:
+    - apiVersion: objectbucket.io/v1alpha1
+      kind: ObjectBucketClaim
+      current: 'status.phase == "Bound"'
+      inProgress: 'has(status.phase) == false || status.phase != "Bound"'
+      failed: 'status.phase == "Failed"'
   decryption:
     provider: sops
     secretRef:

kubernetes/infrastructure/rook-ceph-cluster/objectstore.yaml

@@ -60,20 +60,5 @@ spec:
   store: migration
   displayName: migration
 ---
-apiVersion: objectbucket.io/v1alpha1
-kind: ObjectBucketClaim
-metadata:
-  name: migration-bucket
-  namespace: authentik
-spec:
-  generateBucketName: migration
-  storageClassName: ceph-bucket
----
-apiVersion: objectbucket.io/v1alpha1
-kind: ObjectBucketClaim
-metadata:
-  name: migration-bucket
-  namespace: paperless
-spec:
-  generateBucketName: migration-paperless
-  storageClassName: ceph-bucket
+# Per-app OBCs live in each app's own kustomization so their bind state
+# doesn't cross-gate other apps through infrastructure-ceph-cluster.

scripts/authentik-disable-2fa.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Emergency 2FA removal for a single Authentik user.
+# Deletes every authenticator device (TOTP, WebAuthn, static backup codes,
+# SMS, Duo, mobile, endpoint) directly in Postgres — bypasses the API so it
+# works even if you can't complete the MFA challenge to log in.
+#
+# Usage: ./scripts/authentik-disable-2fa.sh [username]
+#        Defaults to akadmin.
+
+USER_NAME="${1:-akadmin}"
+NS="authentik"
+CLUSTER="ak-pg"
+
+DEVICE_TABLES=(
+  authentik_stages_authenticator_totp_totpdevice
+  authentik_stages_authenticator_webauthn_webauthndevice
+  authentik_stages_authenticator_static_staticdevice
+  authentik_stages_authenticator_sms_smsdevice
+  authentik_stages_authenticator_duo_duodevice
+  authentik_stages_authenticator_mobile_mobiledevice
+  authentik_stages_authenticator_endpoint_gdtc_endpointdevice
+)
+
+# Find the current primary instance
+PRIMARY=$(kubectl get cluster -n "$NS" "$CLUSTER" -o jsonpath='{.status.currentPrimary}' 2>/dev/null || true)
+[[ -n "$PRIMARY" ]] || { echo "Could not find CNPG primary for $NS/$CLUSTER — is the cluster running?"; exit 1; }
+echo "Primary: $PRIMARY"
+
+# Sanity check user exists
+USER_ID=$(kubectl exec -n "$NS" "$PRIMARY" -c postgres -- \
+  psql -At -U postgres authentik \
+  -c "SELECT id FROM authentik_core_user WHERE username = '$USER_NAME';" | tr -d '[:space:]')
+[[ -n "$USER_ID" ]] || { echo "User '$USER_NAME' not found in authentik_core_user"; exit 1; }
+echo "User id: $USER_ID"
+
+# Build a DELETE for every device table, combined in a single transaction
+SQL="BEGIN;"
+for t in "${DEVICE_TABLES[@]}"; do
+  # Skip tables that don't exist in this Authentik version (older/newer schemas)
+  SQL+=" DO \$\$ BEGIN
+           IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '$t') THEN
+             EXECUTE 'DELETE FROM $t WHERE user_id = $USER_ID';
+           END IF;
+         END \$\$;"
+done
+SQL+=" COMMIT;"
+
+echo "Removing all MFA devices for $USER_NAME..."
+kubectl exec -n "$NS" "$PRIMARY" -c postgres -- \
+  psql -v ON_ERROR_STOP=1 -U postgres authentik -c "$SQL"
+
+echo
+echo "Remaining device counts for $USER_NAME:"
+for t in "${DEVICE_TABLES[@]}"; do
+  count=$(kubectl exec -n "$NS" "$PRIMARY" -c postgres -- \
+    psql -At -U postgres authentik \
+    -c "SELECT count(*) FROM information_schema.tables WHERE table_name = '$t';" | tr -d '[:space:]')
+  [[ "$count" == "1" ]] || continue
+  n=$(kubectl exec -n "$NS" "$PRIMARY" -c postgres -- \
+    psql -At -U postgres authentik \
+    -c "SELECT count(*) FROM $t WHERE user_id = $USER_ID;" | tr -d '[:space:]')
+  printf "  %-60s %s\n" "$t" "$n"
+done
+
+echo
+echo "Done. User '$USER_NAME' can now log in with just the password."
+echo "Re-enroll MFA from the user's settings after logging in."