fix(migrate-immich): trap-based cleanup + heartbeat in long polls

Pipeline 751 was killed by Woodpecker's idle-output watchdog after
exactly 32 minutes — the wait loop polled the Job's Complete/Failed
conditions every 30 s but produced no output between polls. With
silent stdout, the agent classifies the step as stuck.

Pipeline 753 had the same pattern.

Refactor:
  - Whole step is now ONE shell script in a single `- |` block. A
    cleanup() function runs on EXIT (any reason: success, error,
    SIGTERM-on-cancel) so a cancelled pipeline doesn't leave Flux
    suspended, the Deployment at replicas=0, or the staging Job +
    SSH Secret stranded.
  - Heartbeat every 6 polls (~3 min) during Job-wait: prints elapsed,
    pod phase, and `du -sh /dst/data` so a 4 h transfer has visible
    progress.
  - velero backup create runs without --wait — poll separately with
    the same heartbeat shape, summing DataUpload bytes_done/total.
  - Job manifest gets ttlSecondsAfterFinished: 600 so even SIGKILL
    on the Woodpecker runner leaves a self-cleaning artefact.

Same pattern still needs to land on paperless / audiobookshelf /
music-assistant / influxdb pipelines.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.woodpecker/migrate-immich.yml

@@ -185,30 +185,56 @@ steps:
         from_secret: ssh_private_key
       VELERO_VERSION: v1.16.0
     commands:
-      - set -eu
-      - apk add --no-cache bash curl ca-certificates sops python3
-      - curl -sfLO "https://dl.k8s.io/release/$(curl -sfL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-      - install kubectl /usr/local/bin/kubectl && rm kubectl
-      - curl -sfL "https://github.com/vmware-tanzu/velero/releases/download/$VELERO_VERSION/velero-$VELERO_VERSION-linux-amd64.tar.gz" | tar -xz -C /tmp
-      - install "/tmp/velero-$VELERO_VERSION-linux-amd64/velero" /usr/local/bin/velero
-      - mkdir -p ~/.config/sops/age ~/.kube
-      - echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt
-      - sops -d --input-type binary --output-type binary kubernetes/access/kubeconfig.sops > ~/.kube/config
-      - chmod 600 ~/.kube/config
-      - mkdir -p /tmp/ssh
-      - python3 -c "import base64,os; open('/tmp/ssh/id_ed25519','wb').write(base64.b64decode(os.environ['SSH_PRIVATE_KEY']))"
-      - chmod 600 /tmp/ssh/id_ed25519
-      - kubectl -n immich create secret generic immich-seed-ssh --from-file=id_ed25519=/tmp/ssh/id_ed25519 --dry-run=client -o yaml | kubectl apply -f -
-      - kubectl -n flux-system patch kustomization immich --type=merge -p '{"spec":{"suspend":true}}'
-      - kubectl -n immich scale deployment immich-server --replicas=0
-      - kubectl -n immich wait --for=delete pod -l app.kubernetes.io/name=immich-server --timeout=300s || true
-      - kubectl -n immich delete job immich-seed --ignore-not-found --wait=true
-      # Library is ~250 GiB — uncompressed tar over ssh, no remote-rsync
-      # dependency. Live copy: photo uploads are write-once at the legacy
-      # so a brief inconsistency window is acceptable (matches the prior
-      # ansible-based backup-immich playbook which also didn't stop the
-      # docker-compose stack).
+      # Single shell script so the EXIT trap runs cleanup on any
+      # termination (success, error, SIGTERM-on-cancel, etc.). Without
+      # this, a cancelled pipeline leaves Flux suspended, the Deployment
+      # at replicas=0, and the staging Job + Secret stranded.
       - |
+        set -eu
+        APP=immich
+        NS=immich
+        DEPLOY=immich-server
+        JOB=immich-seed
+        SECRET=immich-seed-ssh
+        SCHED=immich-daily
+
+        log()      { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
+        elapsed()  { printf '%dm%02ds' $(( ($1) / 60 )) $(( ($1) % 60 )); }
+
+        cleanup() {
+          rc=$?
+          log "=== EXIT cleanup (rc=$rc) ==="
+          kubectl -n "$NS" delete job  "$JOB"    --wait=false 2>/dev/null || true
+          kubectl -n "$NS" delete secret "$SECRET" --wait=false 2>/dev/null || true
+          kubectl -n flux-system patch kustomization "$APP" --type=merge -p '{"spec":{"suspend":false}}' 2>/dev/null || true
+        }
+        trap cleanup EXIT
+
+        # ---- tooling ----
+        apk add --no-cache --quiet bash curl ca-certificates sops python3 >/dev/null
+        curl -sfLO "https://dl.k8s.io/release/$(curl -sfL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+        install kubectl /usr/local/bin/kubectl && rm kubectl
+        curl -sfL "https://github.com/vmware-tanzu/velero/releases/download/$VELERO_VERSION/velero-$VELERO_VERSION-linux-amd64.tar.gz" | tar -xz -C /tmp
+        install "/tmp/velero-$VELERO_VERSION-linux-amd64/velero" /usr/local/bin/velero
+
+        # ---- credentials ----
+        mkdir -p ~/.config/sops/age ~/.kube /tmp/ssh
+        echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt
+        sops -d --input-type binary --output-type binary kubernetes/access/kubeconfig.sops > ~/.kube/config
+        chmod 600 ~/.kube/config
+        python3 -c "import base64,os; open('/tmp/ssh/id_ed25519','wb').write(base64.b64decode(os.environ['SSH_PRIVATE_KEY']))"
+        chmod 600 /tmp/ssh/id_ed25519
+        kubectl -n "$NS" create secret generic "$SECRET" --from-file=id_ed25519=/tmp/ssh/id_ed25519 --dry-run=client -o yaml | kubectl apply -f -
+
+        # ---- quiesce ----
+        kubectl -n flux-system patch kustomization "$APP" --type=merge -p '{"spec":{"suspend":true}}'
+        kubectl -n "$NS" scale deployment "$DEPLOY" --replicas=0
+        kubectl -n "$NS" wait --for=delete pod -l app.kubernetes.io/name="$DEPLOY" --timeout=300s || true
+        kubectl -n "$NS" delete job "$JOB" --ignore-not-found --wait=true
+
+        # ---- staging Job ----
+        # ttlSecondsAfterFinished: belt-and-braces in case the trap
+        # cleanup doesn't run (SIGKILL on the runner, etc.).
         cat <<'YAML' | kubectl apply -f -
         apiVersion: batch/v1
         kind: Job
@@ -217,6 +243,7 @@ steps:
           namespace: immich
         spec:
           backoffLimit: 0
+          ttlSecondsAfterFinished: 600
           template:
             spec:
               restartPolicy: Never
@@ -252,47 +279,87 @@ steps:
                     secretName: immich-seed-ssh
                     defaultMode: 0400
         YAML
-      - |
-        echo "Waiting for immich-seed Job to reach a terminal state ..."
-        deadline=$(( $(date +%s) + 14400 ))  # 4 h cap
-        while [ "$(date +%s)" -lt "$deadline" ]; do
-          c=$(kubectl -n immich get job immich-seed -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || true)
-          f=$(kubectl -n immich get job immich-seed -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
-          [ "$c" = "True" ] && { echo "immich-seed Complete"; break; }
-          [ "$f" = "True" ] && { echo "immich-seed FAILED — describe follows:"; \
-                                 kubectl -n immich describe job immich-seed; \
-                                 kubectl -n immich logs -l job-name=immich-seed --tail=100 || true; exit 1; }
+
+        # ---- wait for Job with heartbeat ----
+        log "Waiting for $JOB Job (4 h cap) ..."
+        START=$(date +%s)
+        DEADLINE=$(( START + 14400 ))
+        ITER=0
+        while [ "$(date +%s)" -lt "$DEADLINE" ]; do
+          ITER=$(( ITER + 1 ))
+          c=$(kubectl -n "$NS" get job "$JOB" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || true)
+          f=$(kubectl -n "$NS" get job "$JOB" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
+          [ "$c" = "True" ] && { log "$JOB Complete"; break; }
+          if [ "$f" = "True" ]; then
+            log "$JOB FAILED — describe + logs follow:"
+            kubectl -n "$NS" describe job "$JOB" || true
+            kubectl -n "$NS" logs -l job-name="$JOB" --tail=200 || true
+            exit 1
+          fi
+          # Heartbeat every 6 polls × 30 s = every 3 min so the Woodpecker
+          # agent's idle-output watchdog doesn't kill the silent step.
+          if [ $(( ITER % 6 )) -eq 0 ]; then
+            ELAPSED=$(( $(date +%s) - START ))
+            POD_PHASE=$(kubectl -n "$NS" get pod -l job-name="$JOB" -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "?")
+            SIZE=$(kubectl -n "$NS" exec -t job/"$JOB" -- du -sh /dst/data 2>/dev/null | awk '{print $1}' || echo "?")
+            log "  heartbeat: elapsed=$(elapsed $ELAPSED) pod=$POD_PHASE extracted=$SIZE"
+          fi
           sleep 30
         done
-        if [ "$c" != "True" ]; then echo "Job did not finish within 4 h"; exit 1; fi
-        kubectl -n immich logs -l job-name=immich-seed --tail=200 || true
-      - |
+        if [ "$c" != "True" ]; then log "Job did not finish within 4 h"; exit 1; fi
+        kubectl -n "$NS" logs -l job-name="$JOB" --tail=50 || true
+
+        # ---- wipe prior schedule-labelled Backups ----
         count_backups() {
-          velero backup get -l velero.io/schedule-name=immich-daily 2>/dev/null \
+          velero backup get -l "velero.io/schedule-name=$SCHED" 2>/dev/null \
             | tail -n +2 | wc -l | tr -d ' '
         }
         existing=$(count_backups)
         if [ "$existing" -gt 0 ]; then
-          echo "Deleting $existing existing immich-daily Backup(s) ..."
-          velero backup delete --selector velero.io/schedule-name=immich-daily --confirm 2>/dev/null || true
+          log "Deleting $existing existing $SCHED Backup(s) ..."
+          velero backup delete --selector "velero.io/schedule-name=$SCHED" --confirm 2>/dev/null || true
           for i in $(seq 1 60); do
             count=$(count_backups)
-            if [ "$count" -eq 0 ]; then echo "All prior Backups deleted."; break; fi
-            echo "  waiting for deletion controller ($count remaining)"
+            if [ "$count" -eq 0 ]; then log "All prior Backups deleted."; break; fi
+            log "  waiting for deletion controller ($count remaining)"
             sleep 5
           done
         else
-          echo "No prior immich-daily Backups to delete."
+          log "No prior $SCHED Backups to delete."
         fi
-      - |
-        BACKUP_NAME="immich-seed-$(date -u +%Y%m%d-%H%M%S)"
+
+        # ---- create the seed Backup, poll progress with heartbeat ----
+        BACKUP_NAME="$APP-seed-$(date -u +%Y%m%d-%H%M%S)"
         velero backup create "$BACKUP_NAME" \
-          --include-namespaces immich \
+          --include-namespaces "$NS" \
           --include-resources persistentvolumeclaims \
           --snapshot-move-data \
-          --labels "velero.io/schedule-name=immich-daily" \
-          --wait
-        velero backup describe "$BACKUP_NAME" --details --colorized=false
-      - kubectl -n immich delete job immich-seed --wait=false
-      - kubectl -n immich delete secret immich-seed-ssh --wait=false
-      - kubectl -n flux-system patch kustomization immich --type=merge -p '{"spec":{"suspend":false}}'
+          --labels "velero.io/schedule-name=$SCHED"
+
+        log "Polling Backup $BACKUP_NAME ..."
+        BSTART=$(date +%s)
+        BDEADLINE=$(( BSTART + 14400 ))
+        BITER=0
+        while [ "$(date +%s)" -lt "$BDEADLINE" ]; do
+          BITER=$(( BITER + 1 ))
+          phase=$(velero backup get "$BACKUP_NAME" 2>/dev/null | tail -n +2 | awk '{print $2}' || echo "?")
+          case "$phase" in
+            Completed)
+              log "Backup $BACKUP_NAME Completed"
+              break
+              ;;
+            Failed|FailedValidation|PartiallyFailed)
+              log "Backup $BACKUP_NAME ended in $phase — describe follows:"
+              velero backup describe "$BACKUP_NAME" --details --colorized=false || true
+              exit 1
+              ;;
+          esac
+          if [ $(( BITER % 6 )) -eq 0 ]; then
+            ELAPSED=$(( $(date +%s) - BSTART ))
+            DU_LINES=$(kubectl -n velero get datauploads --no-headers 2>/dev/null \
+              | awk -v b="$BACKUP_NAME" 'index($0, b) { sum_done+=$4; sum_total+=$5; n++ } END { if (n) printf "%d/%d (n=%d)", sum_done, sum_total, n }' )
+            log "  heartbeat: elapsed=$(elapsed $ELAPSED) phase=$phase data=${DU_LINES:-?}"
+          fi
+          sleep 30
+        done
+        velero backup describe "$BACKUP_NAME" --colorized=false 2>&1 | head -30 || true