From e9792f9afa052f2ee39c2e1ba5e23f47a1130aea Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Tue, 28 Apr 2026 12:30:29 +0200 Subject: [PATCH] fix(migrate-immich): trap-based cleanup + heartbeat in long polls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline 751 was killed by Woodpecker's idle-output watchdog after exactly 32 minutes — the wait loop polled the Job's Complete/Failed conditions every 30 s but produced no output between polls. With silent stdout, the agent classifies the step as stuck. Pipeline 753 had the same pattern. Refactor: - Whole step is now ONE shell script in a single `- |` block. A cleanup() function runs on EXIT (any reason: success, error, SIGTERM-on-cancel) so a cancelled pipeline doesn't leave Flux suspended, the Deployment at replicas=0, or the staging Job + SSH Secret stranded. - Heartbeat every 6 polls (~3 min) during Job-wait: prints elapsed, pod phase, and `du -sh /dst/data` so a 4 h transfer has visible progress. - velero backup create runs without --wait — poll separately with the same heartbeat shape, summing DataUpload bytes_done/total. - Job manifest gets ttlSecondsAfterFinished: 600 so even SIGKILL on the Woodpecker runner leaves a self-cleaning artefact. Same pattern still needs to land on paperless / audiobookshelf / music-assistant / influxdb pipelines. Co-Authored-By: Claude Opus 4.7 (1M context) --- .woodpecker/migrate-immich.yml | 169 +++++++++++++++++++++++---------- 1 file changed, 118 insertions(+), 51 deletions(-) diff --git a/.woodpecker/migrate-immich.yml b/.woodpecker/migrate-immich.yml index 2bc69be0..ba819289 100644 --- a/.woodpecker/migrate-immich.yml +++ b/.woodpecker/migrate-immich.yml @@ -185,30 +185,56 @@ steps: from_secret: ssh_private_key VELERO_VERSION: v1.16.0 commands: - - set -eu - - apk add --no-cache bash curl ca-certificates sops python3 - - curl -sfLO "https://dl.k8s.io/release/$(curl -sfL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - - install kubectl /usr/local/bin/kubectl && rm kubectl - - curl -sfL "https://github.com/vmware-tanzu/velero/releases/download/$VELERO_VERSION/velero-$VELERO_VERSION-linux-amd64.tar.gz" | tar -xz -C /tmp - - install "/tmp/velero-$VELERO_VERSION-linux-amd64/velero" /usr/local/bin/velero - - mkdir -p ~/.config/sops/age ~/.kube - - echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt - - sops -d --input-type binary --output-type binary kubernetes/access/kubeconfig.sops > ~/.kube/config - - chmod 600 ~/.kube/config - - mkdir -p /tmp/ssh - - python3 -c "import base64,os; open('/tmp/ssh/id_ed25519','wb').write(base64.b64decode(os.environ['SSH_PRIVATE_KEY']))" - - chmod 600 /tmp/ssh/id_ed25519 - - kubectl -n immich create secret generic immich-seed-ssh --from-file=id_ed25519=/tmp/ssh/id_ed25519 --dry-run=client -o yaml | kubectl apply -f - - - kubectl -n flux-system patch kustomization immich --type=merge -p '{"spec":{"suspend":true}}' - - kubectl -n immich scale deployment immich-server --replicas=0 - - kubectl -n immich wait --for=delete pod -l app.kubernetes.io/name=immich-server --timeout=300s || true - - kubectl -n immich delete job immich-seed --ignore-not-found --wait=true - # Library is ~250 GiB — uncompressed tar over ssh, no remote-rsync - # dependency. Live copy: photo uploads are write-once at the legacy - # so a brief inconsistency window is acceptable (matches the prior - # ansible-based backup-immich playbook which also didn't stop the - # docker-compose stack). + # Single shell script so the EXIT trap runs cleanup on any + # termination (success, error, SIGTERM-on-cancel, etc.). Without + # this, a cancelled pipeline leaves Flux suspended, the Deployment + # at replicas=0, and the staging Job + Secret stranded. - | + set -eu + APP=immich + NS=immich + DEPLOY=immich-server + JOB=immich-seed + SECRET=immich-seed-ssh + SCHED=immich-daily + + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + elapsed() { printf '%dm%02ds' $(( ($1) / 60 )) $(( ($1) % 60 )); } + + cleanup() { + rc=$? + log "=== EXIT cleanup (rc=$rc) ===" + kubectl -n "$NS" delete job "$JOB" --wait=false 2>/dev/null || true + kubectl -n "$NS" delete secret "$SECRET" --wait=false 2>/dev/null || true + kubectl -n flux-system patch kustomization "$APP" --type=merge -p '{"spec":{"suspend":false}}' 2>/dev/null || true + } + trap cleanup EXIT + + # ---- tooling ---- + apk add --no-cache --quiet bash curl ca-certificates sops python3 >/dev/null + curl -sfLO "https://dl.k8s.io/release/$(curl -sfL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + install kubectl /usr/local/bin/kubectl && rm kubectl + curl -sfL "https://github.com/vmware-tanzu/velero/releases/download/$VELERO_VERSION/velero-$VELERO_VERSION-linux-amd64.tar.gz" | tar -xz -C /tmp + install "/tmp/velero-$VELERO_VERSION-linux-amd64/velero" /usr/local/bin/velero + + # ---- credentials ---- + mkdir -p ~/.config/sops/age ~/.kube /tmp/ssh + echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt + sops -d --input-type binary --output-type binary kubernetes/access/kubeconfig.sops > ~/.kube/config + chmod 600 ~/.kube/config + python3 -c "import base64,os; open('/tmp/ssh/id_ed25519','wb').write(base64.b64decode(os.environ['SSH_PRIVATE_KEY']))" + chmod 600 /tmp/ssh/id_ed25519 + kubectl -n "$NS" create secret generic "$SECRET" --from-file=id_ed25519=/tmp/ssh/id_ed25519 --dry-run=client -o yaml | kubectl apply -f - + + # ---- quiesce ---- + kubectl -n flux-system patch kustomization "$APP" --type=merge -p '{"spec":{"suspend":true}}' + kubectl -n "$NS" scale deployment "$DEPLOY" --replicas=0 + kubectl -n "$NS" wait --for=delete pod -l app.kubernetes.io/name="$DEPLOY" --timeout=300s || true + kubectl -n "$NS" delete job "$JOB" --ignore-not-found --wait=true + + # ---- staging Job ---- + # ttlSecondsAfterFinished: belt-and-braces in case the trap + # cleanup doesn't run (SIGKILL on the runner, etc.). cat <<'YAML' | kubectl apply -f - apiVersion: batch/v1 kind: Job @@ -217,6 +243,7 @@ steps: namespace: immich spec: backoffLimit: 0 + ttlSecondsAfterFinished: 600 template: spec: restartPolicy: Never @@ -252,47 +279,87 @@ steps: secretName: immich-seed-ssh defaultMode: 0400 YAML - - | - echo "Waiting for immich-seed Job to reach a terminal state ..." - deadline=$(( $(date +%s) + 14400 )) # 4 h cap - while [ "$(date +%s)" -lt "$deadline" ]; do - c=$(kubectl -n immich get job immich-seed -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || true) - f=$(kubectl -n immich get job immich-seed -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) - [ "$c" = "True" ] && { echo "immich-seed Complete"; break; } - [ "$f" = "True" ] && { echo "immich-seed FAILED — describe follows:"; \ - kubectl -n immich describe job immich-seed; \ - kubectl -n immich logs -l job-name=immich-seed --tail=100 || true; exit 1; } + + # ---- wait for Job with heartbeat ---- + log "Waiting for $JOB Job (4 h cap) ..." + START=$(date +%s) + DEADLINE=$(( START + 14400 )) + ITER=0 + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + ITER=$(( ITER + 1 )) + c=$(kubectl -n "$NS" get job "$JOB" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || true) + f=$(kubectl -n "$NS" get job "$JOB" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) + [ "$c" = "True" ] && { log "$JOB Complete"; break; } + if [ "$f" = "True" ]; then + log "$JOB FAILED — describe + logs follow:" + kubectl -n "$NS" describe job "$JOB" || true + kubectl -n "$NS" logs -l job-name="$JOB" --tail=200 || true + exit 1 + fi + # Heartbeat every 6 polls × 30 s = every 3 min so the Woodpecker + # agent's idle-output watchdog doesn't kill the silent step. + if [ $(( ITER % 6 )) -eq 0 ]; then + ELAPSED=$(( $(date +%s) - START )) + POD_PHASE=$(kubectl -n "$NS" get pod -l job-name="$JOB" -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "?") + SIZE=$(kubectl -n "$NS" exec -t job/"$JOB" -- du -sh /dst/data 2>/dev/null | awk '{print $1}' || echo "?") + log " heartbeat: elapsed=$(elapsed $ELAPSED) pod=$POD_PHASE extracted=$SIZE" + fi sleep 30 done - if [ "$c" != "True" ]; then echo "Job did not finish within 4 h"; exit 1; fi - kubectl -n immich logs -l job-name=immich-seed --tail=200 || true - - | + if [ "$c" != "True" ]; then log "Job did not finish within 4 h"; exit 1; fi + kubectl -n "$NS" logs -l job-name="$JOB" --tail=50 || true + + # ---- wipe prior schedule-labelled Backups ---- count_backups() { - velero backup get -l velero.io/schedule-name=immich-daily 2>/dev/null \ + velero backup get -l "velero.io/schedule-name=$SCHED" 2>/dev/null \ | tail -n +2 | wc -l | tr -d ' ' } existing=$(count_backups) if [ "$existing" -gt 0 ]; then - echo "Deleting $existing existing immich-daily Backup(s) ..." - velero backup delete --selector velero.io/schedule-name=immich-daily --confirm 2>/dev/null || true + log "Deleting $existing existing $SCHED Backup(s) ..." + velero backup delete --selector "velero.io/schedule-name=$SCHED" --confirm 2>/dev/null || true for i in $(seq 1 60); do count=$(count_backups) - if [ "$count" -eq 0 ]; then echo "All prior Backups deleted."; break; fi - echo " waiting for deletion controller ($count remaining)" + if [ "$count" -eq 0 ]; then log "All prior Backups deleted."; break; fi + log " waiting for deletion controller ($count remaining)" sleep 5 done else - echo "No prior immich-daily Backups to delete." + log "No prior $SCHED Backups to delete." fi - - | - BACKUP_NAME="immich-seed-$(date -u +%Y%m%d-%H%M%S)" + + # ---- create the seed Backup, poll progress with heartbeat ---- + BACKUP_NAME="$APP-seed-$(date -u +%Y%m%d-%H%M%S)" velero backup create "$BACKUP_NAME" \ - --include-namespaces immich \ + --include-namespaces "$NS" \ --include-resources persistentvolumeclaims \ --snapshot-move-data \ - --labels "velero.io/schedule-name=immich-daily" \ - --wait - velero backup describe "$BACKUP_NAME" --details --colorized=false - - kubectl -n immich delete job immich-seed --wait=false - - kubectl -n immich delete secret immich-seed-ssh --wait=false - - kubectl -n flux-system patch kustomization immich --type=merge -p '{"spec":{"suspend":false}}' + --labels "velero.io/schedule-name=$SCHED" + + log "Polling Backup $BACKUP_NAME ..." + BSTART=$(date +%s) + BDEADLINE=$(( BSTART + 14400 )) + BITER=0 + while [ "$(date +%s)" -lt "$BDEADLINE" ]; do + BITER=$(( BITER + 1 )) + phase=$(velero backup get "$BACKUP_NAME" 2>/dev/null | tail -n +2 | awk '{print $2}' || echo "?") + case "$phase" in + Completed) + log "Backup $BACKUP_NAME Completed" + break + ;; + Failed|FailedValidation|PartiallyFailed) + log "Backup $BACKUP_NAME ended in $phase — describe follows:" + velero backup describe "$BACKUP_NAME" --details --colorized=false || true + exit 1 + ;; + esac + if [ $(( BITER % 6 )) -eq 0 ]; then + ELAPSED=$(( $(date +%s) - BSTART )) + DU_LINES=$(kubectl -n velero get datauploads --no-headers 2>/dev/null \ + | awk -v b="$BACKUP_NAME" 'index($0, b) { sum_done+=$4; sum_total+=$5; n++ } END { if (n) printf "%d/%d (n=%d)", sum_done, sum_total, n }' ) + log " heartbeat: elapsed=$(elapsed $ELAPSED) phase=$phase data=${DU_LINES:-?}" + fi + sleep 30 + done + velero backup describe "$BACKUP_NAME" --colorized=false 2>&1 | head -30 || true