fix(migrate): Use PG native archive_mode so end_wal ships to archive

The previous approach — pg_switch_wal + manual loop uploading pg_wal/*
after barman-cloud-backup — lost the end_wal segment. pg_switch_wal
triggers PG's WAL recycling: the just-closed segment file is renamed
in-place to become the next active segment, so by the time the loop
reads pg_wal/, the file named <end_wal> no longer exists.

Configure archive_mode=on with archive_command=barman-cloud-wal-archive
during initdb, then force pg_switch_wal after barman-cloud-backup and
wait for archive_status/*.ready files to drain. PG archives each
segment synchronously on close, so the end_wal segment uploads before
the recycle happens.

Fixes the CNPG recovery error "object storage or file not found
<wal>: WAL not found" with the skipEmptyWalArchiveCheck annotation
already in place.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.woodpecker/migrate-authentik-db.yml

@@ -72,7 +72,31 @@ steps:
       # pipeline simple. PG refuses to run as root, so use `su postgres`.
       - mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA"
       - su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres"
-      - su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -l /tmp/pg.log -o '-c listen_addresses=127.0.0.1' -w start"
+      # Configure continuous WAL archiving so the begin/end_wal of the
+      # base backup actually lands in the Barman WAL archive — without
+      # this, PG recycles the closed segment on pg_switch_wal and the
+      # file named <end_wal> never gets uploaded, causing CNPG recovery
+      # to fail with "object storage or file not found <wal>".
+      - |
+        cat > /usr/local/bin/archive_wal.sh <<'SH'
+        #!/bin/sh
+        exec barman-cloud-wal-archive \
+          --cloud-provider aws-s3 \
+          --endpoint-url "$GARAGE_ENDPOINT" \
+          --gzip -e AES256 \
+          "$BARMAN_DEST" "$BARMAN_SERVER" "$1"
+        SH
+        chmod +x /usr/local/bin/archive_wal.sh
+      - |
+        cat >> "$PGDATA/postgresql.conf" <<CONF
+        archive_mode = on
+        archive_command = '/usr/local/bin/archive_wal.sh %p'
+        archive_timeout = '60s'
+        CONF
+      - chown postgres:postgres "$PGDATA/postgresql.conf"
+      # Pass AWS credentials + Barman vars to the postgres daemon so
+      # archive_command subprocesses inherit them.
+      - su postgres --preserve-environment -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -l /tmp/pg.log -o '-c listen_addresses=127.0.0.1' -w start"
       # Recreate the target role + database and load the legacy dump.
       - psql -h 127.0.0.1 -U postgres -d postgres -c "CREATE ROLE $TARGET_OWNER WITH LOGIN PASSWORD 'seed';"
       - psql -h 127.0.0.1 -U postgres -d postgres -c "CREATE DATABASE $TARGET_DB OWNER $TARGET_OWNER;"
@@ -97,23 +121,17 @@ steps:
         --host 127.0.0.1 --port 5432 --user postgres
         "$BARMAN_DEST" "$BARMAN_SERVER"
       - barman-cloud-backup-list --cloud-provider aws-s3 --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST" "$BARMAN_SERVER"
-      # Archive the WAL segments that cover the base backup so CNPG
-      # bootstrap.recovery can locate the `end_wal` file. The ephemeral
-      # Postgres has no archive_command, so WALs sit in pg_wal/ and
-      # never ship to the Barman WAL archive — recovery then fails with
-      # "object storage or file not found <wal>: WAL not found".
+      # Force a WAL switch so PG archives the segment that covers
+      # the base backup's end_wal (the checkpoint/consistency LSN).
+      # archive_command fires asynchronously — wait for archive_status
+      # to drain so the "*.ready" files are gone before we stop PG.
       - psql -h 127.0.0.1 -U postgres -d postgres -c "CHECKPOINT; SELECT pg_switch_wal();"
       - |
-        for wal in "$PGDATA"/pg_wal/*; do
-          [ -f "$wal" ] || continue
-          name=$(basename "$wal")
-          case "$name" in *.partial) continue ;; esac
-          echo "Archiving WAL $name"
-          barman-cloud-wal-archive \
-            --cloud-provider aws-s3 \
-            --endpoint-url "$GARAGE_ENDPOINT" \
-            --gzip -e AES256 \
-            "$BARMAN_DEST" "$BARMAN_SERVER" "$wal"
+        for i in $(seq 1 30); do
+          ready=$(ls "$PGDATA/pg_wal/archive_status"/*.ready 2>/dev/null | wc -l)
+          echo "archive_status: $ready .ready file(s) remaining"
+          [ "$ready" -eq 0 ] && break
+          sleep 2
         done
       - su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast -w stop"
       - echo "Seed written. Next fresh bootstrap of ak-pg will recover this backup."

.woodpecker/migrate-paperless.yml

@@ -74,7 +74,26 @@ steps:
       # step container, not as a Woodpecker service.
       - mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA"
       - su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres"
-      - su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -l /tmp/pg.log -o '-c listen_addresses=127.0.0.1' -w start"
+      # See migrate-authentik-db.yml for rationale on continuous WAL
+      # archiving during the seed.
+      - |
+        cat > /usr/local/bin/archive_wal.sh <<'SH'
+        #!/bin/sh
+        exec barman-cloud-wal-archive \
+          --cloud-provider aws-s3 \
+          --endpoint-url "$GARAGE_ENDPOINT" \
+          --gzip -e AES256 \
+          "$BARMAN_DEST" "$BARMAN_SERVER" "$1"
+        SH
+        chmod +x /usr/local/bin/archive_wal.sh
+      - |
+        cat >> "$PGDATA/postgresql.conf" <<CONF
+        archive_mode = on
+        archive_command = '/usr/local/bin/archive_wal.sh %p'
+        archive_timeout = '60s'
+        CONF
+      - chown postgres:postgres "$PGDATA/postgresql.conf"
+      - su postgres --preserve-environment -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -l /tmp/pg.log -o '-c listen_addresses=127.0.0.1' -w start"
       - psql -h 127.0.0.1 -U postgres -d postgres -c "CREATE ROLE $TARGET_OWNER WITH LOGIN PASSWORD 'seed';"
       - psql -h 127.0.0.1 -U postgres -d postgres -c "CREATE DATABASE $TARGET_DB OWNER $TARGET_OWNER;"
       - >-
@@ -93,21 +112,13 @@ steps:
         --host 127.0.0.1 --port 5432 --user postgres
         "$BARMAN_DEST" "$BARMAN_SERVER"
       - barman-cloud-backup-list --cloud-provider aws-s3 --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST" "$BARMAN_SERVER"
-      # Archive the WAL segments that cover the base backup so CNPG
-      # bootstrap.recovery can locate the `end_wal` file. See
-      # migrate-authentik-db.yml for the rationale.
       - psql -h 127.0.0.1 -U postgres -d postgres -c "CHECKPOINT; SELECT pg_switch_wal();"
       - |
-        for wal in "$PGDATA"/pg_wal/*; do
-          [ -f "$wal" ] || continue
-          name=$(basename "$wal")
-          case "$name" in *.partial) continue ;; esac
-          echo "Archiving WAL $name"
-          barman-cloud-wal-archive \
-            --cloud-provider aws-s3 \
-            --endpoint-url "$GARAGE_ENDPOINT" \
-            --gzip -e AES256 \
-            "$BARMAN_DEST" "$BARMAN_SERVER" "$wal"
+        for i in $(seq 1 30); do
+          ready=$(ls "$PGDATA/pg_wal/archive_status"/*.ready 2>/dev/null | wc -l)
+          echo "archive_status: $ready .ready file(s) remaining"
+          [ "$ready" -eq 0 ] && break
+          sleep 2
         done
       - su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast -w stop"
       - echo "Seed written. Next fresh bootstrap of pprl-pg will recover this backup."