From e44d86b42f71ae242234c99edaefcd7419c1b3c9 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Sun, 26 Apr 2026 16:52:23 +0200 Subject: [PATCH] fix(paperless): collapse seed Job into paperless pod initContainer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same shape as the prior seed-Job → initContainer refactors: music-assistant (c9ea0af), audiobookshelf (b9717e9), immich (74994aa). Add a `seed` initContainer to the paperless Deployment that owns the S3-wait + extract + chown for the data and media RWX PVCs. paperless- consume is the live drop-folder (not seeded); paperless-redis is mounted by the redis pod (not paperless), so the seed never touched it. Paperless used RWX (ceph-filesystem) for its data PVCs, so the Multi-Attach race that broke music-assistant didn't surface here either — but the refactor removes one more standalone Flux Kustomization, kills the hourly Job churn, and matches the pattern across all migrated apps. Refactor: - paperless.yaml: add `seed` initContainer (the deployment had no initContainers before — the seed Job's sentinel was relied on purely via Flux dependsOn ordering). Mount paths /data + /media match the legacy Job's; the app container's /usr/src/paperless/{data,media} mounts are unchanged - move garage-migration-creds.sops.yaml + pvcs.yaml from kubernetes/apps/paperless-seed/ into kubernetes/apps/paperless/ - delete kubernetes/apps/paperless-seed/ - collapse the Flux Kustomizations: drop paperless-seed, replace `dependsOn: paperless-seed` with velero-bootstrap-restore on the app Kustomization (paperless-db dep stays — separate concern) - raise app Kustomization timeout 1h → 2h to absorb the worst-case tarball wait + extract Also drop the now-stale "during the migration window" comments on the audiobookshelf and music-assistant pvcs.yaml — the seed Kustomizations those comments referenced are gone. Co-Authored-By: Claude Opus 4.7 (1M context) --- kubernetes/apps/audiobookshelf/pvcs.yaml | 6 +- kubernetes/apps/music-assistant/pvcs.yaml | 6 +- .../apps/paperless-seed/kustomization.yaml | 6 - kubernetes/apps/paperless-seed/seed-job.yaml | 104 ------------------ .../garage-migration-creds.sops.yaml | 0 kubernetes/apps/paperless/kustomization.yaml | 2 + kubernetes/apps/paperless/paperless.yaml | 83 ++++++++++++++ .../{paperless-seed => paperless}/pvcs.yaml | 11 +- kubernetes/clusters/homelab/paperless.yaml | 37 +------ 9 files changed, 100 insertions(+), 155 deletions(-) delete mode 100644 kubernetes/apps/paperless-seed/kustomization.yaml delete mode 100644 kubernetes/apps/paperless-seed/seed-job.yaml rename kubernetes/apps/{paperless-seed => paperless}/garage-migration-creds.sops.yaml (100%) rename kubernetes/apps/{paperless-seed => paperless}/pvcs.yaml (70%) diff --git a/kubernetes/apps/audiobookshelf/pvcs.yaml b/kubernetes/apps/audiobookshelf/pvcs.yaml index bc37cf38..be014515 100644 --- a/kubernetes/apps/audiobookshelf/pvcs.yaml +++ b/kubernetes/apps/audiobookshelf/pvcs.yaml @@ -1,7 +1,7 @@ --- -# PVCs live in the seed Kustomization during the migration window (see -# paperless-seed/pvcs.yaml for the rationale). Prune-disabled so the PVCs -# survive retirement of this Kustomization. +# Prune-disabled so the PVCs survive retirement of this Kustomization +# (e.g. when the seed initContainer or the Garage migration creds are +# removed once we're done with the migration phase). apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/kubernetes/apps/music-assistant/pvcs.yaml b/kubernetes/apps/music-assistant/pvcs.yaml index a2dd4f0e..5f8ad0b2 100644 --- a/kubernetes/apps/music-assistant/pvcs.yaml +++ b/kubernetes/apps/music-assistant/pvcs.yaml @@ -1,7 +1,7 @@ --- -# PVC lives in the seed Kustomization during the migration window -# (matches audiobookshelf-seed / paperless-seed). Prune-disabled so the -# PVC survives retirement of this Kustomization. +# Prune-disabled so the PVC survives retirement of this Kustomization +# (e.g. when the seed initContainer or the Garage migration creds are +# removed once we're done with the migration phase). apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/kubernetes/apps/paperless-seed/kustomization.yaml b/kubernetes/apps/paperless-seed/kustomization.yaml deleted file mode 100644 index 04c2e0f1..00000000 --- a/kubernetes/apps/paperless-seed/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - garage-migration-creds.sops.yaml - - pvcs.yaml - - seed-job.yaml diff --git a/kubernetes/apps/paperless-seed/seed-job.yaml b/kubernetes/apps/paperless-seed/seed-job.yaml deleted file mode 100644 index 999f74eb..00000000 --- a/kubernetes/apps/paperless-seed/seed-job.yaml +++ /dev/null @@ -1,104 +0,0 @@ ---- -# Dev-phase seed Job for Paperless PVCs. Reads legacy tarballs from -# Garage (s3://k8s-migration/paperless/...) written by the standalone -# `migrate-paperless` Woodpecker pipeline — no cluster dependency on -# the producer side. Idempotent: sentinel `/data/.seed-done` short- -# circuits on subsequent runs (including after a Velero restore, which -# preserves the sentinel). Legacy `.seed-v3-done` is honored and -# normalized forward on re-run. -apiVersion: batch/v1 -kind: Job -metadata: - name: paperless-data-seed-v3 - namespace: paperless -spec: - ttlSecondsAfterFinished: 3600 - template: - spec: - restartPolicy: Never - securityContext: - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - containers: - - name: seed - image: alpine:3.21 - securityContext: - runAsNonRoot: false - allowPrivilegeEscalation: true - capabilities: - drop: ["ALL"] - add: ["CHOWN", "DAC_OVERRIDE", "FOWNER"] - command: - - sh - - -c - - | - set -eu - if [ -f /data/.seed-done ] || [ -f /data/.seed-v3-done ]; then - echo "Seed already applied (sentinel present); normalizing and exiting." - touch /data/.seed-done - chown 1000:1000 /data/.seed-done 2>/dev/null || true - exit 0 - fi - apk add --no-cache aws-cli tar ca-certificates >/dev/null - export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage - # TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so - # force path-style addressing (bucket in path, not subdomain). - mkdir -p "$HOME/.aws" - printf '[default]\ns3 =\n addressing_style = path\n' > "$HOME/.aws/config" - - echo "Waiting for tarballs at s3://$S3_BUCKET/$S3_PREFIX/ ..." - for i in $(seq 1 360); do - if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/paperless-data.tar.gz" >/dev/null 2>&1 \ - && aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/paperless-media.tar.gz" >/dev/null 2>&1; then - echo "Both tarballs found."; break - fi - echo " not ready yet ($i/360) — sleeping 10s"; sleep 10 - if [ "$i" = "360" ]; then - echo "ERROR: tarballs not found after 60 minutes"; exit 1 - fi - done - - echo "Streaming data tarball → /data ..." - aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/paperless-data.tar.gz" - | tar -xz -C /data - echo "Streaming media tarball → /media ..." - aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/paperless-media.tar.gz" - | tar -xz -C /media - - echo "Normalizing ownership to paperless (1000:1000) ..." - chown -R 1000:1000 /data /media - - echo "Writing seed sentinel ..." - touch /data/.seed-done - chown 1000:1000 /data/.seed-done - - echo "Done. Summary:" - du -sh /data /media - env: - - name: S3_ENDPOINT - value: "https://s3.nas-1.renner.uno" - - name: S3_BUCKET - value: "k8s-migration" - - name: S3_PREFIX - value: "paperless" - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: garage-migration-creds - key: AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: garage-migration-creds - key: AWS_SECRET_ACCESS_KEY - volumeMounts: - - name: data - mountPath: /data - - name: media - mountPath: /media - volumes: - - name: data - persistentVolumeClaim: - claimName: paperless-data - - name: media - persistentVolumeClaim: - claimName: paperless-media diff --git a/kubernetes/apps/paperless-seed/garage-migration-creds.sops.yaml b/kubernetes/apps/paperless/garage-migration-creds.sops.yaml similarity index 100% rename from kubernetes/apps/paperless-seed/garage-migration-creds.sops.yaml rename to kubernetes/apps/paperless/garage-migration-creds.sops.yaml diff --git a/kubernetes/apps/paperless/kustomization.yaml b/kubernetes/apps/paperless/kustomization.yaml index 7dcbf564..ccad1045 100644 --- a/kubernetes/apps/paperless/kustomization.yaml +++ b/kubernetes/apps/paperless/kustomization.yaml @@ -1,6 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - garage-migration-creds.sops.yaml + - pvcs.yaml - secrets.sops.yaml - ftp-scanner-secret.sops.yaml - redis.yaml diff --git a/kubernetes/apps/paperless/paperless.yaml b/kubernetes/apps/paperless/paperless.yaml index 2daa8c90..e1c67740 100644 --- a/kubernetes/apps/paperless/paperless.yaml +++ b/kubernetes/apps/paperless/paperless.yaml @@ -50,6 +50,89 @@ spec: fsGroupChangePolicy: OnRootMismatch seccompProfile: type: RuntimeDefault + initContainers: + # Restore paperless-data + paperless-media from the Garage tarballs + # written by the `migrate-paperless` Woodpecker pipeline. Sentinel- + # guarded so subsequent pod restarts short-circuit in <1s. Lives in + # the app pod (not a separate Job) so no orphan Job churns after + # ttlSecondsAfterFinished. On a fresh cluster rebuild the volumes + # are empty, so this runs the full ~3-4 GB extract before paperless + # starts. paperless-consume isn't seeded (live drop-folder for + # incoming docs) and paperless-redis is mounted by the redis pod. + - name: seed + image: alpine:3.21 + command: + - sh + - -c + - | + set -eu + if [ -f /data/.seed-done ] || [ -f /data/.seed-v3-done ]; then + echo "Seed already applied (sentinel present); normalizing and exiting." + touch /data/.seed-done + chown 1000:1000 /data/.seed-done 2>/dev/null || true + exit 0 + fi + apk add --no-cache aws-cli tar ca-certificates >/dev/null + export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage + # TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so + # force path-style addressing (bucket in path, not subdomain). + mkdir -p "$HOME/.aws" + printf '[default]\ns3 =\n addressing_style = path\n' > "$HOME/.aws/config" + + echo "Waiting for tarballs at s3://$S3_BUCKET/$S3_PREFIX/ ..." + for i in $(seq 1 360); do + if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/paperless-data.tar.gz" >/dev/null 2>&1 \ + && aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/paperless-media.tar.gz" >/dev/null 2>&1; then + echo "Both tarballs found."; break + fi + echo " not ready yet ($i/360) — sleeping 10s"; sleep 10 + if [ "$i" = "360" ]; then + echo "ERROR: tarballs not found after 60 minutes"; exit 1 + fi + done + + echo "Streaming data tarball → /data ..." + aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/paperless-data.tar.gz" - | tar -xz -C /data + echo "Streaming media tarball → /media ..." + aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/paperless-media.tar.gz" - | tar -xz -C /media + + echo "Normalizing ownership to paperless (1000:1000) ..." + chown -R 1000:1000 /data /media + + echo "Writing seed sentinel ..." + touch /data/.seed-done + chown 1000:1000 /data/.seed-done + + echo "Done. Summary:" + du -sh /data /media + env: + - name: S3_ENDPOINT + value: "https://s3.nas-1.renner.uno" + - name: S3_BUCKET + value: "k8s-migration" + - name: S3_PREFIX + value: "paperless" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: garage-migration-creds + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: garage-migration-creds + key: AWS_SECRET_ACCESS_KEY + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: true + capabilities: + drop: ["ALL"] + add: ["CHOWN", "DAC_OVERRIDE", "FOWNER"] + volumeMounts: + - name: data + mountPath: /data + - name: media + mountPath: /media containers: - name: paperless image: ghcr.io/paperless-ngx/paperless-ngx:2.20.14 diff --git a/kubernetes/apps/paperless-seed/pvcs.yaml b/kubernetes/apps/paperless/pvcs.yaml similarity index 70% rename from kubernetes/apps/paperless-seed/pvcs.yaml rename to kubernetes/apps/paperless/pvcs.yaml index 5eea6e7b..84140b3c 100644 --- a/kubernetes/apps/paperless-seed/pvcs.yaml +++ b/kubernetes/apps/paperless/pvcs.yaml @@ -1,11 +1,8 @@ --- -# PVCs live in the seed Kustomization during the migration window so that -# the seed Job can mount them before the app Deployment starts (the main -# `paperless` Kustomization dependsOn this one). Annotated with -# `kustomize.toolkit.fluxcd.io/prune: disabled` so when this seed -# Kustomization is retired post-migration the PVCs survive — Velero's -# bootstrap-restore then owns their contents; moving the PVC manifests -# back into `kubernetes/apps/paperless/` at retirement is the cleanup. +# Prune-disabled so the PVCs survive retirement of this Kustomization +# (e.g. when the seed initContainer or the Garage migration creds are +# removed once we're done with the migration phase). Velero's +# bootstrap-restore then owns their contents. apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index 153460fa..b33a1f09 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -30,43 +30,16 @@ spec: --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization -metadata: - name: paperless-seed - namespace: flux-system -spec: - interval: 10m - retryInterval: 1m - # Seed Job streams ~3-4 GB of documents/media; 2 h leaves margin. - timeout: 2h - sourceRef: - kind: GitRepository - name: flux-system - path: ./kubernetes/apps/paperless-seed - prune: true - wait: true - decryption: - provider: sops - secretRef: - name: sops-age - healthCheckExprs: - - apiVersion: batch/v1 - kind: Job - current: "status.succeeded == 1" - inProgress: "!has(status.succeeded) || status.succeeded == 0" - failed: "has(status.failed) && status.failed >= 1" - dependsOn: - - name: infrastructure-ceph-cluster - - name: velero-bootstrap-restore ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization metadata: name: paperless namespace: flux-system spec: + # Seed Job collapsed into the paperless pod's `seed` initContainer + # (same RWX data + media mounts). Tarballs are ~3-4 GB so 2h timeout + # absorbs upload + extract. interval: 10m retryInterval: 1m - timeout: 1h + timeout: 2h sourceRef: kind: GitRepository name: flux-system @@ -79,4 +52,4 @@ spec: name: sops-age dependsOn: - name: paperless-db - - name: paperless-seed + - name: velero-bootstrap-restore