diff --git a/kubernetes/apps/immich-seed/kustomization.yaml b/kubernetes/apps/immich-seed/kustomization.yaml deleted file mode 100644 index 04c2e0f1..00000000 --- a/kubernetes/apps/immich-seed/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - garage-migration-creds.sops.yaml - - pvcs.yaml - - seed-job.yaml diff --git a/kubernetes/apps/immich-seed/seed-job.yaml b/kubernetes/apps/immich-seed/seed-job.yaml deleted file mode 100644 index ee109578..00000000 --- a/kubernetes/apps/immich-seed/seed-job.yaml +++ /dev/null @@ -1,102 +0,0 @@ ---- -# Dev-phase seed Job for Immich PVCs. Reads the legacy library tarball -# from Garage (s3://k8s-migration/immich/...) written by the standalone -# `migrate-immich` Woodpecker pipeline. The Postgres dump is handled by -# the immich-db Cluster's bootstrap.recovery against the parallel -# s3://k8s-cnpg/im-pg Barman archive seeded by the same pipeline. -# The ML model cache is NOT seeded — models re-download on demand. -# -# Idempotent: sentinel `/library/.restore-done` short-circuits on repeat -# runs, including after a Velero restore (which preserves the sentinel). -# Legacy `.restore-v2-done` is still honored and normalized forward. -apiVersion: batch/v1 -kind: Job -metadata: - name: immich-data-seed-v2 - namespace: immich -spec: - ttlSecondsAfterFinished: 3600 - template: - spec: - restartPolicy: Never - securityContext: - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - containers: - - name: seed - image: alpine:3.21 - securityContext: - runAsNonRoot: false - allowPrivilegeEscalation: true - capabilities: - drop: ["ALL"] - add: ["CHOWN", "DAC_OVERRIDE", "FOWNER"] - command: - - sh - - -c - - | - set -eu - if [ -f /library/.restore-done ] || [ -f /library/.restore-v2-done ]; then - echo "Seed already applied (sentinel present); normalizing and exiting." - touch /library/.restore-done - chown 1000:1000 /library/.restore-done 2>/dev/null || true - exit 0 - fi - apk add --no-cache aws-cli tar ca-certificates >/dev/null - export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage - # TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so - # force path-style addressing (bucket in path, not subdomain). - mkdir -p "$HOME/.aws" - printf '[default]\ns3 =\n addressing_style = path\n' > "$HOME/.aws/config" - - echo "Waiting for tarball at s3://$S3_BUCKET/$S3_PREFIX/ ..." - # ~250 GiB library — allow 4 h for the upload side. - for i in $(seq 1 1440); do - if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/immich-library.tar" >/dev/null 2>&1; then - echo "Library tarball found."; break - fi - echo " not ready yet ($i/1440) — sleeping 10s"; sleep 10 - if [ "$i" = "1440" ]; then - echo "ERROR: tarball not found after 4 hours"; exit 1 - fi - done - - aws s3 ls "s3://$S3_BUCKET/$S3_PREFIX/" - - echo "Streaming library tarball → /library ..." - aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/immich-library.tar" - | tar -x -C /library - - echo "Normalizing ownership to 1000:1000 ..." - chown -R 1000:1000 /library - - echo "Writing seed sentinel ..." - touch /library/.restore-done - chown 1000:1000 /library/.restore-done - - echo "Done. Summary:" - du -sh /library - env: - - name: S3_ENDPOINT - value: "https://s3.nas-1.renner.uno" - - name: S3_BUCKET - value: "k8s-migration" - - name: S3_PREFIX - value: "immich" - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: garage-migration-creds - key: AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: garage-migration-creds - key: AWS_SECRET_ACCESS_KEY - volumeMounts: - - name: library - mountPath: /library - volumes: - - name: library - persistentVolumeClaim: - claimName: immich-library diff --git a/kubernetes/apps/immich-seed/garage-migration-creds.sops.yaml b/kubernetes/apps/immich/garage-migration-creds.sops.yaml similarity index 100% rename from kubernetes/apps/immich-seed/garage-migration-creds.sops.yaml rename to kubernetes/apps/immich/garage-migration-creds.sops.yaml diff --git a/kubernetes/apps/immich/kustomization.yaml b/kubernetes/apps/immich/kustomization.yaml index 869d8a1c..4f696d7b 100644 --- a/kubernetes/apps/immich/kustomization.yaml +++ b/kubernetes/apps/immich/kustomization.yaml @@ -1,6 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - garage-migration-creds.sops.yaml + - pvcs.yaml - secrets.sops.yaml - redis.yaml - machine-learning.yaml diff --git a/kubernetes/apps/immich-seed/pvcs.yaml b/kubernetes/apps/immich/pvcs.yaml similarity index 100% rename from kubernetes/apps/immich-seed/pvcs.yaml rename to kubernetes/apps/immich/pvcs.yaml diff --git a/kubernetes/apps/immich/server.yaml b/kubernetes/apps/immich/server.yaml index e6dd8eb6..251704e5 100644 --- a/kubernetes/apps/immich/server.yaml +++ b/kubernetes/apps/immich/server.yaml @@ -54,26 +54,85 @@ spec: seccompProfile: type: RuntimeDefault initContainers: - # Block until the seed Job writes its sentinel so the server - # doesn't initialize against an empty /data while restore is - # still streaming. - - name: wait-for-restore + # Restore the immich-library PVC from the Garage tarball written + # by the `migrate-immich` Woodpecker pipeline. Sentinel-guarded + # so subsequent pod restarts short-circuit in <1s. Lives in the + # app pod (not a separate Job) so no orphan Job churns after + # ttlSecondsAfterFinished. On a fresh cluster rebuild the volume + # is empty, so this runs the full ~250 GiB extract before the + # server container starts. + # Path is /data because that's the server pod's mount; the seed + # body's references use /data accordingly (same PVC, just a + # different in-container path than the legacy seed Job used). + - name: seed image: alpine:3.21 command: - sh - -c - | - until [ -f /data/.restore-done ]; do - echo "waiting for /data/.restore-done ..." - sleep 10 + set -eu + if [ -f /data/.restore-done ] || [ -f /data/.restore-v2-done ]; then + echo "Seed already applied (sentinel present); normalizing and exiting." + touch /data/.restore-done + chown 1000:1000 /data/.restore-done 2>/dev/null || true + exit 0 + fi + apk add --no-cache aws-cli tar ca-certificates >/dev/null + export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage + # TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so + # force path-style addressing (bucket in path, not subdomain). + mkdir -p "$HOME/.aws" + printf '[default]\ns3 =\n addressing_style = path\n' > "$HOME/.aws/config" + + echo "Waiting for tarball at s3://$S3_BUCKET/$S3_PREFIX/ ..." + # ~250 GiB library — allow 4 h for the upload side. + for i in $(seq 1 1440); do + if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/immich-library.tar" >/dev/null 2>&1; then + echo "Library tarball found."; break + fi + echo " not ready yet ($i/1440) — sleeping 10s"; sleep 10 + if [ "$i" = "1440" ]; then + echo "ERROR: tarball not found after 4 hours"; exit 1 + fi done - echo "restore sentinel present — continuing." + + aws s3 ls "s3://$S3_BUCKET/$S3_PREFIX/" + + echo "Streaming library tarball → /data ..." + aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/immich-library.tar" - | tar -x -C /data + + echo "Normalizing ownership to 1000:1000 ..." + chown -R 1000:1000 /data + + echo "Writing seed sentinel ..." + touch /data/.restore-done + chown 1000:1000 /data/.restore-done + + echo "Done. Summary:" + du -sh /data + env: + - name: S3_ENDPOINT + value: "https://s3.nas-1.renner.uno" + - name: S3_BUCKET + value: "k8s-migration" + - name: S3_PREFIX + value: "immich" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: garage-migration-creds + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: garage-migration-creds + key: AWS_SECRET_ACCESS_KEY securityContext: - runAsNonRoot: true - runAsUser: 1000 - allowPrivilegeEscalation: false + runAsNonRoot: false + allowPrivilegeEscalation: true capabilities: drop: ["ALL"] + add: ["CHOWN", "DAC_OVERRIDE", "FOWNER"] volumeMounts: - name: library mountPath: /data diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index 54d6f4aa..c97d3ed9 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -30,43 +30,16 @@ spec: --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization -metadata: - name: immich-seed - namespace: flux-system -spec: - interval: 10m - retryInterval: 1m - # Library tar (~250 GiB) is the slow path; match audiobookshelf's 4 h ceiling. - timeout: 4h - sourceRef: - kind: GitRepository - name: flux-system - path: ./kubernetes/apps/immich-seed - prune: true - wait: true - decryption: - provider: sops - secretRef: - name: sops-age - healthCheckExprs: - - apiVersion: batch/v1 - kind: Job - current: "status.succeeded == 1" - inProgress: "!has(status.succeeded) || status.succeeded == 0" - failed: "has(status.failed) && status.failed >= 1" - dependsOn: - - name: infrastructure-ceph-cluster - - name: velero-bootstrap-restore ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization metadata: name: immich namespace: flux-system spec: + # Seed Job collapsed into the immich-server pod's `seed` initContainer + # (same RWX library mount). Library tar (~250 GiB) is the slow path, + # so timeout has to absorb the worst-case 4 h tarball wait + extract. interval: 10m retryInterval: 1m - timeout: 1h + timeout: 4h sourceRef: kind: GitRepository name: flux-system @@ -79,4 +52,4 @@ spec: name: sops-age dependsOn: - name: immich-db - - name: immich-seed + - name: velero-bootstrap-restore