fix(immich): collapse seed Job into immich-server pod initContainer
Same shape as music-assistant (c9ea0af) and audiobookshelf (b9717e9): the standalone seed Job had ttlSecondsAfterFinished: 3600 and was being recreated by Flux every hour for no purpose (sentinel-shortcut would re-exit within 1s, but the kubelet/image-pull/PVC-mount cycle still spent resources). Move the seed body into a `seed` initContainer in the immich-server pod. Immich uses RWX (ceph-filesystem) for the library volume, so the Multi-Attach race that broke music-assistant didn't surface here — but the refactor still applies for consistency, kills the hourly Job churn, and removes one more Flux Kustomization. Path note: the legacy seed Job mounted the library at /library; the immich-server pod mounts it at /data. The new initContainer uses /data to match the rest of the pod (it's the same PVC, just a different in-container mount path). Refactor: - replace `wait-for-restore` initContainer in server.yaml with the self-contained `seed` initContainer (preserves runAsRoot + CHOWN/DAC_OVERRIDE/FOWNER for the recursive 1000:1000 chown) - move garage-migration-creds.sops.yaml + pvcs.yaml from kubernetes/apps/immich-seed/ into kubernetes/apps/immich/ - delete kubernetes/apps/immich-seed/ - collapse the standalone immich-seed Flux Kustomization; replace `dependsOn: immich-seed` on the app Kustomization with velero-bootstrap-restore (immich-db dep stays — it's a separate CNPG bootstrap-recovery concern) - raise app Kustomization timeout 1h → 4h to absorb the worst-case tarball wait immich-machine-learning is unchanged — it doesn't touch the library. fetch-le-staging-roots initContainer is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -1,6 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- garage-migration-creds.sops.yaml
|
||||
- pvcs.yaml
|
||||
- seed-job.yaml
|
||||
@@ -1,102 +0,0 @@
|
||||
---
|
||||
# Dev-phase seed Job for Immich PVCs. Reads the legacy library tarball
|
||||
# from Garage (s3://k8s-migration/immich/...) written by the standalone
|
||||
# `migrate-immich` Woodpecker pipeline. The Postgres dump is handled by
|
||||
# the immich-db Cluster's bootstrap.recovery against the parallel
|
||||
# s3://k8s-cnpg/im-pg Barman archive seeded by the same pipeline.
|
||||
# The ML model cache is NOT seeded — models re-download on demand.
|
||||
#
|
||||
# Idempotent: sentinel `/library/.restore-done` short-circuits on repeat
|
||||
# runs, including after a Velero restore (which preserves the sentinel).
|
||||
# Legacy `.restore-v2-done` is still honored and normalized forward.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: immich-data-seed-v2
|
||||
namespace: immich
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: seed
|
||||
image: alpine:3.21
|
||||
securityContext:
|
||||
runAsNonRoot: false
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "DAC_OVERRIDE", "FOWNER"]
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
if [ -f /library/.restore-done ] || [ -f /library/.restore-v2-done ]; then
|
||||
echo "Seed already applied (sentinel present); normalizing and exiting."
|
||||
touch /library/.restore-done
|
||||
chown 1000:1000 /library/.restore-done 2>/dev/null || true
|
||||
exit 0
|
||||
fi
|
||||
apk add --no-cache aws-cli tar ca-certificates >/dev/null
|
||||
export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage
|
||||
# TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so
|
||||
# force path-style addressing (bucket in path, not subdomain).
|
||||
mkdir -p "$HOME/.aws"
|
||||
printf '[default]\ns3 =\n addressing_style = path\n' > "$HOME/.aws/config"
|
||||
|
||||
echo "Waiting for tarball at s3://$S3_BUCKET/$S3_PREFIX/ ..."
|
||||
# ~250 GiB library — allow 4 h for the upload side.
|
||||
for i in $(seq 1 1440); do
|
||||
if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/immich-library.tar" >/dev/null 2>&1; then
|
||||
echo "Library tarball found."; break
|
||||
fi
|
||||
echo " not ready yet ($i/1440) — sleeping 10s"; sleep 10
|
||||
if [ "$i" = "1440" ]; then
|
||||
echo "ERROR: tarball not found after 4 hours"; exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
aws s3 ls "s3://$S3_BUCKET/$S3_PREFIX/"
|
||||
|
||||
echo "Streaming library tarball → /library ..."
|
||||
aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/immich-library.tar" - | tar -x -C /library
|
||||
|
||||
echo "Normalizing ownership to 1000:1000 ..."
|
||||
chown -R 1000:1000 /library
|
||||
|
||||
echo "Writing seed sentinel ..."
|
||||
touch /library/.restore-done
|
||||
chown 1000:1000 /library/.restore-done
|
||||
|
||||
echo "Done. Summary:"
|
||||
du -sh /library
|
||||
env:
|
||||
- name: S3_ENDPOINT
|
||||
value: "https://s3.nas-1.renner.uno"
|
||||
- name: S3_BUCKET
|
||||
value: "k8s-migration"
|
||||
- name: S3_PREFIX
|
||||
value: "immich"
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: garage-migration-creds
|
||||
key: AWS_ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: garage-migration-creds
|
||||
key: AWS_SECRET_ACCESS_KEY
|
||||
volumeMounts:
|
||||
- name: library
|
||||
mountPath: /library
|
||||
volumes:
|
||||
- name: library
|
||||
persistentVolumeClaim:
|
||||
claimName: immich-library
|
||||
@@ -1,6 +1,8 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- garage-migration-creds.sops.yaml
|
||||
- pvcs.yaml
|
||||
- secrets.sops.yaml
|
||||
- redis.yaml
|
||||
- machine-learning.yaml
|
||||
|
||||
@@ -54,26 +54,85 @@ spec:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
# Block until the seed Job writes its sentinel so the server
|
||||
# doesn't initialize against an empty /data while restore is
|
||||
# still streaming.
|
||||
- name: wait-for-restore
|
||||
# Restore the immich-library PVC from the Garage tarball written
|
||||
# by the `migrate-immich` Woodpecker pipeline. Sentinel-guarded
|
||||
# so subsequent pod restarts short-circuit in <1s. Lives in the
|
||||
# app pod (not a separate Job) so no orphan Job churns after
|
||||
# ttlSecondsAfterFinished. On a fresh cluster rebuild the volume
|
||||
# is empty, so this runs the full ~250 GiB extract before the
|
||||
# server container starts.
|
||||
# Path is /data because that's the server pod's mount; the seed
|
||||
# body's references use /data accordingly (same PVC, just a
|
||||
# different in-container path than the legacy seed Job used).
|
||||
- name: seed
|
||||
image: alpine:3.21
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until [ -f /data/.restore-done ]; do
|
||||
echo "waiting for /data/.restore-done ..."
|
||||
sleep 10
|
||||
set -eu
|
||||
if [ -f /data/.restore-done ] || [ -f /data/.restore-v2-done ]; then
|
||||
echo "Seed already applied (sentinel present); normalizing and exiting."
|
||||
touch /data/.restore-done
|
||||
chown 1000:1000 /data/.restore-done 2>/dev/null || true
|
||||
exit 0
|
||||
fi
|
||||
apk add --no-cache aws-cli tar ca-certificates >/dev/null
|
||||
export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage
|
||||
# TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so
|
||||
# force path-style addressing (bucket in path, not subdomain).
|
||||
mkdir -p "$HOME/.aws"
|
||||
printf '[default]\ns3 =\n addressing_style = path\n' > "$HOME/.aws/config"
|
||||
|
||||
echo "Waiting for tarball at s3://$S3_BUCKET/$S3_PREFIX/ ..."
|
||||
# ~250 GiB library — allow 4 h for the upload side.
|
||||
for i in $(seq 1 1440); do
|
||||
if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/immich-library.tar" >/dev/null 2>&1; then
|
||||
echo "Library tarball found."; break
|
||||
fi
|
||||
echo " not ready yet ($i/1440) — sleeping 10s"; sleep 10
|
||||
if [ "$i" = "1440" ]; then
|
||||
echo "ERROR: tarball not found after 4 hours"; exit 1
|
||||
fi
|
||||
done
|
||||
echo "restore sentinel present — continuing."
|
||||
|
||||
aws s3 ls "s3://$S3_BUCKET/$S3_PREFIX/"
|
||||
|
||||
echo "Streaming library tarball → /data ..."
|
||||
aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/immich-library.tar" - | tar -x -C /data
|
||||
|
||||
echo "Normalizing ownership to 1000:1000 ..."
|
||||
chown -R 1000:1000 /data
|
||||
|
||||
echo "Writing seed sentinel ..."
|
||||
touch /data/.restore-done
|
||||
chown 1000:1000 /data/.restore-done
|
||||
|
||||
echo "Done. Summary:"
|
||||
du -sh /data
|
||||
env:
|
||||
- name: S3_ENDPOINT
|
||||
value: "https://s3.nas-1.renner.uno"
|
||||
- name: S3_BUCKET
|
||||
value: "k8s-migration"
|
||||
- name: S3_PREFIX
|
||||
value: "immich"
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: garage-migration-creds
|
||||
key: AWS_ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: garage-migration-creds
|
||||
key: AWS_SECRET_ACCESS_KEY
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: false
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "DAC_OVERRIDE", "FOWNER"]
|
||||
volumeMounts:
|
||||
- name: library
|
||||
mountPath: /data
|
||||
|
||||
@@ -30,43 +30,16 @@ spec:
|
||||
---
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: immich-seed
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
# Library tar (~250 GiB) is the slow path; match audiobookshelf's 4 h ceiling.
|
||||
timeout: 4h
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/apps/immich-seed
|
||||
prune: true
|
||||
wait: true
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
healthCheckExprs:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
current: "status.succeeded == 1"
|
||||
inProgress: "!has(status.succeeded) || status.succeeded == 0"
|
||||
failed: "has(status.failed) && status.failed >= 1"
|
||||
dependsOn:
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: velero-bootstrap-restore
|
||||
---
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: immich
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# Seed Job collapsed into the immich-server pod's `seed` initContainer
|
||||
# (same RWX library mount). Library tar (~250 GiB) is the slow path,
|
||||
# so timeout has to absorb the worst-case 4 h tarball wait + extract.
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
timeout: 1h
|
||||
timeout: 4h
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
@@ -79,4 +52,4 @@ spec:
|
||||
name: sops-age
|
||||
dependsOn:
|
||||
- name: immich-db
|
||||
- name: immich-seed
|
||||
- name: velero-bootstrap-restore
|
||||
|
||||
Reference in New Issue
Block a user