diff --git a/kubernetes/apps/audiobookshelf/deployment.yaml b/kubernetes/apps/audiobookshelf/deployment.yaml index 35e380e7..189f7418 100644 --- a/kubernetes/apps/audiobookshelf/deployment.yaml +++ b/kubernetes/apps/audiobookshelf/deployment.yaml @@ -34,98 +34,12 @@ spec: app.kubernetes.io/name: audiobookshelf spec: enableServiceLinks: false - serviceAccountName: audiobookshelf-seed securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch seccompProfile: type: RuntimeDefault initContainers: - # Atomic seed: idempotency-gated rsync of config + metadata + - # books PVCs from the legacy VM. Two-phase so the legacy - # systemd unit stays up during the long books rsync. - - name: seed - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29 - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SCHED - value: audiobookshelf-daily - - name: LEGACY_HOST - value: audiobookshelf.srv.renner.uno - command: - - bash - - -c - - | - set -euo pipefail - log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - - log "Installing rsync ..." - apk add --no-cache rsync >/dev/null - - EXISTING=$(kubectl -n velero get backups.velero.io \ - -l "velero.io/schedule-name=$SCHED" \ - -o jsonpath='{range .items[?(@.status.phase=="Completed")]}{.metadata.name}{"\n"}{end}' \ - 2>/dev/null | grep -c . || true) - if [ "$EXISTING" -gt 0 ]; then - log "$EXISTING completed Velero Backup(s) for $SCHED — skipping seed" - exit 0 - fi - log "No completed Velero Backup for $SCHED — proceeding with seed" - - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - LEGACY="root@$LEGACY_HOST" - - # Phase 1: books while legacy stays up (long, no --delete). - log "Phase 1: rsync /audiobooks → /audiobooks (legacy stays up) ..." - rsync -aH --info=progress2 \ - "$LEGACY:/audiobooks/" /audiobooks/ - - # Phase 2: stop legacy briefly for SQLite-bearing dirs + delta. - log "Phase 2: stopping audiobookshelf on legacy VM ..." - ssh "$LEGACY" "systemctl stop audiobookshelf" - trap 'log "Restarting audiobookshelf on legacy VM ..."; ssh "$LEGACY" "systemctl start audiobookshelf" || true' EXIT - - log " rsync /usr/share/audiobookshelf/config → /config ..." - rsync -aH --delete --info=progress2 \ - "$LEGACY:/usr/share/audiobookshelf/config/" /config/ - log " rsync /usr/share/audiobookshelf/metadata → /metadata ..." - rsync -aH --delete --info=progress2 \ - "$LEGACY:/usr/share/audiobookshelf/metadata/" /metadata/ - log " rsync /audiobooks delta with --delete ..." - rsync -aH --delete --info=progress2 \ - "$LEGACY:/audiobooks/" /audiobooks/ - chown -R 1000:1000 /config /metadata /audiobooks - du -sh /config /metadata /audiobooks - - BACKUP_NAME="$SCHED-seed-$(date -u +%Y%m%d-%H%M%S)" - log "Creating Velero Backup $BACKUP_NAME ..." - cat </dev/null - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - echo "[$(date -u +%H:%M:%S)] pg_dump on $LEGACY_HOST → /seed/legacy.dump" - ssh "root@$LEGACY_HOST" "su - postgres -c 'pg_dump --clean --if-exists --no-owner --no-privileges --format=custom $PGDATABASE'" > /seed/legacy.dump - ls -lh /seed/legacy.dump - volumeMounts: - - name: ssh - mountPath: /ssh - readOnly: true - - name: seed - mountPath: /seed - containers: - - name: seed - image: postgres:16 - env: - - name: PGDATA - value: /var/lib/postgresql/data - - name: TARGET_DB - value: authentik - - name: TARGET_OWNER - value: authentik - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: ACCESS_SECRET_KEY - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: REGION - - name: GARAGE_ENDPOINT - value: https://s3.nas-1.renner.uno - - name: BARMAN_SERVER - value: ak-pg - - name: BARMAN_DEST - value: s3://k8s-cnpg/ak-pg - command: - - bash - - -c - - | - set -euo pipefail - log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - - log "Installing awscli + barman ..." - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y --no-install-recommends \ - python3 python3-pip python3-venv python3-psycopg2 \ - awscli ca-certificates >/dev/null - - # Idempotency gate: only skip if a real base backup exists at - # //base/. A failed previous attempt - # may leave dangling marker objects under /, but - # not under base/, so this distinguishes "complete backup" from - # "partial-write garbage". - EXISTING=$(aws s3 ls --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST/$BARMAN_SERVER/base/" 2>/dev/null | wc -l || true) - if [ "$EXISTING" -gt 0 ]; then - log "Garage already has $EXISTING base backup entr(ies) at $BARMAN_DEST/$BARMAN_SERVER/base/ — seed skipped." - exit 0 - fi - log "No base backup at $BARMAN_DEST/$BARMAN_SERVER/base/ — running full seed process." - - python3 -m venv --system-site-packages /opt/barman - /opt/barman/bin/pip install --no-cache-dir --quiet 'barman[cloud,aws-snapshots]' - export PATH="/opt/barman/bin:$PATH" - - # Ephemeral Postgres in PGDATA. trust auth, local-only. - mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA" - su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres" - - # WAL archiving so end_wal of the base backup actually lands in - # the archive — without this, CNPG recovery fails on missing WAL. - # postgres daemon sanitizes env on startup, so bake creds into - # the wrapper script. - cat > /usr/local/bin/archive_wal.sh <> /tmp/archive_wal.log 2>&1 - export AWS_ACCESS_KEY_ID='$AWS_ACCESS_KEY_ID' - export AWS_SECRET_ACCESS_KEY='$AWS_SECRET_ACCESS_KEY' - export AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION' - echo "\$(date -Iseconds) archiving \$1" - exec barman-cloud-wal-archive \\ - --cloud-provider aws-s3 \\ - --endpoint-url '$GARAGE_ENDPOINT' \\ - --gzip -e AES256 \\ - '$BARMAN_DEST' '$BARMAN_SERVER' "\$1" - SH - chmod +x /usr/local/bin/archive_wal.sh - : > /tmp/archive_wal.log - chmod 666 /tmp/archive_wal.log - - cat >> "$PGDATA/postgresql.conf" </dev/null | wc -l) - done_count=$(find "$PGDATA/pg_wal/archive_status" -name '*.done' 2>/dev/null | wc -l) - log "archive_status: ready=$ready done=$done_count" - [ "$ready" -eq 0 ] && [ "$done_count" -gt 0 ] && break - sleep 2 - done - log "=== last 20 lines of archive_wal.log ===" - tail -20 /tmp/archive_wal.log || true - - # Confirm the base backup actually landed before we declare success. - barman-cloud-backup-list --cloud-provider aws-s3 \ - --endpoint-url "$GARAGE_ENDPOINT" \ - "$BARMAN_DEST" "$BARMAN_SERVER" - - su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast stop" - log "Seed complete: Garage now has a base backup at $BARMAN_DEST/$BARMAN_SERVER." - volumeMounts: - - name: seed - mountPath: /seed - - name: pgdata - mountPath: /var/lib/postgresql/data - volumes: - - name: ssh - secret: - secretName: authentik-seed-ssh - defaultMode: 0400 - - name: seed - emptyDir: {} - - name: pgdata - emptyDir: {} diff --git a/kubernetes/apps/immich/kustomization.yaml b/kubernetes/apps/immich/kustomization.yaml index cb6fe62c..869d8a1c 100644 --- a/kubernetes/apps/immich/kustomization.yaml +++ b/kubernetes/apps/immich/kustomization.yaml @@ -1,14 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - migration-bucket.yaml - - barman-creds.sops.yaml - - objectstore.yaml - - postgres.yaml - - scheduled-backup.yaml - - seed-ssh.sops.yaml - - seed-job.yaml - - pvcs.yaml - secrets.sops.yaml - redis.yaml - machine-learning.yaml diff --git a/kubernetes/apps/immich/postgres.yaml b/kubernetes/apps/immich/postgres.yaml deleted file mode 100644 index 9cc4cc36..00000000 --- a/kubernetes/apps/immich/postgres.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Cluster -metadata: - name: im-pg - namespace: immich - annotations: - # Same rationale as paperless-db: skip the plugin's pre-recovery - # "Expected empty archive" check so bootstrap.recovery works against a - # pre-populated archive (migration seed or later live backups). - cnpg.io/skipEmptyWalArchiveCheck: "enabled" -spec: - # Single instance during the pgvecto.rs→VectorChord migration window. - # pg_basebackup (used by CNPG for replica join) uses tar format, which - # has a 100-byte filename limit. pgvecto.rs stores indexes under paths - # like `pg_vectors/indexes//segments/` that exceed the - # limit, so any replica join fails with - # "file name too long for tar format: pg_vectors/indexes/..." - # After Immich's in-app migration drops the `vectors` extension and the - # `pg_vectors/` directory goes away, bump back to 2. - instances: 1 - # Dual-extension image: ships pgvector + pgvecto.rs + VectorChord so - # Immich's upstream in-app migration from `vectors` → `vchord` can run on - # first boot against the recovered legacy (pgvecto.rs) dump. Swap to - # `ghcr.io/tensorchord/cloudnative-vectorchord:16-0.3.0` once the - # migration has completed and the `vectors` extension has been dropped. - imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration - storage: - storageClass: ceph-block - size: 20Gi - # Barman Cloud plugin drives both WAL archiving and scheduled base backups - # to Garage (bucket k8s-cnpg). See objectstore.yaml. - plugins: - - name: barman-cloud.cloudnative-pg.io - isWALArchiver: true - parameters: - barmanObjectName: im-pg-store - # Recover from the latest Barman base backup + WAL in s3://k8s-cnpg/im-pg. - # Migration seed is produced by `.woodpecker/migrate-immich.yml` against - # the same path. Producer == consumer path. - bootstrap: - recovery: - owner: immich - database: immich - source: im-pg-backup - externalClusters: - - name: im-pg-backup - plugin: - name: barman-cloud.cloudnative-pg.io - parameters: - barmanObjectName: im-pg-store - serverName: im-pg - postgresql: - # Both extensions preloaded during the migration window. `vectors.so` - # is required for pgvecto.rs catalog objects in the restored dump to - # load at startup; `vchord.so` is required for Immich's in-app - # migration to `CREATE EXTENSION vchord` and rewrite the data. - # CNPG rejects shared_preload_libraries under .parameters because it - # manages that GUC itself — the dedicated list field is the only path. - shared_preload_libraries: - - vectors.so - - vchord.so - parameters: - # Match LXC 225 Postgres tuning: 8 GB RAM / pgvecto.rs workload. - shared_buffers: "512MB" - max_wal_size: "2GB" - wal_compression: "on" - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - memory: 2Gi diff --git a/kubernetes/apps/immich/seed-job.yaml b/kubernetes/apps/immich/seed-job.yaml deleted file mode 100644 index 5378be44..00000000 --- a/kubernetes/apps/immich/seed-job.yaml +++ /dev/null @@ -1,227 +0,0 @@ ---- -# Cold-start seed for im-pg. Same shape as ak-pg-seed: ephemeral -# Postgres + pg_restore from legacy dump + barman-cloud-backup to -# s3://k8s-cnpg/im-pg/. Idempotency-gated on Garage having any -# object under that path. -# -# Legacy immich DB lives in a docker container (label -# com.docker.compose.service=database) on immich.srv.renner.uno. -apiVersion: batch/v1 -kind: Job -metadata: - name: im-pg-seed - namespace: immich -spec: - backoffLimit: 2 - ttlSecondsAfterFinished: 86400 - template: - spec: - restartPolicy: Never - initContainers: - - name: legacy-dump - image: alpine:3.21 - env: - - name: LEGACY_HOST - value: immich.srv.renner.uno - - name: PGDATABASE - value: immich - - name: PGUSER - value: postgres - command: - - sh - - -c - - | - set -eu - apk add --no-cache openssh-client >/dev/null - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - LEGACY="root@$LEGACY_HOST" - echo "[$(date -u +%H:%M:%S)] Locating immich DB container on $LEGACY_HOST ..." - DB_CT=$(ssh "$LEGACY" "docker ps --filter label=com.docker.compose.service=database --format '{{.Names}}' | head -1") - [ -n "$DB_CT" ] || { echo "no DB container found on $LEGACY_HOST"; exit 1; } - echo "[$(date -u +%H:%M:%S)] pg_dump via docker exec $DB_CT → /seed/legacy.dump" - ssh "$LEGACY" "docker exec -i $DB_CT pg_dump --clean --if-exists --no-owner --no-privileges --format=custom -U $PGUSER $PGDATABASE" > /seed/legacy.dump - ls -lh /seed/legacy.dump - volumeMounts: - - name: ssh - mountPath: /ssh - readOnly: true - - name: seed - mountPath: /seed - containers: - - name: seed - # Match the live im-pg Cluster's image: dual-extension build that - # ships pgvector + pgvecto.rs + VectorChord. The ephemeral Postgres - # in this Job needs the same extension files so the legacy dump's - # CREATE EXTENSION statements + the face_search/smart_search vector - # tables actually restore. Without this, the backup written to - # Garage is missing the vector catalog and immich crash-loops with - # "permission denied to create extension vector" once it tries to - # CREATE EXTENSION as the (non-superuser) immich role. - image: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration - # CNPG's base image runs as the postgres user by default; the seed - # script needs root for `apt-get install`, `chown`, and `su postgres`. - securityContext: - runAsUser: 0 - runAsNonRoot: false - env: - - name: PGDATA - value: /var/lib/postgresql/data - - name: TARGET_DB - value: immich - - name: TARGET_OWNER - value: immich - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: ACCESS_SECRET_KEY - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: REGION - - name: GARAGE_ENDPOINT - value: https://s3.nas-1.renner.uno - - name: BARMAN_SERVER - value: im-pg - - name: BARMAN_DEST - value: s3://k8s-cnpg/im-pg - command: - - bash - - -c - - | - set -euo pipefail - log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - - log "Installing awscli + barman ..." - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y --no-install-recommends \ - python3 python3-pip python3-venv python3-psycopg2 \ - awscli ca-certificates >/dev/null - - # Idempotency gate: only skip if a real base backup exists at - # //base/. Failed previous attempts - # may leave dangling marker objects under /. - EXISTING=$(aws s3 ls --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST/$BARMAN_SERVER/base/" 2>/dev/null | wc -l || true) - if [ "$EXISTING" -gt 0 ]; then - log "Garage already has $EXISTING base backup entr(ies) at $BARMAN_DEST/$BARMAN_SERVER/base/ — seed skipped." - exit 0 - fi - log "No base backup at $BARMAN_DEST/$BARMAN_SERVER/base/ — running full seed process." - - python3 -m venv --system-site-packages /opt/barman - /opt/barman/bin/pip install --no-cache-dir --quiet 'barman[cloud,aws-snapshots]' - export PATH="/opt/barman/bin:$PATH" - - mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA" - su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres" - - cat > /usr/local/bin/archive_wal.sh <> /tmp/archive_wal.log 2>&1 - export AWS_ACCESS_KEY_ID='$AWS_ACCESS_KEY_ID' - export AWS_SECRET_ACCESS_KEY='$AWS_SECRET_ACCESS_KEY' - export AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION' - echo "\$(date -Iseconds) archiving \$1" - exec barman-cloud-wal-archive \\ - --cloud-provider aws-s3 \\ - --endpoint-url '$GARAGE_ENDPOINT' \\ - --gzip -e AES256 \\ - '$BARMAN_DEST' '$BARMAN_SERVER' "\$1" - SH - chmod +x /usr/local/bin/archive_wal.sh - : > /tmp/archive_wal.log - chmod 666 /tmp/archive_wal.log - - cat >> "$PGDATA/postgresql.conf" </dev/null | wc -l) - done_count=$(find "$PGDATA/pg_wal/archive_status" -name '*.done' 2>/dev/null | wc -l) - log "archive_status: ready=$ready done=$done_count" - [ "$ready" -eq 0 ] && [ "$done_count" -gt 0 ] && break - sleep 2 - done - log "=== last 20 lines of archive_wal.log ===" - tail -20 /tmp/archive_wal.log || true - - barman-cloud-backup-list --cloud-provider aws-s3 \ - --endpoint-url "$GARAGE_ENDPOINT" \ - "$BARMAN_DEST" "$BARMAN_SERVER" - - su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast stop" - log "Seed complete: Garage now has a base backup at $BARMAN_DEST/$BARMAN_SERVER." - volumeMounts: - - name: seed - mountPath: /seed - - name: pgdata - mountPath: /var/lib/postgresql/data - volumes: - - name: ssh - secret: - secretName: immich-seed-ssh - defaultMode: 0400 - - name: seed - emptyDir: {} - - name: pgdata - emptyDir: {} diff --git a/kubernetes/apps/immich/server.yaml b/kubernetes/apps/immich/server.yaml index 994ad9af..7cb60ecd 100644 --- a/kubernetes/apps/immich/server.yaml +++ b/kubernetes/apps/immich/server.yaml @@ -54,58 +54,11 @@ spec: seccompProfile: type: RuntimeDefault initContainers: - # PVC (library) seed only — the DB seed lives in the standalone - # im-pg-seed Job (seed-job.yaml) which populates Garage so CNPG - # bootstrap.recovery has data to pull from. This initContainer - # rsyncs the legacy Immich LXC's library directory into the - # immich-library PVC and drops a sentinel. - - name: seed - image: alpine:3.21 - env: - - name: LEGACY_HOST - value: immich.srv.renner.uno - command: - - sh - - -c - - | - set -eu - log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - if [ -f /data/.seed-done ]; then - log "PVC seed sentinel present — skipping rsync." - exit 0 - fi - log "No sentinel — rsync from $LEGACY_HOST" - - apk add --no-cache rsync openssh-client >/dev/null - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - LEGACY="root@$LEGACY_HOST" - - log "rsync /mnt/media/immich → /data ..." - set +e - rsync -aH --delete --info=progress2 \ - "$LEGACY:/mnt/media/immich/" /data/ - rsync_rc=$? - set -e - case "$rsync_rc" in - 0) log "rsync OK" ;; - 23) log "rsync exit 23 (partial transfer warnings) — continuing" ;; - *) log "rsync FAILED with code $rsync_rc"; exit "$rsync_rc" ;; - esac - chown -R 1000:1000 /data - du -sh /data - touch /data/.seed-done - log "PVC seed complete; sentinel set." - volumeMounts: - - name: library - mountPath: /data - - name: ssh - mountPath: /ssh - readOnly: true # Pre-fetch LE staging root CAs so Node trusts auth.renner.uno's # staging-issued cert during OIDC token exchange. Same pattern as # audiobookshelf (see its deployment.yaml for the full rationale). + # The library PVC is populated by the seed phase before this + # Deployment reconciles, so no rsync init is needed here. - name: fetch-le-staging-roots image: curlimages/curl:8.10.1 command: @@ -195,10 +148,6 @@ spec: path: immich-config.json - name: le-ca emptyDir: {} - - name: ssh - secret: - secretName: immich-seed-ssh - defaultMode: 0400 --- apiVersion: v1 kind: Service diff --git a/kubernetes/apps/influxdb/helmrelease.yaml b/kubernetes/apps/influxdb/helmrelease.yaml index 14a8375a..a142603b 100644 --- a/kubernetes/apps/influxdb/helmrelease.yaml +++ b/kubernetes/apps/influxdb/helmrelease.yaml @@ -28,113 +28,11 @@ spec: upgrade: remediation: retries: 3 - # The influxdb2 chart's StatefulSet template has no .Values.initContainers - # hook, so the seed initContainer is added via a post-render Kustomize - # patch against the rendered StatefulSet (name: influxdb-influxdb2, - # i.e. {{ template "influxdb.fullname" . }} for release "influxdb" - # with chart "influxdb2"). The SSH volume is added via the chart's - # supported top-level `volumes:` value below. - postRenderers: - - kustomize: - patches: - - target: - version: v1 - kind: StatefulSet - name: influxdb-influxdb2 - patch: | - apiVersion: apps/v1 - kind: StatefulSet - metadata: - name: influxdb-influxdb2 - spec: - template: - spec: - initContainers: - - name: seed - image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29 - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SCHED - value: influxdb-daily - - name: LEGACY_HOST - value: influxdb.srv.renner.uno - command: - - bash - - -c - - | - set -euo pipefail - log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - - log "Installing rsync ..." - apk add --no-cache rsync >/dev/null - - EXISTING=$(kubectl -n velero get backups.velero.io \ - -l "velero.io/schedule-name=$SCHED" \ - -o jsonpath='{range .items[?(@.status.phase=="Completed")]}{.metadata.name}{"\n"}{end}' \ - 2>/dev/null | grep -c . || true) - if [ "$EXISTING" -gt 0 ]; then - log "$EXISTING completed Velero Backup(s) for $SCHED — skipping seed" - exit 0 - fi - log "No completed Velero Backup for $SCHED — proceeding with seed" - - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - LEGACY="root@$LEGACY_HOST" - - log "Stopping legacy influxdb.service ..." - ssh "$LEGACY" "systemctl stop influxdb" - trap 'log "Restarting legacy influxdb.service ..."; ssh "$LEGACY" "systemctl start influxdb" || true' EXIT - - log "rsync /var/lib/influxdb → /var/lib/influxdb2 ..." - rsync -aH --delete --info=progress2 \ - "$LEGACY:/var/lib/influxdb/" /var/lib/influxdb2/ - chown -R 1500:1500 /var/lib/influxdb2 || true - du -sh /var/lib/influxdb2 - - BACKUP_NAME="$SCHED-seed-$(date -u +%Y%m%d-%H%M%S)" - log "Creating Velero Backup $BACKUP_NAME ..." - cat </dev/null | grep -c . || true) - if [ "$EXISTING" -gt 0 ]; then - log "$EXISTING completed Velero Backup(s) for $SCHED — skipping seed" - exit 0 - fi - log "No completed Velero Backup for $SCHED — proceeding with seed" - - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - LEGACY="root@$LEGACY_HOST" - - log "Stopping music-assistant docker-compose stack on legacy LXC ..." - ssh "$LEGACY" "cd /opt/music-assistant && docker compose stop" - trap 'log "Restarting music-assistant on legacy LXC ..."; ssh "$LEGACY" "cd /opt/music-assistant && docker compose up -d" || true' EXIT - - # Legacy MA LXC ships without rsync, so use ssh+tar streaming. - log "Clearing /data ..." - find /data -mindepth 1 -delete - log "Streaming /opt/music-assistant/data via ssh+tar ..." - ssh "$LEGACY" "cd /opt/music-assistant/data && tar -cf - ." | tar -xf - -C /data/ - du -sh /data - - BACKUP_NAME="$SCHED-seed-$(date -u +%Y%m%d-%H%M%S)" - log "Creating Velero Backup $BACKUP_NAME ..." - cat </dev/null - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - LEGACY="root@$LEGACY_HOST" - - log "rsync paperless_data → /usr/src/paperless/data ..." - rsync -aH --delete --info=progress2 \ - "$LEGACY:/var/lib/docker/volumes/paperless_data/_data/" /usr/src/paperless/data/ - log "rsync paperless_media → /usr/src/paperless/media ..." - rsync -aH --delete --info=progress2 \ - "$LEGACY:/var/lib/docker/volumes/paperless_media/_data/" /usr/src/paperless/media/ - chown -R 1000:1000 /usr/src/paperless/data /usr/src/paperless/media - du -sh /usr/src/paperless/data /usr/src/paperless/media - touch /usr/src/paperless/data/.seed-done - log "PVC seed complete; sentinel set." - volumeMounts: - - name: data - mountPath: /usr/src/paperless/data - - name: media - mountPath: /usr/src/paperless/media - - name: ssh - mountPath: /ssh - readOnly: true containers: - name: paperless image: ghcr.io/paperless-ngx/paperless-ngx:2.20.14 @@ -170,10 +123,6 @@ spec: - name: consume persistentVolumeClaim: claimName: paperless-consume - - name: ssh - secret: - secretName: paperless-seed-ssh - defaultMode: 0400 --- apiVersion: v1 kind: Service diff --git a/kubernetes/apps/paperless/postgres.yaml b/kubernetes/apps/paperless/postgres.yaml deleted file mode 100644 index 2f5246b1..00000000 --- a/kubernetes/apps/paperless/postgres.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Cluster -metadata: - name: pprl-pg - namespace: paperless - annotations: - # See authentik-db/postgres.yaml — same rationale: skip the - # plugin's pre-recovery "Expected empty archive" check so - # bootstrap.recovery works against a pre-populated archive - # (migration seed or later live backups). - cnpg.io/skipEmptyWalArchiveCheck: "enabled" -spec: - instances: 2 - imageName: ghcr.io/cloudnative-pg/postgresql:16 - storage: - storageClass: ceph-block - size: 20Gi - # The Barman Cloud plugin drives both WAL archiving and - # scheduled base backups to Garage on nas-1 (bucket k8s-cnpg). - # See objectstore.yaml for the destination config. - plugins: - - name: barman-cloud.cloudnative-pg.io - isWALArchiver: true - parameters: - barmanObjectName: pprl-pg-store - # Bootstrap always recovers from the latest Barman base backup + WAL in - # s3://k8s-cnpg/pprl-pg. Producer path == consumer path: the live cluster - # writes to the same location via the plugin above (serverName defaults to - # the Cluster name, i.e. `pprl-pg`), and the migration pipeline seeds the - # same path using an ephemeral Postgres + barman-cloud-backup. Unpinned - # recovery target = latest WAL. - bootstrap: - recovery: - owner: paperless - database: paperless - source: pprl-pg-backup - externalClusters: - - name: pprl-pg-backup - plugin: - name: barman-cloud.cloudnative-pg.io - parameters: - barmanObjectName: pprl-pg-store - serverName: pprl-pg - postgresql: - parameters: - shared_buffers: "256MB" - max_connections: "200" - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - memory: 1Gi diff --git a/kubernetes/apps/paperless/seed-job.yaml b/kubernetes/apps/paperless/seed-job.yaml deleted file mode 100644 index c5c1d8a9..00000000 --- a/kubernetes/apps/paperless/seed-job.yaml +++ /dev/null @@ -1,196 +0,0 @@ ---- -# Cold-start seed for pprl-pg. Same shape as ak-pg-seed: ephemeral -# Postgres + pg_restore from legacy dump + barman-cloud-backup to -# s3://k8s-cnpg/pprl-pg/. Idempotency-gated on Garage having any -# object under that path. -# -# Legacy paperless DB lives in a docker container on -# paperless.srv.renner.uno; the init container locates it via -# `docker ps` label filter and pg_dumps from inside. -apiVersion: batch/v1 -kind: Job -metadata: - name: pprl-pg-seed - namespace: paperless -spec: - backoffLimit: 2 - ttlSecondsAfterFinished: 86400 - template: - spec: - restartPolicy: Never - initContainers: - - name: legacy-dump - image: alpine:3.21 - env: - - name: LEGACY_HOST - value: paperless.srv.renner.uno - - name: PGDATABASE - value: paperless - - name: PGUSER - value: paperless - command: - - sh - - -c - - | - set -eu - apk add --no-cache openssh-client >/dev/null - mkdir -p /root/.ssh - install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 - printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config - LEGACY="root@$LEGACY_HOST" - echo "[$(date -u +%H:%M:%S)] Locating paperless DB container on $LEGACY_HOST ..." - DB_CT=$(ssh "$LEGACY" "docker ps --filter label=com.docker.compose.service=db --format '{{.Names}}' | head -1") - [ -n "$DB_CT" ] || { echo "no DB container found on $LEGACY_HOST"; exit 1; } - echo "[$(date -u +%H:%M:%S)] pg_dump via docker exec $DB_CT → /seed/legacy.dump" - ssh "$LEGACY" "docker exec -i $DB_CT pg_dump --clean --if-exists --no-owner --no-privileges --format=custom -U $PGUSER $PGDATABASE" > /seed/legacy.dump - ls -lh /seed/legacy.dump - volumeMounts: - - name: ssh - mountPath: /ssh - readOnly: true - - name: seed - mountPath: /seed - containers: - - name: seed - image: postgres:16 - env: - - name: PGDATA - value: /var/lib/postgresql/data - - name: TARGET_DB - value: paperless - - name: TARGET_OWNER - value: paperless - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: ACCESS_SECRET_KEY - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: barman-cloud-creds - key: REGION - - name: GARAGE_ENDPOINT - value: https://s3.nas-1.renner.uno - - name: BARMAN_SERVER - value: pprl-pg - - name: BARMAN_DEST - value: s3://k8s-cnpg/pprl-pg - command: - - bash - - -c - - | - set -euo pipefail - log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - - log "Installing awscli + barman ..." - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y --no-install-recommends \ - python3 python3-pip python3-venv python3-psycopg2 \ - awscli ca-certificates >/dev/null - - # Idempotency gate: only skip if a real base backup exists at - # //base/. Failed previous attempts - # may leave dangling marker objects under /. - EXISTING=$(aws s3 ls --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST/$BARMAN_SERVER/base/" 2>/dev/null | wc -l || true) - if [ "$EXISTING" -gt 0 ]; then - log "Garage already has $EXISTING base backup entr(ies) at $BARMAN_DEST/$BARMAN_SERVER/base/ — seed skipped." - exit 0 - fi - log "No base backup at $BARMAN_DEST/$BARMAN_SERVER/base/ — running full seed process." - - python3 -m venv --system-site-packages /opt/barman - /opt/barman/bin/pip install --no-cache-dir --quiet 'barman[cloud,aws-snapshots]' - export PATH="/opt/barman/bin:$PATH" - - mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA" - su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres" - - cat > /usr/local/bin/archive_wal.sh <> /tmp/archive_wal.log 2>&1 - export AWS_ACCESS_KEY_ID='$AWS_ACCESS_KEY_ID' - export AWS_SECRET_ACCESS_KEY='$AWS_SECRET_ACCESS_KEY' - export AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION' - echo "\$(date -Iseconds) archiving \$1" - exec barman-cloud-wal-archive \\ - --cloud-provider aws-s3 \\ - --endpoint-url '$GARAGE_ENDPOINT' \\ - --gzip -e AES256 \\ - '$BARMAN_DEST' '$BARMAN_SERVER' "\$1" - SH - chmod +x /usr/local/bin/archive_wal.sh - : > /tmp/archive_wal.log - chmod 666 /tmp/archive_wal.log - - cat >> "$PGDATA/postgresql.conf" </dev/null | wc -l) - done_count=$(find "$PGDATA/pg_wal/archive_status" -name '*.done' 2>/dev/null | wc -l) - log "archive_status: ready=$ready done=$done_count" - [ "$ready" -eq 0 ] && [ "$done_count" -gt 0 ] && break - sleep 2 - done - log "=== last 20 lines of archive_wal.log ===" - tail -20 /tmp/archive_wal.log || true - - barman-cloud-backup-list --cloud-provider aws-s3 \ - --endpoint-url "$GARAGE_ENDPOINT" \ - "$BARMAN_DEST" "$BARMAN_SERVER" - - su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast stop" - log "Seed complete: Garage now has a base backup at $BARMAN_DEST/$BARMAN_SERVER." - volumeMounts: - - name: seed - mountPath: /seed - - name: pgdata - mountPath: /var/lib/postgresql/data - volumes: - - name: ssh - secret: - secretName: paperless-seed-ssh - defaultMode: 0400 - - name: seed - emptyDir: {} - - name: pgdata - emptyDir: {} diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 44a2b93b..2e269dda 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -285,32 +285,22 @@ if phase_runs wait-apps; then return 1 } - # Order is largely cosmetic — Flux reconciles all Kustomizations in parallel, - # so wall-clock time is dominated by the slowest one regardless of wait - # order. Listed roughly fast → slow so progress lands earlier in CI logs. - # Timeouts are sized for the *worst-case under contention* (concurrent - # Velero restores + immich's 250GB extract starving network/IO), not the - # happy-path. Bumping individual timeouts costs nothing on a healthy - # cluster since reconciliations run in parallel and the wait just - # observes. + # The seed phase populates every CNPG database and every migrating + # PVC from legacy hosts before any app Kustomization reconciles. + # Wall-clock dominated by immich-library-seed (~250 GiB rsync over + # LAN, ~30-45 min). Every app KS dependsOn `seed`, so the per-app + # waits below should be near-instant once seed lands Ready. + wait_kustomization seed "Seed phase (DB + PVC populate from legacy)" 1440 # apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up # is genuinely 5–10 min on a fresh OSD-wiped cluster). wait_kustomization apps "apps" 90 - # authentik waits on the ak-pg-seed Job populating Garage AND the CNPG - # cluster's bootstrap.recovery succeeding from that seed. Allow 20 min. - wait_kustomization authentik "Authentik" 120 + wait_kustomization authentik "Authentik" 60 wait_kustomization monitoring "kube-prometheus-stack (Loki + Prom + AM + Grafana)" 180 - # On cold-start, velero-bootstrap-restore takes ~15min before this KS - # can even start reconciling, so the wait timer can burn most of its - # budget on the dependency. 30min covers that + the actual reconcile. - wait_kustomization music-assistant "Music Assistant (small Garage tarball)" 180 - wait_kustomization paperless "Paperless (DB + ~3.5GB media Velero restore)" 180 + wait_kustomization music-assistant "Music Assistant" 60 + wait_kustomization paperless "Paperless" 60 wait_kustomization home-assistant "Home Assistant (HAOS backup → native restore)" 180 - # Measured ~70 min on a 70 GB library (17 min upload + 52 min extract). - wait_kustomization audiobookshelf "Audiobookshelf (~70GB library tar)" 480 - # ~250 GiB library — the longest wait. Mirrors the prior immich-seed - # Kustomization's 4 h ceiling (1440 × 10s = 240 min). - wait_kustomization immich "Immich (~250GB library tar)" 1440 + wait_kustomization audiobookshelf "Audiobookshelf" 60 + wait_kustomization immich "Immich" 60 # Wait for cert-manager to finish issuing every Certificate. Kustomization # Ready only means the Ingress + Certificate manifests are applied; cert- diff --git a/kubernetes/clusters/homelab/audiobookshelf.yaml b/kubernetes/clusters/homelab/audiobookshelf.yaml index a963e74a..69dfb331 100644 --- a/kubernetes/clusters/homelab/audiobookshelf.yaml +++ b/kubernetes/clusters/homelab/audiobookshelf.yaml @@ -4,11 +4,8 @@ metadata: name: audiobookshelf namespace: flux-system spec: - # PVC contents come from velero-bootstrap-restore reading from the - # audiobookshelf-daily Schedule's latest Backup (or the seed Backup - # written by migrate-audiobookshelf when no daily exists yet). - # dependsOn enforces the order; this Kustomization just reconciles - # manifests, no in-pod 70 GB extract anymore. + # PVC contents are populated by the seed phase (rsync from legacy + # audiobookshelf VM); this Kustomization reconciles app manifests only. interval: 10m retryInterval: 1m timeout: 30m @@ -26,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: velero-bootstrap-restore + - name: seed diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index ba410be6..695d61e4 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -21,3 +21,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster + - name: seed diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index d271e142..8b343db7 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -4,9 +4,8 @@ metadata: name: immich namespace: flux-system spec: - # PVC contents come from velero-bootstrap-restore; this Kustomization - # reconciles manifests only. Library tar (~250 GiB) is restored by - # Velero CSI Snapshot Data Movement, not in-pod. + # DB cluster + library PVC are populated by the seed phase; this + # Kustomization reconciles app manifests only. interval: 10m retryInterval: 1m timeout: 30m @@ -24,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: velero-bootstrap-restore + - name: seed diff --git a/kubernetes/clusters/homelab/influxdb.yaml b/kubernetes/clusters/homelab/influxdb.yaml index 7f141abc..cebe0369 100644 --- a/kubernetes/clusters/homelab/influxdb.yaml +++ b/kubernetes/clusters/homelab/influxdb.yaml @@ -4,12 +4,8 @@ metadata: name: influxdb namespace: flux-system spec: - # PVC contents come from velero-bootstrap-restore reading the latest - # influxdb-daily Backup (or the seed Backup written by migrate- - # influxdb when no daily exists yet). Replaces the prior API-restore - # seed-Job model — Velero captures the data files directly so the - # pod starts against a populated PVC and the helm chart's empty- - # bucket bootstrap is skipped. + # PVC contents are populated by the seed phase (rsync from legacy + # influxdb VM); this Kustomization reconciles app manifests only. interval: 10m retryInterval: 1m timeout: 30m @@ -27,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: velero-bootstrap-restore + - name: seed diff --git a/kubernetes/clusters/homelab/music-assistant.yaml b/kubernetes/clusters/homelab/music-assistant.yaml index 6e954015..917b70e5 100644 --- a/kubernetes/clusters/homelab/music-assistant.yaml +++ b/kubernetes/clusters/homelab/music-assistant.yaml @@ -4,8 +4,8 @@ metadata: name: music-assistant namespace: flux-system spec: - # PVC contents come from velero-bootstrap-restore; this Kustomization - # reconciles manifests only. + # PVC contents are populated by the seed phase (ssh+tar from legacy + # music-assistant LXC); this Kustomization reconciles app manifests only. interval: 10m retryInterval: 1m timeout: 30m @@ -23,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: velero-bootstrap-restore + - name: seed diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index 1f41d395..1f85c3f4 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -4,9 +4,8 @@ metadata: name: paperless namespace: flux-system spec: - # PVC contents are restored by velero-bootstrap-restore from the seed - # Backup written by the migrate-paperless pipeline (and from later - # paperless-daily Schedule runs). dependsOn enforces the order. + # DB cluster + data/media PVCs are populated by the seed phase; this + # Kustomization reconciles app manifests only. interval: 10m retryInterval: 1m timeout: 30m @@ -24,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: velero-bootstrap-restore + - name: seed diff --git a/kubernetes/clusters/homelab/seed.yaml b/kubernetes/clusters/homelab/seed.yaml new file mode 100644 index 00000000..8a56448b --- /dev/null +++ b/kubernetes/clusters/homelab/seed.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed + namespace: flux-system +spec: + # Migration phase: pulls data from legacy hosts into in-cluster CNPG + # databases and PVCs so app Kustomizations can reconcile against + # populated state. Lives between infrastructure and apps. Removed + # entirely once all services have cut over from legacy. + interval: 10m + retryInterval: 2m + # Long enough to cover the slowest seed (immich library: ~250 GiB + # rsync over LAN ~ 30-45 min) plus all the smaller seeds in parallel. + timeout: 4h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed + prune: true + wait: true + decryption: + provider: sops + secretRef: + name: sops-age + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman + # kstatus already handles Job (succeeded == 1) by default. CNPG Cluster + # readiness needs an explicit expression because kstatus doesn't know + # the Cluster CRD's status semantics. + healthCheckExprs: + - apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + failed: "has(status.phase) && status.phase.startsWith('Failed')" + current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)" diff --git a/kubernetes/clusters/homelab/velero-bootstrap-restore.yaml b/kubernetes/clusters/homelab/velero-bootstrap-restore.yaml deleted file mode 100644 index b22b4f86..00000000 --- a/kubernetes/clusters/homelab/velero-bootstrap-restore.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: velero-bootstrap-restore - namespace: flux-system -spec: - interval: 10m - retryInterval: 2m - # Audiobookshelf library is ~70 GB; data-movement restore is the slow - # path. The Job polls with `--wait` so this timeout must cover the - # longest expected restore, not just the Job submission. - timeout: 4h - sourceRef: - kind: GitRepository - name: flux-system - path: ./kubernetes/infrastructure/velero-bootstrap-restore - prune: true - wait: true - healthCheckExprs: - - apiVersion: batch/v1 - kind: Job - current: "status.succeeded == 1" - inProgress: "!has(status.succeeded) || status.succeeded == 0" - failed: "has(status.failed) && status.failed >= 3" - dependsOn: - - name: infrastructure - - name: infrastructure-ceph-cluster diff --git a/kubernetes/seed/audiobookshelf/kustomization.yaml b/kubernetes/seed/audiobookshelf/kustomization.yaml new file mode 100644 index 00000000..43395653 --- /dev/null +++ b/kubernetes/seed/audiobookshelf/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - migration-bucket.yaml + - pvcs.yaml + - seed-ssh.sops.yaml + - library-seed-job.yaml diff --git a/kubernetes/seed/audiobookshelf/library-seed-job.yaml b/kubernetes/seed/audiobookshelf/library-seed-job.yaml new file mode 100644 index 00000000..0d4e86ed --- /dev/null +++ b/kubernetes/seed/audiobookshelf/library-seed-job.yaml @@ -0,0 +1,75 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: audiobookshelf-library-seed + namespace: audiobookshelf +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: rsync + image: alpine:3.21 + env: + - { name: LEGACY_HOST, value: audiobookshelf.srv.renner.uno } + command: + - sh + - -c + - | + set -eu + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + if [ -f /audiobooks/.seed-done ]; then + log "PVC seed sentinel present — skipping rsync." + exit 0 + fi + + apk add --no-cache rsync openssh-client >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + LEGACY="root@$LEGACY_HOST" + + # Phase 1: bulk books while legacy stays up (no --delete). + log "Phase 1: rsync /audiobooks (legacy stays up) ..." + rsync -aH --info=progress2 \ + "$LEGACY:/audiobooks/" /audiobooks/ + + # Phase 2: stop legacy briefly for SQLite-bearing dirs + delta. + log "Phase 2: stopping audiobookshelf on legacy ..." + ssh "$LEGACY" "systemctl stop audiobookshelf" + trap 'log "Restarting audiobookshelf on legacy ..."; ssh "$LEGACY" "systemctl start audiobookshelf" || true' EXIT + + log " rsync /usr/share/audiobookshelf/config → /config ..." + rsync -aH --delete --info=progress2 \ + "$LEGACY:/usr/share/audiobookshelf/config/" /config/ + log " rsync /usr/share/audiobookshelf/metadata → /metadata ..." + rsync -aH --delete --info=progress2 \ + "$LEGACY:/usr/share/audiobookshelf/metadata/" /metadata/ + log " rsync /audiobooks delta with --delete ..." + rsync -aH --delete --info=progress2 \ + "$LEGACY:/audiobooks/" /audiobooks/ + chown -R 1000:1000 /config /metadata /audiobooks + du -sh /config /metadata /audiobooks + touch /audiobooks/.seed-done + log "PVC seed complete; sentinel set." + volumeMounts: + - { name: config, mountPath: /config } + - { name: metadata, mountPath: /metadata } + - { name: books, mountPath: /audiobooks } + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: config + persistentVolumeClaim: + claimName: audiobookshelf-config + - name: metadata + persistentVolumeClaim: + claimName: audiobookshelf-metadata + - name: books + persistentVolumeClaim: + claimName: audiobookshelf-books + - name: ssh + secret: + secretName: audiobookshelf-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/apps/audiobookshelf/migration-bucket.yaml b/kubernetes/seed/audiobookshelf/migration-bucket.yaml similarity index 100% rename from kubernetes/apps/audiobookshelf/migration-bucket.yaml rename to kubernetes/seed/audiobookshelf/migration-bucket.yaml diff --git a/kubernetes/apps/audiobookshelf/pvcs.yaml b/kubernetes/seed/audiobookshelf/pvcs.yaml similarity index 100% rename from kubernetes/apps/audiobookshelf/pvcs.yaml rename to kubernetes/seed/audiobookshelf/pvcs.yaml diff --git a/kubernetes/apps/audiobookshelf/seed-ssh.sops.yaml b/kubernetes/seed/audiobookshelf/seed-ssh.sops.yaml similarity index 100% rename from kubernetes/apps/audiobookshelf/seed-ssh.sops.yaml rename to kubernetes/seed/audiobookshelf/seed-ssh.sops.yaml diff --git a/kubernetes/apps/authentik/barman-creds.sops.yaml b/kubernetes/seed/authentik/barman-creds.sops.yaml similarity index 100% rename from kubernetes/apps/authentik/barman-creds.sops.yaml rename to kubernetes/seed/authentik/barman-creds.sops.yaml diff --git a/kubernetes/seed/authentik/cluster.yaml b/kubernetes/seed/authentik/cluster.yaml new file mode 100644 index 00000000..3c04a94d --- /dev/null +++ b/kubernetes/seed/authentik/cluster.yaml @@ -0,0 +1,31 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ak-pg + namespace: authentik +spec: + instances: 2 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + enableSuperuserAccess: true + storage: + storageClass: ceph-block + size: 10Gi + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: ak-pg-store + bootstrap: + initdb: + database: authentik + owner: authentik + postgresql: + parameters: + shared_buffers: "256MB" + max_connections: "200" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 1Gi diff --git a/kubernetes/seed/authentik/kustomization.yaml b/kubernetes/seed/authentik/kustomization.yaml new file mode 100644 index 00000000..d1058bb8 --- /dev/null +++ b/kubernetes/seed/authentik/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - migration-bucket.yaml + - barman-creds.sops.yaml + - objectstore.yaml + - cluster.yaml + - scheduled-backup.yaml + - seed-ssh.sops.yaml + - seed-job.yaml diff --git a/kubernetes/apps/authentik/migration-bucket.yaml b/kubernetes/seed/authentik/migration-bucket.yaml similarity index 100% rename from kubernetes/apps/authentik/migration-bucket.yaml rename to kubernetes/seed/authentik/migration-bucket.yaml diff --git a/kubernetes/apps/authentik/objectstore.yaml b/kubernetes/seed/authentik/objectstore.yaml similarity index 100% rename from kubernetes/apps/authentik/objectstore.yaml rename to kubernetes/seed/authentik/objectstore.yaml diff --git a/kubernetes/apps/authentik/scheduled-backup.yaml b/kubernetes/seed/authentik/scheduled-backup.yaml similarity index 100% rename from kubernetes/apps/authentik/scheduled-backup.yaml rename to kubernetes/seed/authentik/scheduled-backup.yaml diff --git a/kubernetes/seed/authentik/seed-job.yaml b/kubernetes/seed/authentik/seed-job.yaml new file mode 100644 index 00000000..529a48be --- /dev/null +++ b/kubernetes/seed/authentik/seed-job.yaml @@ -0,0 +1,80 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ak-pg-seed + namespace: authentik +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-cluster + image: postgres:16 + env: + - { name: PGHOST, value: ak-pg-rw } + - { name: PGDATABASE, value: authentik } + - name: PGUSER + valueFrom: + secretKeyRef: { name: ak-pg-superuser, key: username } + - name: PGPASSWORD + valueFrom: + secretKeyRef: { name: ak-pg-superuser, key: password } + command: + - sh + - -c + - | + set -eu + until psql -tAc 'SELECT 1' >/dev/null 2>&1; do + echo "[$(date -u +%H:%M:%S)] waiting for $PGHOST..." + sleep 5 + done + echo "[$(date -u +%H:%M:%S)] $PGHOST ready" + containers: + - name: seed + image: postgres:16 + env: + - { name: LEGACY_HOST, value: authentik-db-1.srv.renner.uno } + - { name: LEGACY_DB, value: authentik } + - { name: PGHOST, value: ak-pg-rw } + - { name: PGDATABASE, value: authentik } + - name: PGUSER + valueFrom: + secretKeyRef: { name: ak-pg-superuser, key: username } + - name: PGPASSWORD + valueFrom: + secretKeyRef: { name: ak-pg-superuser, key: password } + command: + - bash + - -c + - | + set -euo pipefail + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + + # Idempotent: skip if target DB already has user tables. + count=$(psql -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public'") + if [ "$count" -gt 0 ]; then + log "$PGDATABASE already has $count tables in public — seed skipped." + exit 0 + fi + + apt-get update -qq + apt-get install -y --no-install-recommends openssh-client >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + + log "pg_dump $LEGACY_HOST/$LEGACY_DB → pg_restore $PGHOST/$PGDATABASE ..." + ssh "root@$LEGACY_HOST" "su - postgres -c 'pg_dump -Fc --clean --if-exists $LEGACY_DB'" \ + | pg_restore -d "$PGDATABASE" \ + || log "pg_restore completed with warnings" + + log "Seed complete." + volumeMounts: + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: ssh + secret: + secretName: authentik-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/apps/authentik/seed-ssh.sops.yaml b/kubernetes/seed/authentik/seed-ssh.sops.yaml similarity index 100% rename from kubernetes/apps/authentik/seed-ssh.sops.yaml rename to kubernetes/seed/authentik/seed-ssh.sops.yaml diff --git a/kubernetes/apps/immich/barman-creds.sops.yaml b/kubernetes/seed/immich/barman-creds.sops.yaml similarity index 100% rename from kubernetes/apps/immich/barman-creds.sops.yaml rename to kubernetes/seed/immich/barman-creds.sops.yaml diff --git a/kubernetes/seed/immich/cluster.yaml b/kubernetes/seed/immich/cluster.yaml new file mode 100644 index 00000000..143efa84 --- /dev/null +++ b/kubernetes/seed/immich/cluster.yaml @@ -0,0 +1,39 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: im-pg + namespace: immich +spec: + # Single instance during pgvecto.rs → VectorChord migration window; + # pg_basebackup tar format chokes on the long pg_vectors/indexes/ + # paths, blocking replica join. Bump back to 2 once Immich's in-app + # vchord migration drops the vectors extension. + instances: 1 + imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration + enableSuperuserAccess: true + storage: + storageClass: ceph-block + size: 20Gi + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: im-pg-store + bootstrap: + initdb: + database: immich + owner: immich + postgresql: + shared_preload_libraries: + - vectors.so + - vchord.so + parameters: + shared_buffers: "512MB" + max_wal_size: "2GB" + wal_compression: "on" + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + memory: 2Gi diff --git a/kubernetes/seed/immich/kustomization.yaml b/kubernetes/seed/immich/kustomization.yaml new file mode 100644 index 00000000..e03a425a --- /dev/null +++ b/kubernetes/seed/immich/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - migration-bucket.yaml + - barman-creds.sops.yaml + - objectstore.yaml + - cluster.yaml + - scheduled-backup.yaml + - seed-ssh.sops.yaml + - pvcs.yaml + - pg-seed-job.yaml + - library-seed-job.yaml diff --git a/kubernetes/seed/immich/library-seed-job.yaml b/kubernetes/seed/immich/library-seed-job.yaml new file mode 100644 index 00000000..8dd53392 --- /dev/null +++ b/kubernetes/seed/immich/library-seed-job.yaml @@ -0,0 +1,58 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: immich-library-seed + namespace: immich +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: rsync + image: alpine:3.21 + env: + - { name: LEGACY_HOST, value: immich.srv.renner.uno } + command: + - sh + - -c + - | + set -eu + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + if [ -f /data/.seed-done ]; then + log "PVC seed sentinel present — skipping rsync." + exit 0 + fi + + apk add --no-cache rsync openssh-client >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + + log "rsync /mnt/media/immich → /data ..." + set +e + rsync -aH --delete --info=progress2 \ + "root@$LEGACY_HOST:/mnt/media/immich/" /data/ + rc=$? + set -e + case "$rc" in + 0) log "rsync OK" ;; + 23) log "rsync exit 23 (partial transfer warnings) — continuing" ;; + *) log "rsync FAILED with code $rc"; exit "$rc" ;; + esac + chown -R 1000:1000 /data + du -sh /data + touch /data/.seed-done + log "PVC seed complete; sentinel set." + volumeMounts: + - { name: library, mountPath: /data } + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: library + persistentVolumeClaim: + claimName: immich-library + - name: ssh + secret: + secretName: immich-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/apps/immich/migration-bucket.yaml b/kubernetes/seed/immich/migration-bucket.yaml similarity index 100% rename from kubernetes/apps/immich/migration-bucket.yaml rename to kubernetes/seed/immich/migration-bucket.yaml diff --git a/kubernetes/apps/immich/objectstore.yaml b/kubernetes/seed/immich/objectstore.yaml similarity index 100% rename from kubernetes/apps/immich/objectstore.yaml rename to kubernetes/seed/immich/objectstore.yaml diff --git a/kubernetes/seed/immich/pg-seed-job.yaml b/kubernetes/seed/immich/pg-seed-job.yaml new file mode 100644 index 00000000..776374b2 --- /dev/null +++ b/kubernetes/seed/immich/pg-seed-job.yaml @@ -0,0 +1,96 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: im-pg-seed + namespace: immich +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-cluster + image: postgres:16 + env: + - { name: PGHOST, value: im-pg-rw } + - { name: PGDATABASE, value: immich } + - name: PGUSER + valueFrom: + secretKeyRef: { name: im-pg-superuser, key: username } + - name: PGPASSWORD + valueFrom: + secretKeyRef: { name: im-pg-superuser, key: password } + command: + - sh + - -c + - | + set -eu + until psql -tAc 'SELECT 1' >/dev/null 2>&1; do + echo "[$(date -u +%H:%M:%S)] waiting for $PGHOST..." + sleep 5 + done + echo "[$(date -u +%H:%M:%S)] $PGHOST ready" + containers: + - name: seed + image: postgres:16 + env: + - { name: LEGACY_HOST, value: immich.srv.renner.uno } + - { name: LEGACY_DB, value: immich } + - { name: PGHOST, value: im-pg-rw } + - { name: PGDATABASE, value: immich } + - name: PGUSER + valueFrom: + secretKeyRef: { name: im-pg-superuser, key: username } + - name: PGPASSWORD + valueFrom: + secretKeyRef: { name: im-pg-superuser, key: password } + command: + - bash + - -c + - | + set -euo pipefail + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + + count=$(psql -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public'") + if [ "$count" -gt 0 ]; then + log "$PGDATABASE has $count tables in public — seed skipped." + exit 0 + fi + + apt-get update -qq + apt-get install -y --no-install-recommends openssh-client >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + + # Pre-create non-trusted extensions immich needs at runtime. + # The legacy dump only ships pgvecto.rs (`vectors`); newer + # immich also CREATEs `vector` (pgvector) and `vchord` at + # startup. Both are non-trusted, so the immich role can't + # CREATE them — pre-create as superuser with IF NOT EXISTS so + # the dump's own CREATE EXTENSION calls and immich's runtime + # CREATE all short-circuit. + log "Pre-creating extensions ..." + for ext in cube earthdistance vector vectors vchord; do + psql -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS \"$ext\" CASCADE;" \ + || log "extension $ext: failed (continuing)" + done + + log "pg_dump $LEGACY_HOST/$LEGACY_DB → pg_restore $PGHOST/$PGDATABASE ..." + # Legacy immich DB lives in a docker container on the LXC; + # locate via compose service label. + DB_CT=$(ssh "root@$LEGACY_HOST" "docker ps --filter label=com.docker.compose.service=database --format '{{.Names}}' | head -1") + [ -n "$DB_CT" ] || { log "no DB container on $LEGACY_HOST"; exit 1; } + ssh "root@$LEGACY_HOST" "docker exec -i $DB_CT pg_dump -Fc --clean --if-exists -U postgres $LEGACY_DB" \ + | pg_restore -d "$PGDATABASE" \ + || log "pg_restore completed with warnings" + + log "Seed complete." + volumeMounts: + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: ssh + secret: + secretName: immich-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/apps/immich/pvcs.yaml b/kubernetes/seed/immich/pvcs.yaml similarity index 100% rename from kubernetes/apps/immich/pvcs.yaml rename to kubernetes/seed/immich/pvcs.yaml diff --git a/kubernetes/apps/immich/scheduled-backup.yaml b/kubernetes/seed/immich/scheduled-backup.yaml similarity index 100% rename from kubernetes/apps/immich/scheduled-backup.yaml rename to kubernetes/seed/immich/scheduled-backup.yaml diff --git a/kubernetes/apps/immich/seed-ssh.sops.yaml b/kubernetes/seed/immich/seed-ssh.sops.yaml similarity index 100% rename from kubernetes/apps/immich/seed-ssh.sops.yaml rename to kubernetes/seed/immich/seed-ssh.sops.yaml diff --git a/kubernetes/seed/influxdb/data-seed-job.yaml b/kubernetes/seed/influxdb/data-seed-job.yaml new file mode 100644 index 00000000..7e65fe86 --- /dev/null +++ b/kubernetes/seed/influxdb/data-seed-job.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: influxdb-data-seed + namespace: influxdb +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: rsync + image: alpine:3.21 + env: + - { name: LEGACY_HOST, value: influxdb.srv.renner.uno } + command: + - sh + - -c + - | + set -eu + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + if [ -f /var/lib/influxdb2/.seed-done ]; then + log "PVC seed sentinel present — skipping." + exit 0 + fi + + apk add --no-cache rsync openssh-client >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + LEGACY="root@$LEGACY_HOST" + + log "Stopping legacy influxdb.service ..." + ssh "$LEGACY" "systemctl stop influxdb" + trap 'log "Restarting legacy influxdb.service ..."; ssh "$LEGACY" "systemctl start influxdb" || true' EXIT + + log "rsync /var/lib/influxdb → /var/lib/influxdb2 ..." + rsync -aH --delete --info=progress2 \ + "$LEGACY:/var/lib/influxdb/" /var/lib/influxdb2/ + chown -R 1500:1500 /var/lib/influxdb2 || true + du -sh /var/lib/influxdb2 + touch /var/lib/influxdb2/.seed-done + log "PVC seed complete; sentinel set." + volumeMounts: + - { name: data, mountPath: /var/lib/influxdb2 } + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: data + persistentVolumeClaim: + claimName: data-influxdb-influxdb2-0 + - name: ssh + secret: + secretName: influxdb-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/seed/influxdb/kustomization.yaml b/kubernetes/seed/influxdb/kustomization.yaml new file mode 100644 index 00000000..3e490cb2 --- /dev/null +++ b/kubernetes/seed/influxdb/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - seed-ssh.sops.yaml + - pvc.yaml + - data-seed-job.yaml diff --git a/kubernetes/seed/influxdb/pvc.yaml b/kubernetes/seed/influxdb/pvc.yaml new file mode 100644 index 00000000..54d8fd10 --- /dev/null +++ b/kubernetes/seed/influxdb/pvc.yaml @@ -0,0 +1,19 @@ +--- +# Pre-create the PVC the influxdb-influxdb2 StatefulSet would otherwise +# auto-provision via volumeClaimTemplates, so the seed Job can populate +# it before the StatefulSet pod ever starts. Name + storageClass + size +# match the chart values. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-influxdb-influxdb2-0 + namespace: influxdb + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + accessModes: + - ReadWriteOnce + storageClassName: ceph-block + resources: + requests: + storage: 50Gi diff --git a/kubernetes/apps/influxdb/seed-ssh.sops.yaml b/kubernetes/seed/influxdb/seed-ssh.sops.yaml similarity index 100% rename from kubernetes/apps/influxdb/seed-ssh.sops.yaml rename to kubernetes/seed/influxdb/seed-ssh.sops.yaml diff --git a/kubernetes/seed/kustomization.yaml b/kubernetes/seed/kustomization.yaml new file mode 100644 index 00000000..0f0725e9 --- /dev/null +++ b/kubernetes/seed/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - authentik/ + - immich/ + - paperless/ + - audiobookshelf/ + - music-assistant/ + - influxdb/ diff --git a/kubernetes/seed/music-assistant/data-seed-job.yaml b/kubernetes/seed/music-assistant/data-seed-job.yaml new file mode 100644 index 00000000..53134b22 --- /dev/null +++ b/kubernetes/seed/music-assistant/data-seed-job.yaml @@ -0,0 +1,66 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: music-assistant-data-seed + namespace: music-assistant +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + # Music Assistant's data PVC lives on edge (RWO ceph-block scoped to + # the edge node where the app runs); seed Job must schedule on the + # same node so it can mount the volume. + nodeSelector: + topology.kubernetes.io/zone: edge + tolerations: + - key: workload-class + operator: Equal + value: pinned + effect: NoSchedule + restartPolicy: OnFailure + containers: + - name: seed + image: alpine:3.21 + env: + - { name: LEGACY_HOST, value: music-assistant.iot.renner.uno } + command: + - sh + - -c + - | + set -eu + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + if [ -f /data/.seed-done ]; then + log "PVC seed sentinel present — skipping." + exit 0 + fi + + apk add --no-cache openssh-client tar >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + LEGACY="root@$LEGACY_HOST" + + log "Stopping music-assistant docker-compose stack on legacy ..." + ssh "$LEGACY" "cd /opt/music-assistant && docker compose stop" + trap 'log "Restarting music-assistant on legacy ..."; ssh "$LEGACY" "cd /opt/music-assistant && docker compose up -d" || true' EXIT + + # Legacy MA LXC ships without rsync; ssh+tar streaming. + log "Clearing /data ..." + find /data -mindepth 1 -delete + log "Streaming /opt/music-assistant/data via ssh+tar ..." + ssh "$LEGACY" "cd /opt/music-assistant/data && tar -cf - ." | tar -xf - -C /data/ + du -sh /data + touch /data/.seed-done + log "PVC seed complete; sentinel set." + volumeMounts: + - { name: data, mountPath: /data } + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: data + persistentVolumeClaim: + claimName: music-assistant-data + - name: ssh + secret: + secretName: music-assistant-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/seed/music-assistant/kustomization.yaml b/kubernetes/seed/music-assistant/kustomization.yaml new file mode 100644 index 00000000..ee94c0fd --- /dev/null +++ b/kubernetes/seed/music-assistant/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - pvcs.yaml + - seed-ssh.sops.yaml + - data-seed-job.yaml diff --git a/kubernetes/apps/music-assistant/pvcs.yaml b/kubernetes/seed/music-assistant/pvcs.yaml similarity index 100% rename from kubernetes/apps/music-assistant/pvcs.yaml rename to kubernetes/seed/music-assistant/pvcs.yaml diff --git a/kubernetes/apps/music-assistant/seed-ssh.sops.yaml b/kubernetes/seed/music-assistant/seed-ssh.sops.yaml similarity index 100% rename from kubernetes/apps/music-assistant/seed-ssh.sops.yaml rename to kubernetes/seed/music-assistant/seed-ssh.sops.yaml diff --git a/kubernetes/apps/paperless/barman-creds.sops.yaml b/kubernetes/seed/paperless/barman-creds.sops.yaml similarity index 100% rename from kubernetes/apps/paperless/barman-creds.sops.yaml rename to kubernetes/seed/paperless/barman-creds.sops.yaml diff --git a/kubernetes/seed/paperless/cluster.yaml b/kubernetes/seed/paperless/cluster.yaml new file mode 100644 index 00000000..0248f2ed --- /dev/null +++ b/kubernetes/seed/paperless/cluster.yaml @@ -0,0 +1,31 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: pprl-pg + namespace: paperless +spec: + instances: 2 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + enableSuperuserAccess: true + storage: + storageClass: ceph-block + size: 20Gi + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: pprl-pg-store + bootstrap: + initdb: + database: paperless + owner: paperless + postgresql: + parameters: + shared_buffers: "256MB" + max_connections: "200" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 1Gi diff --git a/kubernetes/seed/paperless/data-seed-job.yaml b/kubernetes/seed/paperless/data-seed-job.yaml new file mode 100644 index 00000000..a5be7786 --- /dev/null +++ b/kubernetes/seed/paperless/data-seed-job.yaml @@ -0,0 +1,57 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: paperless-data-seed + namespace: paperless +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: rsync + image: alpine:3.21 + env: + - { name: LEGACY_HOST, value: paperless.srv.renner.uno } + command: + - sh + - -c + - | + set -eu + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + if [ -f /data/.seed-done ]; then + log "PVC seed sentinel present — skipping rsync." + exit 0 + fi + + apk add --no-cache rsync openssh-client >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + + log "rsync paperless_data → /data ..." + rsync -aH --delete --info=progress2 \ + "root@$LEGACY_HOST:/var/lib/docker/volumes/paperless_data/_data/" /data/ + log "rsync paperless_media → /media ..." + rsync -aH --delete --info=progress2 \ + "root@$LEGACY_HOST:/var/lib/docker/volumes/paperless_media/_data/" /media/ + chown -R 1000:1000 /data /media + du -sh /data /media + touch /data/.seed-done + log "PVC seed complete; sentinel set." + volumeMounts: + - { name: data, mountPath: /data } + - { name: media, mountPath: /media } + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: data + persistentVolumeClaim: + claimName: paperless-data + - name: media + persistentVolumeClaim: + claimName: paperless-media + - name: ssh + secret: + secretName: paperless-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/seed/paperless/kustomization.yaml b/kubernetes/seed/paperless/kustomization.yaml new file mode 100644 index 00000000..451ba6e2 --- /dev/null +++ b/kubernetes/seed/paperless/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - migration-bucket.yaml + - barman-creds.sops.yaml + - objectstore.yaml + - cluster.yaml + - scheduled-backup.yaml + - seed-ssh.sops.yaml + - pvcs.yaml + - pg-seed-job.yaml + - data-seed-job.yaml diff --git a/kubernetes/apps/paperless/migration-bucket.yaml b/kubernetes/seed/paperless/migration-bucket.yaml similarity index 100% rename from kubernetes/apps/paperless/migration-bucket.yaml rename to kubernetes/seed/paperless/migration-bucket.yaml diff --git a/kubernetes/apps/paperless/objectstore.yaml b/kubernetes/seed/paperless/objectstore.yaml similarity index 100% rename from kubernetes/apps/paperless/objectstore.yaml rename to kubernetes/seed/paperless/objectstore.yaml diff --git a/kubernetes/seed/paperless/pg-seed-job.yaml b/kubernetes/seed/paperless/pg-seed-job.yaml new file mode 100644 index 00000000..71a38ce4 --- /dev/null +++ b/kubernetes/seed/paperless/pg-seed-job.yaml @@ -0,0 +1,84 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: pprl-pg-seed + namespace: paperless +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-cluster + image: postgres:16 + env: + - { name: PGHOST, value: pprl-pg-rw } + - { name: PGDATABASE, value: paperless } + - name: PGUSER + valueFrom: + secretKeyRef: { name: pprl-pg-superuser, key: username } + - name: PGPASSWORD + valueFrom: + secretKeyRef: { name: pprl-pg-superuser, key: password } + command: + - sh + - -c + - | + set -eu + until psql -tAc 'SELECT 1' >/dev/null 2>&1; do + echo "[$(date -u +%H:%M:%S)] waiting for $PGHOST..." + sleep 5 + done + echo "[$(date -u +%H:%M:%S)] $PGHOST ready" + containers: + - name: seed + image: postgres:16 + env: + - { name: LEGACY_HOST, value: paperless.srv.renner.uno } + - { name: LEGACY_DB, value: paperless } + - { name: LEGACY_DB_USER, value: paperless } + - { name: PGHOST, value: pprl-pg-rw } + - { name: PGDATABASE, value: paperless } + - name: PGUSER + valueFrom: + secretKeyRef: { name: pprl-pg-superuser, key: username } + - name: PGPASSWORD + valueFrom: + secretKeyRef: { name: pprl-pg-superuser, key: password } + command: + - bash + - -c + - | + set -euo pipefail + log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + + count=$(psql -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public'") + if [ "$count" -gt 0 ]; then + log "$PGDATABASE has $count tables in public — seed skipped." + exit 0 + fi + + apt-get update -qq + apt-get install -y --no-install-recommends openssh-client >/dev/null + mkdir -p /root/.ssh + install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519 + printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config + + log "pg_dump $LEGACY_HOST/$LEGACY_DB → pg_restore $PGHOST/$PGDATABASE ..." + # Legacy paperless DB lives in a docker container; locate via + # compose service label. + DB_CT=$(ssh "root@$LEGACY_HOST" "docker ps --filter label=com.docker.compose.service=db --format '{{.Names}}' | head -1") + [ -n "$DB_CT" ] || { log "no DB container on $LEGACY_HOST"; exit 1; } + ssh "root@$LEGACY_HOST" "docker exec -i $DB_CT pg_dump -Fc --clean --if-exists -U $LEGACY_DB_USER $LEGACY_DB" \ + | pg_restore -d "$PGDATABASE" \ + || log "pg_restore completed with warnings" + + log "Seed complete." + volumeMounts: + - { name: ssh, mountPath: /ssh, readOnly: true } + volumes: + - name: ssh + secret: + secretName: paperless-seed-ssh + defaultMode: 0400 diff --git a/kubernetes/apps/paperless/pvcs.yaml b/kubernetes/seed/paperless/pvcs.yaml similarity index 100% rename from kubernetes/apps/paperless/pvcs.yaml rename to kubernetes/seed/paperless/pvcs.yaml diff --git a/kubernetes/apps/paperless/scheduled-backup.yaml b/kubernetes/seed/paperless/scheduled-backup.yaml similarity index 100% rename from kubernetes/apps/paperless/scheduled-backup.yaml rename to kubernetes/seed/paperless/scheduled-backup.yaml diff --git a/kubernetes/apps/paperless/seed-ssh.sops.yaml b/kubernetes/seed/paperless/seed-ssh.sops.yaml similarity index 100% rename from kubernetes/apps/paperless/seed-ssh.sops.yaml rename to kubernetes/seed/paperless/seed-ssh.sops.yaml