feat(seed): dedicated migration phase between infra and apps
ci/woodpecker/manual/rebuild-k8s Pipeline failed
ci/woodpecker/manual/rebuild-k8s Pipeline failed
Replaces the per-app race-y CNPG bootstrap.recovery + velero-bootstrap-restore
pattern with a single seed Flux KS that populates databases and PVCs from
legacy hosts before any app KS reconciles.
Changes:
- New kubernetes/seed/<app>/ tree owning, per app:
- CNPG Cluster CRs with bootstrap.initdb (was: bootstrap.recovery) +
enableSuperuserAccess so seed Jobs can connect as postgres
- DB seed Jobs: ssh legacy 'pg_dump -Fc' | pg_restore -d <db> against
the running cluster (was: ephemeral PG + barman-cloud-backup → S3)
- PVC seed Jobs: rsync/ssh+tar from legacy → app PVCs as standalone
Jobs (was: in-pod init containers, blocking the app Deployment's
progressDeadlineSeconds for hours on a 250 GiB rsync)
- All ObjectStores, ScheduledBackups, barman-creds, PVCs, seed-ssh
Secrets relocated here
- New top-level kubernetes/clusters/homelab/seed.yaml Flux KS:
dependsOn infrastructure-cnpg-barman, timeout 4h, healthCheckExprs
cover CNPG Cluster phase + native Job succeeded.
- Stripped from kubernetes/apps/<app>/: Cluster CRs, seed Jobs, init
container rsync blocks, ssh volumes, seed ServiceAccount references.
~454 lines deleted, ~33 added.
- App KSs (immich, authentik, paperless, audiobookshelf,
music-assistant, influxdb): dependsOn velero-bootstrap-restore →
dependsOn seed.
- bootstrap.sh wait-apps: new wait_kustomization seed (4h) before
per-app waits; per-app timeouts dropped to 60s since they're
near-instant once seed is Ready.
- Deleted kubernetes/clusters/homelab/velero-bootstrap-restore.yaml;
manifests stay in kubernetes/infrastructure/velero-bootstrap-restore/
for post-cutover re-add (it remains the cold-rebuild path once
legacy hosts are decommissioned and seed phase is removed).
Eliminates the documented race failure modes:
- cnpg-bootstrap-recovery-no-initdb-fallback (recovery Job pounds empty
bucket until seed lands → multiple Error pods)
- velero-bootstrap-restore-bsl-sync-race (Job iterates schedules in an
empty bucket for ~15 min)
- immich-server progressDeadlineSeconds blowup (250 GiB rsync init
exceeds the 600s default and Flux marks the Deployment Failed)
Seed is migration-only: removed entirely once all services have cut
over from legacy. At that point the Cluster + ObjectStore +
ScheduledBackup + PVC manifests move back into kubernetes/apps/<app>/
and velero-bootstrap-restore.yaml is re-added at the cluster level.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -34,98 +34,12 @@ spec:
|
||||
app.kubernetes.io/name: audiobookshelf
|
||||
spec:
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: audiobookshelf-seed
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
# Atomic seed: idempotency-gated rsync of config + metadata +
|
||||
# books PVCs from the legacy VM. Two-phase so the legacy
|
||||
# systemd unit stays up during the long books rsync.
|
||||
- name: seed
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29
|
||||
env:
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SCHED
|
||||
value: audiobookshelf-daily
|
||||
- name: LEGACY_HOST
|
||||
value: audiobookshelf.srv.renner.uno
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
log "Installing rsync ..."
|
||||
apk add --no-cache rsync >/dev/null
|
||||
|
||||
EXISTING=$(kubectl -n velero get backups.velero.io \
|
||||
-l "velero.io/schedule-name=$SCHED" \
|
||||
-o jsonpath='{range .items[?(@.status.phase=="Completed")]}{.metadata.name}{"\n"}{end}' \
|
||||
2>/dev/null | grep -c . || true)
|
||||
if [ "$EXISTING" -gt 0 ]; then
|
||||
log "$EXISTING completed Velero Backup(s) for $SCHED — skipping seed"
|
||||
exit 0
|
||||
fi
|
||||
log "No completed Velero Backup for $SCHED — proceeding with seed"
|
||||
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
# Phase 1: books while legacy stays up (long, no --delete).
|
||||
log "Phase 1: rsync /audiobooks → /audiobooks (legacy stays up) ..."
|
||||
rsync -aH --info=progress2 \
|
||||
"$LEGACY:/audiobooks/" /audiobooks/
|
||||
|
||||
# Phase 2: stop legacy briefly for SQLite-bearing dirs + delta.
|
||||
log "Phase 2: stopping audiobookshelf on legacy VM ..."
|
||||
ssh "$LEGACY" "systemctl stop audiobookshelf"
|
||||
trap 'log "Restarting audiobookshelf on legacy VM ..."; ssh "$LEGACY" "systemctl start audiobookshelf" || true' EXIT
|
||||
|
||||
log " rsync /usr/share/audiobookshelf/config → /config ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/usr/share/audiobookshelf/config/" /config/
|
||||
log " rsync /usr/share/audiobookshelf/metadata → /metadata ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/usr/share/audiobookshelf/metadata/" /metadata/
|
||||
log " rsync /audiobooks delta with --delete ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/audiobooks/" /audiobooks/
|
||||
chown -R 1000:1000 /config /metadata /audiobooks
|
||||
du -sh /config /metadata /audiobooks
|
||||
|
||||
BACKUP_NAME="$SCHED-seed-$(date -u +%Y%m%d-%H%M%S)"
|
||||
log "Creating Velero Backup $BACKUP_NAME ..."
|
||||
cat <<YAML | kubectl apply -f -
|
||||
apiVersion: velero.io/v1
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: $BACKUP_NAME
|
||||
namespace: velero
|
||||
labels:
|
||||
velero.io/schedule-name: $SCHED
|
||||
spec:
|
||||
includedNamespaces:
|
||||
- $NAMESPACE
|
||||
includedResources:
|
||||
- persistentvolumeclaims
|
||||
snapshotMoveData: true
|
||||
itemOperationTimeout: 4h0m0s
|
||||
YAML
|
||||
log "Seed complete: PVCs populated; Velero Backup submitted."
|
||||
volumeMounts:
|
||||
- { name: config, mountPath: /config }
|
||||
- { name: metadata, mountPath: /metadata }
|
||||
- { name: books, mountPath: /audiobooks }
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
# Fetch the Let's Encrypt staging root CAs at pod start so
|
||||
# Node (openid-client) trusts the staging-issued auth.renner.uno
|
||||
# cert when doing server-side OIDC token exchange. Bundle both
|
||||
@@ -208,10 +122,6 @@ spec:
|
||||
claimName: audiobookshelf-books
|
||||
- name: le-ca
|
||||
emptyDir: {}
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: audiobookshelf-seed-ssh
|
||||
defaultMode: 0400
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- pvcs.yaml
|
||||
- migration-bucket.yaml
|
||||
- seed-rbac.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# RBAC for the seed initContainer in the audiobookshelf Pod. Cross-namespace
|
||||
# because Velero Backup CRs live in the velero namespace.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: audiobookshelf-seed
|
||||
namespace: audiobookshelf
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: audiobookshelf-seed-velero
|
||||
namespace: velero
|
||||
rules:
|
||||
- apiGroups: [velero.io]
|
||||
resources: [backups]
|
||||
verbs: [get, list, watch, create]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: audiobookshelf-seed-velero
|
||||
namespace: velero
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: audiobookshelf-seed
|
||||
namespace: audiobookshelf
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: audiobookshelf-seed-velero
|
||||
@@ -1,13 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- migration-bucket.yaml
|
||||
- barman-creds.sops.yaml
|
||||
- objectstore.yaml
|
||||
- postgres.yaml
|
||||
- scheduled-backup.yaml
|
||||
- secret.sops.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- seed-job.yaml
|
||||
- helmrelease.yaml
|
||||
- ingress-test.yaml
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ak-pg
|
||||
namespace: authentik
|
||||
annotations:
|
||||
# Archive is pre-populated by the migration seed (base + WAL) and,
|
||||
# on later reboots, by this cluster's own continuous archiving. The
|
||||
# plugin's pre-recovery "Expected empty archive" check conflicts
|
||||
# with the producer==consumer invariant of the symmetric-restore
|
||||
# design, so skip it. bootstrap.recovery still reads latest backup
|
||||
# from the archive; if none, it recovers the migration seed.
|
||||
cnpg.io/skipEmptyWalArchiveCheck: "enabled"
|
||||
spec:
|
||||
instances: 2
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
size: 10Gi
|
||||
# The Barman Cloud plugin drives both WAL archiving and
|
||||
# scheduled base backups to Garage on nas-1 (bucket k8s-cnpg).
|
||||
# See objectstore.yaml for the destination config.
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: ak-pg-store
|
||||
# Bootstrap always recovers from the latest Barman base backup + WAL in
|
||||
# s3://k8s-cnpg/ak-pg. Producer path == consumer path: the live cluster
|
||||
# writes to the same location via the plugin above (serverName defaults to
|
||||
# the Cluster name, i.e. `ak-pg`), and the migration pipeline seeds the
|
||||
# same path using an ephemeral Postgres + barman-cloud-backup. Unpinned
|
||||
# recovery target = latest WAL.
|
||||
bootstrap:
|
||||
recovery:
|
||||
# Explicit owner/database — CNPG will create/adopt the
|
||||
# authentik user + DB from the recovered archive and reset its
|
||||
# password to match the managed ak-pg-app Secret referenced by
|
||||
# the authentik HelmRelease.
|
||||
owner: authentik
|
||||
database: authentik
|
||||
source: ak-pg-backup
|
||||
externalClusters:
|
||||
- name: ak-pg-backup
|
||||
plugin:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
parameters:
|
||||
barmanObjectName: ak-pg-store
|
||||
serverName: ak-pg
|
||||
postgresql:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_connections: "200"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
@@ -1,207 +0,0 @@
|
||||
---
|
||||
# Cold-start seed for ak-pg. Runs an ephemeral Postgres in this Job,
|
||||
# pg_restores the legacy authentik DB into it, and writes a Barman
|
||||
# base backup (+ end-WAL) to s3://k8s-cnpg/ak-pg/. CNPG's
|
||||
# bootstrap.recovery on the ak-pg Cluster reads from that path; the
|
||||
# Cluster's recovery Job retries until this seed populates Garage,
|
||||
# then succeeds.
|
||||
#
|
||||
# Idempotency: gates on Garage having any object under the cluster
|
||||
# path. Once a backup exists (whether from this seed, a prior seed, or
|
||||
# the live cluster's continuous archiving), the Job exits 0.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ak-pg-seed
|
||||
namespace: authentik
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
initContainers:
|
||||
- name: legacy-dump
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- name: LEGACY_HOST
|
||||
value: authentik-db-1.srv.renner.uno
|
||||
- name: PGDATABASE
|
||||
value: authentik
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
echo "[$(date -u +%H:%M:%S)] pg_dump on $LEGACY_HOST → /seed/legacy.dump"
|
||||
ssh "root@$LEGACY_HOST" "su - postgres -c 'pg_dump --clean --if-exists --no-owner --no-privileges --format=custom $PGDATABASE'" > /seed/legacy.dump
|
||||
ls -lh /seed/legacy.dump
|
||||
volumeMounts:
|
||||
- name: ssh
|
||||
mountPath: /ssh
|
||||
readOnly: true
|
||||
- name: seed
|
||||
mountPath: /seed
|
||||
containers:
|
||||
- name: seed
|
||||
image: postgres:16
|
||||
env:
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data
|
||||
- name: TARGET_DB
|
||||
value: authentik
|
||||
- name: TARGET_OWNER
|
||||
value: authentik
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: ACCESS_SECRET_KEY
|
||||
- name: AWS_DEFAULT_REGION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: REGION
|
||||
- name: GARAGE_ENDPOINT
|
||||
value: https://s3.nas-1.renner.uno
|
||||
- name: BARMAN_SERVER
|
||||
value: ak-pg
|
||||
- name: BARMAN_DEST
|
||||
value: s3://k8s-cnpg/ak-pg
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
log "Installing awscli + barman ..."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 python3-pip python3-venv python3-psycopg2 \
|
||||
awscli ca-certificates >/dev/null
|
||||
|
||||
# Idempotency gate: only skip if a real base backup exists at
|
||||
# <BARMAN_DEST>/<BARMAN_SERVER>/base/. A failed previous attempt
|
||||
# may leave dangling marker objects under <BARMAN_DEST>/, but
|
||||
# not under base/, so this distinguishes "complete backup" from
|
||||
# "partial-write garbage".
|
||||
EXISTING=$(aws s3 ls --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST/$BARMAN_SERVER/base/" 2>/dev/null | wc -l || true)
|
||||
if [ "$EXISTING" -gt 0 ]; then
|
||||
log "Garage already has $EXISTING base backup entr(ies) at $BARMAN_DEST/$BARMAN_SERVER/base/ — seed skipped."
|
||||
exit 0
|
||||
fi
|
||||
log "No base backup at $BARMAN_DEST/$BARMAN_SERVER/base/ — running full seed process."
|
||||
|
||||
python3 -m venv --system-site-packages /opt/barman
|
||||
/opt/barman/bin/pip install --no-cache-dir --quiet 'barman[cloud,aws-snapshots]'
|
||||
export PATH="/opt/barman/bin:$PATH"
|
||||
|
||||
# Ephemeral Postgres in PGDATA. trust auth, local-only.
|
||||
mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA"
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres"
|
||||
|
||||
# WAL archiving so end_wal of the base backup actually lands in
|
||||
# the archive — without this, CNPG recovery fails on missing WAL.
|
||||
# postgres daemon sanitizes env on startup, so bake creds into
|
||||
# the wrapper script.
|
||||
cat > /usr/local/bin/archive_wal.sh <<SH
|
||||
#!/bin/sh
|
||||
exec >> /tmp/archive_wal.log 2>&1
|
||||
export AWS_ACCESS_KEY_ID='$AWS_ACCESS_KEY_ID'
|
||||
export AWS_SECRET_ACCESS_KEY='$AWS_SECRET_ACCESS_KEY'
|
||||
export AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION'
|
||||
echo "\$(date -Iseconds) archiving \$1"
|
||||
exec barman-cloud-wal-archive \\
|
||||
--cloud-provider aws-s3 \\
|
||||
--endpoint-url '$GARAGE_ENDPOINT' \\
|
||||
--gzip -e AES256 \\
|
||||
'$BARMAN_DEST' '$BARMAN_SERVER' "\$1"
|
||||
SH
|
||||
chmod +x /usr/local/bin/archive_wal.sh
|
||||
: > /tmp/archive_wal.log
|
||||
chmod 666 /tmp/archive_wal.log
|
||||
|
||||
cat >> "$PGDATA/postgresql.conf" <<CONF
|
||||
listen_addresses = 'localhost'
|
||||
wal_level = replica
|
||||
archive_mode = on
|
||||
archive_command = '/usr/local/bin/archive_wal.sh %p'
|
||||
archive_timeout = '60s'
|
||||
CONF
|
||||
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -l /tmp/pg.log -w start"
|
||||
|
||||
psql -U postgres -c "CREATE ROLE $TARGET_OWNER LOGIN;" || true
|
||||
psql -U postgres -c "CREATE DATABASE $TARGET_DB OWNER $TARGET_OWNER;" || true
|
||||
|
||||
# Run pg_restore as postgres (superuser) AND let the dump's own
|
||||
# ALTER OWNER + GRANT statements run — the dump already says
|
||||
# "OWNER TO authentik" / "GRANT ... TO authentik" for every
|
||||
# user object, and as postgres those run cleanly regardless of
|
||||
# extension schemas or anything else. No ownership fumbling
|
||||
# needed afterwards: PGDATA ends up exactly matching the
|
||||
# legacy DB's ownership/ACL state.
|
||||
log "pg_restore /seed/legacy.dump → $TARGET_DB ..."
|
||||
pg_restore -U postgres -d "$TARGET_DB" \
|
||||
/seed/legacy.dump \
|
||||
|| log "pg_restore completed with warnings (expected)"
|
||||
|
||||
# barman-cloud-backup connects to PG to start/stop the backup
|
||||
# via SQL. Without --host/--port/--user it falls back to the
|
||||
# OS user (root), which has no PG role. Use 127.0.0.1 (matches
|
||||
# listen_addresses=localhost) and the postgres superuser.
|
||||
log "barman-cloud-backup → $BARMAN_DEST/$BARMAN_SERVER ..."
|
||||
barman-cloud-backup \
|
||||
--cloud-provider aws-s3 \
|
||||
--endpoint-url "$GARAGE_ENDPOINT" \
|
||||
--gzip -e AES256 \
|
||||
--host 127.0.0.1 --port 5432 --user postgres \
|
||||
"$BARMAN_DEST" "$BARMAN_SERVER"
|
||||
|
||||
# Force end_wal to flush; archive_command fires asynchronously,
|
||||
# so wait for archive_status/*.ready files to drain before we
|
||||
# stop PG (otherwise the segment that holds end_wal never
|
||||
# uploads and CNPG recovery fails on "WAL not found").
|
||||
psql -h 127.0.0.1 -U postgres -d postgres -c "CHECKPOINT; SELECT pg_switch_wal();"
|
||||
for i in $(seq 1 60); do
|
||||
ready=$(find "$PGDATA/pg_wal/archive_status" -name '*.ready' 2>/dev/null | wc -l)
|
||||
done_count=$(find "$PGDATA/pg_wal/archive_status" -name '*.done' 2>/dev/null | wc -l)
|
||||
log "archive_status: ready=$ready done=$done_count"
|
||||
[ "$ready" -eq 0 ] && [ "$done_count" -gt 0 ] && break
|
||||
sleep 2
|
||||
done
|
||||
log "=== last 20 lines of archive_wal.log ==="
|
||||
tail -20 /tmp/archive_wal.log || true
|
||||
|
||||
# Confirm the base backup actually landed before we declare success.
|
||||
barman-cloud-backup-list --cloud-provider aws-s3 \
|
||||
--endpoint-url "$GARAGE_ENDPOINT" \
|
||||
"$BARMAN_DEST" "$BARMAN_SERVER"
|
||||
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast stop"
|
||||
log "Seed complete: Garage now has a base backup at $BARMAN_DEST/$BARMAN_SERVER."
|
||||
volumeMounts:
|
||||
- name: seed
|
||||
mountPath: /seed
|
||||
- name: pgdata
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: authentik-seed-ssh
|
||||
defaultMode: 0400
|
||||
- name: seed
|
||||
emptyDir: {}
|
||||
- name: pgdata
|
||||
emptyDir: {}
|
||||
@@ -1,14 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- migration-bucket.yaml
|
||||
- barman-creds.sops.yaml
|
||||
- objectstore.yaml
|
||||
- postgres.yaml
|
||||
- scheduled-backup.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- seed-job.yaml
|
||||
- pvcs.yaml
|
||||
- secrets.sops.yaml
|
||||
- redis.yaml
|
||||
- machine-learning.yaml
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: im-pg
|
||||
namespace: immich
|
||||
annotations:
|
||||
# Same rationale as paperless-db: skip the plugin's pre-recovery
|
||||
# "Expected empty archive" check so bootstrap.recovery works against a
|
||||
# pre-populated archive (migration seed or later live backups).
|
||||
cnpg.io/skipEmptyWalArchiveCheck: "enabled"
|
||||
spec:
|
||||
# Single instance during the pgvecto.rs→VectorChord migration window.
|
||||
# pg_basebackup (used by CNPG for replica join) uses tar format, which
|
||||
# has a 100-byte filename limit. pgvecto.rs stores indexes under paths
|
||||
# like `pg_vectors/indexes/<uuid>/segments/<uuid>` that exceed the
|
||||
# limit, so any replica join fails with
|
||||
# "file name too long for tar format: pg_vectors/indexes/..."
|
||||
# After Immich's in-app migration drops the `vectors` extension and the
|
||||
# `pg_vectors/` directory goes away, bump back to 2.
|
||||
instances: 1
|
||||
# Dual-extension image: ships pgvector + pgvecto.rs + VectorChord so
|
||||
# Immich's upstream in-app migration from `vectors` → `vchord` can run on
|
||||
# first boot against the recovered legacy (pgvecto.rs) dump. Swap to
|
||||
# `ghcr.io/tensorchord/cloudnative-vectorchord:16-0.3.0` once the
|
||||
# migration has completed and the `vectors` extension has been dropped.
|
||||
imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
size: 20Gi
|
||||
# Barman Cloud plugin drives both WAL archiving and scheduled base backups
|
||||
# to Garage (bucket k8s-cnpg). See objectstore.yaml.
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: im-pg-store
|
||||
# Recover from the latest Barman base backup + WAL in s3://k8s-cnpg/im-pg.
|
||||
# Migration seed is produced by `.woodpecker/migrate-immich.yml` against
|
||||
# the same path. Producer == consumer path.
|
||||
bootstrap:
|
||||
recovery:
|
||||
owner: immich
|
||||
database: immich
|
||||
source: im-pg-backup
|
||||
externalClusters:
|
||||
- name: im-pg-backup
|
||||
plugin:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
parameters:
|
||||
barmanObjectName: im-pg-store
|
||||
serverName: im-pg
|
||||
postgresql:
|
||||
# Both extensions preloaded during the migration window. `vectors.so`
|
||||
# is required for pgvecto.rs catalog objects in the restored dump to
|
||||
# load at startup; `vchord.so` is required for Immich's in-app
|
||||
# migration to `CREATE EXTENSION vchord` and rewrite the data.
|
||||
# CNPG rejects shared_preload_libraries under .parameters because it
|
||||
# manages that GUC itself — the dedicated list field is the only path.
|
||||
shared_preload_libraries:
|
||||
- vectors.so
|
||||
- vchord.so
|
||||
parameters:
|
||||
# Match LXC 225 Postgres tuning: 8 GB RAM / pgvecto.rs workload.
|
||||
shared_buffers: "512MB"
|
||||
max_wal_size: "2GB"
|
||||
wal_compression: "on"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
@@ -1,227 +0,0 @@
|
||||
---
|
||||
# Cold-start seed for im-pg. Same shape as ak-pg-seed: ephemeral
|
||||
# Postgres + pg_restore from legacy dump + barman-cloud-backup to
|
||||
# s3://k8s-cnpg/im-pg/. Idempotency-gated on Garage having any
|
||||
# object under that path.
|
||||
#
|
||||
# Legacy immich DB lives in a docker container (label
|
||||
# com.docker.compose.service=database) on immich.srv.renner.uno.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: im-pg-seed
|
||||
namespace: immich
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
initContainers:
|
||||
- name: legacy-dump
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- name: LEGACY_HOST
|
||||
value: immich.srv.renner.uno
|
||||
- name: PGDATABASE
|
||||
value: immich
|
||||
- name: PGUSER
|
||||
value: postgres
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
echo "[$(date -u +%H:%M:%S)] Locating immich DB container on $LEGACY_HOST ..."
|
||||
DB_CT=$(ssh "$LEGACY" "docker ps --filter label=com.docker.compose.service=database --format '{{.Names}}' | head -1")
|
||||
[ -n "$DB_CT" ] || { echo "no DB container found on $LEGACY_HOST"; exit 1; }
|
||||
echo "[$(date -u +%H:%M:%S)] pg_dump via docker exec $DB_CT → /seed/legacy.dump"
|
||||
ssh "$LEGACY" "docker exec -i $DB_CT pg_dump --clean --if-exists --no-owner --no-privileges --format=custom -U $PGUSER $PGDATABASE" > /seed/legacy.dump
|
||||
ls -lh /seed/legacy.dump
|
||||
volumeMounts:
|
||||
- name: ssh
|
||||
mountPath: /ssh
|
||||
readOnly: true
|
||||
- name: seed
|
||||
mountPath: /seed
|
||||
containers:
|
||||
- name: seed
|
||||
# Match the live im-pg Cluster's image: dual-extension build that
|
||||
# ships pgvector + pgvecto.rs + VectorChord. The ephemeral Postgres
|
||||
# in this Job needs the same extension files so the legacy dump's
|
||||
# CREATE EXTENSION statements + the face_search/smart_search vector
|
||||
# tables actually restore. Without this, the backup written to
|
||||
# Garage is missing the vector catalog and immich crash-loops with
|
||||
# "permission denied to create extension vector" once it tries to
|
||||
# CREATE EXTENSION as the (non-superuser) immich role.
|
||||
image: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration
|
||||
# CNPG's base image runs as the postgres user by default; the seed
|
||||
# script needs root for `apt-get install`, `chown`, and `su postgres`.
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
runAsNonRoot: false
|
||||
env:
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data
|
||||
- name: TARGET_DB
|
||||
value: immich
|
||||
- name: TARGET_OWNER
|
||||
value: immich
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: ACCESS_SECRET_KEY
|
||||
- name: AWS_DEFAULT_REGION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: REGION
|
||||
- name: GARAGE_ENDPOINT
|
||||
value: https://s3.nas-1.renner.uno
|
||||
- name: BARMAN_SERVER
|
||||
value: im-pg
|
||||
- name: BARMAN_DEST
|
||||
value: s3://k8s-cnpg/im-pg
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
log "Installing awscli + barman ..."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 python3-pip python3-venv python3-psycopg2 \
|
||||
awscli ca-certificates >/dev/null
|
||||
|
||||
# Idempotency gate: only skip if a real base backup exists at
|
||||
# <BARMAN_DEST>/<BARMAN_SERVER>/base/. Failed previous attempts
|
||||
# may leave dangling marker objects under <BARMAN_DEST>/.
|
||||
EXISTING=$(aws s3 ls --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST/$BARMAN_SERVER/base/" 2>/dev/null | wc -l || true)
|
||||
if [ "$EXISTING" -gt 0 ]; then
|
||||
log "Garage already has $EXISTING base backup entr(ies) at $BARMAN_DEST/$BARMAN_SERVER/base/ — seed skipped."
|
||||
exit 0
|
||||
fi
|
||||
log "No base backup at $BARMAN_DEST/$BARMAN_SERVER/base/ — running full seed process."
|
||||
|
||||
python3 -m venv --system-site-packages /opt/barman
|
||||
/opt/barman/bin/pip install --no-cache-dir --quiet 'barman[cloud,aws-snapshots]'
|
||||
export PATH="/opt/barman/bin:$PATH"
|
||||
|
||||
mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA"
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres"
|
||||
|
||||
cat > /usr/local/bin/archive_wal.sh <<SH
|
||||
#!/bin/sh
|
||||
exec >> /tmp/archive_wal.log 2>&1
|
||||
export AWS_ACCESS_KEY_ID='$AWS_ACCESS_KEY_ID'
|
||||
export AWS_SECRET_ACCESS_KEY='$AWS_SECRET_ACCESS_KEY'
|
||||
export AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION'
|
||||
echo "\$(date -Iseconds) archiving \$1"
|
||||
exec barman-cloud-wal-archive \\
|
||||
--cloud-provider aws-s3 \\
|
||||
--endpoint-url '$GARAGE_ENDPOINT' \\
|
||||
--gzip -e AES256 \\
|
||||
'$BARMAN_DEST' '$BARMAN_SERVER' "\$1"
|
||||
SH
|
||||
chmod +x /usr/local/bin/archive_wal.sh
|
||||
: > /tmp/archive_wal.log
|
||||
chmod 666 /tmp/archive_wal.log
|
||||
|
||||
cat >> "$PGDATA/postgresql.conf" <<CONF
|
||||
listen_addresses = 'localhost'
|
||||
wal_level = replica
|
||||
archive_mode = on
|
||||
archive_command = '/usr/local/bin/archive_wal.sh %p'
|
||||
archive_timeout = '60s'
|
||||
# Match the live im-pg Cluster's preload list so pgvecto.rs
|
||||
# catalog objects from the legacy dump load at startup AND
|
||||
# the dump's CREATE EXTENSION vectors / vchord succeed.
|
||||
shared_preload_libraries = 'vectors.so,vchord.so'
|
||||
CONF
|
||||
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -l /tmp/pg.log -w start"
|
||||
|
||||
psql -U postgres -c "CREATE ROLE $TARGET_OWNER LOGIN;" || true
|
||||
psql -U postgres -c "CREATE DATABASE $TARGET_DB OWNER $TARGET_OWNER;" || true
|
||||
|
||||
# Pre-create non-trusted extensions immich needs at runtime.
|
||||
# The legacy dump only ships pgvecto.rs (`vectors`) — newer
|
||||
# immich versions also CREATE EXTENSION `vector` (pgvector)
|
||||
# at startup for the vchord migration path. pgvector is not a
|
||||
# trusted extension, so the immich role can't CREATE it. Pre-
|
||||
# creating as postgres with IF NOT EXISTS lets the runtime
|
||||
# `CREATE EXTENSION IF NOT EXISTS vector` short-circuit (the
|
||||
# privilege check is skipped when the extension already
|
||||
# exists). cube/earthdistance/vectors/vchord get the same
|
||||
# treatment so the dump's own CREATE EXTENSION calls
|
||||
# (with IF NOT EXISTS, emitted by pg_dump) are no-ops too.
|
||||
for ext in cube earthdistance vector vectors vchord; do
|
||||
psql -U postgres -d "$TARGET_DB" -c "CREATE EXTENSION IF NOT EXISTS \"$ext\" CASCADE;" || log "extension $ext: failed (continuing)"
|
||||
done
|
||||
|
||||
# Run pg_restore as postgres + let the dump's own ALTER OWNER
|
||||
# + GRANT statements run. See authentik seed-job.yaml for the
|
||||
# rationale.
|
||||
log "pg_restore /seed/legacy.dump → $TARGET_DB ..."
|
||||
pg_restore -U postgres -d "$TARGET_DB" \
|
||||
/seed/legacy.dump \
|
||||
|| log "pg_restore completed with warnings (expected)"
|
||||
|
||||
# barman-cloud-backup connects to PG via SQL; without
|
||||
# --host/--port/--user it falls back to the OS user (root)
|
||||
# which has no PG role.
|
||||
log "barman-cloud-backup → $BARMAN_DEST/$BARMAN_SERVER ..."
|
||||
barman-cloud-backup \
|
||||
--cloud-provider aws-s3 \
|
||||
--endpoint-url "$GARAGE_ENDPOINT" \
|
||||
--gzip -e AES256 \
|
||||
--host 127.0.0.1 --port 5432 --user postgres \
|
||||
"$BARMAN_DEST" "$BARMAN_SERVER"
|
||||
|
||||
# archive_command fires async; wait for ready files to drain
|
||||
# so end_wal of the base backup actually uploads.
|
||||
psql -h 127.0.0.1 -U postgres -d postgres -c "CHECKPOINT; SELECT pg_switch_wal();"
|
||||
for i in $(seq 1 60); do
|
||||
ready=$(find "$PGDATA/pg_wal/archive_status" -name '*.ready' 2>/dev/null | wc -l)
|
||||
done_count=$(find "$PGDATA/pg_wal/archive_status" -name '*.done' 2>/dev/null | wc -l)
|
||||
log "archive_status: ready=$ready done=$done_count"
|
||||
[ "$ready" -eq 0 ] && [ "$done_count" -gt 0 ] && break
|
||||
sleep 2
|
||||
done
|
||||
log "=== last 20 lines of archive_wal.log ==="
|
||||
tail -20 /tmp/archive_wal.log || true
|
||||
|
||||
barman-cloud-backup-list --cloud-provider aws-s3 \
|
||||
--endpoint-url "$GARAGE_ENDPOINT" \
|
||||
"$BARMAN_DEST" "$BARMAN_SERVER"
|
||||
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast stop"
|
||||
log "Seed complete: Garage now has a base backup at $BARMAN_DEST/$BARMAN_SERVER."
|
||||
volumeMounts:
|
||||
- name: seed
|
||||
mountPath: /seed
|
||||
- name: pgdata
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: immich-seed-ssh
|
||||
defaultMode: 0400
|
||||
- name: seed
|
||||
emptyDir: {}
|
||||
- name: pgdata
|
||||
emptyDir: {}
|
||||
@@ -54,58 +54,11 @@ spec:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
# PVC (library) seed only — the DB seed lives in the standalone
|
||||
# im-pg-seed Job (seed-job.yaml) which populates Garage so CNPG
|
||||
# bootstrap.recovery has data to pull from. This initContainer
|
||||
# rsyncs the legacy Immich LXC's library directory into the
|
||||
# immich-library PVC and drops a sentinel.
|
||||
- name: seed
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- name: LEGACY_HOST
|
||||
value: immich.srv.renner.uno
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
if [ -f /data/.seed-done ]; then
|
||||
log "PVC seed sentinel present — skipping rsync."
|
||||
exit 0
|
||||
fi
|
||||
log "No sentinel — rsync from $LEGACY_HOST"
|
||||
|
||||
apk add --no-cache rsync openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
log "rsync /mnt/media/immich → /data ..."
|
||||
set +e
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/mnt/media/immich/" /data/
|
||||
rsync_rc=$?
|
||||
set -e
|
||||
case "$rsync_rc" in
|
||||
0) log "rsync OK" ;;
|
||||
23) log "rsync exit 23 (partial transfer warnings) — continuing" ;;
|
||||
*) log "rsync FAILED with code $rsync_rc"; exit "$rsync_rc" ;;
|
||||
esac
|
||||
chown -R 1000:1000 /data
|
||||
du -sh /data
|
||||
touch /data/.seed-done
|
||||
log "PVC seed complete; sentinel set."
|
||||
volumeMounts:
|
||||
- name: library
|
||||
mountPath: /data
|
||||
- name: ssh
|
||||
mountPath: /ssh
|
||||
readOnly: true
|
||||
# Pre-fetch LE staging root CAs so Node trusts auth.renner.uno's
|
||||
# staging-issued cert during OIDC token exchange. Same pattern as
|
||||
# audiobookshelf (see its deployment.yaml for the full rationale).
|
||||
# The library PVC is populated by the seed phase before this
|
||||
# Deployment reconciles, so no rsync init is needed here.
|
||||
- name: fetch-le-staging-roots
|
||||
image: curlimages/curl:8.10.1
|
||||
command:
|
||||
@@ -195,10 +148,6 @@ spec:
|
||||
path: immich-config.json
|
||||
- name: le-ca
|
||||
emptyDir: {}
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: immich-seed-ssh
|
||||
defaultMode: 0400
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -28,113 +28,11 @@ spec:
|
||||
upgrade:
|
||||
remediation:
|
||||
retries: 3
|
||||
# The influxdb2 chart's StatefulSet template has no .Values.initContainers
|
||||
# hook, so the seed initContainer is added via a post-render Kustomize
|
||||
# patch against the rendered StatefulSet (name: influxdb-influxdb2,
|
||||
# i.e. {{ template "influxdb.fullname" . }} for release "influxdb"
|
||||
# with chart "influxdb2"). The SSH volume is added via the chart's
|
||||
# supported top-level `volumes:` value below.
|
||||
postRenderers:
|
||||
- kustomize:
|
||||
patches:
|
||||
- target:
|
||||
version: v1
|
||||
kind: StatefulSet
|
||||
name: influxdb-influxdb2
|
||||
patch: |
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: influxdb-influxdb2
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
initContainers:
|
||||
- name: seed
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29
|
||||
env:
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SCHED
|
||||
value: influxdb-daily
|
||||
- name: LEGACY_HOST
|
||||
value: influxdb.srv.renner.uno
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
log "Installing rsync ..."
|
||||
apk add --no-cache rsync >/dev/null
|
||||
|
||||
EXISTING=$(kubectl -n velero get backups.velero.io \
|
||||
-l "velero.io/schedule-name=$SCHED" \
|
||||
-o jsonpath='{range .items[?(@.status.phase=="Completed")]}{.metadata.name}{"\n"}{end}' \
|
||||
2>/dev/null | grep -c . || true)
|
||||
if [ "$EXISTING" -gt 0 ]; then
|
||||
log "$EXISTING completed Velero Backup(s) for $SCHED — skipping seed"
|
||||
exit 0
|
||||
fi
|
||||
log "No completed Velero Backup for $SCHED — proceeding with seed"
|
||||
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
log "Stopping legacy influxdb.service ..."
|
||||
ssh "$LEGACY" "systemctl stop influxdb"
|
||||
trap 'log "Restarting legacy influxdb.service ..."; ssh "$LEGACY" "systemctl start influxdb" || true' EXIT
|
||||
|
||||
log "rsync /var/lib/influxdb → /var/lib/influxdb2 ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/var/lib/influxdb/" /var/lib/influxdb2/
|
||||
chown -R 1500:1500 /var/lib/influxdb2 || true
|
||||
du -sh /var/lib/influxdb2
|
||||
|
||||
BACKUP_NAME="$SCHED-seed-$(date -u +%Y%m%d-%H%M%S)"
|
||||
log "Creating Velero Backup $BACKUP_NAME ..."
|
||||
cat <<YAML | kubectl apply -f -
|
||||
apiVersion: velero.io/v1
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: $BACKUP_NAME
|
||||
namespace: velero
|
||||
labels:
|
||||
velero.io/schedule-name: $SCHED
|
||||
spec:
|
||||
includedNamespaces:
|
||||
- $NAMESPACE
|
||||
includedResources:
|
||||
- persistentvolumeclaims
|
||||
snapshotMoveData: true
|
||||
itemOperationTimeout: 4h0m0s
|
||||
YAML
|
||||
log "Seed complete: PVC populated; Velero Backup submitted."
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/influxdb2
|
||||
- name: ssh
|
||||
mountPath: /ssh
|
||||
readOnly: true
|
||||
# The PVC (data-influxdb-influxdb2-0) is pre-populated by the seed
|
||||
# phase before this HelmRelease reconciles, so the chart's StatefulSet
|
||||
# template adopts the existing PVC + populated data on first start —
|
||||
# no init container needed.
|
||||
values:
|
||||
serviceAccount:
|
||||
create: false
|
||||
name: influxdb-seed
|
||||
|
||||
# The chart's StatefulSet template merges this into the pod's
|
||||
# .spec.volumes via the chart-supported top-level `volumes:` key.
|
||||
# The seed initContainer (added by postRenderer above) references it.
|
||||
volumes:
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: influxdb-seed-ssh
|
||||
defaultMode: 0400
|
||||
|
||||
image:
|
||||
repository: influxdb
|
||||
# Match VM 108 for format compatibility with `influx restore`.
|
||||
|
||||
@@ -3,6 +3,4 @@ kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- bootstrap-secret.sops.yaml
|
||||
- seed-rbac.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- helmrelease.yaml
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# RBAC for the seed initContainer in the influxdb StatefulSet Pod.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: influxdb-seed
|
||||
namespace: influxdb
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: influxdb-seed-velero
|
||||
namespace: velero
|
||||
rules:
|
||||
- apiGroups: [velero.io]
|
||||
resources: [backups]
|
||||
verbs: [get, list, watch, create]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: influxdb-seed-velero
|
||||
namespace: velero
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: influxdb-seed
|
||||
namespace: influxdb
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: influxdb-seed-velero
|
||||
@@ -23,7 +23,6 @@ spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: music-assistant-seed
|
||||
nodeSelector:
|
||||
topology.kubernetes.io/zone: edge
|
||||
tolerations:
|
||||
@@ -31,78 +30,6 @@ spec:
|
||||
operator: Equal
|
||||
value: pinned
|
||||
effect: NoSchedule
|
||||
initContainers:
|
||||
# Atomic seed: idempotency-gated ssh+tar of /opt/music-assistant/
|
||||
# data from the legacy LXC (which ships without rsync). Same Pod
|
||||
# mounts the data PVC; main container starts only after seed.
|
||||
- name: seed
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-04-29
|
||||
env:
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SCHED
|
||||
value: music-assistant-daily
|
||||
- name: LEGACY_HOST
|
||||
value: music-assistant.iot.renner.uno
|
||||
command:
|
||||
- sh
|
||||
- -eu
|
||||
- -c
|
||||
- |
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
# ci-k8s has openssh + kubectl + tar baked in.
|
||||
|
||||
EXISTING=$(kubectl -n velero get backups.velero.io \
|
||||
-l "velero.io/schedule-name=$SCHED" \
|
||||
-o jsonpath='{range .items[?(@.status.phase=="Completed")]}{.metadata.name}{"\n"}{end}' \
|
||||
2>/dev/null | grep -c . || true)
|
||||
if [ "$EXISTING" -gt 0 ]; then
|
||||
log "$EXISTING completed Velero Backup(s) for $SCHED — skipping seed"
|
||||
exit 0
|
||||
fi
|
||||
log "No completed Velero Backup for $SCHED — proceeding with seed"
|
||||
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
log "Stopping music-assistant docker-compose stack on legacy LXC ..."
|
||||
ssh "$LEGACY" "cd /opt/music-assistant && docker compose stop"
|
||||
trap 'log "Restarting music-assistant on legacy LXC ..."; ssh "$LEGACY" "cd /opt/music-assistant && docker compose up -d" || true' EXIT
|
||||
|
||||
# Legacy MA LXC ships without rsync, so use ssh+tar streaming.
|
||||
log "Clearing /data ..."
|
||||
find /data -mindepth 1 -delete
|
||||
log "Streaming /opt/music-assistant/data via ssh+tar ..."
|
||||
ssh "$LEGACY" "cd /opt/music-assistant/data && tar -cf - ." | tar -xf - -C /data/
|
||||
du -sh /data
|
||||
|
||||
BACKUP_NAME="$SCHED-seed-$(date -u +%Y%m%d-%H%M%S)"
|
||||
log "Creating Velero Backup $BACKUP_NAME ..."
|
||||
cat <<YAML | kubectl apply -f -
|
||||
apiVersion: velero.io/v1
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: $BACKUP_NAME
|
||||
namespace: velero
|
||||
labels:
|
||||
velero.io/schedule-name: $SCHED
|
||||
spec:
|
||||
includedNamespaces:
|
||||
- $NAMESPACE
|
||||
includedResources:
|
||||
- persistentvolumeclaims
|
||||
snapshotMoveData: true
|
||||
itemOperationTimeout: 4h0m0s
|
||||
YAML
|
||||
log "Seed complete: PVC populated; Velero Backup submitted."
|
||||
volumeMounts:
|
||||
- { name: data, mountPath: /data }
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
containers:
|
||||
- name: music-assistant
|
||||
image: ghcr.io/music-assistant/server:2.8.5
|
||||
@@ -158,10 +85,6 @@ spec:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: music-assistant-data
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: music-assistant-seed-ssh
|
||||
defaultMode: 0400
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- pvcs.yaml
|
||||
- seed-rbac.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# RBAC for the seed initContainer in the music-assistant Pod.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: music-assistant-seed
|
||||
namespace: music-assistant
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: music-assistant-seed-velero
|
||||
namespace: velero
|
||||
rules:
|
||||
- apiGroups: [velero.io]
|
||||
resources: [backups]
|
||||
verbs: [get, list, watch, create]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: music-assistant-seed-velero
|
||||
namespace: velero
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: music-assistant-seed
|
||||
namespace: music-assistant
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: music-assistant-seed-velero
|
||||
@@ -1,14 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- migration-bucket.yaml
|
||||
- barman-creds.sops.yaml
|
||||
- objectstore.yaml
|
||||
- postgres.yaml
|
||||
- scheduled-backup.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- seed-job.yaml
|
||||
- pvcs.yaml
|
||||
- secrets.sops.yaml
|
||||
- ftp-scanner-secret.sops.yaml
|
||||
- redis.yaml
|
||||
|
||||
@@ -50,53 +50,6 @@ spec:
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
# PVC seed only — the DB seed lives in the standalone pprl-pg-seed
|
||||
# Job (seed-job.yaml) which populates Garage so CNPG bootstrap.recovery
|
||||
# has data to pull from. This initContainer rsyncs the legacy
|
||||
# paperless VM's docker-volume directories into the RWO PVCs and
|
||||
# drops a sentinel so subsequent Pod restarts skip the rsync.
|
||||
- name: seed
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- name: LEGACY_HOST
|
||||
value: paperless.srv.renner.uno
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
if [ -f /usr/src/paperless/data/.seed-done ]; then
|
||||
log "PVC seed sentinel present — skipping rsync."
|
||||
exit 0
|
||||
fi
|
||||
log "No sentinel — rsync from $LEGACY_HOST"
|
||||
|
||||
apk add --no-cache rsync openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
log "rsync paperless_data → /usr/src/paperless/data ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/var/lib/docker/volumes/paperless_data/_data/" /usr/src/paperless/data/
|
||||
log "rsync paperless_media → /usr/src/paperless/media ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/var/lib/docker/volumes/paperless_media/_data/" /usr/src/paperless/media/
|
||||
chown -R 1000:1000 /usr/src/paperless/data /usr/src/paperless/media
|
||||
du -sh /usr/src/paperless/data /usr/src/paperless/media
|
||||
touch /usr/src/paperless/data/.seed-done
|
||||
log "PVC seed complete; sentinel set."
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /usr/src/paperless/data
|
||||
- name: media
|
||||
mountPath: /usr/src/paperless/media
|
||||
- name: ssh
|
||||
mountPath: /ssh
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: paperless
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:2.20.14
|
||||
@@ -170,10 +123,6 @@ spec:
|
||||
- name: consume
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-consume
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: paperless-seed-ssh
|
||||
defaultMode: 0400
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: pprl-pg
|
||||
namespace: paperless
|
||||
annotations:
|
||||
# See authentik-db/postgres.yaml — same rationale: skip the
|
||||
# plugin's pre-recovery "Expected empty archive" check so
|
||||
# bootstrap.recovery works against a pre-populated archive
|
||||
# (migration seed or later live backups).
|
||||
cnpg.io/skipEmptyWalArchiveCheck: "enabled"
|
||||
spec:
|
||||
instances: 2
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
size: 20Gi
|
||||
# The Barman Cloud plugin drives both WAL archiving and
|
||||
# scheduled base backups to Garage on nas-1 (bucket k8s-cnpg).
|
||||
# See objectstore.yaml for the destination config.
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: pprl-pg-store
|
||||
# Bootstrap always recovers from the latest Barman base backup + WAL in
|
||||
# s3://k8s-cnpg/pprl-pg. Producer path == consumer path: the live cluster
|
||||
# writes to the same location via the plugin above (serverName defaults to
|
||||
# the Cluster name, i.e. `pprl-pg`), and the migration pipeline seeds the
|
||||
# same path using an ephemeral Postgres + barman-cloud-backup. Unpinned
|
||||
# recovery target = latest WAL.
|
||||
bootstrap:
|
||||
recovery:
|
||||
owner: paperless
|
||||
database: paperless
|
||||
source: pprl-pg-backup
|
||||
externalClusters:
|
||||
- name: pprl-pg-backup
|
||||
plugin:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
parameters:
|
||||
barmanObjectName: pprl-pg-store
|
||||
serverName: pprl-pg
|
||||
postgresql:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_connections: "200"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
@@ -1,196 +0,0 @@
|
||||
---
|
||||
# Cold-start seed for pprl-pg. Same shape as ak-pg-seed: ephemeral
|
||||
# Postgres + pg_restore from legacy dump + barman-cloud-backup to
|
||||
# s3://k8s-cnpg/pprl-pg/. Idempotency-gated on Garage having any
|
||||
# object under that path.
|
||||
#
|
||||
# Legacy paperless DB lives in a docker container on
|
||||
# paperless.srv.renner.uno; the init container locates it via
|
||||
# `docker ps` label filter and pg_dumps from inside.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: pprl-pg-seed
|
||||
namespace: paperless
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
initContainers:
|
||||
- name: legacy-dump
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- name: LEGACY_HOST
|
||||
value: paperless.srv.renner.uno
|
||||
- name: PGDATABASE
|
||||
value: paperless
|
||||
- name: PGUSER
|
||||
value: paperless
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
echo "[$(date -u +%H:%M:%S)] Locating paperless DB container on $LEGACY_HOST ..."
|
||||
DB_CT=$(ssh "$LEGACY" "docker ps --filter label=com.docker.compose.service=db --format '{{.Names}}' | head -1")
|
||||
[ -n "$DB_CT" ] || { echo "no DB container found on $LEGACY_HOST"; exit 1; }
|
||||
echo "[$(date -u +%H:%M:%S)] pg_dump via docker exec $DB_CT → /seed/legacy.dump"
|
||||
ssh "$LEGACY" "docker exec -i $DB_CT pg_dump --clean --if-exists --no-owner --no-privileges --format=custom -U $PGUSER $PGDATABASE" > /seed/legacy.dump
|
||||
ls -lh /seed/legacy.dump
|
||||
volumeMounts:
|
||||
- name: ssh
|
||||
mountPath: /ssh
|
||||
readOnly: true
|
||||
- name: seed
|
||||
mountPath: /seed
|
||||
containers:
|
||||
- name: seed
|
||||
image: postgres:16
|
||||
env:
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data
|
||||
- name: TARGET_DB
|
||||
value: paperless
|
||||
- name: TARGET_OWNER
|
||||
value: paperless
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: ACCESS_SECRET_KEY
|
||||
- name: AWS_DEFAULT_REGION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: barman-cloud-creds
|
||||
key: REGION
|
||||
- name: GARAGE_ENDPOINT
|
||||
value: https://s3.nas-1.renner.uno
|
||||
- name: BARMAN_SERVER
|
||||
value: pprl-pg
|
||||
- name: BARMAN_DEST
|
||||
value: s3://k8s-cnpg/pprl-pg
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
log "Installing awscli + barman ..."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 python3-pip python3-venv python3-psycopg2 \
|
||||
awscli ca-certificates >/dev/null
|
||||
|
||||
# Idempotency gate: only skip if a real base backup exists at
|
||||
# <BARMAN_DEST>/<BARMAN_SERVER>/base/. Failed previous attempts
|
||||
# may leave dangling marker objects under <BARMAN_DEST>/.
|
||||
EXISTING=$(aws s3 ls --endpoint-url "$GARAGE_ENDPOINT" "$BARMAN_DEST/$BARMAN_SERVER/base/" 2>/dev/null | wc -l || true)
|
||||
if [ "$EXISTING" -gt 0 ]; then
|
||||
log "Garage already has $EXISTING base backup entr(ies) at $BARMAN_DEST/$BARMAN_SERVER/base/ — seed skipped."
|
||||
exit 0
|
||||
fi
|
||||
log "No base backup at $BARMAN_DEST/$BARMAN_SERVER/base/ — running full seed process."
|
||||
|
||||
python3 -m venv --system-site-packages /opt/barman
|
||||
/opt/barman/bin/pip install --no-cache-dir --quiet 'barman[cloud,aws-snapshots]'
|
||||
export PATH="/opt/barman/bin:$PATH"
|
||||
|
||||
mkdir -p "$PGDATA" && chown postgres:postgres "$PGDATA"
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/initdb -D $PGDATA --auth=trust --encoding=UTF8 --no-locale --username=postgres"
|
||||
|
||||
cat > /usr/local/bin/archive_wal.sh <<SH
|
||||
#!/bin/sh
|
||||
exec >> /tmp/archive_wal.log 2>&1
|
||||
export AWS_ACCESS_KEY_ID='$AWS_ACCESS_KEY_ID'
|
||||
export AWS_SECRET_ACCESS_KEY='$AWS_SECRET_ACCESS_KEY'
|
||||
export AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION'
|
||||
echo "\$(date -Iseconds) archiving \$1"
|
||||
exec barman-cloud-wal-archive \\
|
||||
--cloud-provider aws-s3 \\
|
||||
--endpoint-url '$GARAGE_ENDPOINT' \\
|
||||
--gzip -e AES256 \\
|
||||
'$BARMAN_DEST' '$BARMAN_SERVER' "\$1"
|
||||
SH
|
||||
chmod +x /usr/local/bin/archive_wal.sh
|
||||
: > /tmp/archive_wal.log
|
||||
chmod 666 /tmp/archive_wal.log
|
||||
|
||||
cat >> "$PGDATA/postgresql.conf" <<CONF
|
||||
listen_addresses = 'localhost'
|
||||
wal_level = replica
|
||||
archive_mode = on
|
||||
archive_command = '/usr/local/bin/archive_wal.sh %p'
|
||||
archive_timeout = '60s'
|
||||
CONF
|
||||
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -l /tmp/pg.log -w start"
|
||||
|
||||
psql -U postgres -c "CREATE ROLE $TARGET_OWNER LOGIN;" || true
|
||||
psql -U postgres -c "CREATE DATABASE $TARGET_DB OWNER $TARGET_OWNER;" || true
|
||||
|
||||
# Run pg_restore as postgres + let the dump's own ALTER OWNER
|
||||
# + GRANT statements run. See authentik seed-job.yaml for the
|
||||
# rationale.
|
||||
log "pg_restore /seed/legacy.dump → $TARGET_DB ..."
|
||||
pg_restore -U postgres -d "$TARGET_DB" \
|
||||
/seed/legacy.dump \
|
||||
|| log "pg_restore completed with warnings (expected)"
|
||||
|
||||
# barman-cloud-backup connects to PG via SQL; without
|
||||
# --host/--port/--user it falls back to the OS user (root)
|
||||
# which has no PG role.
|
||||
log "barman-cloud-backup → $BARMAN_DEST/$BARMAN_SERVER ..."
|
||||
barman-cloud-backup \
|
||||
--cloud-provider aws-s3 \
|
||||
--endpoint-url "$GARAGE_ENDPOINT" \
|
||||
--gzip -e AES256 \
|
||||
--host 127.0.0.1 --port 5432 --user postgres \
|
||||
"$BARMAN_DEST" "$BARMAN_SERVER"
|
||||
|
||||
# archive_command fires async; wait for ready files to drain
|
||||
# so end_wal of the base backup actually uploads.
|
||||
psql -h 127.0.0.1 -U postgres -d postgres -c "CHECKPOINT; SELECT pg_switch_wal();"
|
||||
for i in $(seq 1 60); do
|
||||
ready=$(find "$PGDATA/pg_wal/archive_status" -name '*.ready' 2>/dev/null | wc -l)
|
||||
done_count=$(find "$PGDATA/pg_wal/archive_status" -name '*.done' 2>/dev/null | wc -l)
|
||||
log "archive_status: ready=$ready done=$done_count"
|
||||
[ "$ready" -eq 0 ] && [ "$done_count" -gt 0 ] && break
|
||||
sleep 2
|
||||
done
|
||||
log "=== last 20 lines of archive_wal.log ==="
|
||||
tail -20 /tmp/archive_wal.log || true
|
||||
|
||||
barman-cloud-backup-list --cloud-provider aws-s3 \
|
||||
--endpoint-url "$GARAGE_ENDPOINT" \
|
||||
"$BARMAN_DEST" "$BARMAN_SERVER"
|
||||
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $PGDATA -m fast stop"
|
||||
log "Seed complete: Garage now has a base backup at $BARMAN_DEST/$BARMAN_SERVER."
|
||||
volumeMounts:
|
||||
- name: seed
|
||||
mountPath: /seed
|
||||
- name: pgdata
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: paperless-seed-ssh
|
||||
defaultMode: 0400
|
||||
- name: seed
|
||||
emptyDir: {}
|
||||
- name: pgdata
|
||||
emptyDir: {}
|
||||
+11
-21
@@ -285,32 +285,22 @@ if phase_runs wait-apps; then
|
||||
return 1
|
||||
}
|
||||
|
||||
# Order is largely cosmetic — Flux reconciles all Kustomizations in parallel,
|
||||
# so wall-clock time is dominated by the slowest one regardless of wait
|
||||
# order. Listed roughly fast → slow so progress lands earlier in CI logs.
|
||||
# Timeouts are sized for the *worst-case under contention* (concurrent
|
||||
# Velero restores + immich's 250GB extract starving network/IO), not the
|
||||
# happy-path. Bumping individual timeouts costs nothing on a healthy
|
||||
# cluster since reconciliations run in parallel and the wait just
|
||||
# observes.
|
||||
# The seed phase populates every CNPG database and every migrating
|
||||
# PVC from legacy hosts before any app Kustomization reconciles.
|
||||
# Wall-clock dominated by immich-library-seed (~250 GiB rsync over
|
||||
# LAN, ~30-45 min). Every app KS dependsOn `seed`, so the per-app
|
||||
# waits below should be near-instant once seed lands Ready.
|
||||
wait_kustomization seed "Seed phase (DB + PVC populate from legacy)" 1440
|
||||
# apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up
|
||||
# is genuinely 5–10 min on a fresh OSD-wiped cluster).
|
||||
wait_kustomization apps "apps" 90
|
||||
# authentik waits on the ak-pg-seed Job populating Garage AND the CNPG
|
||||
# cluster's bootstrap.recovery succeeding from that seed. Allow 20 min.
|
||||
wait_kustomization authentik "Authentik" 120
|
||||
wait_kustomization authentik "Authentik" 60
|
||||
wait_kustomization monitoring "kube-prometheus-stack (Loki + Prom + AM + Grafana)" 180
|
||||
# On cold-start, velero-bootstrap-restore takes ~15min before this KS
|
||||
# can even start reconciling, so the wait timer can burn most of its
|
||||
# budget on the dependency. 30min covers that + the actual reconcile.
|
||||
wait_kustomization music-assistant "Music Assistant (small Garage tarball)" 180
|
||||
wait_kustomization paperless "Paperless (DB + ~3.5GB media Velero restore)" 180
|
||||
wait_kustomization music-assistant "Music Assistant" 60
|
||||
wait_kustomization paperless "Paperless" 60
|
||||
wait_kustomization home-assistant "Home Assistant (HAOS backup → native restore)" 180
|
||||
# Measured ~70 min on a 70 GB library (17 min upload + 52 min extract).
|
||||
wait_kustomization audiobookshelf "Audiobookshelf (~70GB library tar)" 480
|
||||
# ~250 GiB library — the longest wait. Mirrors the prior immich-seed
|
||||
# Kustomization's 4 h ceiling (1440 × 10s = 240 min).
|
||||
wait_kustomization immich "Immich (~250GB library tar)" 1440
|
||||
wait_kustomization audiobookshelf "Audiobookshelf" 60
|
||||
wait_kustomization immich "Immich" 60
|
||||
|
||||
# Wait for cert-manager to finish issuing every Certificate. Kustomization
|
||||
# Ready only means the Ingress + Certificate manifests are applied; cert-
|
||||
|
||||
@@ -4,11 +4,8 @@ metadata:
|
||||
name: audiobookshelf
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# PVC contents come from velero-bootstrap-restore reading from the
|
||||
# audiobookshelf-daily Schedule's latest Backup (or the seed Backup
|
||||
# written by migrate-audiobookshelf when no daily exists yet).
|
||||
# dependsOn enforces the order; this Kustomization just reconciles
|
||||
# manifests, no in-pod 70 GB extract anymore.
|
||||
# PVC contents are populated by the seed phase (rsync from legacy
|
||||
# audiobookshelf VM); this Kustomization reconciles app manifests only.
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
timeout: 30m
|
||||
@@ -26,4 +23,4 @@ spec:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: velero-bootstrap-restore
|
||||
- name: seed
|
||||
|
||||
@@ -21,3 +21,4 @@ spec:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: seed
|
||||
|
||||
@@ -4,9 +4,8 @@ metadata:
|
||||
name: immich
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# PVC contents come from velero-bootstrap-restore; this Kustomization
|
||||
# reconciles manifests only. Library tar (~250 GiB) is restored by
|
||||
# Velero CSI Snapshot Data Movement, not in-pod.
|
||||
# DB cluster + library PVC are populated by the seed phase; this
|
||||
# Kustomization reconciles app manifests only.
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
timeout: 30m
|
||||
@@ -24,4 +23,4 @@ spec:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: velero-bootstrap-restore
|
||||
- name: seed
|
||||
|
||||
@@ -4,12 +4,8 @@ metadata:
|
||||
name: influxdb
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# PVC contents come from velero-bootstrap-restore reading the latest
|
||||
# influxdb-daily Backup (or the seed Backup written by migrate-
|
||||
# influxdb when no daily exists yet). Replaces the prior API-restore
|
||||
# seed-Job model — Velero captures the data files directly so the
|
||||
# pod starts against a populated PVC and the helm chart's empty-
|
||||
# bucket bootstrap is skipped.
|
||||
# PVC contents are populated by the seed phase (rsync from legacy
|
||||
# influxdb VM); this Kustomization reconciles app manifests only.
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
timeout: 30m
|
||||
@@ -27,4 +23,4 @@ spec:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: velero-bootstrap-restore
|
||||
- name: seed
|
||||
|
||||
@@ -4,8 +4,8 @@ metadata:
|
||||
name: music-assistant
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# PVC contents come from velero-bootstrap-restore; this Kustomization
|
||||
# reconciles manifests only.
|
||||
# PVC contents are populated by the seed phase (ssh+tar from legacy
|
||||
# music-assistant LXC); this Kustomization reconciles app manifests only.
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
timeout: 30m
|
||||
@@ -23,4 +23,4 @@ spec:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: velero-bootstrap-restore
|
||||
- name: seed
|
||||
|
||||
@@ -4,9 +4,8 @@ metadata:
|
||||
name: paperless
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# PVC contents are restored by velero-bootstrap-restore from the seed
|
||||
# Backup written by the migrate-paperless pipeline (and from later
|
||||
# paperless-daily Schedule runs). dependsOn enforces the order.
|
||||
# DB cluster + data/media PVCs are populated by the seed phase; this
|
||||
# Kustomization reconciles app manifests only.
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
timeout: 30m
|
||||
@@ -24,4 +23,4 @@ spec:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: velero-bootstrap-restore
|
||||
- name: seed
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: seed
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# Migration phase: pulls data from legacy hosts into in-cluster CNPG
|
||||
# databases and PVCs so app Kustomizations can reconcile against
|
||||
# populated state. Lives between infrastructure and apps. Removed
|
||||
# entirely once all services have cut over from legacy.
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
# Long enough to cover the slowest seed (immich library: ~250 GiB
|
||||
# rsync over LAN ~ 30-45 min) plus all the smaller seeds in parallel.
|
||||
timeout: 4h
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/seed
|
||||
prune: true
|
||||
wait: true
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
dependsOn:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: infrastructure-cnpg-barman
|
||||
# kstatus already handles Job (succeeded == 1) by default. CNPG Cluster
|
||||
# readiness needs an explicit expression because kstatus doesn't know
|
||||
# the Cluster CRD's status semantics.
|
||||
healthCheckExprs:
|
||||
- apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
failed: "has(status.phase) && status.phase.startsWith('Failed')"
|
||||
current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"
|
||||
@@ -1,27 +0,0 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: velero-bootstrap-restore
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
# Audiobookshelf library is ~70 GB; data-movement restore is the slow
|
||||
# path. The Job polls with `--wait` so this timeout must cover the
|
||||
# longest expected restore, not just the Job submission.
|
||||
timeout: 4h
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/infrastructure/velero-bootstrap-restore
|
||||
prune: true
|
||||
wait: true
|
||||
healthCheckExprs:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
current: "status.succeeded == 1"
|
||||
inProgress: "!has(status.succeeded) || status.succeeded == 0"
|
||||
failed: "has(status.failed) && status.failed >= 3"
|
||||
dependsOn:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-ceph-cluster
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- migration-bucket.yaml
|
||||
- pvcs.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- library-seed-job.yaml
|
||||
@@ -0,0 +1,75 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: audiobookshelf-library-seed
|
||||
namespace: audiobookshelf
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: rsync
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: audiobookshelf.srv.renner.uno }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
if [ -f /audiobooks/.seed-done ]; then
|
||||
log "PVC seed sentinel present — skipping rsync."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apk add --no-cache rsync openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
# Phase 1: bulk books while legacy stays up (no --delete).
|
||||
log "Phase 1: rsync /audiobooks (legacy stays up) ..."
|
||||
rsync -aH --info=progress2 \
|
||||
"$LEGACY:/audiobooks/" /audiobooks/
|
||||
|
||||
# Phase 2: stop legacy briefly for SQLite-bearing dirs + delta.
|
||||
log "Phase 2: stopping audiobookshelf on legacy ..."
|
||||
ssh "$LEGACY" "systemctl stop audiobookshelf"
|
||||
trap 'log "Restarting audiobookshelf on legacy ..."; ssh "$LEGACY" "systemctl start audiobookshelf" || true' EXIT
|
||||
|
||||
log " rsync /usr/share/audiobookshelf/config → /config ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/usr/share/audiobookshelf/config/" /config/
|
||||
log " rsync /usr/share/audiobookshelf/metadata → /metadata ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/usr/share/audiobookshelf/metadata/" /metadata/
|
||||
log " rsync /audiobooks delta with --delete ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/audiobooks/" /audiobooks/
|
||||
chown -R 1000:1000 /config /metadata /audiobooks
|
||||
du -sh /config /metadata /audiobooks
|
||||
touch /audiobooks/.seed-done
|
||||
log "PVC seed complete; sentinel set."
|
||||
volumeMounts:
|
||||
- { name: config, mountPath: /config }
|
||||
- { name: metadata, mountPath: /metadata }
|
||||
- { name: books, mountPath: /audiobooks }
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: audiobookshelf-config
|
||||
- name: metadata
|
||||
persistentVolumeClaim:
|
||||
claimName: audiobookshelf-metadata
|
||||
- name: books
|
||||
persistentVolumeClaim:
|
||||
claimName: audiobookshelf-books
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: audiobookshelf-seed-ssh
|
||||
defaultMode: 0400
|
||||
@@ -0,0 +1,31 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ak-pg
|
||||
namespace: authentik
|
||||
spec:
|
||||
instances: 2
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
enableSuperuserAccess: true
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
size: 10Gi
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: ak-pg-store
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: authentik
|
||||
owner: authentik
|
||||
postgresql:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_connections: "200"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- migration-bucket.yaml
|
||||
- barman-creds.sops.yaml
|
||||
- objectstore.yaml
|
||||
- cluster.yaml
|
||||
- scheduled-backup.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- seed-job.yaml
|
||||
@@ -0,0 +1,80 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ak-pg-seed
|
||||
namespace: authentik
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
initContainers:
|
||||
- name: wait-for-cluster
|
||||
image: postgres:16
|
||||
env:
|
||||
- { name: PGHOST, value: ak-pg-rw }
|
||||
- { name: PGDATABASE, value: authentik }
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef: { name: ak-pg-superuser, key: username }
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: { name: ak-pg-superuser, key: password }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
until psql -tAc 'SELECT 1' >/dev/null 2>&1; do
|
||||
echo "[$(date -u +%H:%M:%S)] waiting for $PGHOST..."
|
||||
sleep 5
|
||||
done
|
||||
echo "[$(date -u +%H:%M:%S)] $PGHOST ready"
|
||||
containers:
|
||||
- name: seed
|
||||
image: postgres:16
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: authentik-db-1.srv.renner.uno }
|
||||
- { name: LEGACY_DB, value: authentik }
|
||||
- { name: PGHOST, value: ak-pg-rw }
|
||||
- { name: PGDATABASE, value: authentik }
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef: { name: ak-pg-superuser, key: username }
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: { name: ak-pg-superuser, key: password }
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
# Idempotent: skip if target DB already has user tables.
|
||||
count=$(psql -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public'")
|
||||
if [ "$count" -gt 0 ]; then
|
||||
log "$PGDATABASE already has $count tables in public — seed skipped."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
|
||||
log "pg_dump $LEGACY_HOST/$LEGACY_DB → pg_restore $PGHOST/$PGDATABASE ..."
|
||||
ssh "root@$LEGACY_HOST" "su - postgres -c 'pg_dump -Fc --clean --if-exists $LEGACY_DB'" \
|
||||
| pg_restore -d "$PGDATABASE" \
|
||||
|| log "pg_restore completed with warnings"
|
||||
|
||||
log "Seed complete."
|
||||
volumeMounts:
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: authentik-seed-ssh
|
||||
defaultMode: 0400
|
||||
@@ -0,0 +1,39 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: im-pg
|
||||
namespace: immich
|
||||
spec:
|
||||
# Single instance during pgvecto.rs → VectorChord migration window;
|
||||
# pg_basebackup tar format chokes on the long pg_vectors/indexes/<uuid>
|
||||
# paths, blocking replica join. Bump back to 2 once Immich's in-app
|
||||
# vchord migration drops the vectors extension.
|
||||
instances: 1
|
||||
imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration
|
||||
enableSuperuserAccess: true
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
size: 20Gi
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: im-pg-store
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: immich
|
||||
owner: immich
|
||||
postgresql:
|
||||
shared_preload_libraries:
|
||||
- vectors.so
|
||||
- vchord.so
|
||||
parameters:
|
||||
shared_buffers: "512MB"
|
||||
max_wal_size: "2GB"
|
||||
wal_compression: "on"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- migration-bucket.yaml
|
||||
- barman-creds.sops.yaml
|
||||
- objectstore.yaml
|
||||
- cluster.yaml
|
||||
- scheduled-backup.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- pvcs.yaml
|
||||
- pg-seed-job.yaml
|
||||
- library-seed-job.yaml
|
||||
@@ -0,0 +1,58 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: immich-library-seed
|
||||
namespace: immich
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: rsync
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: immich.srv.renner.uno }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
if [ -f /data/.seed-done ]; then
|
||||
log "PVC seed sentinel present — skipping rsync."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apk add --no-cache rsync openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
|
||||
log "rsync /mnt/media/immich → /data ..."
|
||||
set +e
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"root@$LEGACY_HOST:/mnt/media/immich/" /data/
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) log "rsync OK" ;;
|
||||
23) log "rsync exit 23 (partial transfer warnings) — continuing" ;;
|
||||
*) log "rsync FAILED with code $rc"; exit "$rc" ;;
|
||||
esac
|
||||
chown -R 1000:1000 /data
|
||||
du -sh /data
|
||||
touch /data/.seed-done
|
||||
log "PVC seed complete; sentinel set."
|
||||
volumeMounts:
|
||||
- { name: library, mountPath: /data }
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: library
|
||||
persistentVolumeClaim:
|
||||
claimName: immich-library
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: immich-seed-ssh
|
||||
defaultMode: 0400
|
||||
@@ -0,0 +1,96 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: im-pg-seed
|
||||
namespace: immich
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
initContainers:
|
||||
- name: wait-for-cluster
|
||||
image: postgres:16
|
||||
env:
|
||||
- { name: PGHOST, value: im-pg-rw }
|
||||
- { name: PGDATABASE, value: immich }
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef: { name: im-pg-superuser, key: username }
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: { name: im-pg-superuser, key: password }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
until psql -tAc 'SELECT 1' >/dev/null 2>&1; do
|
||||
echo "[$(date -u +%H:%M:%S)] waiting for $PGHOST..."
|
||||
sleep 5
|
||||
done
|
||||
echo "[$(date -u +%H:%M:%S)] $PGHOST ready"
|
||||
containers:
|
||||
- name: seed
|
||||
image: postgres:16
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: immich.srv.renner.uno }
|
||||
- { name: LEGACY_DB, value: immich }
|
||||
- { name: PGHOST, value: im-pg-rw }
|
||||
- { name: PGDATABASE, value: immich }
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef: { name: im-pg-superuser, key: username }
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: { name: im-pg-superuser, key: password }
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
count=$(psql -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public'")
|
||||
if [ "$count" -gt 0 ]; then
|
||||
log "$PGDATABASE has $count tables in public — seed skipped."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
|
||||
# Pre-create non-trusted extensions immich needs at runtime.
|
||||
# The legacy dump only ships pgvecto.rs (`vectors`); newer
|
||||
# immich also CREATEs `vector` (pgvector) and `vchord` at
|
||||
# startup. Both are non-trusted, so the immich role can't
|
||||
# CREATE them — pre-create as superuser with IF NOT EXISTS so
|
||||
# the dump's own CREATE EXTENSION calls and immich's runtime
|
||||
# CREATE all short-circuit.
|
||||
log "Pre-creating extensions ..."
|
||||
for ext in cube earthdistance vector vectors vchord; do
|
||||
psql -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS \"$ext\" CASCADE;" \
|
||||
|| log "extension $ext: failed (continuing)"
|
||||
done
|
||||
|
||||
log "pg_dump $LEGACY_HOST/$LEGACY_DB → pg_restore $PGHOST/$PGDATABASE ..."
|
||||
# Legacy immich DB lives in a docker container on the LXC;
|
||||
# locate via compose service label.
|
||||
DB_CT=$(ssh "root@$LEGACY_HOST" "docker ps --filter label=com.docker.compose.service=database --format '{{.Names}}' | head -1")
|
||||
[ -n "$DB_CT" ] || { log "no DB container on $LEGACY_HOST"; exit 1; }
|
||||
ssh "root@$LEGACY_HOST" "docker exec -i $DB_CT pg_dump -Fc --clean --if-exists -U postgres $LEGACY_DB" \
|
||||
| pg_restore -d "$PGDATABASE" \
|
||||
|| log "pg_restore completed with warnings"
|
||||
|
||||
log "Seed complete."
|
||||
volumeMounts:
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: immich-seed-ssh
|
||||
defaultMode: 0400
|
||||
@@ -0,0 +1,55 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: influxdb-data-seed
|
||||
namespace: influxdb
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: rsync
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: influxdb.srv.renner.uno }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
if [ -f /var/lib/influxdb2/.seed-done ]; then
|
||||
log "PVC seed sentinel present — skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apk add --no-cache rsync openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
log "Stopping legacy influxdb.service ..."
|
||||
ssh "$LEGACY" "systemctl stop influxdb"
|
||||
trap 'log "Restarting legacy influxdb.service ..."; ssh "$LEGACY" "systemctl start influxdb" || true' EXIT
|
||||
|
||||
log "rsync /var/lib/influxdb → /var/lib/influxdb2 ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"$LEGACY:/var/lib/influxdb/" /var/lib/influxdb2/
|
||||
chown -R 1500:1500 /var/lib/influxdb2 || true
|
||||
du -sh /var/lib/influxdb2
|
||||
touch /var/lib/influxdb2/.seed-done
|
||||
log "PVC seed complete; sentinel set."
|
||||
volumeMounts:
|
||||
- { name: data, mountPath: /var/lib/influxdb2 }
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: data-influxdb-influxdb2-0
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: influxdb-seed-ssh
|
||||
defaultMode: 0400
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- seed-ssh.sops.yaml
|
||||
- pvc.yaml
|
||||
- data-seed-job.yaml
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
# Pre-create the PVC the influxdb-influxdb2 StatefulSet would otherwise
|
||||
# auto-provision via volumeClaimTemplates, so the seed Job can populate
|
||||
# it before the StatefulSet pod ever starts. Name + storageClass + size
|
||||
# match the chart values.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: data-influxdb-influxdb2-0
|
||||
namespace: influxdb
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- authentik/
|
||||
- immich/
|
||||
- paperless/
|
||||
- audiobookshelf/
|
||||
- music-assistant/
|
||||
- influxdb/
|
||||
@@ -0,0 +1,66 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: music-assistant-data-seed
|
||||
namespace: music-assistant
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
# Music Assistant's data PVC lives on edge (RWO ceph-block scoped to
|
||||
# the edge node where the app runs); seed Job must schedule on the
|
||||
# same node so it can mount the volume.
|
||||
nodeSelector:
|
||||
topology.kubernetes.io/zone: edge
|
||||
tolerations:
|
||||
- key: workload-class
|
||||
operator: Equal
|
||||
value: pinned
|
||||
effect: NoSchedule
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: seed
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: music-assistant.iot.renner.uno }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
if [ -f /data/.seed-done ]; then
|
||||
log "PVC seed sentinel present — skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apk add --no-cache openssh-client tar >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
LEGACY="root@$LEGACY_HOST"
|
||||
|
||||
log "Stopping music-assistant docker-compose stack on legacy ..."
|
||||
ssh "$LEGACY" "cd /opt/music-assistant && docker compose stop"
|
||||
trap 'log "Restarting music-assistant on legacy ..."; ssh "$LEGACY" "cd /opt/music-assistant && docker compose up -d" || true' EXIT
|
||||
|
||||
# Legacy MA LXC ships without rsync; ssh+tar streaming.
|
||||
log "Clearing /data ..."
|
||||
find /data -mindepth 1 -delete
|
||||
log "Streaming /opt/music-assistant/data via ssh+tar ..."
|
||||
ssh "$LEGACY" "cd /opt/music-assistant/data && tar -cf - ." | tar -xf - -C /data/
|
||||
du -sh /data
|
||||
touch /data/.seed-done
|
||||
log "PVC seed complete; sentinel set."
|
||||
volumeMounts:
|
||||
- { name: data, mountPath: /data }
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: music-assistant-data
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: music-assistant-seed-ssh
|
||||
defaultMode: 0400
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- pvcs.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- data-seed-job.yaml
|
||||
@@ -0,0 +1,31 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: pprl-pg
|
||||
namespace: paperless
|
||||
spec:
|
||||
instances: 2
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
enableSuperuserAccess: true
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
size: 20Gi
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: pprl-pg-store
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: paperless
|
||||
owner: paperless
|
||||
postgresql:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_connections: "200"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
@@ -0,0 +1,57 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: paperless-data-seed
|
||||
namespace: paperless
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: rsync
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: paperless.srv.renner.uno }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
if [ -f /data/.seed-done ]; then
|
||||
log "PVC seed sentinel present — skipping rsync."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apk add --no-cache rsync openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
|
||||
log "rsync paperless_data → /data ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"root@$LEGACY_HOST:/var/lib/docker/volumes/paperless_data/_data/" /data/
|
||||
log "rsync paperless_media → /media ..."
|
||||
rsync -aH --delete --info=progress2 \
|
||||
"root@$LEGACY_HOST:/var/lib/docker/volumes/paperless_media/_data/" /media/
|
||||
chown -R 1000:1000 /data /media
|
||||
du -sh /data /media
|
||||
touch /data/.seed-done
|
||||
log "PVC seed complete; sentinel set."
|
||||
volumeMounts:
|
||||
- { name: data, mountPath: /data }
|
||||
- { name: media, mountPath: /media }
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-data
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-media
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: paperless-seed-ssh
|
||||
defaultMode: 0400
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- migration-bucket.yaml
|
||||
- barman-creds.sops.yaml
|
||||
- objectstore.yaml
|
||||
- cluster.yaml
|
||||
- scheduled-backup.yaml
|
||||
- seed-ssh.sops.yaml
|
||||
- pvcs.yaml
|
||||
- pg-seed-job.yaml
|
||||
- data-seed-job.yaml
|
||||
@@ -0,0 +1,84 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: pprl-pg-seed
|
||||
namespace: paperless
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
initContainers:
|
||||
- name: wait-for-cluster
|
||||
image: postgres:16
|
||||
env:
|
||||
- { name: PGHOST, value: pprl-pg-rw }
|
||||
- { name: PGDATABASE, value: paperless }
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef: { name: pprl-pg-superuser, key: username }
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: { name: pprl-pg-superuser, key: password }
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
until psql -tAc 'SELECT 1' >/dev/null 2>&1; do
|
||||
echo "[$(date -u +%H:%M:%S)] waiting for $PGHOST..."
|
||||
sleep 5
|
||||
done
|
||||
echo "[$(date -u +%H:%M:%S)] $PGHOST ready"
|
||||
containers:
|
||||
- name: seed
|
||||
image: postgres:16
|
||||
env:
|
||||
- { name: LEGACY_HOST, value: paperless.srv.renner.uno }
|
||||
- { name: LEGACY_DB, value: paperless }
|
||||
- { name: LEGACY_DB_USER, value: paperless }
|
||||
- { name: PGHOST, value: pprl-pg-rw }
|
||||
- { name: PGDATABASE, value: paperless }
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef: { name: pprl-pg-superuser, key: username }
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: { name: pprl-pg-superuser, key: password }
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
count=$(psql -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public'")
|
||||
if [ "$count" -gt 0 ]; then
|
||||
log "$PGDATABASE has $count tables in public — seed skipped."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends openssh-client >/dev/null
|
||||
mkdir -p /root/.ssh
|
||||
install -m 600 /ssh/id_ed25519 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
|
||||
log "pg_dump $LEGACY_HOST/$LEGACY_DB → pg_restore $PGHOST/$PGDATABASE ..."
|
||||
# Legacy paperless DB lives in a docker container; locate via
|
||||
# compose service label.
|
||||
DB_CT=$(ssh "root@$LEGACY_HOST" "docker ps --filter label=com.docker.compose.service=db --format '{{.Names}}' | head -1")
|
||||
[ -n "$DB_CT" ] || { log "no DB container on $LEGACY_HOST"; exit 1; }
|
||||
ssh "root@$LEGACY_HOST" "docker exec -i $DB_CT pg_dump -Fc --clean --if-exists -U $LEGACY_DB_USER $LEGACY_DB" \
|
||||
| pg_restore -d "$PGDATABASE" \
|
||||
|| log "pg_restore completed with warnings"
|
||||
|
||||
log "Seed complete."
|
||||
volumeMounts:
|
||||
- { name: ssh, mountPath: /ssh, readOnly: true }
|
||||
volumes:
|
||||
- name: ssh
|
||||
secret:
|
||||
secretName: paperless-seed-ssh
|
||||
defaultMode: 0400
|
||||
Reference in New Issue
Block a user