diff --git a/kubernetes/clusters/homelab/seed-woodpecker.yaml b/kubernetes/clusters/homelab/seed-woodpecker.yaml new file mode 100644 index 00000000..e954c274 --- /dev/null +++ b/kubernetes/clusters/homelab/seed-woodpecker.yaml @@ -0,0 +1,31 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed-woodpecker + namespace: flux-system +spec: + # Owns the wp-pg CNPG Cluster, its ObjectStore + ScheduledBackup, and the + # Hetzner S3 creds. Provisions the Postgres backend that woodpecker-server + # migrates onto (from its old sqlite-on-PVC), which removes the fragile + # ceph-block snapshot backup path that broke #295/#324. Bootstraps an empty + # DB via initdb; the sqlite->pg data migration is a one-time manual runbook + # (kubernetes/seed/woodpecker/MIGRATION.md). + interval: 10m + retryInterval: 2m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed/woodpecker + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman + healthCheckExprs: + - apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + failed: "has(status.phase) && status.phase.startsWith('Failed')" + current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)" diff --git a/kubernetes/seed/woodpecker/MIGRATION.md b/kubernetes/seed/woodpecker/MIGRATION.md new file mode 100644 index 00000000..b050f59f --- /dev/null +++ b/kubernetes/seed/woodpecker/MIGRATION.md @@ -0,0 +1,142 @@ +# Woodpecker: SQLite → CNPG Postgres migration runbook + +One-time migration of `woodpecker-server` off its `data-woodpecker-server-0` +sqlite-on-ceph-block PVC onto the `wp-pg` CNPG cluster. Motivation: the ext4 +CSI-snapshot backup of that PVC fails permanently (`structure needs cleaning`, +#295/#324); moving to CNPG gives WAL-based barman backups and removes the PVC +entirely. Woodpecker has **no built-in migrator** (woodpecker-ci#924); this is +the community-proven **schema-first / data-only** pgloader path. + +**Rollback is trivial at every step until PR 3:** the sqlite PVC is untouched, +so reverting the two env vars in `server.yaml` and scaling back up returns to +sqlite immediately. + +Keep woodpecker on **v3.13.0** throughout (schema-build and cutover) — do NOT +bump the image during the migration, or the `migrations` bookkeeping diverges. + +## Staging +- **PR 1 (this dir):** provision the empty `wp-pg` cluster + backups. No + woodpecker impact. Merge and let it reach `Cluster in healthy state`. +- **Runbook below:** one-time offline data migration. +- **PR 2:** flip `server.yaml` `WOODPECKER_DATABASE_DRIVER/DATASOURCE` to + postgres (cutover). +- **PR 3 (after functional verify):** drop `volumeClaimTemplates` + the + `/var/lib/woodpecker` mount, convert StatefulSet → Deployment, delete the old + PVC. + +## Prerequisites +- `wp-pg` healthy: `kubectl -n apps get cluster wp-pg` → `Cluster in healthy state`. +- App secret exists: `kubectl -n apps get secret wp-pg-app` (keys `username`, + `password`, `dbname=woodpecker`). Connect host is `wp-pg-rw.apps.svc:5432`. + +## Runbook + +```sh +NS=apps +PGPASS=$(kubectl -n $NS get secret wp-pg-app -o jsonpath='{.data.password}' | base64 -d) +DSN="postgres://woodpecker:${PGPASS}@wp-pg-rw.apps.svc.cluster.local:5432/woodpecker?sslmode=require" +``` + +### 1. Quiesce woodpecker (stop writes to sqlite) +```sh +flux suspend kustomization apps # stop Flux re-scaling +kubectl -n $NS scale statefulset woodpecker-server --replicas=0 +kubectl -n $NS rollout status statefulset woodpecker-server --timeout=90s +kubectl -n $NS scale deployment woodpecker-agent --replicas=0 # stop new work +``` + +### 2. Extract + integrity-check the sqlite DB +The PVC is now unmounted. Mount it into a helper pod and copy the file out. +```sh +kubectl -n $NS apply -f - <<'EOF' +apiVersion: v1 +kind: Pod +metadata: { name: wp-sqlite-grab, namespace: apps } +spec: + containers: + - { name: c, image: alpine:3.20, command: ["sleep","3600"], + volumeMounts: [{ name: d, mountPath: /data }] } + volumes: + - { name: d, persistentVolumeClaim: { claimName: data-woodpecker-server-0 } } +EOF +kubectl -n $NS wait --for=condition=Ready pod/wp-sqlite-grab --timeout=60s +kubectl -n $NS exec wp-sqlite-grab -- sh -c 'apk add --no-cache sqlite >/dev/null && sqlite3 /data/woodpecker.sqlite "PRAGMA integrity_check;"' +kubectl -n $NS cp wp-sqlite-grab:/data/woodpecker.sqlite ./woodpecker.sqlite +cp woodpecker.sqlite woodpecker.sqlite.bak +``` + +### 3. Build the Postgres schema WITH woodpecker itself +Run one throwaway v3.13.0 server against the empty DB so it creates every table +and populates `migrations`, then delete it. (Env-only; no PVC, no traffic.) +```sh +kubectl -n $NS run wp-schemabuild --restart=Never --image=woodpeckerci/woodpecker-server:v3.13.0 \ + --env "WOODPECKER_DATABASE_DRIVER=postgres" \ + --env "WOODPECKER_DATABASE_DATASOURCE=${DSN}" \ + --env "WOODPECKER_AGENT_SECRET=schemabuild" --env "WOODPECKER_OPEN=false" +# wait until logs show migrations applied + server listening, then: +kubectl -n $NS logs wp-schemabuild | grep -iE 'migrat|listen|starting' +kubectl -n $NS delete pod wp-schemabuild +# sanity: tables + populated migrations +kubectl -n $NS run pgcheck --rm -it --restart=Never --image=postgres:16 -- \ + psql "$DSN" -c '\dt' -c 'SELECT count(*) FROM migrations;' +``` + +### 4. pgloader — DATA ONLY, excluding `migrations`, int→bool +First enumerate the boolean columns so the cast is right: +`sqlite3 woodpecker.sqlite ".schema" | grep -iE 'bool|active|allow|require|trusted|gated'`. +`woodpecker.load`: +``` +load database + from sqlite:///work/woodpecker.sqlite + into + with data only, truncate, disable triggers, batch rows = 2500, prefetch rows = 2500 + set work_mem to '64MB', maintenance_work_mem to '256MB' + excluding table names matching 'migrations' + cast type integer when (= precision 1) to boolean drop typemod using integer-to-boolean, + type tinyint to boolean using integer-to-boolean; +``` +Run in a throwaway pod (emptyDir; `kubectl cp` the `.sqlite` + `.load` in): +```sh +kubectl -n $NS run pgloader --restart=Never --image=ghcr.io/dimitri/pgloader:latest --command -- sleep 3600 +kubectl -n $NS cp woodpecker.sqlite pgloader:/work/woodpecker.sqlite +kubectl -n $NS cp woodpecker.load pgloader:/work/woodpecker.load +kubectl -n $NS exec pgloader -- pgloader /work/woodpecker.load +kubectl -n $NS delete pod pgloader +``` +If you hit `type number does not exist`, add `cast type number to bigint` and rerun. + +### 5. Reset sequences (mandatory after data-only load) +```sh +kubectl -n $NS run pgseq --rm -it --restart=Never --image=postgres:16 -- psql "$DSN" -v ON_ERROR_STOP=1 <<'SQL' +DO $$ DECLARE r record; BEGIN + FOR r IN + SELECT pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname) AS seq, + n.nspname AS sch, c.relname AS tbl, a.attname AS col + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + JOIN pg_attribute a ON a.attrelid=c.oid + WHERE c.relkind='r' AND n.nspname='public' + AND pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname) IS NOT NULL + LOOP + EXECUTE format('SELECT setval(%L, COALESCE((SELECT MAX(%I) FROM %I.%I),0)+1, false)', r.seq, r.col, r.sch, r.tbl); + END LOOP; +END $$; +SQL +``` + +### 6. Verify data parity +Compare `SELECT count(*)` per table (repos, users, secrets, crons, pipelines, +steps) between sqlite and postgres; spot-check a boolean returns `t/f`. + +### 7. Cutover (PR 2) +Merge PR 2 (`server.yaml` → postgres). Then: +```sh +kubectl -n $NS delete pod wp-sqlite-grab +flux resume kustomization apps +kubectl -n $NS scale deployment woodpecker-agent --replicas=1 +kubectl -n $NS logs -f statefulset/woodpecker-server # expect: no pending migrations, healthy +``` + +### 8. Functional verify → then PR 3 +Log in; confirm activated repos + Gitea webhooks fire, secrets/crons present, +history visible; trigger one end-to-end pipeline. **Only then** land PR 3 (drop +the PVC + StatefulSet→Deployment). diff --git a/kubernetes/seed/woodpecker/cluster.yaml b/kubernetes/seed/woodpecker/cluster.yaml new file mode 100644 index 00000000..e0ae2b38 --- /dev/null +++ b/kubernetes/seed/woodpecker/cluster.yaml @@ -0,0 +1,34 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: wp-pg + namespace: apps +spec: + # Single instance: the Woodpecker CI DB is small and non-critical (state is + # largely reconstructable), so HA isn't warranted — but WAL archiving to + # object storage still gives point-in-time backups. Mirrors grafana-db's + # single-instance shape, with barman enabled like immich/paperless. + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + storage: + storageClass: ceph-block + size: 5Gi + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: wp-pg-store + bootstrap: + initdb: + database: woodpecker + owner: woodpecker + postgresql: + parameters: + shared_buffers: "128MB" + max_connections: "50" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi diff --git a/kubernetes/seed/woodpecker/hetzner-creds-externalsecret.yaml b/kubernetes/seed/woodpecker/hetzner-creds-externalsecret.yaml new file mode 100644 index 00000000..18b77e42 --- /dev/null +++ b/kubernetes/seed/woodpecker/hetzner-creds-externalsecret.yaml @@ -0,0 +1,20 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: hetzner-s3-creds + namespace: apps +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + name: hetzner-s3-creds + dataFrom: + # Reuses the paperless copy of the shared Hetzner S3 credentials (identical + # ACCESS_KEY_ID / ACCESS_SECRET_KEY / REGION) to avoid seeding a new OpenBao + # path for this migration. FOLLOW-UP: give woodpecker its own + # seed/woodpecker/hetzner-creds path once OpenBao can be written, and repoint + # this here (the value is the same, so it's a drop-in swap). + - extract: + key: seed/paperless/hetzner-creds diff --git a/kubernetes/seed/woodpecker/kustomization.yaml b/kubernetes/seed/woodpecker/kustomization.yaml new file mode 100644 index 00000000..bb863c2b --- /dev/null +++ b/kubernetes/seed/woodpecker/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - hetzner-creds-externalsecret.yaml + - objectstore.yaml + - cluster.yaml + - scheduled-backup.yaml diff --git a/kubernetes/seed/woodpecker/objectstore.yaml b/kubernetes/seed/woodpecker/objectstore.yaml new file mode 100644 index 00000000..f1efae45 --- /dev/null +++ b/kubernetes/seed/woodpecker/objectstore.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: wp-pg-store + namespace: apps +spec: + configuration: + # Same offsite Hetzner Object Storage bucket as the other CNPG clusters, + # under a per-cluster prefix. See seed/paperless/objectstore.yaml for the + # SSE / addressing rationale (Hetzner is SSE-C only; transit is TLS, + # bucket private). + destinationPath: s3://renner-uno-k8s-cnpg/wp-pg + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" diff --git a/kubernetes/seed/woodpecker/scheduled-backup.yaml b/kubernetes/seed/woodpecker/scheduled-backup.yaml new file mode 100644 index 00000000..7b19336a --- /dev/null +++ b/kubernetes/seed/woodpecker/scheduled-backup.yaml @@ -0,0 +1,19 @@ +--- +# Daily base backup at 02:35 UTC (offset from the other clusters' 02:30 to +# avoid a thundering herd against the Hetzner bucket). WAL archiving runs +# continuously (see cluster.yaml spec.plugins[].isWALArchiver). immediate:true +# takes the first base backup as soon as this CR applies. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: wp-pg-daily + namespace: apps +spec: + schedule: "0 35 2 * * *" + immediate: true + backupOwnerReference: self + cluster: + name: wp-pg + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io