fix(music-assistant): move seed into pod initContainer (kill Multi-Attach race)

The standalone seed Job had `ttlSecondsAfterFinished: 3600`, so Flux recreated it ~1h after each successful run. The recreated pod had no node affinity and got scheduled to k8s-3, while the music-assistant app is pinned to k8s-edge (`topology.kubernetes.io/zone: edge` for hostNetwork on the IoT VLAN). The shared PVC is RWO Ceph-RBD — cross-node attach is refused, so the seed pod stuck in Multi-Attach forever and the music-assistant-seed Kustomization stayed Reconciling. Move the seed body verbatim into a `seed` initContainer in the app Deployment. Same pod = same kubelet RBD mount, no Multi-Attach is even possible. The existing `[ -f /data/.restore-done ]` sentinel keeps ordinary pod restarts to <1s; on a fresh cluster rebuild the initContainer does the full extract before the app container starts — exactly the per-rebuild semantics we want. Refactor: - replace `wait-for-restore` initContainer with a self-contained `seed` initContainer that owns the s3 wait/extract/sentinel logic - move garage-migration-creds.sops.yaml + pvcs.yaml from kubernetes/apps/music-assistant-seed/ into kubernetes/apps/music-assistant/ - delete kubernetes/apps/music-assistant-seed/ - collapse the two Flux Kustomizations into one; replace the `dependsOn: music-assistant-seed` chain with the standard infrastructure deps it inherits - raise the app Kustomization timeout to 4h to cover the worst-case 3h-tarball-wait window the seed Job had The same latent race exists in audiobookshelf-seed and paperless-seed (they happen to co-locate on k8s-3 with their app pods today, so they work by luck). Worth a follow-up to apply the same pattern there. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-26 15:02:59 +02:00
parent 92a1733959
commit c9ea0af308
7 changed files with 70 additions and 131 deletions
@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - garage-migration-creds.sops.yaml
-  - pvcs.yaml
-  - seed-job.yaml
@@ -1,84 +0,0 @@
---
-# Seed Job for Music Assistant's data PVC. Reads the legacy tarball
-# from Garage (s3://k8s-migration/music-assistant/music-assistant-data.tar.gz)
-# written by the `migrate-music-assistant` Woodpecker pipeline.
-# Sentinel is `/data/.restore-done`; legacy `.restore-v1-done` is still
-# honored on previously-seeded PVCs and normalized forward on re-run.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: music-assistant-data-seed-v1
-  namespace: music-assistant
-spec:
-  ttlSecondsAfterFinished: 3600
-  template:
-    spec:
-      restartPolicy: Never
-      containers:
-        - name: seed
-          image: alpine:3.21
-          command:
-            - sh
-            - -c
-            - |
-              set -eu
-              if [ -f /data/.restore-done ] || [ -f /data/.restore-v1-done ]; then
-                echo "Seed already applied (sentinel present); normalizing and exiting."
-                touch /data/.restore-done
-                exit 0
-              fi
-              apk add --no-cache aws-cli tar ca-certificates >/dev/null
-              export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage
-              # TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so
-              # force path-style addressing (bucket in path, not subdomain).
-              mkdir -p "$HOME/.aws"
-              printf '[default]\ns3 =\n    addressing_style = path\n' > "$HOME/.aws/config"
-
-              echo "Waiting for tarball at s3://$S3_BUCKET/$S3_PREFIX/music-assistant-data.tar.gz ..."
-              for i in $(seq 1 1080); do
-                if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/music-assistant-data.tar.gz" >/dev/null 2>&1; then
-                  echo "Tarball found."; break
-                fi
-                echo "  not ready yet ($i/1080) — sleeping 10s"; sleep 10
-                if [ "$i" = "1080" ]; then
-                  echo "ERROR: tarball not found after 3 hours"; exit 1
-                fi
-              done
-
-              aws s3 ls "s3://$S3_BUCKET/$S3_PREFIX/"
-
-              echo "Wiping any first-run MA state under /data ..."
-              find /data -mindepth 1 -maxdepth 1 -exec rm -rf {} +
-
-              echo "Streaming data tarball → /data ..."
-              aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/music-assistant-data.tar.gz" - | tar -xz -C /data
-
-              echo "Writing seed sentinel ..."
-              touch /data/.restore-done
-
-              echo "Done. Summary:"
-              du -sh /data
-          env:
-            - name: S3_ENDPOINT
-              value: "https://s3.nas-1.renner.uno"
-            - name: S3_BUCKET
-              value: "k8s-migration"
-            - name: S3_PREFIX
-              value: "music-assistant"
-            - name: AWS_ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-                  name: garage-migration-creds
-                  key: AWS_ACCESS_KEY_ID
-            - name: AWS_SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: garage-migration-creds
-                  key: AWS_SECRET_ACCESS_KEY
-          volumeMounts:
-            - name: data
-              mountPath: /data
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: music-assistant-data
@@ -31,20 +31,73 @@ spec:
          value: pinned
          effect: NoSchedule
      initContainers:
-        # Block until the seed Job extracts LXC 229's data tarball and
-        # writes the sentinel. Without this, MA would boot against an
-        # empty /data and re-initialize default config.
-        - name: wait-for-restore
+        # Restore /data from the Garage tarball written by the
+        # `migrate-music-assistant` Woodpecker pipeline. Sentinel-guarded
+        # so subsequent pod restarts short-circuit in <1s. Lives in the
+        # app pod (not a separate Job) so the RWO Ceph-RBD PVC is shared
+        # via the same kubelet mount — Multi-Attach is impossible by
+        # construction. On a fresh cluster rebuild the volume is empty,
+        # so this runs the full extract before the app container starts.
+        - name: seed
          image: alpine:3.21
          command:
            - sh
            - -c
            - |
-              until [ -f /data/.restore-done ]; do
-                echo "waiting for /data/.restore-done ..."
-                sleep 10
+              set -eu
+              if [ -f /data/.restore-done ] || [ -f /data/.restore-v1-done ]; then
+                echo "Seed already applied (sentinel present); normalizing and exiting."
+                touch /data/.restore-done
+                exit 0
+              fi
+              apk add --no-cache aws-cli tar ca-certificates >/dev/null
+              export HOME=/tmp AWS_ENDPOINT_URL="$S3_ENDPOINT" AWS_DEFAULT_REGION=garage
+              # TLS cert is for s3.nas-1.renner.uno only, not wildcarded, so
+              # force path-style addressing (bucket in path, not subdomain).
+              mkdir -p "$HOME/.aws"
+              printf '[default]\ns3 =\n    addressing_style = path\n' > "$HOME/.aws/config"
+
+              echo "Waiting for tarball at s3://$S3_BUCKET/$S3_PREFIX/music-assistant-data.tar.gz ..."
+              for i in $(seq 1 1080); do
+                if aws s3api head-object --bucket "$S3_BUCKET" --key "$S3_PREFIX/music-assistant-data.tar.gz" >/dev/null 2>&1; then
+                  echo "Tarball found."; break
+                fi
+                echo "  not ready yet ($i/1080) — sleeping 10s"; sleep 10
+                if [ "$i" = "1080" ]; then
+                  echo "ERROR: tarball not found after 3 hours"; exit 1
+                fi
              done
-              echo "restore sentinel present — continuing."
+
+              aws s3 ls "s3://$S3_BUCKET/$S3_PREFIX/"
+
+              echo "Wiping any first-run MA state under /data ..."
+              find /data -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+
+              echo "Streaming data tarball → /data ..."
+              aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/music-assistant-data.tar.gz" - | tar -xz -C /data
+
+              echo "Writing seed sentinel ..."
+              touch /data/.restore-done
+
+              echo "Done. Summary:"
+              du -sh /data
+          env:
+            - name: S3_ENDPOINT
+              value: "https://s3.nas-1.renner.uno"
+            - name: S3_BUCKET
+              value: "k8s-migration"
+            - name: S3_PREFIX
+              value: "music-assistant"
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: garage-migration-creds
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: garage-migration-creds
+                  key: AWS_SECRET_ACCESS_KEY
          volumeMounts:
            - name: data
              mountPath: /data
@@ -1,5 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - garage-migration-creds.sops.yaml
+  - pvcs.yaml
  - deployment.yaml
  - ingress.yaml
@@ -1,43 +1,15 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
-metadata:
-  name: music-assistant-seed
-  namespace: flux-system
-spec:
-  interval: 10m
-  retryInterval: 1m
-  # Seed Job waits up to 3 h for the migrate pipeline to upload the tarball.
-  timeout: 4h
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./kubernetes/apps/music-assistant-seed
-  prune: true
-  wait: true
-  decryption:
-    provider: sops
-    secretRef:
-      name: sops-age
-  healthCheckExprs:
-    - apiVersion: batch/v1
-      kind: Job
-      current: "status.succeeded == 1"
-      inProgress: "!has(status.succeeded) || status.succeeded == 0"
-      failed: "has(status.failed) && status.failed >= 1"
-  dependsOn:
-    - name: infrastructure
-    - name: infrastructure-crds
-    - name: infrastructure-ceph-cluster
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
 metadata:
  name: music-assistant
  namespace: flux-system
 spec:
+  # Seed Job-vs-app race fixed by moving the restore work into an
+  # initContainer of the app pod (same RWO PVC, no Multi-Attach). Up to
+  # 3 h waiting on the migrate pipeline tarball, so timeout is generous.
  interval: 10m
  retryInterval: 1m
-  timeout: 30m
+  timeout: 4h
  sourceRef:
    kind: GitRepository
    name: flux-system
@@ -49,4 +21,6 @@ spec:
    secretRef:
      name: sops-age
  dependsOn:
-    - name: music-assistant-seed
+    - name: infrastructure
+    - name: infrastructure-crds
+    - name: infrastructure-ceph-cluster