From e50c622da36e08c890d595542ae94e824c872055 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Sun, 28 Jun 2026 11:14:52 +0200 Subject: [PATCH] =?UTF-8?q?fix(velero):=20bump=20data-mover=20timeouts=204?= =?UTF-8?q?h=E2=86=928h=20(immich-library=20hits=204h=20cap)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The immich-library DataUpload (~250 GiB, millions of small files) now runs ~3h50m–4h+ and was intermittently Canceled at exactly the 4h `defaultItemOperationTimeout`. A canceled upload writes no kopia snapshot, so the daily backup ends PartiallyFailed; the weekly restore-test then selects that backup and fails restoring immich-library with "Unable to load snapshot: snapshot not found", firing KubeJobFailed. Observed cancellations: 2026-06-26 immich-library DataUpload Canceled 4h00m09s 2026-06-27 immich-library DataUpload Canceled 4h00m12s 2026-06-28 immich-library DataUpload Completed 3h50m18s Raise both dataMoverPrepareTimeout and defaultItemOperationTimeout to 8h for headroom as the library keeps growing. Co-Authored-By: Claude Opus 4.8 --- kubernetes/infrastructure/velero/helmrelease.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/kubernetes/infrastructure/velero/helmrelease.yaml b/kubernetes/infrastructure/velero/helmrelease.yaml index 6c43aafe..036406b4 100644 --- a/kubernetes/infrastructure/velero/helmrelease.yaml +++ b/kubernetes/infrastructure/velero/helmrelease.yaml @@ -56,9 +56,16 @@ spec: # the immich library (~250 GiB, millions of small files) — CSI # VolumeSnapshotContent provisioning + Kopia repo prep on the # node-agent can take longer than that on this cluster's - # ceph-cluster-csi pace. Bump server flag to 4 h. - dataMoverPrepareTimeout: 4h - defaultItemOperationTimeout: 4h + # ceph-cluster-csi pace. + # + # itemOperationTimeout caps the async DataUpload/DataDownload + # itself. The immich-library upload now runs ~3h50m–4h+, so at the + # old 4 h cap it was intermittently Canceled at exactly 4 h (no + # snapshot written → PartiallyFailed backup → restore-test fails + # with "snapshot not found"). The volume is still growing, so both + # timeouts are at 8 h to leave real headroom. + dataMoverPrepareTimeout: 8h + defaultItemOperationTimeout: 8h backupStorageLocation: - name: default provider: aws -- 2.54.0