From 428d9aa94e98941f829bc91b45f04b0f519de775 Mon Sep 17 00:00:00 2001 From: Alert Fix Agent Date: Thu, 16 Jul 2026 08:49:46 +0000 Subject: [PATCH] fix: [alert] VeleroBackupFailed (velero) (#251) --- .../velero/node-agent-config.yaml | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/kubernetes/infrastructure/velero/node-agent-config.yaml b/kubernetes/infrastructure/velero/node-agent-config.yaml index 927128a1..708ee4d8 100644 --- a/kubernetes/infrastructure/velero/node-agent-config.yaml +++ b/kubernetes/infrastructure/velero/node-agent-config.yaml @@ -6,18 +6,24 @@ # that clone is metadata-bound and ran ~2.5 h for a 250 GiB immich # library on this cluster — pure overhead, since Kopia only reads. # -# `readOnly: true` tells Velero's node-agent to mount the snapshot -# directly as ReadOnlyMany and skip the clone entirely. Requires -# ceph-csi >= 3.10 (cluster runs 3.16.2) and Velero >= 1.15 -# (cluster runs 1.18.0). +# `readOnly: true` for ceph-filesystem tells Velero's node-agent to +# mount the CephFS snapshot subvolume directly as ReadOnlyMany and +# skip the clone entirely. Requires ceph-csi >= 3.10 (cluster runs +# 3.16.2) and Velero >= 1.15 (cluster runs 1.18.0). # -# `storageClass: ceph-block-snapshot-readonly` redirects the exposer -# pod's snapshot PVC to a clone of ceph-block that adds -# `mountOptions: [noload]`. ext4 RO mounts of RBD snapshots fail when -# the source workload was actively writing (dirty journal cannot be -# replayed under RO); noload skips journal recovery on mount. -# Required for influxdb2 and protonmail-bridge style continuously- -# written volumes. Defined in snapshot-readonly-storageclass.yaml. +# ceph-block (RBD) volumes intentionally use the default clone path +# (no readOnly here). The previous approach used readOnly: true with +# storageClass: ceph-block-snapshot-readonly (noload) to skip ext4 +# journal replay on the RO snapshot mount. However, noload only +# bypasses journal replay — if the snapshot captures the filesystem in +# a structurally inconsistent state (e.g. mid-write to a directory +# entry), kopia returns "structure needs cleaning" (ENOTCLEAN) even +# though the mount succeeds. This manifested as two consecutive failed +# DataUploads for data-woodpecker-server-0 (apps-daily 2026-07-15 and +# 2026-07-16). RBD CoW clones are provisioned instantly by Ceph, so +# the RW clone path has negligible overhead while allowing ext4 to +# replay the journal on mount and resolve any inconsistencies before +# kopia reads. # # Referenced via `--node-agent-configmap=node-agent-config` on the # node-agent (set in helmrelease.yaml nodeAgent.extraArgs). @@ -30,10 +36,6 @@ data: node-agent-config: | { "backupPVC": { - "ceph-block": { - "storageClass": "ceph-block-snapshot-readonly", - "readOnly": true - }, "ceph-filesystem": { "readOnly": true } -- 2.54.0