Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7666d220c5 | ||
|
|
baa5e62bbe | ||
|
|
503d2a4876 | ||
|
|
48407aa202 | ||
|
|
ce6e8ba2cb | ||
|
|
96387108ff |
@@ -27,7 +27,7 @@ spec:
|
||||
# Pinned to 7.4-alpine to match the LXC 225 compose exactly.
|
||||
# Any redis major-version bump is a separate migration.
|
||||
- name: redis
|
||||
image: redis:7.4.10-alpine # {"$imagepolicy": "flux-system:redis-immich"}
|
||||
image: redis:7.4.11-alpine # {"$imagepolicy": "flux-system:redis-immich"}
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:8.10.0-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
image: redis:8.10.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
|
||||
@@ -50,12 +50,39 @@ spec:
|
||||
# clone + subsequent steps share the workspace PVC and the k8s backend
|
||||
# can schedule them on different nodes -> an RWO volume then fails to
|
||||
# mount on the second node. CephFS (RWX) lets the workspace follow
|
||||
# the pipeline across nodes. The -ephemeral variant (reclaim Delete,
|
||||
# rook-ceph-cluster/storageclass.yaml) is REQUIRED here: workspaces
|
||||
# are throwaway, and the Retain class leaked one Released PV per
|
||||
# finished pipeline (issue #262 follow-up).
|
||||
# the pipeline across nodes. Reclaim MUST be Delete: workspaces are
|
||||
# throwaway, and a Retain class leaked one Released PV per finished
|
||||
# pipeline (issue #262 follow-up).
|
||||
#
|
||||
# ceph-filesystem-ci, not ceph-filesystem-ephemeral: a dedicated
|
||||
# CephFS whose pools are BOTH size 1 (infrastructure/
|
||||
# rook-ceph-cluster/cephfs-ci.yaml). The shared filesystem
|
||||
# replicates data0 2x and metadata 3x, which is the correct trade
|
||||
# for real data and pure waste for a git clone that is deleted
|
||||
# minutes later.
|
||||
#
|
||||
# Measured 2026-08-21, polling both pools every 15s (a 5-minute
|
||||
# interval misses an entire pipeline -- PVC created, used and
|
||||
# deleted inside ~2 minutes):
|
||||
# CI active 1.5min: data0 143.2 MB, metadata 17.2 MB
|
||||
# CI idle 16.0min: data0 0.0 MB, metadata 11.0 MB
|
||||
# data0 write volume is 100% CI. Per pipeline this moves the drive
|
||||
# cost from 71.6x2 + 8.2x3 = ~168 MB to ~80 MB.
|
||||
#
|
||||
# ceph-filesystem-ephemeral is NOT repointed, and the reason is a
|
||||
# technical constraint rather than a safety one. Its other consumer
|
||||
# is velero/restore-test.sh, whose probe PVCs are snapshot CLONES:
|
||||
# `ceph fs subvolume snapshot clone` operates within one filesystem,
|
||||
# so a snapshot taken in `cephfs` cannot be cloned into `cephfs-ci`.
|
||||
#
|
||||
# Those probes are just as throwaway as a CI workspace -- the
|
||||
# authoritative copy is the backup on Hetzner, and a lost probe just
|
||||
# means re-running the test -- so they are equally undeserving of
|
||||
# replication. Landing them unreplicated would mean a second size-1
|
||||
# data pool inside `cephfs`, and restore-test runs WEEKLY against
|
||||
# CI's ~96 pipelines/day, so it is not worth the complexity.
|
||||
- name: WOODPECKER_BACKEND_K8S_STORAGE_CLASS
|
||||
value: ceph-filesystem-ephemeral
|
||||
value: ceph-filesystem-ci
|
||||
- name: WOODPECKER_BACKEND_K8S_STORAGE_RWX
|
||||
value: "true"
|
||||
# MUST be a JSON object literal — the backend json.Unmarshal's this
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
---
|
||||
# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces, with both
|
||||
# pools at size 1.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# Woodpecker's Kubernetes backend creates a PVC per pipeline, fills it with a
|
||||
# checkout plus build output, and deletes it when the pipeline ends. Measured
|
||||
# 2026-08-21 by polling both pools at 15s (a 5-minute interval misses a whole
|
||||
# pipeline — its PVC is created, used and deleted inside ~2 minutes):
|
||||
#
|
||||
# one pipeline, a single 15s interval: data0 +71.6 MB metadata +8.2 MB
|
||||
# every interval without a CI pod: data0 ~0 metadata ~0
|
||||
#
|
||||
# CI drove essentially all of it. On the shared filesystem that costs
|
||||
# 71.6x2 + 8.2x3 = ~168 MB of drive writes per pipeline. Here it costs ~80 MB.
|
||||
#
|
||||
# WHY size 1 IS CORRECT HERE, HAVING BEEN REJECTED FOR THE MAIN FILESYSTEM
|
||||
# `cephfs` keeps metadataPool at size 3 because losing it destroys the whole
|
||||
# filesystem, not the objects on one OSD, and 550 GiB in cephfs-data0 depends
|
||||
# on it. None of that applies to this filesystem. Every byte in it is a fresh
|
||||
# git clone; the worst case of total loss is that in-flight pipelines fail and
|
||||
# get re-run. Replicating throwaway data is the thing being paid for, so it is
|
||||
# the thing removed.
|
||||
#
|
||||
# Losing an OSD takes these PGs offline and kills running pipelines. That is
|
||||
# not a regression: CI already breaks during Ceph maintenance.
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephFilesystem
|
||||
metadata:
|
||||
name: cephfs-ci
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
metadataPool:
|
||||
replicated:
|
||||
size: 1
|
||||
# Rook refuses size 1 without this — it is asserting that an unreplicated
|
||||
# pool is deliberate rather than a typo. It is.
|
||||
requireSafeReplicaSize: false
|
||||
dataPools:
|
||||
- name: data0
|
||||
replicated:
|
||||
size: 1
|
||||
requireSafeReplicaSize: false
|
||||
metadataServer:
|
||||
activeCount: 1
|
||||
# No standby-replay, unlike `cephfs`. A standby exists there so a failed
|
||||
# MDS does not hang every CephFS mount cluster-wide; here the only casualty
|
||||
# is a running pipeline, and a second daemon to protect throwaway work is
|
||||
# not worth its memory.
|
||||
activeStandby: false
|
||||
resources:
|
||||
# Deliberately NOT the 4Gi that `cephfs` needs. That figure is driven by
|
||||
# immich's millions of cached inodes; a CI workspace is a few thousand
|
||||
# short-lived files that vanish with the pipeline.
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
---
|
||||
# CI-only class. `ceph-filesystem-ephemeral` is deliberately left pointing at
|
||||
# the main filesystem: velero/restore-test.sh uses it to restore REAL data for
|
||||
# verification, which must not land on unreplicated pools.
|
||||
#
|
||||
# A new class rather than repointing the old one — StorageClass.parameters are
|
||||
# immutable, so `fsName` cannot be edited in place.
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ceph-filesystem-ci
|
||||
provisioner: rook-ceph.cephfs.csi.ceph.com
|
||||
parameters:
|
||||
clusterID: rook-ceph
|
||||
fsName: cephfs-ci
|
||||
pool: cephfs-ci-data0
|
||||
csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
|
||||
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
# Delete, for the same reason ceph-filesystem-ephemeral uses it: with Retain,
|
||||
# every finished pipeline leaks a Released PV pinning a subvolume (191 of them
|
||||
# by 2026-07-16, issue #262 follow-up).
|
||||
reclaimPolicy: Delete
|
||||
allowVolumeExpansion: true
|
||||
@@ -4,5 +4,6 @@ resources:
|
||||
- cephcluster.yaml
|
||||
- toolbox.yaml
|
||||
- storageclass.yaml
|
||||
- cephfs-ci.yaml
|
||||
- volumesnapshotclass.yaml
|
||||
- objectstore.yaml
|
||||
|
||||
Reference in New Issue
Block a user