From 433719bfc0cfbb1f4c3cde82914482ce7fbc2234 Mon Sep 17 00:00:00 2001 From: atlas-bot Date: Sat, 3 Oct 2026 21:16:14 +0000 Subject: [PATCH] gitea: grow data volume 40Gi -> 60Gi via a Flux-owned PVC, add low-space alert (REN-583) The volume filled on REN-580. volumeClaimTemplates is immutable, so own the live claim data-gitea-0 in apps/gitea/pvc.yaml (prune disabled) and let Flux server-side apply grow it; ceph-block expands online. Add GiteaDataVolumeLow (<25% free for 15m) with a promtool suite and two mutation controls. Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/gitea/kustomization.yaml | 2 + kubernetes/apps/gitea/pvc.yaml | 34 ++++++++++++++ kubernetes/apps/gitea/statefulset.yaml | 14 ++---- .../apps/monitoring-rules/kustomization.yaml | 3 ++ .../prometheusrule-gitea.yaml | 42 +++++++++++++++++ .../apps/monitoring-rules/tests/README.md | 1 + .../monitoring-rules/tests/gitea_test.yaml | 46 +++++++++++++++++++ .../apps/monitoring-rules/tests/mutations.txt | 5 ++ 8 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 kubernetes/apps/gitea/pvc.yaml create mode 100644 kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml create mode 100644 kubernetes/apps/monitoring-rules/tests/gitea_test.yaml diff --git a/kubernetes/apps/gitea/kustomization.yaml b/kubernetes/apps/gitea/kustomization.yaml index 6166ef86..9808f012 100644 --- a/kubernetes/apps/gitea/kustomization.yaml +++ b/kubernetes/apps/gitea/kustomization.yaml @@ -6,6 +6,8 @@ kind: Kustomization # and verified. See MIGRATION-gitea-woodpecker.md. resources: - statefulset.yaml + # The live size of the data volume; see the file (REN-583). + - pvc.yaml - service.yaml - ingressroute-ssh.yaml - ingress.yaml diff --git a/kubernetes/apps/gitea/pvc.yaml b/kubernetes/apps/gitea/pvc.yaml new file mode 100644 index 00000000..738dc57f --- /dev/null +++ b/kubernetes/apps/gitea/pvc.yaml @@ -0,0 +1,34 @@ +# The live data volume of gitea-0, owned here so its size is set in git +# (REN-583). The StatefulSet controller created this claim from the +# volumeClaimTemplates in statefulset.yaml, but that template is immutable on +# an existing StatefulSet, so changing the size there cannot be applied. A +# PVC's resources.requests.storage CAN grow in place (ceph-block has +# allowVolumeExpansion: true; ceph-csi grows the RBD image and the filesystem +# online, no pod restart). Flux applies this over the existing claim with +# server-side apply and the resize follows. +# +# - Name, namespace, accessModes, storageClassName and volumeMode must match +# the live claim exactly: they are immutable, a mismatch fails the whole +# gitea Kustomization (no change to the live claim). +# - Grow only. A smaller value is rejected by the API server. +# - prune: disabled, so removing this file from git never deletes the claim +# and its data. To hand ownership back, delete the file; the claim stays. +# +# Size history: 20Gi -> 40Gi 2026-07-22 (#336, live resize by hand); +# 40Gi -> 60Gi 2026-10-03 (REN-583, after the volume filled again on REN-580). +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-gitea-0 + namespace: gitea + labels: + app.kubernetes.io/name: gitea + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: ceph-block + volumeMode: Filesystem + resources: + requests: + storage: 60Gi diff --git a/kubernetes/apps/gitea/statefulset.yaml b/kubernetes/apps/gitea/statefulset.yaml index 916c815d..4968740f 100644 --- a/kubernetes/apps/gitea/statefulset.yaml +++ b/kubernetes/apps/gitea/statefulset.yaml @@ -147,14 +147,10 @@ spec: accessModes: ["ReadWriteOnce"] storageClassName: ceph-block resources: - # Expanded 20Gi -> 40Gi on 2026-07-22 (ceph-block online resize) to - # clear a 100%-full outage (#336). The live PVC was resized directly. - # NOTE: volumeClaimTemplates is immutable on an existing StatefulSet, - # so bumping this value cannot be `kubectl apply`-ed in place — Flux - # will reject the whole object. Landing this size change requires a - # one-time `kubectl delete statefulset gitea -n gitea --cascade=orphan` - # (pod + PVC keep running) so Flux re-creates the STS from this - # template and re-adopts them. After the cleanup_packages GC reclaims - # the ~10G of orphaned blobs, real usage is ~9G, so 40Gi is headroom. + # This template only applies when the claim does not exist yet. The + # live size of data-gitea-0 is set in pvc.yaml (REN-583), not here: + # volumeClaimTemplates is immutable on an existing StatefulSet, so a + # change to this value cannot be applied and Flux would reject the + # whole object. Leave it at 40Gi. requests: storage: 40Gi diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index 18d42414..43e3ed03 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -77,4 +77,7 @@ resources: # customResourceState block in apps/monitoring/helmrelease-prometheus.yaml — # these two files must land and be reverted together. - prometheusrule-flux.yaml + # Gitea's data volume. It filled twice (#336, REN-580) and the chart's + # KubePersistentVolumeFillingUp warned 33 minutes before the second (REN-583). + - prometheusrule-gitea.yaml - probes-blackbox.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml new file mode 100644 index 00000000..ed1e889c --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml @@ -0,0 +1,42 @@ +# Gitea's data volume (data-gitea-0) running low (REN-583). +# +# It filled twice: 2026-07-22 (#336) and 2026-10-03 (REN-580), both times +# because the package registry grew faster than cleanup_packages reclaimed it. +# A full volume fails image pushes (HTTP 500) and Gitea's own SQLite writes. +# +# The chart's KubePersistentVolumeFillingUp did not give usable lead time on +# REN-580: its warning needs under 15% free AND a 4-day linear projection to +# zero AND one hour of both, so it fired at 20:26Z; the volume was full at +# 20:59Z. Growth here comes in steps (a run of image builds added 8 GiB in a +# few hours that day), which a linear projection lags. +# +# So this is a plain floor: under 25% free (15 GiB at 60Gi) for 15m. That is +# about two of the 8 GiB steps seen on REN-580, and time for someone to run the +# package cleanup or grow pvc.yaml in apps/gitea. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: gitea + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + - name: gitea.rules + rules: + - alert: GiteaDataVolumeLow + expr: >- + kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"} + / kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"} + < 0.25 + for: 15m + labels: + severity: warning + annotations: + summary: "Gitea data volume has {{ $value | humanizePercentage }} free" + description: >- + data-gitea-0 is under 25% free. When it fills, image pushes fail + with HTTP 500 and Gitea cannot write its database. Run the + cleanup_packages cron in Gitea's site admin and delete old + package versions, or grow the claim in apps/gitea/pvc.yaml + (ceph-block resizes online). diff --git a/kubernetes/apps/monitoring-rules/tests/README.md b/kubernetes/apps/monitoring-rules/tests/README.md index 7bff6713..2aae4a37 100644 --- a/kubernetes/apps/monitoring-rules/tests/README.md +++ b/kubernetes/apps/monitoring-rules/tests/README.md @@ -31,6 +31,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`). | `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). | | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). | | `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). | +| `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). | | `mutations.txt` | Negative controls. Each must make the suite fail. | ## Adding a suite diff --git a/kubernetes/apps/monitoring-rules/tests/gitea_test.yaml b/kubernetes/apps/monitoring-rules/tests/gitea_test.yaml new file mode 100644 index 00000000..ded17f3a --- /dev/null +++ b/kubernetes/apps/monitoring-rules/tests/gitea_test.yaml @@ -0,0 +1,46 @@ +# Suite for prometheusrule-gitea.yaml (REN-583). Capacity 60 GiB; free space +# in GiB. 15 GiB free is exactly 25% and must stay silent. +rule_files: + - ../prometheusrule-gitea.yaml + +evaluation_interval: 1m + +tests: + # 20 GiB free, then 14 GiB free from minute 10 on: pending at 10m, fires + # at 25m (15m later), not before. + - interval: 1m + input_series: + - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}' + values: '64424509440+0x40' + - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}' + values: '21474836480+0x9 15032385536+0x30' + alert_rule_test: + - eval_time: 24m + alertname: GiteaDataVolumeLow + exp_alerts: [] + - eval_time: 25m + alertname: GiteaDataVolumeLow + exp_alerts: + - exp_labels: + severity: warning + namespace: gitea + persistentvolumeclaim: data-gitea-0 + exp_annotations: + summary: "Gitea data volume has 23.33% free" + description: "data-gitea-0 is under 25% free. When it fills, image pushes fail with HTTP 500 and Gitea cannot write its database. Run the cleanup_packages cron in Gitea's site admin and delete old package versions, or grow the claim in apps/gitea/pvc.yaml (ceph-block resizes online)." + + # Exactly 25% free stays silent; another claim at 1% free is not this alert. + - interval: 1m + input_series: + - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}' + values: '64424509440+0x40' + - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}' + values: '16106127360+0x40' + - series: 'kubelet_volume_stats_capacity_bytes{namespace="other",persistentvolumeclaim="data-other-0"}' + values: '64424509440+0x40' + - series: 'kubelet_volume_stats_available_bytes{namespace="other",persistentvolumeclaim="data-other-0"}' + values: '644245094+0x40' + alert_rule_test: + - eval_time: 40m + alertname: GiteaDataVolumeLow + exp_alerts: [] diff --git a/kubernetes/apps/monitoring-rules/tests/mutations.txt b/kubernetes/apps/monitoring-rules/tests/mutations.txt index ad2eaaca..24aa4daa 100644 --- a/kubernetes/apps/monitoring-rules/tests/mutations.txt +++ b/kubernetes/apps/monitoring-rules/tests/mutations.txt @@ -167,3 +167,8 @@ disk-wear-critical-device-only|kubernetes/apps/monitoring-rules/prometheusrule-d # exclusion matches nothing. # Detected by: cases A and B. disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|s/n\[0-9\]+(p\[0-9\]+)?"/n[0-9]+"/ + +# prometheusrule-gitea.yaml (REN-583). Detected by the 24m/25m pair and the +# exactly-25% case in gitea_test.yaml. +gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/ +gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/ -- 2.54.0