kubernetes/apps/gitea/kustomization.yaml

@@ -6,6 +6,8 @@ kind: Kustomization
 # and verified. See MIGRATION-gitea-woodpecker.md.
 resources:
   - statefulset.yaml
+  # The live size of the data volume; see the file (REN-583).
+  - pvc.yaml
   - service.yaml
   - ingressroute-ssh.yaml
   - ingress.yaml

kubernetes/apps/gitea/pvc.yaml

@@ -0,0 +1,34 @@
+# The live data volume of gitea-0, owned here so its size is set in git
+# (REN-583). The StatefulSet controller created this claim from the
+# volumeClaimTemplates in statefulset.yaml, but that template is immutable on
+# an existing StatefulSet, so changing the size there cannot be applied. A
+# PVC's resources.requests.storage CAN grow in place (ceph-block has
+# allowVolumeExpansion: true; ceph-csi grows the RBD image and the filesystem
+# online, no pod restart). Flux applies this over the existing claim with
+# server-side apply and the resize follows.
+#
+# - Name, namespace, accessModes, storageClassName and volumeMode must match
+#   the live claim exactly: they are immutable, a mismatch fails the whole
+#   gitea Kustomization (no change to the live claim).
+# - Grow only. A smaller value is rejected by the API server.
+# - prune: disabled, so removing this file from git never deletes the claim
+#   and its data. To hand ownership back, delete the file; the claim stays.
+#
+# Size history: 20Gi -> 40Gi 2026-07-22 (#336, live resize by hand);
+# 40Gi -> 60Gi 2026-10-03 (REN-583, after the volume filled again on REN-580).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: data-gitea-0
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes: ["ReadWriteOnce"]
+  storageClassName: ceph-block
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: 60Gi

kubernetes/apps/gitea/statefulset.yaml

@@ -147,14 +147,10 @@ spec:
         accessModes: ["ReadWriteOnce"]
         storageClassName: ceph-block
         resources:
-          # Expanded 20Gi -> 40Gi on 2026-07-22 (ceph-block online resize) to
-          # clear a 100%-full outage (#336). The live PVC was resized directly.
-          # NOTE: volumeClaimTemplates is immutable on an existing StatefulSet,
-          # so bumping this value cannot be `kubectl apply`-ed in place — Flux
-          # will reject the whole object. Landing this size change requires a
-          # one-time `kubectl delete statefulset gitea -n gitea --cascade=orphan`
-          # (pod + PVC keep running) so Flux re-creates the STS from this
-          # template and re-adopts them. After the cleanup_packages GC reclaims
-          # the ~10G of orphaned blobs, real usage is ~9G, so 40Gi is headroom.
+          # This template only applies when the claim does not exist yet. The
+          # live size of data-gitea-0 is set in pvc.yaml (REN-583), not here:
+          # volumeClaimTemplates is immutable on an existing StatefulSet, so a
+          # change to this value cannot be applied and Flux would reject the
+          # whole object. Leave it at 40Gi.
           requests:
             storage: 40Gi

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -77,4 +77,7 @@ resources:
   # customResourceState block in apps/monitoring/helmrelease-prometheus.yaml —
   # these two files must land and be reverted together.
   - prometheusrule-flux.yaml
+  # Gitea's data volume. It filled twice (#336, REN-580) and the chart's
+  # KubePersistentVolumeFillingUp warned 33 minutes before the second (REN-583).
+  - prometheusrule-gitea.yaml
   - probes-blackbox.yaml

kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml

@@ -0,0 +1,42 @@
+# Gitea's data volume (data-gitea-0) running low (REN-583).
+#
+# It filled twice: 2026-07-22 (#336) and 2026-10-03 (REN-580), both times
+# because the package registry grew faster than cleanup_packages reclaimed it.
+# A full volume fails image pushes (HTTP 500) and Gitea's own SQLite writes.
+#
+# The chart's KubePersistentVolumeFillingUp did not give usable lead time on
+# REN-580: its warning needs under 15% free AND a 4-day linear projection to
+# zero AND one hour of both, so it fired at 20:26Z; the volume was full at
+# 20:59Z. Growth here comes in steps (a run of image builds added 8 GiB in a
+# few hours that day), which a linear projection lags.
+#
+# So this is a plain floor: under 25% free (15 GiB at 60Gi) for 15m. That is
+# about two of the 8 GiB steps seen on REN-580, and time for someone to run the
+# package cleanup or grow pvc.yaml in apps/gitea.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: gitea
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: gitea.rules
+      rules:
+        - alert: GiteaDataVolumeLow
+          expr: >-
+            kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}
+              / kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}
+            < 0.25
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gitea data volume has {{ $value | humanizePercentage }} free"
+            description: >-
+              data-gitea-0 is under 25% free. When it fills, image pushes fail
+              with HTTP 500 and Gitea cannot write its database. Run the
+              cleanup_packages cron in Gitea's site admin and delete old
+              package versions, or grow the claim in apps/gitea/pvc.yaml
+              (ceph-block resizes online).

kubernetes/apps/monitoring-rules/tests/README.md

@@ -31,6 +31,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
 | `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). |
 | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
 | `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
+| `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |
 
 ## Adding a suite

kubernetes/apps/monitoring-rules/tests/gitea_test.yaml

@@ -0,0 +1,46 @@
+# Suite for prometheusrule-gitea.yaml (REN-583). Capacity 60 GiB; free space
+# in GiB. 15 GiB free is exactly 25% and must stay silent.
+rule_files:
+  - ../prometheusrule-gitea.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # 20 GiB free, then 14 GiB free from minute 10 on: pending at 10m, fires
+  # at 25m (15m later), not before.
+  - interval: 1m
+    input_series:
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '21474836480+0x9 15032385536+0x30'
+    alert_rule_test:
+      - eval_time: 24m
+        alertname: GiteaDataVolumeLow
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: GiteaDataVolumeLow
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              namespace: gitea
+              persistentvolumeclaim: data-gitea-0
+            exp_annotations:
+              summary: "Gitea data volume has 23.33% free"
+              description: "data-gitea-0 is under 25% free. When it fills, image pushes fail with HTTP 500 and Gitea cannot write its database. Run the cleanup_packages cron in Gitea's site admin and delete old package versions, or grow the claim in apps/gitea/pvc.yaml (ceph-block resizes online)."
+
+  # Exactly 25% free stays silent; another claim at 1% free is not this alert.
+  - interval: 1m
+    input_series:
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="gitea",persistentvolumeclaim="data-gitea-0"}'
+        values: '16106127360+0x40'
+      - series: 'kubelet_volume_stats_capacity_bytes{namespace="other",persistentvolumeclaim="data-other-0"}'
+        values: '64424509440+0x40'
+      - series: 'kubelet_volume_stats_available_bytes{namespace="other",persistentvolumeclaim="data-other-0"}'
+        values: '644245094+0x40'
+    alert_rule_test:
+      - eval_time: 40m
+        alertname: GiteaDataVolumeLow
+        exp_alerts: []

kubernetes/apps/monitoring-rules/tests/mutations.txt

@@ -167,3 +167,8 @@ disk-wear-critical-device-only|kubernetes/apps/monitoring-rules/prometheusrule-d
 # exclusion matches nothing.
 # Detected by: cases A and B.
 disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|s/n\[0-9\]+(p\[0-9\]+)?"/n[0-9]+"/
+
+# prometheusrule-gitea.yaml (REN-583). Detected by the 24m/25m pair and the
+# exactly-25% case in gitea_test.yaml.
+gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/
+gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/