diff --git a/kubernetes/apps/grafana-db/postgres.yaml b/kubernetes/apps/grafana-db/postgres.yaml index c4026196..dc2e098e 100644 --- a/kubernetes/apps/grafana-db/postgres.yaml +++ b/kubernetes/apps/grafana-db/postgres.yaml @@ -53,6 +53,28 @@ spec: # and slower than the lz4/zstd this build also accepts). Compresses the # full-page images that dominate WAL volume after each checkpoint. wal_compression: "zstd" + # THE dominant write source on this cluster, by three orders of magnitude. + # + # A WAL segment is a fixed 16 MiB file. archive_timeout forces a segment + # switch on a timer so the archive never lags reality by more than that + # interval -- and the switched-out segment is written and archived at its + # full 16 MiB regardless of how few bytes of real WAL it contains. + # + # CNPG defaults this to 5min, i.e. up to 288 segments/day = 4.6 GB/day. + # Measured here: this database is 21 MB and generates 4.5 MB/day of actual + # WAL, while its RBD volume takes 211 MB/hour. Essentially every byte + # written is padding -- roughly 1100x amplification. + # + # 30min cuts that to 48 segments/day. The cost is RPO: on total loss of + # the cluster, up to 30min of Grafana state is unrecoverable rather than + # 5min. Grafana's state here is dashboards, annotations and preferences, + # and the write rate says it changes ~4.5 MB/day -- so that window is + # nearly always empty anyway. + # + # Applied to gf-pg FIRST, alone, as a staged rollout: it is the lowest-risk + # cluster and has the cleanest signal, since its real WAL is small enough + # that any change in volume is unambiguously this parameter. + archive_timeout: "30min" resources: requests: cpu: 100m