fix: [alert] CephHealthError (rook-ceph) (#525)

kubernetes/apps/audiobookshelf/deployment.yaml

@@ -121,9 +121,7 @@ spec:
           persistentVolumeClaim:
             claimName: audiobookshelf-books
         - name: le-ca
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/authentik/cluster.yaml

@@ -43,30 +43,6 @@ spec:
     parameters:
       shared_buffers: "256MB"
       max_connections: "400"
-      # Write-reduction tuning. These databases are largely idle, and the
-      # Postgres defaults favour throughput over storage that wears out.
-      #
-      # checkpoint_timeout is the lever. The 5min default pushes every dirty
-      # page to disk 288 times a day, so a small database whose same few pages
-      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
-      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
-      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
-      # day. At 15min those pages coalesce and are written a third as often.
-      #
-      # max_wal_size must rise with it, or checkpoints trigger on size instead
-      # and the timeout change does nothing. It also bounds crash recovery to
-      # at most 2GB of WAL replay.
-      #
-      # NOT changed: synchronous_commit stays at the default `on`. It is off for
-      # ha-pg because a state recorder is telemetry; these hold credentials,
-      # documents and job state, where losing the last few hundred ms of commits
-      # on an unclean shutdown is not an acceptable trade.
-      checkpoint_timeout: "15min"
-      max_wal_size: "2GB"
-      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
-      # and slower than the lz4/zstd this build also accepts). Compresses the
-      # full-page images that dominate WAL volume after each checkpoint.
-      wal_compression: "zstd"
     # Async replication makes CNPG switchover slow because the standby has
     # to chase missing WAL via barman-cloud and replay queued records
     # before it can promote. With one synchronous standby, the standby is

kubernetes/apps/grafana-db/postgres.yaml

@@ -29,52 +29,6 @@ spec:
     parameters:
       shared_buffers: "128MB"
       max_connections: "50"
-      # Write-reduction tuning. These databases are largely idle, and the
-      # Postgres defaults favour throughput over storage that wears out.
-      #
-      # checkpoint_timeout is the lever. The 5min default pushes every dirty
-      # page to disk 288 times a day, so a small database whose same few pages
-      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
-      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
-      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
-      # day. At 15min those pages coalesce and are written a third as often.
-      #
-      # max_wal_size must rise with it, or checkpoints trigger on size instead
-      # and the timeout change does nothing. It also bounds crash recovery to
-      # at most 2GB of WAL replay.
-      #
-      # NOT changed: synchronous_commit stays at the default `on`. It is off for
-      # ha-pg because a state recorder is telemetry; these hold credentials,
-      # documents and job state, where losing the last few hundred ms of commits
-      # on an unclean shutdown is not an acceptable trade.
-      checkpoint_timeout: "15min"
-      max_wal_size: "2GB"
-      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
-      # and slower than the lz4/zstd this build also accepts). Compresses the
-      # full-page images that dominate WAL volume after each checkpoint.
-      wal_compression: "zstd"
-      # THE dominant write source on this cluster, by three orders of magnitude.
-      #
-      # A WAL segment is a fixed 16 MiB file. archive_timeout forces a segment
-      # switch on a timer so the archive never lags reality by more than that
-      # interval -- and the switched-out segment is written and archived at its
-      # full 16 MiB regardless of how few bytes of real WAL it contains.
-      #
-      # CNPG defaults this to 5min, i.e. up to 288 segments/day = 4.6 GB/day.
-      # Measured here: this database is 21 MB and generates 4.5 MB/day of actual
-      # WAL, while its RBD volume takes 211 MB/hour. Essentially every byte
-      # written is padding -- roughly 1100x amplification.
-      #
-      # 30min cuts that to 48 segments/day. The cost is RPO: on total loss of
-      # the cluster, up to 30min of Grafana state is unrecoverable rather than
-      # 5min. Grafana's state here is dashboards, annotations and preferences,
-      # and the write rate says it changes ~4.5 MB/day -- so that window is
-      # nearly always empty anyway.
-      #
-      # Applied to gf-pg FIRST, alone, as a staged rollout: it is the lowest-risk
-      # cluster and has the cleanest signal, since its real WAL is small enough
-      # that any change in volume is unambiguously this parameter.
-      archive_timeout: "30min"
   resources:
     requests:
       cpu: 100m

kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md

@@ -66,52 +66,7 @@ Confirm the backup actually landed in the bucket rather than trusting the CR
 status — the WordPress backup failed nightly for three days against a bucket
 that was declared but never created (see `opentofu/object_storage.tf`).
 
-## Phase 2b — bulk statistics copy: DONE 2026-08-19
-
-Completed with **HA running** — the SQLite file was read through a read-only
-mount, and SQLite's WAL mode gives readers a consistent snapshot, so no downtime
-was needed for the bulk copy.
-
-```
-statistics_meta:        sqlite=1449       pg=1449       OK
-statistics:             sqlite=10799904   pg=10799904   OK   (153.9s)
-statistics_short_term:  sqlite=1969804    pg=1969804    OK   ( 27.0s)
-
-statistics span in Postgres: 2024-03-13 -> 2026-08-19
-entities with long-term history: 1449
-rows carrying sum (energy dashboard): 2627859
-database size: 2252 MB   (PVC is 20Gi)
-```
-
-`ANALYZE` was run on all three tables. That is not housekeeping: after a bulk
-`COPY` the planner has no statistics, and HA queries these tables constantly for
-history and energy views, so without it every view seq-scans 10.8M rows.
-
-Two schema details that cost a failed first attempt, recorded so the top-up does
-not repeat them:
-
-- `has_mean` / `has_sum` are INTEGER in SQLite and BOOLEAN in Postgres. Postgres
-  will not coerce; convert explicitly and preserve NULL.
-- The legacy `created` / `start` / `last_reset` DATETIME columns are 100% NULL in
-  both bulk tables (superseded by the `_ts` variants), which is what makes a
-  tab-separated `COPY` safe — nothing in those tables needs escaping.
-
-The schema itself was created with HA's own models
-(`Base.metadata.create_all`), **not** hand-written DDL, and `schema_changes` was
-left EMPTY on purpose. HA's `_initialize_database` detects a new database by
-looking for an index on `events.time_fired_ts`, then writes `SCHEMA_VERSION`
-itself. Pre-stamping that row would skip HA's own initialisation path.
-
-### High-water marks for the cutover top-up
-
-Rows written after the bulk copy are not yet in Postgres. Resume from:
-
-```
-statistics             max_id=13323904    max_start_ts=1787162400.0
-statistics_short_term  max_id=159205762   max_start_ts=1787166000.0
-```
-
-## Phase 2c — cutover (REMAINING)
+## Phase 2 — migrate the data (NOT in this PR)
 
 Do not start until phase 1 is verified.
 
@@ -135,18 +90,7 @@ HA outage, and buys a table that is discarded within 7 days anyway.
    SELECT setval('statistics_short_term_id_seq', (SELECT MAX(id) FROM statistics_short_term));
    ```
 5. **Repoint** `recorder.db_url` in the `homeassistant` repo — that change lives
-   there, not here. The `recorder_db_url` secret it references is already
-   rendered into `secrets.yaml` by the `render-db-url` init container in
-   `deployment.yaml` (phase 2a), so this order is mandatory: **the init
-   container must be deployed first**, or HA refuses to start on a missing
-   secret.
-
-   The init container percent-encodes the credentials and validates the
-   resulting YAML rather than counting lines. That is not incidental — the CNPG
-   password is random, and a naive `"postgresql://user:pass@host/db"` string
-   makes `secrets.yaml` unparseable the moment the password contains a quote,
-   which takes out *every* secret HA loads, not just this one. Verified against
-   a deliberately hostile password before merge.
+   there, not here:
    ```yaml
    recorder:
      db_url: !secret recorder_db_url    # postgresql://homeassistant:<pw>@ha-pg-rw.home-assistant.svc.cluster.local/homeassistant

kubernetes/apps/home-assistant/deployment.yaml

@@ -36,64 +36,6 @@ spec:
       # (sml2mqtt bridge owns the USB device on edge), so it no longer needs
       # the edge USB device and schedules on the general cluster pool. This
       # gets the heavy HA pod (8Gi) off the soon-to-be-tiny edge box.
-      initContainers:
-        # Puts CNPG's connection URI into /config/secrets.yaml as
-        # `recorder_db_url`, for the recorder's move off SQLite
-        # (MIGRATION-recorder-postgres.md).
-        #
-        # HA has no way to read a db_url from the environment — `!env_var` was
-        # removed from its YAML loader — so `!secret` is the only mechanism, and
-        # secrets.yaml is PVC state that config-sync deliberately does not
-        # manage. Hence a small init container rather than a env: entry.
-        #
-        # The URI comes STRAIGHT from the ha-pg-app Secret. CNPG builds and
-        # percent-encodes it, so there is nothing to escape here: the value is
-        # restricted to [A-Za-z0-9:/@._%-], verified against the live secret.
-        # An earlier version of this assembled the URI from user/password/dbname
-        # and needed percent-encoding plus YAML round-trip validation to be
-        # safe. That was solving a problem CNPG had already solved.
-        #
-        # Re-rendered on every pod start, so a CNPG password rotation takes
-        # effect rather than silently leaving a stale URI behind.
-        #
-        # INERT until the recorder is repointed: nothing reads this key until
-        # `db_url: !secret recorder_db_url` lands in configuration.yaml in the
-        # homeassistant repo. That ordering is required — HA refuses to start
-        # when a referenced secret is missing.
-        - name: render-db-url
-          image: busybox:1.36
-          command:
-            - sh
-            - -c
-            - |
-              set -eu
-              S=/config/secrets.yaml
-              T=/config/.secrets.render.tmp
-              [ -f "$S" ] || : > "$S"
-              # Filter-then-append, never rewrite in place: this file holds
-              # every other HA secret (influxdb_token and friends).
-              grep -v '^recorder_db_url:' "$S" > "$T" || true
-              printf 'recorder_db_url: "%s"\n' "$DB_URI" >> "$T"
-              # Refuse to shrink the file. Cheap guard against the one failure
-              # that would actually hurt: losing the other keys.
-              [ "$(wc -l < "$T")" -ge "$(wc -l < "$S")" ] || {
-                echo "FATAL: rewrite would shrink secrets.yaml; aborting" >&2; exit 1; }
-              chmod 600 "$T"
-              mv "$T" "$S"
-              echo "recorder_db_url rendered"
-          env:
-            - name: DB_URI
-              valueFrom:
-                secretKeyRef: { name: ha-pg-app, key: fqdn-uri }
-          volumeMounts:
-            - name: config
-              mountPath: /config
-          resources:
-            requests:
-              cpu: 10m
-              memory: 16Mi
-            limits:
-              memory: 32Mi
       containers:
         - name: home-assistant
           image: ghcr.io/home-assistant/home-assistant:2026.8.2 # {"$imagepolicy": "flux-system:home-assistant"}
@@ -279,13 +221,9 @@ spec:
         # Writable emptyDir for the nginx entrypoint's envsubst output
         # (/etc/nginx/conf.d/default.conf rendered from the .template).
         - name: proxy-conf-d
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
         # code-server's settings/extensions cache. Not on the config PVC
         # so installed extensions don't pollute HA's /config listing.
         # Trade-off: extensions reset across pod restarts.
         - name: vscode-state
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}

kubernetes/apps/home-assistant/ha-pg-cluster.yaml

@@ -55,21 +55,7 @@ spec:
       owner: homeassistant
   postgresql:
     parameters:
-      # 512MB, not the 256MB this shipped with. Measured after 15h on Postgres:
-      # buffers_backend was 629,704 of 939,025 total page writes — 67% written
-      # by backend processes rather than the checkpointer or bgwriter. That
-      # ratio means the working set does not fit in shared_buffers, so backends
-      # evict dirty pages themselves and the same pages are written out
-      # repeatedly instead of coalescing in memory.
-      #
-      # The recorder touches a wide set of pages continuously (~30 state
-      # changes/sec across states, state_attributes and five indexes), which is
-      # exactly the workload that punishes a small buffer pool.
-      #
-      # Memory headroom checked before raising: the instance was using 383Mi of
-      # a 1Gi limit with 256MB of that being shared_buffers, so the limit goes
-      # to 1.5Gi to keep the same margin.
-      shared_buffers: "512MB"
+      shared_buffers: "256MB"
       max_connections: "50"
       # The rest of this block is tuned to MINIMISE WRITE VOLUME, which is the
       # entire reason for the migration. Defaults would give a correct database
@@ -85,26 +71,12 @@ spec:
       # Compress full-page images in WAL. Recorder writes are many small row
       # inserts scattered across pages, so full-page writes after each
       # checkpoint dominate WAL volume; compression cuts that directly.
-      #
-      # Measured on this cluster: full-page images are 62.4% of WAL bytes, so
-      # this is the single largest lever on recorder write volume.
-      #
-      # zstd, not "on": the boolean spelling resolves to pglz, the original
-      # 2016-era algorithm, which is both slower and weaker than the two
-      # alternatives PostgreSQL 16 accepts here (lz4, zstd). zstd gives the
-      # best ratio on 8 KiB page images, and the CPU cost lands on a cluster
-      # that is IO-bound on QLC NVMe, not CPU-bound.
-      wal_compression: "zstd"
+      wal_compression: "on"
       # Fewer checkpoints => fewer full-page writes, since a page is only
-      # fully-logged on its first modification after a checkpoint. Doubling the
-      # interval halves how often each hot page pays that 8 KiB toll.
-      #
-      # Recovery cost is bounded by how much WAL a single interval produces,
-      # not by the interval itself: at the measured rate 30min is well under
-      # 200MB of replay, i.e. seconds. max_wal_size stays 2GB precisely so the
-      # timeout stays the binding trigger -- if size were the binding one, the
-      # interval change would do nothing.
-      checkpoint_timeout: "30min"
+      # fully-logged on its first modification after a checkpoint. 15min with a
+      # 2GB ceiling keeps recovery time sane while cutting repeat full-page
+      # images for the hot rows the recorder rewrites constantly.
+      checkpoint_timeout: "15min"
       max_wal_size: "2GB"
       min_wal_size: "512MB"
       # full_page_writes is deliberately LEFT ON (the default). RBD can tear a
@@ -115,4 +87,4 @@ spec:
       cpu: 200m
       memory: 512Mi
     limits:
-      memory: 1536Mi
+      memory: 1Gi

kubernetes/apps/home-assistant/mosquitto-deployment.yaml

@@ -313,9 +313,7 @@ spec:
           configMap:
             name: mosquitto-config
         - name: auth
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
         - name: tls
           secret:
             secretName: mosquitto-tls

kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml

@@ -344,13 +344,9 @@ spec:
           configMap:
             name: zigbee2mqtt-interview-watch-script
         - name: interview-watch-state
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
         - name: discovery-guard-script
           configMap:
             name: zigbee2mqtt-discovery-guard-script
         - name: discovery-guard-state
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}

kubernetes/apps/immich/cluster.yaml

@@ -42,29 +42,7 @@ spec:
     parameters:
       shared_buffers: "512MB"
       max_wal_size: "2GB"
-      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
-      # and slower than the lz4/zstd this build also accepts). Compresses the
-      # full-page images that dominate WAL volume after each checkpoint.
-      wal_compression: "zstd"
-      # Write-reduction tuning. These databases are largely idle, and the
-      # Postgres defaults favour throughput over storage that wears out.
-      #
-      # checkpoint_timeout is the lever. The 5min default pushes every dirty
-      # page to disk 288 times a day, so a small database whose same few pages
-      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
-      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
-      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
-      # day. At 15min those pages coalesce and are written a third as often.
-      #
-      # max_wal_size must rise with it, or checkpoints trigger on size instead
-      # and the timeout change does nothing. It also bounds crash recovery to
-      # at most 2GB of WAL replay.
-      #
-      # NOT changed: synchronous_commit stays at the default `on`. It is off for
-      # ha-pg because a state recorder is telemetry; these hold credentials,
-      # documents and job state, where losing the last few hundred ms of commits
-      # on an unclean shutdown is not an acceptable trade.
-      checkpoint_timeout: "15min"
+      wal_compression: "on"
   resources:
     requests:
       cpu: 500m

kubernetes/apps/immich/server.yaml

@@ -147,9 +147,7 @@ spec:
               - key: immich-config.json
                 path: immich-config.json
         - name: le-ca
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -13,10 +13,6 @@ resources:
   - prometheusrule-netbird-ingress.yaml
   - prometheusrule-velero.yaml
   - prometheusrule-woodpecker-rotation.yaml
-  # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
-  # alerted if that stopped — HetznerBackupEndpointDown only probes the S3
-  # endpoint, which proves reachability, not that anything was written.
-  - prometheusrule-cnpg-backup.yaml
   - alloy-servicemonitor.yaml
   - cnpg-podmonitors.yaml
   - home-assistant-servicemonitor.yaml

kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -26,8 +26,32 @@ spec:
             description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
 
         - alert: CephHealthError
-          # HEALTH_ERR means data availability or redundancy is at risk.
-          expr: ceph_health_status == 2
+          # HEALTH_ERR used to mean "data availability or redundancy is at
+          # risk". Ceph 19.2.6 broke that assumption: it added the
+          # AUTH_INSECURE_* cephx checks, two of them ERR-severity, which flag
+          # the legacy `aes` key type every keyring in this cluster still uses.
+          # That is a security-posture finding, not a durability one — see
+          # CephAuthInsecureKeys below, which is what actually tracks it.
+          #
+          # The problem with leaving `ceph_health_status == 2` alone is not
+          # noise, it is blindness: while the auth checks hold the cluster at
+          # HEALTH_ERR, a genuine PG_AVAILABILITY or OSD_FULL is
+          # indistinguishable from them, because the aggregate is already
+          # saturated. So page on the ERR checks that are *not* auth ones.
+          #
+          # Written as "unless every active ERR check is an AUTH_INSECURE_ one"
+          # rather than a direct match so that it fails open: if
+          # ceph_health_detail goes missing or Ceph renames a check, both counts
+          # come back empty, the `unless` removes nothing, and this still pages.
+          # The `> 0` matters — the mgr exports all 31 known checks, the
+          # inactive ones at 0, so an unfiltered count would compare constants.
+          expr: |
+            ceph_health_status == 2
+              unless on (cluster) (
+                count by (cluster) (ceph_health_detail{severity="HEALTH_ERR"} > 0)
+                  ==
+                count by (cluster) (ceph_health_detail{severity="HEALTH_ERR", name=~"AUTH_INSECURE_.*"} > 0)
+              )
           for: 5m
           labels:
             severity: critical
@@ -35,6 +59,22 @@ spec:
             summary: "Ceph is HEALTH_ERR"
             description: "The Ceph cluster is in HEALTH_ERR — availability or redundancy is at risk. Inspect immediately with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` and `ceph osd df tree`."
 
+        - alert: CephAuthInsecureKeys
+          # Carries the condition CephHealthError deliberately stops paging for,
+          # so excluding it there narrows the alert rather than silencing it.
+          # Warning, not critical: nothing is at risk of being lost, but this
+          # does not self-clear either — it needs a planned cephx key rotation
+          # across client.admin, the CSI provisioner/node keys and the RGW key.
+          # Rotating those carelessly breaks RBD/CephFS provisioning
+          # cluster-wide, so this stays a human decision, not a 3am one.
+          expr: count(ceph_health_detail{name=~"AUTH_INSECURE_.*"} > 0) > 0
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ceph cephx keys use insecure key types"
+            description: "Ceph reports {{ $value }} active AUTH_INSECURE_* health checks — cephx entities still using the legacy `aes` key type, flagged since the 19.2.6 upgrade. No data is at risk. Enumerate with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` and plan rotation against the 19.2.6 release notes; durable settings belong in `spec.cephConfig` in kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml."
+
         - alert: CephMonDiskLow
           # Named explicitly rather than left to CephHealthWarn, because the
           # mons sit on the node's /var alongside container storage: this

kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml

@@ -1,78 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: cnpg-backup
-  namespace: monitoring
-  labels:
-    # kube-prometheus-stack's Prometheus resource selects on this label;
-    # without it the rule is created but never loaded into Prometheus.
-    release: kube-prometheus-stack
-spec:
-  groups:
-    - name: cnpg-backup.rules
-      rules:
-        # WHY THIS FILE EXISTS
-        # Seven Postgres clusters back up to Hetzner and NOTHING alerted if that
-        # stopped. The only related rule was HetznerBackupEndpointDown, which
-        # probes the S3 endpoint — it proves the bucket is reachable, not that
-        # anything was written to it. Same failure shape as the WordPress backup
-        # that failed nightly for three days against a bucket that was declared
-        # in tofu but never applied: the status surface looked fine because
-        # nothing checked the thing that mattered.
-
-        - alert: CNPGWalArchiveBacklog
-          # `ready` counts WAL segments sitting in pg_wal/archive_status waiting
-          # to be archived. Steady state is 0 across every cluster (verified
-          # 2026-08-19). A sustained backlog means the archiver is failing:
-          # point-in-time recovery is silently broken from that moment, and
-          # pg_wal keeps growing until it fills the PVC and takes Postgres down.
-          #
-          # This is the load-bearing alert of the two. It depends only on the
-          # instance-level collector, so unlike the backup-age metrics below it
-          # is unaffected by the missing Backup CRs (#522).
-          #
-          # Threshold: >5 segments sustained for 30m. Brief non-zero values are
-          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
-          # 80 MiB of un-archived WAL, well below anything that threatens the
-          # volume but far above idle noise.
-          expr: cnpg_collector_pg_wal_archive_status{value="ready"} > 5
-          for: 30m
-          labels:
-            severity: warning
-          annotations:
-            summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
-            description: "{{ $value }} WAL segments have been waiting to archive for over 30 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."
-
-        - alert: CNPGBackupFailed
-          # cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0
-          # when no failure is recorded. Guard on > 0 so the epoch-zero default
-          # is not read as "failed in 1970".
-          expr: |
-            cnpg_collector_last_failed_backup_timestamp > 0
-            and
-            (time() - cnpg_collector_last_failed_backup_timestamp) < 86400
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "CNPG backup failed in the last 24h on {{ $labels.namespace }}/{{ $labels.pod }}"
-            description: "A base backup for {{ $labels.namespace }}/{{ $labels.pod }} failed within the last 24 hours. Verify what is actually in the bucket rather than trusting CR status — see the barman-cloud-backup-list recipe in kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md."
-
-        # DELIBERATELY ABSENT: a backup-age alert, which is the one you would
-        # most want here.
-        #
-        # The natural expression is
-        #   (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600
-        # and it CANNOT be used today: that metric reads exactly 0 on every
-        # instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2,
-        # pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any
-        # successful backup anywhere, because the Backup CRs it derives that
-        # from are being deleted within a minute of completing — see #522.
-        #
-        # Backups themselves DO work; that was verified independently by
-        # listing the bucket with barman-cloud-backup-list and restoring a WAL
-        # segment back out of S3. The gap is in the record, not the data.
-        #
-        # Adding the rule now would fire permanently for all seven clusters and
-        # be silenced within a day, which is worse than not having it. Add it as
-        # part of resolving #522, once the metric reflects reality.

kubernetes/apps/monitoring/helmrelease-prometheus.yaml

@@ -402,52 +402,6 @@ spec:
         limits:
           memory: 256Mi
 
-    # Drop the apiserver's high-cardinality latency histograms.
-    #
-    # Measured 2026-08-20: Prometheus held 298,236 active series, of which
-    # ~108,000 — over a third — were apiserver/etcd `_bucket` histograms. Each
-    # histogram fans out to one series per bucket x verb x resource x scope, so
-    # a single metric like apiserver_request_duration_seconds_bucket was 23,810
-    # series on its own. For comparison, every Home Assistant metric combined is
-    # ~4,100.
-    #
-    # That cardinality is the dominant driver of Prometheus write volume
-    # (~11 GB/day of TSDB + WAL, doubled again by Ceph replication), and none of
-    # it is observable value here: per-bucket apiserver latency quantiles are for
-    # clusters with SLAs and dozens of nodes, not three.
-    #
-    # ONLY metrics with ZERO rule dependencies are dropped. Verified against the
-    # live rule set before writing this — each metric checked individually for
-    # references in alert and recording rules:
-    #
-    #   apiserver_request_duration_seconds_bucket        rules=0   23,810 series
-    #   apiserver_request_body_size_bytes_bucket         rules=0   23,200
-    #   etcd_request_duration_seconds_bucket             rules=0   15,840
-    #   apiserver_watch_list_duration_seconds_bucket     rules=0   11,538
-    #   apiserver_watch_cache_read_wait_seconds_bucket   rules=0    8,232
-    #   apiserver_response_sizes_bucket                  rules=0    7,592
-    #   apiserver_watch_events_sizes_bucket              rules=0    5,220
-    #
-    # apiserver_request_sli_duration_seconds_bucket is DELIBERATELY KEPT: 23
-    # recording rules depend on it, including the apiserver availability SLO and
-    # the KubeAPIErrorBudgetBurn burn-rate series. Dropping it would silently
-    # break that alerting, which is the failure mode this comment exists to
-    # prevent if someone later "tidies up" this regex.
-    #
-    # Only the `_bucket` series are dropped. The matching `_count` and `_sum`
-    # remain, so request rates and mean latencies still work — what is lost is
-    # per-bucket quantiles.
-    #
-    # etcd_* appears here even though kubeEtcd is disabled below: these are the
-    # apiserver's own etcd-client metrics, exposed on the apiserver endpoint, not
-    # a separate etcd scrape.
-    kubeApiServer:
-      serviceMonitor:
-        metricRelabelings:
-          - action: drop
-            sourceLabels: [__name__]
-            regex: (apiserver_request_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|etcd_request_duration_seconds_bucket|apiserver_watch_list_duration_seconds_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket)
-
     # Disable etcd/scheduler/controller-manager scraping (Talos doesn't expose them by default)
     kubeEtcd:
       enabled: false

kubernetes/apps/monitoring/pve-exporter-deployment.yaml

@@ -64,9 +64,7 @@ spec:
           secret:
             secretName: pve-exporter-config
         - name: tmp
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/paperless/MIGRATION-data-to-block.md

@@ -1,97 +0,0 @@
-# paperless-data: CephFS → RBD
-
-## Why
-
-`paperless-data` is `ReadWriteMany` on `ceph-filesystem`, but **only the paperless
-Deployment mounts it**. Verified 2026-08-20:
-
-```
-$ pods mounting paperless-data
-paperless-68cb8b8744-7hjb8      <- one, and only one
-
-paperless-data     ReadWriteMany  ceph-filesystem
-paperless-media    ReadWriteMany  ceph-filesystem   <- genuinely shared (ftp-scanner)
-paperless-consume  ReadWriteMany  ceph-filesystem   <- genuinely shared (ftp-scanner)
-```
-
-Its contents are single-writer state — the 50 MB Whoosh index,
-`classification_model.pickle`, `celerybeat-schedule.db`, `log/`,
-`migration_lock`, `protonmail.pem`. 75 files. Nothing else reads or writes them.
-
-It is on CephFS by default, not by requirement.
-
-## What it costs
-
-Every access goes through the MDS. Measured with **zero CI pipelines running**:
-
-```
-paperless-data client   request_load_avg 22   (all other sessions 0-1)
-MDS over 180s           4,052 client requests  = 22.5/s
-                        2,718 journal events   = 15/s
-                            2 dir commits      <- almost none: not file writes
-```
-
-So it is cap/inode churn from repeated access, not data being written. That is
-most of the constant **7-8 GB/day** written to `cephfs-metadata`, and since that
-pool is `size: 3`, roughly **21-24 GB/day** reaches the drives.
-
-`cephfs-metadata` deliberately stays at size 3. Losing it destroys the entire
-filesystem rather than the objects on one OSD, and the 275 GiB in `cephfs-data0`
-depends on it. Cutting the writes at source is correct; cutting the replication
-is not.
-
-## Sequence
-
-Staged deliberately: the PVC lands first and empty, so merging it changes
-nothing. The copy and the repoint are separate.
-
-### Phase 1 — provision (this PR)
-
-`paperless-data-block` (RWO, `ceph-block`, 10Gi) is created and mounted by
-nothing. Inert.
-
-Verify: `kubectl -n paperless get pvc paperless-data-block` → Bound.
-
-### Phase 2 — copy
-
-The Whoosh index must come across intact. Paperless *can* rebuild it
-(`document_index reindex`), but that reprocesses every document, so copying is
-both faster and less disruptive.
-
-1. Scale paperless to 0. The source must be quiescent — `celerybeat-schedule.db`
-   and the Whoosh index are both live-written, and copying them under an active
-   writer gives a torn read.
-2. Run a one-off pod that mounts BOTH PVCs and copies:
-   ```
-   cp -a /src/. /dst/
-   ```
-   The RWO target pins the pod to one node; the RWX source will follow.
-3. Verify file count and index size match:
-   ```
-   find /src -type f | wc -l   ==   find /dst -type f | wc -l
-   du -sh /src/index           ==   du -sh /dst/index
-   ```
-
-### Phase 3 — repoint
-
-Change the Deployment's `data` volume to `claimName: paperless-data-block`,
-merge, let it roll.
-
-Verify afterwards:
-- paperless starts, search returns results (proves the Whoosh index survived)
-- `ceph tell mds.cephfs-b session ls` no longer shows a busy `paperless-data`
-  client
-- `rate(ceph_pool_wr_bytes{pool_id="3"})` drops from its 7-8 GB/day baseline
-
-**Do not delete `paperless-data` until search has been confirmed working.** It is
-the rollback, and it is `prune: disabled` so Flux will not remove it.
-
-## Expected effect
-
-~7-8 GB/day off `cephfs-metadata`, so ~21-24 GB/day off the drives once
-replication is counted. Against ~432 GB/day cluster-wide that is roughly 5% —
-worth doing, but it is a correctness fix (a single-writer volume does not belong
-on a shared filesystem) more than a large win.
-
-It also removes 22 req/sec of MDS load, which benefits the workloads that
-genuinely need CephFS — Woodpecker CI workspaces share the same MDS.

kubernetes/apps/paperless/cluster.yaml

@@ -34,30 +34,6 @@ spec:
     parameters:
       shared_buffers: "256MB"
       max_connections: "200"
-      # Write-reduction tuning. These databases are largely idle, and the
-      # Postgres defaults favour throughput over storage that wears out.
-      #
-      # checkpoint_timeout is the lever. The 5min default pushes every dirty
-      # page to disk 288 times a day, so a small database whose same few pages
-      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
-      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
-      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
-      # day. At 15min those pages coalesce and are written a third as often.
-      #
-      # max_wal_size must rise with it, or checkpoints trigger on size instead
-      # and the timeout change does nothing. It also bounds crash recovery to
-      # at most 2GB of WAL replay.
-      #
-      # NOT changed: synchronous_commit stays at the default `on`. It is off for
-      # ha-pg because a state recorder is telemetry; these hold credentials,
-      # documents and job state, where losing the last few hundred ms of commits
-      # on an unclean shutdown is not an acceptable trade.
-      checkpoint_timeout: "15min"
-      max_wal_size: "2GB"
-      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
-      # and slower than the lz4/zstd this build also accepts). Compresses the
-      # full-page images that dominate WAL volume after each checkpoint.
-      wal_compression: "zstd"
     # See ak-pg cluster.yaml for the rationale on synchronous replication.
     synchronous:
       method: any

kubernetes/apps/paperless/paperless.yaml

@@ -208,25 +208,9 @@ spec:
             seccompProfile:
               type: RuntimeDefault
       volumes:
-        # paperless-data-block (RWO, ceph-block), NOT paperless-data
-        # (RWX, ceph-filesystem). Only this Deployment ever mounted the volume,
-        # and its contents are single-writer state — the Whoosh index,
-        # classification model, celerybeat schedule, logs. Being on CephFS meant
-        # every access went through the MDS: 22 requests/sec sustained with zero
-        # CI running, ~15 journal events/sec against 2 directory commits per
-        # 180s, i.e. cap/inode churn rather than file writes. That was most of
-        # the constant 7-8 GB/day on cephfs-metadata, and that pool is size 3.
-        #
-        # `consume` and `media` below STAY on CephFS: the ftp-scanner writes to
-        # those from another node, so they genuinely need RWX.
-        #
-        # paperless-data is retained (prune: disabled) as the rollback. Do not
-        # delete it until search has been confirmed working — that is what
-        # proves the Whoosh index survived the copy, not merely that paperless
-        # starts. Migration record: MIGRATION-data-to-block.md.
         - name: data
           persistentVolumeClaim:
-            claimName: paperless-data-block
+            claimName: paperless-data
         - name: media
           persistentVolumeClaim:
             claimName: paperless-media
@@ -234,9 +218,7 @@ spec:
           persistentVolumeClaim:
             claimName: paperless-consume
         - name: bridge-cert
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/paperless/pvcs.yaml

@@ -18,45 +18,6 @@ spec:
     requests:
       storage: 10Gi
 ---
-# Replacement for paperless-data, on RBD instead of CephFS.
-#
-# WHY: paperless-data is declared ReadWriteMany on ceph-filesystem, but only ONE
-# pod ever mounts it — the paperless Deployment itself. Its contents are all
-# single-writer state: the 50 MB Whoosh index, classification_model.pickle,
-# celerybeat-schedule.db, logs, migration_lock. Nothing shares it. Only `consume`
-# and `media` genuinely need RWX, because the FTP scanner writes to those from a
-# different node.
-#
-# COST OF LEAVING IT: every access goes through the MDS. Measured 2026-08-20 with
-# zero CI running, this volume's client was doing 22 MDS requests/sec sustained,
-# and the MDS logged ~15 journal events/sec against only 2 directory commits per
-# 180s — i.e. cap and inode churn from repeated access, not file writes. That is
-# most of the constant 7-8 GB/day written to cephfs-metadata, which is a size 3
-# pool, so ~21-24 GB/day actually hits the drives.
-#
-# cephfs-metadata deliberately STAYS at size 3 (see storageclass.yaml): losing
-# the metadata pool destroys the whole filesystem, not just the objects on one
-# OSD, so the 275 GiB in cephfs-data0 rides on it. Reducing the writes at source
-# is the right fix; reducing the replication is not.
-#
-# MIGRATION: this PVC is created EMPTY and nothing mounts it yet. The data copy
-# and the deployment repoint are a separate, staged step — see
-# MIGRATION-data-to-block.md. Merging this alone is inert.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-data-block
-  namespace: paperless
-  annotations:
-    kustomize.toolkit.fluxcd.io/prune: disabled
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block
-  resources:
-    requests:
-      storage: 10Gi
----
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:

kubernetes/apps/trmnl/cluster.yaml

@@ -32,30 +32,7 @@ spec:
     parameters:
       shared_buffers: "128MB"
       max_connections: "50"
-      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
-      # and slower than the lz4/zstd this build also accepts). Compresses the
-      # full-page images that dominate WAL volume after each checkpoint.
-      wal_compression: "zstd"
-      # Write-reduction tuning. These databases are largely idle, and the
-      # Postgres defaults favour throughput over storage that wears out.
-      #
-      # checkpoint_timeout is the lever. The 5min default pushes every dirty
-      # page to disk 288 times a day, so a small database whose same few pages
-      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
-      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
-      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
-      # day. At 15min those pages coalesce and are written a third as often.
-      #
-      # max_wal_size must rise with it, or checkpoints trigger on size instead
-      # and the timeout change does nothing. It also bounds crash recovery to
-      # at most 2GB of WAL replay.
-      #
-      # NOT changed: synchronous_commit stays at the default `on`. It is off for
-      # ha-pg because a state recorder is telemetry; these hold credentials,
-      # documents and job state, where losing the last few hundred ms of commits
-      # on an unclean shutdown is not an acceptable trade.
-      checkpoint_timeout: "15min"
-      max_wal_size: "2GB"
+      wal_compression: "on"
   resources:
     requests:
       cpu: 100m

kubernetes/apps/trmnl/renderer.yaml

@@ -362,22 +362,6 @@ spec:
             name: trmnl-renderer-schedule
         - name: tmp
           emptyDir:
-            # RAM-backed, NOT disk. Without `medium: Memory` an emptyDir lands
-            # on the node's filesystem, and this one was the single largest
-            # writer in the cluster: 249 GB over 7.3 days = ~34 GB/day onto
-            # /dev/vda5, more than every Ceph OSD and every database combined
-            # (measured 2026-08-20 from the container's cgroup io.stat).
-            #
-            # The cause is cadence, not volume. schedules.template.json renders
-            # every 60 s, so Chrome is launched 1440 times a day and each launch
-            # creates, fills and deletes a /tmp/puppeteer_dev_chrome_profile-*
-            # directory. At rest /tmp holds under 8 MB; it is pure churn for
-            # data that never outlives the render.
-            #
-            # tmpfs pages count against the container's memory limit, which is
-            # why the cap stays at 512Mi: measured usage is 129Mi against a 2Gi
-            # limit, so even the full cap leaves ample headroom.
-            medium: Memory
             sizeLimit: 512Mi
 ---
 apiVersion: v1

kubernetes/apps/woodpecker/cluster.yaml

@@ -56,30 +56,6 @@ spec:
     parameters:
       shared_buffers: "128MB"
       max_connections: "50"
-      # Write-reduction tuning. These databases are largely idle, and the
-      # Postgres defaults favour throughput over storage that wears out.
-      #
-      # checkpoint_timeout is the lever. The 5min default pushes every dirty
-      # page to disk 288 times a day, so a small database whose same few pages
-      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
-      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
-      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
-      # day. At 15min those pages coalesce and are written a third as often.
-      #
-      # max_wal_size must rise with it, or checkpoints trigger on size instead
-      # and the timeout change does nothing. It also bounds crash recovery to
-      # at most 2GB of WAL replay.
-      #
-      # NOT changed: synchronous_commit stays at the default `on`. It is off for
-      # ha-pg because a state recorder is telemetry; these hold credentials,
-      # documents and job state, where losing the last few hundred ms of commits
-      # on an unclean shutdown is not an acceptable trade.
-      checkpoint_timeout: "15min"
-      max_wal_size: "2GB"
-      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
-      # and slower than the lz4/zstd this build also accepts). Compresses the
-      # full-page images that dominate WAL volume after each checkpoint.
-      wal_compression: "zstd"
   resources:
     requests:
       cpu: 100m

kubernetes/infrastructure/cnpg-barman-plugin/manifest.yaml

@@ -917,27 +917,11 @@ subjects:
   name: plugin-barman-cloud
   namespace: database
 ---
-# stringData, NOT base64 `data`. An image reference is not a secret, and
-# encoding it here had two real costs: no reviewer sees a version change in a
-# diff, and no $imagepolicy marker can be attached to a base64 blob. The result
-# was that automation advanced the controller to v0.14.0 while this sidecar sat
-# at v0.12.0 — a two-minor skew in the component that performs every base backup
-# and WAL archive, invisible until someone read the pod spec (#535).
-#
-# The Secret KIND is kept because the controller reads it via secretKeyRef
-# (deployment env SIDECAR_IMAGE); changing it to a ConfigMap would need a matching
-# edit to that reference and diverge further from upstream. stringData is
-# accepted identically by the API server and serialised into `data` on write.
-#
-# The name carries a kustomize hash suffix from upstream's secretGenerator. It is
-# static here, so editing the value does NOT change the name and the deployment's
-# secretKeyRef stays valid.
-#
-# IF YOU RE-IMPORT THE UPSTREAM MANIFEST: upstream ships this as base64 `data`
-# with no marker. Re-apply this block, or the skew silently returns.
 apiVersion: v1
-stringData:
-  SIDECAR_IMAGE: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar:v0.14.0 # {"$imagepolicy": "flux-system:cnpg-barman-plugin-sidecar"}
+data:
+  SIDECAR_IMAGE: |
+    Z2hjci5pby9jbG91ZG5hdGl2ZS1wZy9wbHVnaW4tYmFybWFuLWNsb3VkLXNpZGVjYXI6dj
+    AuMTIuMA==
 kind: Secret
 metadata:
   name: plugin-barman-cloud-kg5c2795f7

kubernetes/infrastructure/image-automation/cnpg-barman-plugin.yaml

@@ -28,40 +28,3 @@ spec:
   policy:
     semver:
       range: "0.x"
----
-# The SIDECAR image, tracked separately from the controller above.
-#
-# These are two DIFFERENT images that must move together:
-#   plugin-barman-cloud          the controller, in the database namespace
-#   plugin-barman-cloud-sidecar  injected into every Postgres pod
-#
-# Only the controller was tracked, so automation walked it from v0.12.0 to
-# v0.14.0 while the sidecar stayed at v0.12.0 — a two-minor skew in the
-# component that performs every base backup and WAL archive (#535). The skew was
-# invisible because the sidecar tag was base64-encoded inside a Secret, where no
-# $imagepolicy marker can reach it and no reviewer would see it in a diff.
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImageRepository
-metadata:
-  name: cnpg-barman-plugin-sidecar
-  namespace: flux-system
-spec:
-  image: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar
-  interval: 1h
----
-# Same filter and range as the controller policy deliberately: the two images
-# are released together and must not be allowed to drift apart again.
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImagePolicy
-metadata:
-  name: cnpg-barman-plugin-sidecar
-  namespace: flux-system
-spec:
-  imageRepositoryRef:
-    name: cnpg-barman-plugin-sidecar
-  filterTags:
-    pattern: '^v(?P<ver>\d+\.\d+\.\d+)$'
-    extract: '$ver'
-  policy:
-    semver:
-      range: "0.x"

kubernetes/infrastructure/netbird-ingress/daemonset.yaml

@@ -156,6 +156,4 @@ spec:
         # each pod enrolls its own peer and a restart re-enrolls cleanly
         # (the old peer is reaped).
         - name: state
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}

kubernetes/infrastructure/rook-ceph-cluster/toolbox.yaml

@@ -164,9 +164,7 @@ spec:
             name: rook-config-override
             optional: true
         - name: ceph-config
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
+          emptyDir: {}
       tolerations:
         - key: "node.kubernetes.io/unreachable"
           operator: "Exists"

kubernetes/infrastructure/velero/restore-test-cronjob.yaml

@@ -85,6 +85,4 @@ spec:
                 name: restore-test-script
                 defaultMode: 0755
             - name: tmp
-              emptyDir:
-                medium: Memory
-                sizeLimit: 16Mi
+              emptyDir: {}