fix: [alert] KubePodNotReady (paperless) (#896 )

2026-09-21 09:45:31 +00:00
1 changed files with 24 additions and 23 deletions
@@ -27,22 +27,22 @@ data:
  PAPERLESS_ALLOWED_HOSTS: "*"
  PAPERLESS_TIME_ZONE: "Europe/Berlin"
  PAPERLESS_OCR_LANGUAGE: "deu+eng"
-  # OCR concurrency, and it is a memory setting rather than a speed one.
+  # OCR concurrency, pinned rather than derived. paperless hands
+  # THREADS_PER_WORKER to ocrmypdf as its job count, and the default is
+  # cpu_count() / TASK_WORKERS. The container has no CPU limit, so
+  # multiprocessing.cpu_count() reports the NODE's cores — 20 on k8s-1 —
+  # and OCR fans out to "Starting processing with 20 workers concurrently".
+  # Twenty pages of a 300dpi colour scan rasterised at once blew past
+  # limits.memory below and the kernel OOMKilled the container (exit 137)
+  # mid-consume. Because celery re-picks the same document on every restart,
+  # that is a crash loop and not a one-off: the pod OOMs ~37s in, before the
+  # webserver ever finishes booting, so it never goes Ready and the archive
+  # stays down.
  #
-  # ocrmypdf sizes its worker pool from os.cpu_count(), which reports the
-  # HOST's CPU count and does not observe cgroup limits. k8s-1 has 20 cores,
-  # so an unset THREADS_PER_WORKER meant "Starting processing with 20 workers
-  # concurrently" against a 2Gi limit: 20 pages rasterised at once, over the
-  # limit ~4s later, exitCode 137 / OOMKilled. One multi-page scan therefore
-  # crash-looped the whole Deployment: the container died ~37s into each
-  # start, so the pod sat in CrashLoopBackOff and the web UI was unavailable
-  # too, not just consumption. Adding resources.limits.cpu does NOT fix this
-  # — os.cpu_count() ignores CPU quota.
-  #
-  # Peak memory scales with the PRODUCT TASK_WORKERS * THREADS_PER_WORKER, so
-  # both are pinned; leaving either to an upstream default leaves the bound
-  # undefined. TASK_WORKERS "1" is the value this version already resolved to
-  # (celery logged `concurrency: 1 (prefork)`) — explicit, not a change.
+  # TASK_WORKERS is already effectively 1 (20 cores / 1 worker = the 20 jobs
+  # observed); pinning it keeps that explicit so THREADS_PER_WORKER stays the
+  # only knob, and neither value silently changes when this lands on a node
+  # with a different core count.
  PAPERLESS_TASK_WORKERS: "1"
  PAPERLESS_THREADS_PER_WORKER: "4"
  PAPERLESS_CONSUMER_RECURSIVE: "true"
@@ -291,14 +291,15 @@ spec:
              cpu: 200m
              memory: 512Mi
            limits:
-              # 4Gi, raised from 2Gi: margin behind the OCR concurrency cap in
-              # paperless-config, not a fix on its own. Bounding the worker
-              # pool is what removes the cliff; this just stops a single
-              # unusually large scan from sitting right at the edge of it.
-              # No CPU limit on purpose — the container's boot can run a
-              # checksum backfill and a full index rebuild, and throttling
-              # that risks losing the startupProbe budget below.
-              memory: 4Gi
+              # Headroom, not the fix — capping OCR jobs in paperless-config
+              # is. Sized from what a healthy pod actually uses: the last
+              # long-lived container held a steady ~1.0Gi working set, so 2Gi
+              # left barely 1Gi for an OCR burst. 3Gi keeps a 4-job burst
+              # clear of the ceiling. The crashlooping containers never showed
+              # this in cadvisor (~820Mi peak) because they lived 37s against
+              # a ~30s scrape interval — the spike fell between samples, so
+              # the OOMKilled status, not the metric, is the reliable signal.
+              memory: 3Gi
          securityContext:
            runAsNonRoot: false
            allowPrivilegeEscalation: true