Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
939028fadd | ||
|
|
19824a176a | ||
|
|
eec1f36c44 | ||
|
|
7ef86e3948 | ||
|
|
4c723015ed | ||
|
|
a467e2925c |
@@ -16,7 +16,6 @@ resources:
|
||||
- prometheus-gate-egress.yaml
|
||||
- wordpress-gate-egress.yaml
|
||||
- llama-gate-egress.yaml
|
||||
- outpost-servicemonitors.yaml
|
||||
- ldap-outpost-cert.yaml
|
||||
- ldap-outpost-lb.yaml
|
||||
- ldap-outpost-cnp.yaml
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# Cilium's agent ServiceMonitor, declared here rather than by the chart.
|
||||
#
|
||||
# Reproduces what cilium 1.20.1 renders with prometheus.serviceMonitor.enabled,
|
||||
# taken field for field off the live object so scraping is unchanged: same name,
|
||||
# same namespace, same selector, same 10s interval and node relabeling.
|
||||
#
|
||||
# It lives here because the chart's template is gated on monitoring.coreos.com/v1
|
||||
# existing at install time, and on an empty cluster it does not — those CRDs
|
||||
# arrive with kube-prometheus-stack, which reconciles after infrastructure.
|
||||
# Cilium is one of infrastructure's healthChecks, so the chart aborting on the
|
||||
# missing CRD held the whole cluster: CNI waiting on monitoring waiting on CNI.
|
||||
#
|
||||
# velero/helmrelease.yaml already does exactly this for its own SM/PM and says
|
||||
# why; this is the same move for the component where it mattered most.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: cilium-agent
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app.kubernetes.io/name: cilium-agent
|
||||
app.kubernetes.io/part-of: cilium
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: cilium-agent
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- kube-system
|
||||
# The chart's Service keeps rendering (prometheus.enabled), so the target and
|
||||
# its `metrics` port are there regardless of who declares the ServiceMonitor.
|
||||
targetLabels:
|
||||
- k8s-app
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
interval: 10s
|
||||
honorLabels: true
|
||||
relabelings:
|
||||
# Agent metrics are per-node; without this every series lands without a
|
||||
# node label and BPF map pressure cannot be attributed to a machine.
|
||||
- action: replace
|
||||
sourceLabels:
|
||||
- __meta_kubernetes_pod_node_name
|
||||
targetLabel: node
|
||||
replacement: ${1}
|
||||
@@ -22,8 +22,19 @@ resources:
|
||||
# endpoint, which proves reachability, not that anything was written.
|
||||
- prometheusrule-cnpg-backup.yaml
|
||||
- alloy-servicemonitor.yaml
|
||||
- cilium-servicemonitor.yaml
|
||||
- cnpg-podmonitors.yaml
|
||||
- crowdsec-servicemonitor.yaml
|
||||
# Velero's own monitors: monitoring.coreos.com CRDs arrive with
|
||||
# kube-prometheus-stack, which reconciles after infrastructure, so these
|
||||
# cannot live beside the Velero HelmRelease.
|
||||
- velero-podmonitor.yaml
|
||||
- velero-servicemonitor.yaml
|
||||
- authentik-outpost-servicemonitors.yaml
|
||||
# Rook's exporter/mgr monitors: the operator aborts its node-daemon reconcile
|
||||
# when it cannot create these, which held the CephCluster in Progressing and
|
||||
# deadlocked the whole apps layer. See the file for the cycle.
|
||||
- rook-servicemonitors.yaml
|
||||
- home-assistant-servicemonitor.yaml
|
||||
- immich-servicemonitor.yaml
|
||||
- loki-servicemonitor.yaml
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# Rook's two ServiceMonitors, declared here rather than by the operator.
|
||||
#
|
||||
# Reproduced field for field off the live objects in the production cluster, so
|
||||
# scraping is unchanged: same names, same namespace, same selectors, same 10s
|
||||
# interval and `cluster` relabeling. The only thing dropped is the CephCluster
|
||||
# ownerReference, which Rook set and Flux cannot.
|
||||
#
|
||||
# They live here because CephCluster.spec.monitoring.enabled is now false, and
|
||||
# that is not a monitoring decision — it breaks a deadlock. Rook's node-daemon
|
||||
# reconcile creates this ServiceMonitor and, unlike the exporter DaemonSet next
|
||||
# to it, treats a failure as fatal (pkg/operator/ceph/cluster/nodedaemon/
|
||||
# reconcile.go, v1.20.7):
|
||||
#
|
||||
# if !cephCluster.Spec.Monitoring.MetricsDisabled {
|
||||
# // exporter reconciled; its errors are logged and reconcile continues
|
||||
# if cephCluster.Spec.Monitoring.Enabled {
|
||||
# if err := EnableCephExporterServiceMonitor(...); err != nil {
|
||||
# return errors.Wrap(err, "failed to enable service monitor")
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# On an empty cluster monitoring.coreos.com/v1 does not exist yet, so that call
|
||||
# fails with "the server could not find the requested resource (post
|
||||
# servicemonitors.monitoring.coreos.com)", the whole node-daemon reconcile
|
||||
# aborts, and the CephCluster never leaves status.phase Progressing. That is
|
||||
# what infrastructure-ceph-cluster's healthCheckExpr waits on — and the chain
|
||||
# closes on itself:
|
||||
#
|
||||
# infrastructure-ceph-cluster → CephCluster phase Ready
|
||||
# → Rook reconcile settles
|
||||
# → monitoring.coreos.com CRDs exist
|
||||
# → kube-prometheus-stack (monitoring)
|
||||
# → grafana-db
|
||||
# → infrastructure-ceph-cluster
|
||||
#
|
||||
# Nothing outside that loop can break it, so a cold rebuild stopped here with
|
||||
# every one of the twenty app Kustomizations behind it.
|
||||
#
|
||||
# Same treatment velero/helmrelease.yaml and cilium/helmrelease.yaml already
|
||||
# get, and for the same reason: the metrics stay where they are, only the
|
||||
# scrape wiring moves to the layer where its CRDs exist.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: rook-ceph-exporter
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
team: rook
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: rook-ceph-exporter
|
||||
rook_cluster: rook-ceph
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- rook-ceph
|
||||
endpoints:
|
||||
- port: ceph-exporter-http-metrics
|
||||
path: /metrics
|
||||
interval: 10s
|
||||
honorLabels: true
|
||||
# Rook stamps the cluster name so series from several Ceph clusters stay
|
||||
# separable. One cluster here, but dropping it would rename every series.
|
||||
relabelings:
|
||||
- action: replace
|
||||
replacement: rook-ceph
|
||||
targetLabel: cluster
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: rook-ceph-mgr
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
team: rook
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: rook-ceph-mgr
|
||||
rook_cluster: rook-ceph
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- rook-ceph
|
||||
endpoints:
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 10s
|
||||
honorLabels: true
|
||||
relabelings:
|
||||
- action: replace
|
||||
replacement: rook-ceph
|
||||
targetLabel: cluster
|
||||
@@ -1,116 +0,0 @@
|
||||
---
|
||||
# A read-only view of paperless's backup store, used as a recovery source and
|
||||
# nothing else.
|
||||
#
|
||||
# It points at the production prefix, which is the whole point — the eval
|
||||
# instance is a copy of the real archive's metadata — and that makes what is
|
||||
# ABSENT here the safety-critical part:
|
||||
#
|
||||
# no retentionPolicy The production store has "30d". Without one here
|
||||
# there is no policy to enforce, and enforcement is not
|
||||
# tied to taking a backup: the plugin runs it from the
|
||||
# instance sidecar on a timer
|
||||
# (instanceSidecarConfiguration.retentionPolicyIntervalSeconds,
|
||||
# default 1800). So "this cluster never backs up" would
|
||||
# not have been enough on its own.
|
||||
#
|
||||
# A second property makes it belt-and-braces. Retention runs
|
||||
# barman-cloud-backup-delete, which takes a server name and works inside
|
||||
# <destinationPath>/<serverName> — the store's own status keys its recovery
|
||||
# window by "pprl-pg", confirming that layout. This cluster is pprleval-pg, so
|
||||
# even an accidental retention policy would address a prefix that holds nothing
|
||||
# rather than production's backups.
|
||||
#
|
||||
# The cluster below completes the guarantee with isWALArchiver: false, so
|
||||
# nothing is written back either. It does load the plugin — that is required
|
||||
# for the restore to work at all — and read-only comes from the flag rather
|
||||
# than from leaving the plugin out.
|
||||
apiVersion: barmancloud.cnpg.io/v1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: pprleval-pg-source
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
configuration:
|
||||
destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg
|
||||
endpointURL: https://fsn1.your-objectstorage.com
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_SECRET_KEY
|
||||
region:
|
||||
name: hetzner-s3-creds
|
||||
key: REGION
|
||||
wal:
|
||||
compression: gzip
|
||||
data:
|
||||
compression: gzip
|
||||
---
|
||||
# A disposable clone of the paperless database, for measuring how well the
|
||||
# local LLM classifies documents without running the experiment against the
|
||||
# live archive.
|
||||
#
|
||||
# Losing this cluster costs a re-restore and nothing else, which is the intent
|
||||
# — it is rebuilt whenever a test needs a clean starting point.
|
||||
#
|
||||
# One instance, not two. Replication buys availability, and an eval instance
|
||||
# that is down is simply an eval that waits.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: pprleval-pg
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
enableSuperuserAccess: true
|
||||
storage:
|
||||
# Delete-reclaim, unlike the Retain default: this volume is a copy that
|
||||
# gets rebuilt, so it should vanish with the cluster rather than leave a
|
||||
# Released PV and an orphaned RBD image behind (issue #262).
|
||||
storageClass: ceph-block-ephemeral
|
||||
size: 10Gi
|
||||
# Declared with isWALArchiver: false, which is what kubernetes/RECOVERY.md
|
||||
# arrived at after an earlier version of that runbook said to omit it: the
|
||||
# instance manager loads this plugin to replay WAL during the restore, and
|
||||
# without it the instance never starts ("Unknown plugin:
|
||||
# barman-cloud.cloudnative-pg.io").
|
||||
#
|
||||
# externalClusters below only says WHERE the backup is; it does not get the
|
||||
# plugin injected into the pod. So "no plugin" is not the way to keep this
|
||||
# read-only — "plugin, not archiving" is, and it is the combination verified
|
||||
# on im-pg on 2026-07-25.
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
enabled: true
|
||||
isWALArchiver: false
|
||||
parameters:
|
||||
barmanObjectName: pprleval-pg-source
|
||||
bootstrap:
|
||||
recovery:
|
||||
source: pprl-pg
|
||||
database: paperless
|
||||
owner: paperless
|
||||
externalClusters:
|
||||
- name: pprl-pg
|
||||
plugin:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
parameters:
|
||||
barmanObjectName: pprleval-pg-source
|
||||
serverName: pprl-pg
|
||||
postgresql:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_connections: "100"
|
||||
# The production tuning for write reduction is deliberately not copied.
|
||||
# It exists to spare the SSDs under a database that runs for years; this
|
||||
# one is read-only for hours at a time and then deleted.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
@@ -1,13 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: paperless-eval
|
||||
# The instance only. The measurement harness lives in scripts/paperless-eval/
|
||||
# and is applied by hand — a Job is immutable, re-running one means deleting and
|
||||
# recreating it, and Flux would just put the old one back. Nothing here should
|
||||
# converge on a finished measurement.
|
||||
resources:
|
||||
- secrets.yaml
|
||||
- cluster.yaml
|
||||
- pvcs.yaml
|
||||
- redis.yaml
|
||||
- paperless.yaml
|
||||
@@ -1,203 +0,0 @@
|
||||
---
|
||||
# Deliberately much smaller than the production ConfigMap. Everything that
|
||||
# exists to serve people is gone: no remote-user auth, no CSRF origins, no
|
||||
# barcode or mail consumption, no Tika/Gotenberg. Nothing consumes documents
|
||||
# here, and nothing signs in.
|
||||
#
|
||||
# What remains is the database connection and the AI settings, because those
|
||||
# are what the evaluation measures.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: paperless-eval-config
|
||||
namespace: paperless-eval
|
||||
data:
|
||||
PAPERLESS_URL: "http://paperless.paperless-eval.svc.cluster.local"
|
||||
PAPERLESS_ALLOWED_HOSTS: "*"
|
||||
PAPERLESS_TIME_ZONE: "Europe/Berlin"
|
||||
PAPERLESS_REDIS: "redis://redis:6379"
|
||||
PAPERLESS_DBENGINE: "postgresql"
|
||||
PAPERLESS_DBHOST: "pprleval-pg-rw"
|
||||
PAPERLESS_DBNAME: "paperless"
|
||||
# Polling off, so a pod that finds an empty consume volume does not spend a
|
||||
# thread looking at it every ten seconds for the length of a run.
|
||||
PAPERLESS_CONSUMER_POLLING_INTERVAL: "0"
|
||||
|
||||
# The AI settings under test. The endpoints are placeholders that each eval
|
||||
# job overrides for the model it is measuring — a job runs its own
|
||||
# llama-server beside paperless in the same pod, so the value that matters at
|
||||
# run time is the one in the job, not the one here.
|
||||
#
|
||||
# Same caveat as production, and sharper here: paperless reads these from an
|
||||
# ApplicationConfiguration row and falls back to the environment only when
|
||||
# that row is empty. The row comes across in the restored database, so
|
||||
# anything configured in the production admin UI arrives with it and silently
|
||||
# wins over this ConfigMap — an eval would then measure production's settings
|
||||
# while appearing to measure the job's.
|
||||
#
|
||||
# Check it before trusting any result, and after every fresh restore:
|
||||
#
|
||||
# select llm_backend, llm_model, llm_embedding_backend, llm_embedding_model
|
||||
# from paperless_applicationconfiguration;
|
||||
#
|
||||
# Every column must be null for the environment to apply.
|
||||
PAPERLESS_AI_ENABLED: "true"
|
||||
PAPERLESS_AI_LLM_BACKEND: "openai-like"
|
||||
PAPERLESS_AI_LLM_ENDPOINT: "http://127.0.0.1:8080/v1"
|
||||
PAPERLESS_AI_LLM_MODEL: "qwen3.6-35b-a3b"
|
||||
PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS: "true"
|
||||
PAPERLESS_AI_LLM_CONTEXT_SIZE: "12288"
|
||||
PAPERLESS_AI_LLM_REQUEST_TIMEOUT: "900"
|
||||
PAPERLESS_AI_LLM_OUTPUT_LANGUAGE: "de-de"
|
||||
PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
|
||||
PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
|
||||
PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
|
||||
PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
|
||||
---
|
||||
# A base instance that proves the restored database is usable and that the
|
||||
# search index and classifier build against it. The evaluation itself does not
|
||||
# run here — each job brings its own paperless container so it can point at its
|
||||
# own model — but this is what makes a broken clone visible before a job spends
|
||||
# hours on it.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: paperless
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
app.kubernetes.io/name: paperless
|
||||
annotations:
|
||||
configmap.reloader.stakater.com/reload: paperless-eval-config
|
||||
# Both, like production: the secret refreshes on the same 10m cycle, and
|
||||
# without this the pod would keep a rotated key until something else
|
||||
# restarted it — which defeats the point of matching the interval.
|
||||
secret.reloader.stakater.com/reload: paperless-eval-secrets
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: paperless
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: paperless
|
||||
spec:
|
||||
enableServiceLinks: false
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: paperless
|
||||
# No $imagepolicy marker, unlike apps/paperless: a measurement is
|
||||
# only comparable against a fixed version, so this tag moves when
|
||||
# somebody decides to re-measure, not when a release lands.
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: paperless-eval-config
|
||||
env:
|
||||
- name: PAPERLESS_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: paperless-eval-secrets
|
||||
key: secret-key
|
||||
- name: PAPERLESS_DBUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: pprleval-pg-app
|
||||
key: username
|
||||
- name: PAPERLESS_DBPASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: pprleval-pg-app
|
||||
key: password
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /usr/src/paperless/data
|
||||
- name: media
|
||||
mountPath: /usr/src/paperless/media
|
||||
- name: consume
|
||||
mountPath: /usr/src/paperless/consume
|
||||
# 60 minutes, against production's 10, and matching this app's Flux
|
||||
# Kustomization timeout.
|
||||
#
|
||||
# Production's budget was sized for a from-scratch boot on this same
|
||||
# corpus and held, so 10 would probably do — but there the volume is
|
||||
# Retain and the index survives, making that a one-off. Here the
|
||||
# volume is ephemeral and the index is rebuilt on every re-test, so
|
||||
# the expensive path is the normal one. A killed pod starts the work
|
||||
# over rather than resuming, which turns a slow boot into a
|
||||
# crash-loop that stalls Flux behind wait: true.
|
||||
#
|
||||
# The LLM index is not in this budget: it is a celery task, and boot
|
||||
# only runs init-llmindex-migrate, a migration check.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /accounts/login/
|
||||
port: http
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 180
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /accounts/login/
|
||||
port: http
|
||||
initialDelaySeconds: 120
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /accounts/login/
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
# Production's context verbatim, and it has to be: the paperless-ngx
|
||||
# entrypoint starts as root, chowns the mounted data/media/consume
|
||||
# volumes and then drops to uid 1000 via setgid/setuid. Without these
|
||||
# the same image fails that sequence with EPERM.
|
||||
#
|
||||
# It bites harder here than in production, where those volumes were
|
||||
# chowned once years ago and keep their ownership. These are fresh on
|
||||
# every rebuild, so the chown is not a no-op.
|
||||
securityContext:
|
||||
runAsNonRoot: false
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-eval-golden
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-eval-media
|
||||
- name: consume
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-eval-consume
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: paperless
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: paperless
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: http
|
||||
@@ -1,129 +0,0 @@
|
||||
---
|
||||
# All four use ceph-block-ephemeral rather than the Retain default. Everything
|
||||
# here is derived — a restored database, an index built from it, a cache — so
|
||||
# the volumes should go when the namespace does instead of leaving Released PVs
|
||||
# and orphaned RBD images to reclaim by hand (issue #262).
|
||||
#
|
||||
# /data holds the Tantivy search index and the classifier pickle, both rebuilt
|
||||
# from the database, and the LLM index — which is the expensive one at roughly
|
||||
# three hours for the full corpus.
|
||||
#
|
||||
# Measured on production for the same ~3900 documents: 36 MiB search index,
|
||||
# 125 MiB LLM index. 10Gi is far more than that needs, and deliberately so —
|
||||
# an embedding model with wider vectors than bge-m3's 1024 dimensions is
|
||||
# exactly what phase two varies, and running out of space mid-run would cost
|
||||
# the whole rebuild.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
# Named for its role rather than its mount point: the smoke-test deployment
|
||||
# builds /data here, and every eval job copies it out. Replaces the earlier
|
||||
# RWO paperless-eval-data, whose access mode could not be widened in place —
|
||||
# a bound claim's spec is immutable.
|
||||
name: paperless-eval-golden
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
# RWX, because the eval jobs read it too — each copies it into its own
|
||||
# emptyDir rather than mounting it live. The vector store is SQLite, and
|
||||
# SQLite locking across pods over CephFS is not something to rely on.
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ceph-filesystem-ephemeral
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
# Empty, and stays that way. Paperless refuses to start without the path, but
|
||||
# the AI path reads title, content and filename out of the database and never
|
||||
# opens a file — so the documents themselves are not copied here. That is what
|
||||
# keeps this instance cheap and keeps a second copy of the archive off the
|
||||
# cluster.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: paperless-eval-media
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block-ephemeral
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
# Also empty: nothing is consumed here, but the path has to exist.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: paperless-eval-consume
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block-ephemeral
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
# The eval jobs' model cache, RWX so several jobs can read the same weights at
|
||||
# once — that is the whole point of measuring two candidates in parallel.
|
||||
#
|
||||
# Separate from the ai namespace's llama-server-models despite holding some of
|
||||
# the same files: PVCs do not cross namespaces. A model therefore downloads
|
||||
# once here and is then shared by every job, which is the cost a new candidate
|
||||
# carries anyway.
|
||||
#
|
||||
# NOT ephemeral, unlike everything else in this namespace: re-downloading tens
|
||||
# of gigabytes on a slow uplink is a worse outcome than an orphaned volume.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: llama-models
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ceph-filesystem
|
||||
resources:
|
||||
requests:
|
||||
storage: 120Gi
|
||||
---
|
||||
# Results outlive the jobs that produce them: a Job is deleted and recreated to
|
||||
# re-run a configuration, and comparing configurations is the entire exercise.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: paperless-eval-results
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ceph-filesystem
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: paperless-eval-redis
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block-ephemeral
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
@@ -1,81 +0,0 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redis
|
||||
namespace: paperless-eval
|
||||
labels:
|
||||
app.kubernetes.io/name: redis
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: redis
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
fsGroup: 999
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: redis
|
||||
# No $imagepolicy marker; see the note on the paperless image.
|
||||
image: redis:8.10.1-alpine
|
||||
# No --save: this holds the celery queue and the suggestion cache,
|
||||
# both of which are regenerated. Persisting them across a restart of
|
||||
# a throwaway instance would only preserve stale suggestions.
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
- ""
|
||||
- --loglevel
|
||||
- warning
|
||||
ports:
|
||||
- name: redis
|
||||
containerPort: 6379
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: redis
|
||||
initialDelaySeconds: 10
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-eval-redis
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: redis
|
||||
ports:
|
||||
- port: 6379
|
||||
targetPort: redis
|
||||
@@ -1,45 +0,0 @@
|
||||
---
|
||||
# The same Hetzner S3 key every CNPG-using namespace pulls, needed here only so
|
||||
# the recovery source can be read. See apps/paperless for why one shared key
|
||||
# rather than per-app paths.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: hetzner-s3-creds
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: infrastructure/hetzner-s3
|
||||
---
|
||||
# Django's secret key, shared with the production instance rather than given
|
||||
# its own vault entry.
|
||||
#
|
||||
# It signs sessions and CSRF tokens, and this instance has no ingress and no
|
||||
# users — nothing signs in, so nothing is at stake in sharing it. A separate
|
||||
# entry would mean a vault write before the namespace can start, for a
|
||||
# throwaway instance that gets rebuilt.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: paperless-eval-secrets
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
# 10m, matching apps/paperless rather than the 1h default: the point of
|
||||
# sharing the key is that the two hold the same value, and a slower refresh
|
||||
# here would let them diverge for up to an hour after a rotation.
|
||||
refreshInterval: 10m
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: paperless-eval-secrets
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/paperless
|
||||
+10
-2
@@ -159,8 +159,16 @@ if [[ "$PHASE" == "vmtest" ]]; then
|
||||
|
||||
# sun_path caps a unix socket path at 108 bytes and the provisioner puts a
|
||||
# qemu monitor socket under the state directory, so this has to stay short.
|
||||
# A path under the repo or a long mktemp dir silently fails to boot.
|
||||
VM_STATE=/tmp/bootstrap-vmtest
|
||||
# A path under the repo or a long mktemp dir silently fails to boot. The
|
||||
# caller may override it to aim the qcow2 images at a sized volume; the
|
||||
# guard turns "too long" from a silent boot failure into an error.
|
||||
VM_STATE="${VM_STATE:-/tmp/bootstrap-vmtest}"
|
||||
[[ ${#VM_STATE} -le 50 ]] \
|
||||
|| fail "VM_STATE '$VM_STATE' is ${#VM_STATE} bytes; keep it under 50 so the qemu monitor socket fits in sun_path's 108"
|
||||
# cleanup_vmtest runs `sudo rm -rf "$VM_STATE"`, so refuse anything that is
|
||||
# not a nested absolute path.
|
||||
[[ $VM_STATE == /*/?* ]] \
|
||||
|| fail "VM_STATE '$VM_STATE' must be an absolute path at least two levels deep"
|
||||
VM_CLUSTER=bootstrap-vmtest
|
||||
VM_DIR=$(mktemp -d)
|
||||
TALOSCTL_BIN=$(command -v talosctl)
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: paperless-eval
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# A throwaway copy of paperless for measuring the local LLM against the real
|
||||
# corpus. Its Cluster restores from paperless's backup store and archives
|
||||
# nothing back; see kubernetes/apps/paperless-eval/cluster.yaml.
|
||||
#
|
||||
# prune: true and no prune-disabled annotations anywhere underneath — the
|
||||
# data here is a copy, and being able to delete the whole namespace by
|
||||
# removing this file is the point.
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
# 30m is not enough: the first start restores the database and then builds
|
||||
# the search index over ~3900 documents before the webserver answers.
|
||||
timeout: 60m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/apps/paperless-eval
|
||||
prune: true
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
- name: infrastructure-cnpg-barman
|
||||
healthCheckExprs:
|
||||
- apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
failed: "has(status.phase) && status.phase.startsWith('Failed')"
|
||||
current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"
|
||||
@@ -70,9 +70,23 @@ spec:
|
||||
# map only shows up as "no space left on device" in the log while pod
|
||||
# sandbox creation fails (#452). cilium_bpf_map_pressure makes it
|
||||
# alertable beforehand.
|
||||
#
|
||||
# That reasoning is about the metrics, which this line provides. The
|
||||
# ServiceMonitor is only the scrape wiring and is declared standalone in
|
||||
# apps/monitoring-rules instead — the same treatment velero/helmrelease
|
||||
# already gives its own, and for the same reason: the chart's template is
|
||||
# gated on monitoring.coreos.com/v1 existing at install time.
|
||||
#
|
||||
# Here it is worse than for velero. Cilium's chart does not skip the
|
||||
# template, it aborts rendering — "Service Monitor requires
|
||||
# monitoring.coreos.com/v1 CRDs" — and cilium is one of the seven
|
||||
# healthChecks the infrastructure Kustomization waits on. So on an empty
|
||||
# cluster the CNI blocked infrastructure, which blocked grafana-db, which
|
||||
# blocked monitoring, which is where those CRDs come from. The lowest
|
||||
# layer in the cluster was waiting on the highest.
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
enabled: false
|
||||
bpf:
|
||||
# Per-endpoint policy map size: one entry per (identity, direction,
|
||||
# protocol, port) allowed. When it fills, endpoint regeneration fails
|
||||
|
||||
@@ -99,7 +99,7 @@ spec:
|
||||
# required.
|
||||
#
|
||||
# NOTE: enabling this makes the LAPI serve HTTPS-only. The Traefik ingress
|
||||
# bouncer is currently INERT (detached, see traefik/crowdsec-middleware.yaml)
|
||||
# bouncer is currently INERT (detached, see post-crds/traefik-crowdsec-middleware.yaml)
|
||||
# so there is no live impact, but re-enabling it will require pointing the
|
||||
# plugin at https + trusting this CA (or giving it a bouncer-ou client cert).
|
||||
tls:
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- bouncer-key-externalsecret.yaml
|
||||
- edge-bouncer-key-externalsecret.yaml
|
||||
- lapi-ingress.yaml
|
||||
- helmrelease.yaml
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- externalsecret.yaml
|
||||
- helmrelease.yaml
|
||||
|
||||
@@ -73,11 +73,6 @@ metadata:
|
||||
# classifies documents without experimenting on the live archive.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: paperless-eval
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: audiobookshelf
|
||||
---
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- externalsecret.yaml
|
||||
- daemonset.yaml
|
||||
|
||||
@@ -34,22 +34,6 @@ spec:
|
||||
- ports:
|
||||
- port: "8080"
|
||||
protocol: TCP
|
||||
# The paperless-eval namespace reaches the embedding endpoint for the same
|
||||
# reason paperless does; its chat model runs inside each eval job's own pod
|
||||
# and needs no grant here.
|
||||
#
|
||||
# Scoped to the namespace rather than to an app label, unlike the grants
|
||||
# around it: that namespace exists only to run evaluations, and its
|
||||
# workload is jobs created per measurement whose labels vary with what is
|
||||
# being measured. A label match would be a trap that presents as an
|
||||
# embedding call silently dropped.
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": paperless-eval
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8080"
|
||||
protocol: TCP
|
||||
# Home Assistant's llama.cpp conversation agent calls the chat model only —
|
||||
# it has no use for the embedding endpoint, so unlike paperless above this
|
||||
# grant does not extend to llama-embed.
|
||||
|
||||
@@ -6,7 +6,6 @@ resources:
|
||||
- ingress-isolation.yaml
|
||||
- authentik-ns.yaml
|
||||
- paperless-ns.yaml
|
||||
- paperless-eval-ns.yaml
|
||||
- audiobookshelf-ns.yaml
|
||||
- home-assistant-ns.yaml
|
||||
- spegel-ns.yaml
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
# Narrower than paperless-ns.yaml, because this namespace serves nobody.
|
||||
#
|
||||
# There is no ingress, no gate outpost and no hostname: the only things that
|
||||
# talk to it are its own pods. Monitoring is admitted so a run can be watched,
|
||||
# and host/remote-node for the kubelet probes.
|
||||
#
|
||||
# Egress reaches the ai namespace for the embedding endpoint, which is the one
|
||||
# dependency an eval job has outside its own pod, and world:443 so the CNPG
|
||||
# recovery can read the backup store on Hetzner.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-ingress
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
endpointSelector: {}
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": paperless-eval
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": monitoring
|
||||
- fromEntities:
|
||||
- host
|
||||
- remote-node
|
||||
# ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See
|
||||
# home-assistant-ns.yaml for the rationale.
|
||||
- fromEndpoints:
|
||||
- matchExpressions:
|
||||
- key: io.kubernetes.pod.namespace
|
||||
operator: Exists
|
||||
icmps:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": paperless-eval
|
||||
# The embedding endpoint. The chat model runs inside each job's own pod, so
|
||||
# it needs no grant; the embedder is shared with production because it is
|
||||
# read-only and cheap, and swapping it is a phase-two concern.
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": ai
|
||||
app.kubernetes.io/name: llama-embed
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8080"
|
||||
protocol: TCP
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": rook-ceph
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": kube-system
|
||||
k8s-app: kube-dns
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "53"
|
||||
protocol: UDP
|
||||
- toEntities:
|
||||
- host
|
||||
- remote-node
|
||||
# world:443 — the CNPG recovery reads the backup store on Hetzner Object
|
||||
# Storage, and image pulls.
|
||||
- toEntities:
|
||||
- world
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "443"
|
||||
protocol: TCP
|
||||
# ICMP — egress mirror; see ingress block above.
|
||||
- toEndpoints:
|
||||
- matchExpressions:
|
||||
- key: io.kubernetes.pod.namespace
|
||||
operator: Exists
|
||||
icmps:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
---
|
||||
# The CNPG operator polls each instance's status endpoint over HTTPS
|
||||
# (https://<pod-ip>:8000/pg/status) to populate Cluster status. Without it the
|
||||
# Cluster reports "Instance Status Extraction Error: HTTP communication issue"
|
||||
# while Postgres serves normally — issue #433, where the operator's dial timed
|
||||
# out because the apps baseline never allowed the database namespace.
|
||||
#
|
||||
# It matters here even for a throwaway: the Flux Kustomization waits on
|
||||
# Cluster status, and a smoke test that cannot report its own health is not one.
|
||||
#
|
||||
# Scoped to the pprleval-pg endpoints and that one port, like the equivalent
|
||||
# rules for wp-pg and ha-pg: the barman and mariadb operators sharing the
|
||||
# database namespace have no reason to reach anything here.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: pprleval-pg-allow-cnpg-operator
|
||||
namespace: paperless-eval
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: pprleval-pg
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": database
|
||||
app.kubernetes.io/name: cloudnative-pg
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8000"
|
||||
protocol: TCP
|
||||
@@ -10,33 +10,8 @@
|
||||
# name hairpins off the ingress LB from a pod. Needs BOTH netpol halves
|
||||
# (gitea-allow-flux-rotator + rotator-gitea-flux-egress).
|
||||
#
|
||||
# The ExternalSecret MERGES into Flux's existing flux-system secret (does not
|
||||
# own it) so a sync failure leaves the last-good value in place; Flux's
|
||||
# source-controller reads it fresh each fetch, no restart. OpenBao policy/role
|
||||
# rotator-gitea-flux + the gitea-flux bootstrap (flux account password) are
|
||||
# provisioned by hand (see policies/rotator-gitea-flux.hcl).
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: flux-system-git-auth
|
||||
namespace: flux-system
|
||||
spec:
|
||||
refreshInterval: 10m
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: flux-system
|
||||
creationPolicy: Merge
|
||||
data:
|
||||
- secretKey: username
|
||||
remoteRef:
|
||||
key: infrastructure/flux-git
|
||||
property: username
|
||||
- secretKey: password
|
||||
remoteRef:
|
||||
key: infrastructure/flux-git
|
||||
property: token
|
||||
# The ExternalSecret that publishes the rotated token to Flux lives in
|
||||
# ../../post-crds/flux-git-externalsecret.yaml (needs ESO's CRDs).
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# Keeps Flux's flux-system secret supplied with the rotated Gitea credential
|
||||
# WITHOUT owning it, so a sync failure leaves the last-good value in place;
|
||||
# Flux's source-controller reads it fresh each fetch, no restart. OpenBao
|
||||
# policy/role rotator-gitea-flux + the gitea-flux bootstrap (flux account
|
||||
# password) are provisioned by hand (see ../openbao/policies/rotator-gitea-flux.hcl).
|
||||
#
|
||||
# See creationPolicy below for why that is Orphan and not Merge — the
|
||||
# distinction is what makes a recovery from the break-glass source possible.
|
||||
#
|
||||
# Split out of ../openbao/rotation/flux-git.yaml, which still holds the
|
||||
# rotation CronJob: external-secrets.io CRDs are installed by the ESO
|
||||
# HelmRelease in the infrastructure Kustomization, so this manifest cannot
|
||||
# pass server-side dry-run there on a fresh cluster.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: flux-system-git-auth
|
||||
namespace: flux-system
|
||||
spec:
|
||||
refreshInterval: 10m
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: flux-system
|
||||
# Orphan, not Merge: create the Secret if it is absent, but never own it.
|
||||
#
|
||||
# Merge cannot create — it only writes into a Secret that already exists,
|
||||
# on the assumption that `flux bootstrap` made one. That assumption breaks
|
||||
# in exactly the situation this cluster is rebuilt from. The break-glass
|
||||
# bootstrap pulls from the GitHub mirror and keeps its token in
|
||||
# flux-system-breakglass (bootstrap.sh), so `flux-system` is never created,
|
||||
# so this ExternalSecret has nothing to merge into — and reports Ready while
|
||||
# producing no Secret at all. The operator then performs the documented
|
||||
# handover back to Gitea and finds the GitRepository pointing at a Secret
|
||||
# that does not exist.
|
||||
#
|
||||
# Owner would create it but attach an ownerReference, so deleting this
|
||||
# object would take Flux's credential with it. Orphan creates and updates
|
||||
# without ownership, which is what the paragraph at the top of this file
|
||||
# actually wanted: a sync failure, or the loss of this object, leaves the
|
||||
# last-good value in place.
|
||||
#
|
||||
# Safe on a live cluster: the Secret holds exactly the two keys written
|
||||
# below (verified against production), so writing it whole is the same
|
||||
# result as merging into it.
|
||||
creationPolicy: Orphan
|
||||
# Explicit rather than relying on the default, because it is the half that
|
||||
# matters here: if the OpenBao entry ever disappears, keep serving the
|
||||
# credential Flux already has rather than leaving it with none.
|
||||
deletionPolicy: Retain
|
||||
data:
|
||||
- secretKey: username
|
||||
remoteRef:
|
||||
key: infrastructure/flux-git
|
||||
property: username
|
||||
- secretKey: password
|
||||
remoteRef:
|
||||
key: infrastructure/flux-git
|
||||
property: token
|
||||
@@ -10,4 +10,25 @@ resources:
|
||||
# configured (2026-07-13): a store that can't turn Ready would block this
|
||||
# KS's wait gate and freeze the dependsOn chain.
|
||||
- openbao-clustersecretstore.yaml
|
||||
# ExternalSecrets. external-secrets.io CRDs come from the ESO HelmRelease in
|
||||
# the infrastructure Kustomization, so these cannot pass its server-side
|
||||
# dry-run on a fresh cluster — Flux rejects the whole apply set, including
|
||||
# the ESO HelmRelease that would have installed the CRDs. They sat next to
|
||||
# their consumers until a cold-rebuild test hit that deadlock (2026-09-01);
|
||||
# nothing had ever applied this layer against an empty API server before.
|
||||
- crowdsec-bouncer-key-externalsecret.yaml
|
||||
- crowdsec-edge-bouncer-key-externalsecret.yaml
|
||||
- external-dns-unifi-externalsecret.yaml
|
||||
- flux-git-externalsecret.yaml
|
||||
- netbird-ingress-externalsecret.yaml
|
||||
- velero-externalsecret.yaml
|
||||
- velero-hetzner-creds-externalsecret.yaml
|
||||
# traefik.io CRDs, same reasoning — installed by the Traefik HelmRelease in
|
||||
# infrastructure. Traefik is deliberately absent from that Kustomization's
|
||||
# healthChecks, so this KS may dry-run before the chart lands; it retries on
|
||||
# retryInterval rather than deadlocking, because nothing here installs the
|
||||
# CRDs it is waiting for.
|
||||
- traefik-crowdsec-middleware.yaml
|
||||
- traefik-rate-limit-middleware.yaml
|
||||
- traefik-security-headers-middleware.yaml
|
||||
- ../network-policies/
|
||||
|
||||
@@ -27,7 +27,6 @@ spec:
|
||||
- ingress
|
||||
- monitoring
|
||||
- paperless
|
||||
- paperless-eval # same apps/paperless key plus the shared barman S3 creds
|
||||
- trmnl
|
||||
- velero
|
||||
- woodpecker # wp-pg barman creds; woodpecker moving out of `apps` (#433)
|
||||
|
||||
@@ -78,7 +78,20 @@ spec:
|
||||
enabled: true
|
||||
ssl: false
|
||||
monitoring:
|
||||
enabled: true
|
||||
# false, and this is not a decision about metrics. `metricsDisabled` is the
|
||||
# switch that turns metrics off; this one only decides whether Rook creates
|
||||
# the ServiceMonitors — and creating them is fatal to the reconcile when it
|
||||
# fails. On an empty cluster monitoring.coreos.com/v1 does not exist, so
|
||||
# Rook's node-daemon reconcile aborted, the CephCluster never left
|
||||
# status.phase Progressing, and infrastructure-ceph-cluster's healthCheckExpr
|
||||
# waits on exactly that — with grafana-db → monitoring → those same CRDs
|
||||
# behind it, closing the loop.
|
||||
#
|
||||
# The ceph-exporter DaemonSet and the mgr metrics endpoint are untouched;
|
||||
# only the scrape wiring moved, to apps/monitoring-rules/
|
||||
# rook-servicemonitors.yaml, where the CRDs exist by the time it applies.
|
||||
# Both objects are reproduced there field for field off the live cluster.
|
||||
enabled: false
|
||||
# Dedicated Ceph cluster network on VLAN 49, attached to the OSDs through
|
||||
# Multus. The attachment is san-network-attachment.yaml.
|
||||
#
|
||||
|
||||
@@ -2,6 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- helmrelease.yaml
|
||||
- crowdsec-middleware.yaml
|
||||
- security-headers-middleware.yaml
|
||||
- rate-limit-middleware.yaml
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- externalsecret.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
- snapshot-readonly-storageclass.yaml
|
||||
- node-agent-config.yaml
|
||||
- helmrelease.yaml
|
||||
- servicemonitor.yaml
|
||||
- podmonitor.yaml
|
||||
- restore-test-rbac.yaml
|
||||
- restore-test-cronjob.yaml
|
||||
# Velero Schedule CRs live in kubernetes/infrastructure/post-crds/
|
||||
|
||||
Reference in New Issue
Block a user