Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  claude-botandClaude Opus 5
|
2f25145cd1 | ||
|
claude-botandClaude Opus 5
|
e2fa885fa3 |
@@ -42,7 +42,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: fetch-alerts
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
@@ -135,38 +135,8 @@ steps:
|
||||
TITLES=$$(jq -c '[.[].title]' .ci-alert-monitor/open-issues.json)
|
||||
NOISE_BODY=$$(jq -r 'map(select(.title == "Alert monitor: noise log")) | .[0].body // ""' \
|
||||
.ci-alert-monitor/open-issues.json)
|
||||
# Suppression entries are exactly the structured lines the write-back
|
||||
# step emits: "- <fp> <ALERTNAME> — <YYYY-MM-DD>: <reason>". Parse THAT
|
||||
# shape rather than grepping the whole body for 16-hex tokens. A reason
|
||||
# routinely cites another alert's fingerprint in prose ("see the issue
|
||||
# filed for fingerprint <fp>"), and the body-wide grep turned every such
|
||||
# citation into a silent, permanent suppression of an alert that was
|
||||
# never classified as noise — verified against the live noise log, which
|
||||
# yielded 22 fingerprints for 21 real entries, the extra one being
|
||||
# 2ddc41edb689fc33, an alert that HAD been filed as a genuine issue.
|
||||
#
|
||||
# Entries also expire after NOISE_TTL_DAYS. A noise verdict says "this
|
||||
# occurrence was benign", not "this alert can never be interesting
|
||||
# again" — but the fingerprint is sha256(alertname|scope) and therefore
|
||||
# stable across recurrences, so without an expiry a single blip mutes
|
||||
# that alertname+scope pair forever. Re-triaging a recurrence weeks
|
||||
# later is cheap; missing a real outage because a July blip muted
|
||||
# openbao's KubePodNotReady is not.
|
||||
#
|
||||
# Lines that fail to parse are simply dropped, i.e. they stop
|
||||
# suppressing — this fails toward MORE alerting, never less.
|
||||
#
|
||||
# date is BusyBox here, not GNU: `date -d '30 days ago'` errors out, so
|
||||
# the cutoff goes through epoch arithmetic and `date -d @<epoch>`, which
|
||||
# this image does support.
|
||||
NOISE_TTL_DAYS=$${NOISE_TTL_DAYS:-30}
|
||||
CUTOFF=$$(date -u -d @$$(( $$(date -u +%s) - NOISE_TTL_DAYS * 86400 )) +%F)
|
||||
FPS=$$(printf '%s\n' "$$NOISE_BODY" \
|
||||
| sed -nE 's/^- ([0-9a-f]{16}) [^ ]+ — ([0-9]{4}-[0-9]{2}-[0-9]{2}):.*/\2 \1/p' \
|
||||
| awk -v c="$$CUTOFF" 'BEGIN{gsub(/-/,"",c)} { d=$$1; gsub(/-/,"",d); if (d+0 >= c+0) print $$2 }' \
|
||||
FPS=$$(printf '%s' "$$NOISE_BODY" | grep -oE '\b[0-9a-f]{16}\b' \
|
||||
| jq -R -s -c 'split("\n") | map(select(length > 0))')
|
||||
echo "Noise suppressions in effect: $$(printf '%s' "$${FPS:-[]}" | jq length)" \
|
||||
"(TTL $${NOISE_TTL_DAYS}d, entries dated >= $$CUTOFF)"
|
||||
# Synthesize pseudo-alerts for BLIPS: lookback series whose label set
|
||||
# is not currently firing at all (checked against the PRE-filter raw
|
||||
# snapshot, so live-but-null-routed alerts don't reappear as blips).
|
||||
@@ -290,7 +260,7 @@ steps:
|
||||
'{body: ($$old + "\n" + $$add)}' > /tmp/gitea-patch.json
|
||||
gitea_write PATCH "$$API/issues/$$NOISE_NUM" /tmp/gitea-patch.json
|
||||
else
|
||||
printf '%s\n\n' "Fingerprints the alert-monitor triage agent classified as noise. The dedup step skips any alert whose fingerprint appears below — delete a line to make that alert eligible for triage again. Keep this issue open; it is the suppression state. Entries EXPIRE after NOISE_TTL_DAYS (default 30) counted from the date in the line, so a one-off blip stops suppressing its alertname+scope pair once the window passes; re-add happens automatically if it is triaged as noise again. Only lines in the exact '- <fingerprint> <ALERTNAME> — <YYYY-MM-DD>: <reason>' form suppress anything — a fingerprint mentioned in prose does not." > /tmp/noise-header.txt
|
||||
printf '%s\n\n' "Fingerprints the alert-monitor triage agent classified as noise. The dedup step skips any alert whose fingerprint appears below — delete a line to make that alert eligible for triage again. Keep this issue open; it is the suppression state." > /tmp/noise-header.txt
|
||||
jq -n --rawfile hdr /tmp/noise-header.txt --rawfile add /tmp/noise-lines.txt \
|
||||
'{title: "Alert monitor: noise log", body: ($$hdr + $$add)}' > /tmp/gitea-create.json
|
||||
gitea_write POST "$$API/issues" /tmp/gitea-create.json
|
||||
|
||||
@@ -60,7 +60,7 @@ steps:
|
||||
# run that never touched them. Tracked in #461.
|
||||
- name: kustomize-build
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-07-30
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
@@ -115,7 +115,7 @@ steps:
|
||||
# An earlier revision of this comment also asserted that several deploy-*.yml
|
||||
# files emit "Additional property timeout is not allowed" warnings. That was
|
||||
# observed under the v3.9.0 CLI this step used to pin, and it is NOT known to
|
||||
# still hold on current CLIs — review raised the likelihood that the warning was
|
||||
# still hold on v3.16.0 — review raised the likelihood that the warning was
|
||||
# schema drift in a 7-minor-old CLI rather than real debt in those files, which
|
||||
# is one of the reasons this pin moved to latest stable. It cannot be confirmed
|
||||
# from a green run either, because the loop below sends lint output to /tmp/out
|
||||
@@ -123,15 +123,9 @@ steps:
|
||||
# The claim is dropped rather than restated: what this step guarantees is the
|
||||
# exit-code contract above, which is what the gate actually rests on.
|
||||
# Latest stable, per the standing rule that any software uses the latest
|
||||
# stable version. Matching the deployed woodpecker-server is a CONSEQUENCE of
|
||||
# that, not a separate requirement — the server is bumped by image-automation
|
||||
# within the 3.x range, so both should land on the same release.
|
||||
#
|
||||
# They do not land together, though: the server is a Kubernetes manifest and
|
||||
# is bumped automatically, this pipeline file is not and cannot be (see
|
||||
# infrastructure/image-automation/README.md). So the server moves first and
|
||||
# this trails until someone bumps it by hand — which is what happened at
|
||||
# v3.16.0 -> v3.17.0, caught by the update audit (#426) rather than by CI.
|
||||
# stable version. Matching the deployed woodpecker-server (v3.16.0) is a
|
||||
# CONSEQUENCE of that, not a separate requirement — the server is bumped by
|
||||
# image-automation within the 3.x range, so both land on the same release.
|
||||
#
|
||||
# It matters here because the CLI compiles the config against its own schema:
|
||||
# one behind the server can pass a config the server then rejects, or reject
|
||||
@@ -158,7 +152,7 @@ steps:
|
||||
# whenever it trails the deployed server. If you bump the server, bump here.
|
||||
- name: woodpecker-lint
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: woodpeckerci/woodpecker-cli:v3.17.0-alpine
|
||||
image: woodpeckerci/woodpecker-cli:v3.16.0-alpine
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
@@ -246,7 +240,7 @@ steps:
|
||||
# this step fails permanently on its own documentation.
|
||||
- name: no-digest-pins
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-07-30
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
@@ -13,7 +13,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: validate-secrets
|
||||
image: alpine:3.24.1
|
||||
image: alpine:3.21
|
||||
environment:
|
||||
TF_VAR_proxmox_api_url:
|
||||
from_secret: proxmox_api_url
|
||||
@@ -129,20 +129,13 @@ steps:
|
||||
- echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
|
||||
- export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
|
||||
- |
|
||||
# Ensure Talos image exists on all Proxmox nodes via API.
|
||||
#
|
||||
# Only nodes still running Proxmox belong in this list. jupiter and
|
||||
# saturn run Talos on bare metal now, so the storage API returns
|
||||
# nothing for them and the download-wait loop below would burn its full
|
||||
# 60x10s per node before giving up. Bare-metal nodes boot the
|
||||
# metal-amd64 image from their own installed disk instead, and the
|
||||
# whole step goes away once neptune is converted too.
|
||||
# Ensure Talos image exists on all Proxmox nodes via API
|
||||
TALOS_VERSION="v1.12.6"
|
||||
TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
|
||||
TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
|
||||
TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
|
||||
API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
|
||||
for node in neptune; do
|
||||
for node in jupiter saturn neptune; do
|
||||
EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
|
||||
-H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
|
||||
if [ "$EXISTS" -gt 0 ]; then
|
||||
|
||||
@@ -4,7 +4,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: e2e
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ steps:
|
||||
- sh scripts/ci/test-model-selection.sh
|
||||
|
||||
- name: e2e-test
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
@@ -49,7 +49,7 @@ steps:
|
||||
# (the issue-pick logic needs jq, which only the ci-claude image has);
|
||||
# minting a 1h read-only token per run is cheap. Degrades to
|
||||
# snapshot-only if the ci-diagnostics ServiceAccount isn't reconciled.
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-05-01
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
@@ -167,7 +167,7 @@ permanent recovery.
|
||||
|
||||
## CNPG restore-test (verify backups are restorable — non-destructive)
|
||||
|
||||
The monthly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
|
||||
The weekly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
|
||||
PVC restores but **explicitly does not touch CNPG** (barman-cloud is a separate
|
||||
code path). Run this drill periodically to prove the CNPG backups are actually
|
||||
restorable, **without touching the live clusters**. It restores into a throwaway
|
||||
|
||||
@@ -1,411 +0,0 @@
|
||||
# Runbook — converting a control-plane node from Proxmox VM to bare metal
|
||||
|
||||
Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
|
||||
metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
|
||||
after doing jupiter on 2026-08-25; saturn and neptune remain.
|
||||
|
||||
Everything here that reads like an over-specific warning is something that
|
||||
actually happened. The conversion itself is straightforward — the failures were
|
||||
all in state that outlived the VM.
|
||||
|
||||
## What the node ends up as
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| System disk | Kingston OM8PGP41024Q-A0, selected by serial |
|
||||
| EPHEMERAL (`/var`) | the retired Crucial P3, selected by serial |
|
||||
| Ceph OSD | Samsung PM983, selected by `/dev/disk/by-id` |
|
||||
| Management | VLAN 40 tagged on X710 port 0 |
|
||||
| v30iot | VLAN 30 on the same port, named `ens19` |
|
||||
| Ceph cluster net | X710 port 1, `10.99.0.x/29`, MTU 9000 |
|
||||
|
||||
## Before you start
|
||||
|
||||
- [ ] All three OSDs up, all mons in quorum, PGs `active+clean`. Do not start
|
||||
from a degraded cluster; the window itself removes one of three.
|
||||
- [ ] Cluster fits on two nodes. Check `kubectl describe node` allocation on the
|
||||
two survivors. Take immich down (`kubernetes/apps/immich`, both
|
||||
Deployments to `replicas: 0`) — it frees ~1000m against a shortfall of
|
||||
roughly 200m and takes its image pulls off the remaining nodes.
|
||||
- [ ] `cephfs-metadata` to size 2 / min_size 1 for the duration. At size 3 on
|
||||
two hosts every PG is undersized with min_size 2, so one further OSD loss
|
||||
blocks IO on every CephFS mount cluster-wide.
|
||||
- [ ] Talos metal ISO on a USB stick. Schematic
|
||||
`a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64`
|
||||
(`net.ifnames=1`, intel-ucode, thunderbolt; no qemu-guest-agent).
|
||||
Verify by comparing partition offsets and sizes against the ISO with
|
||||
`fdisk -l` — blkid labels lie after a rewrite.
|
||||
- [ ] Physical access. Secure Boot must be turned off (see below), and that is
|
||||
the one step nothing can do remotely.
|
||||
|
||||
## 1. Evacuate the OSD
|
||||
|
||||
```sh
|
||||
ceph osd out <id>
|
||||
```
|
||||
|
||||
Wait until no PGs are `backfilling` or `backfill_wait`. Do **not** wait for
|
||||
`ceph osd safe-to-destroy` — pools at size 3 keep the OSD in their acting set
|
||||
because three copies cannot be placed on two hosts, so it never returns clean.
|
||||
|
||||
Then remove it. The node must already be out of `cephcluster.yaml`'s
|
||||
`storage.nodes` list, or the operator rebuilds the OSD within one reconcile and
|
||||
starts backfilling onto the drive you are about to pull:
|
||||
|
||||
```sh
|
||||
kubectl -n rook-ceph scale deployment rook-ceph-osd-<id> --replicas=0
|
||||
ceph osd purge <id> --yes-i-really-mean-it
|
||||
kubectl -n rook-ceph delete deployment rook-ceph-osd-<id>
|
||||
```
|
||||
|
||||
## 2. Drain, and leave the cluster
|
||||
|
||||
```sh
|
||||
kubectl drain <node> --ignore-daemonsets --delete-emptydir-data --timeout=300s
|
||||
```
|
||||
|
||||
Single-replica workloads with `minAvailable: 1` PDBs will refuse eviction — that
|
||||
is deliberate, they exist to surface the pod rather than let it be yanked. On
|
||||
this cluster that is `influxdb-influxdb2-0`, `woodpecker-server` and the
|
||||
CNPG primary. Move each one deliberately with `kubectl delete pod`; they
|
||||
reschedule onto the survivors and their RBD volumes follow.
|
||||
|
||||
```sh
|
||||
talosctl -n <node> reset --graceful --wipe-mode system-disk
|
||||
kubectl delete node <node>
|
||||
```
|
||||
|
||||
`--wipe-mode system-disk` and not the default `all`: the OSD drive is being
|
||||
pulled anyway and wiping a terabyte of it is wasted time. `--graceful` leaves
|
||||
etcd cleanly. Delete the Node object — it carries the `unschedulable` flag from
|
||||
the drain, which the rebuilt node would otherwise inherit.
|
||||
|
||||
## 2a. Destroy the VM in OpenTofu — while the host is still a PVE node
|
||||
|
||||
Remove `opentofu/vm_15N_k8s_N.tf`, then apply. This **must** happen before the
|
||||
host leaves the Proxmox cluster: afterwards the provider cannot reach the node,
|
||||
the plan errors, and the only escape is `tofu state rm`, which is off the table.
|
||||
|
||||
Point `proxmox_api_url` at a host that will *survive* the conversion, not the one
|
||||
being converted.
|
||||
|
||||
Manual triggers are currently broken by the Woodpecker v3.18.0 regression
|
||||
(issue #582). Until that is resolved, run it locally:
|
||||
|
||||
```sh
|
||||
AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… AWS_REGION=fsn1 \
|
||||
tofu -chdir=opentofu init -backend-config=<file>
|
||||
… plan -target=module.k8s_N -var-file=<file> -out=<plan>
|
||||
… apply <plan>
|
||||
```
|
||||
|
||||
The `AWS_*` environment variables are required *in addition to*
|
||||
`-backend-config`, or the S3 backend fails `HeadObject` with 403 even with
|
||||
credentials that can read the object fine via the aws CLI.
|
||||
|
||||
## 2b. Proxmox quorum — matters from the second conversion onward
|
||||
|
||||
Two nodes remain after jupiter, with `expected_votes: 2` and quorum 2. Removing
|
||||
one leaves the last node inquorate and `/etc/pve` read-only, which blocks the
|
||||
final VM destroy. On the surviving node:
|
||||
|
||||
```sh
|
||||
pvecm expected 1
|
||||
```
|
||||
|
||||
This is the official recovery. Proxmox documents it in the storage replication
|
||||
chapter (`/usr/share/pve-docs/chapter-pvesr.html`), verbatim:
|
||||
|
||||
> If you have no quorum, we strongly advise to fix this first and make the node
|
||||
> operable again. Only if this is not possible at the moment, you may use the
|
||||
> following command to enforce quorum on the current node: `# pvecm expected 1`
|
||||
|
||||
The caveat there is about restoring the missing node instead. Here the missing
|
||||
node is deliberately gone for good, so enforcing quorum is the correct branch
|
||||
rather than a workaround.
|
||||
|
||||
Run it only once the node is actually alone — attempting it while both nodes are
|
||||
up fails with `CS_ERR_INVALID_PARAM`, since corosync will not set expected votes
|
||||
below the members it can currently see. Which is exactly why it becomes valid
|
||||
when one remains.
|
||||
|
||||
Runtime only; it does not survive a reboot, so re-run it if the survivor
|
||||
restarts before its own conversion.
|
||||
|
||||
## 3. Hardware
|
||||
|
||||
Power the host off, then:
|
||||
|
||||
1. Kingston boot drive out of slot 1, into slot 2.
|
||||
2. Crucial P3 out of slot 2 — it becomes EPHEMERAL, so put it in slot 3 rather
|
||||
than setting it aside.
|
||||
3. Slot 1 switch to **U.2**. Set it before the drive goes in; there is nothing
|
||||
to record afterwards, the installed drive type implies the position.
|
||||
4. PM983 onto the included 7mm M.2-to-U.2 adapter, into slot 1.
|
||||
|
||||
Boot the host's existing PVE once before touching Talos and check the drive:
|
||||
|
||||
```sh
|
||||
nvme smart-log /dev/nvmeXn1 | grep -iE 'percentage_used|media_errors|critical'
|
||||
nvme id-ns /dev/nvmeXn1 -H | grep -i 'in use' # want 512b data, 0 metadata
|
||||
wipefs /dev/nvmeYn1 # Crucial: clear the old bluestore sig
|
||||
ip link set <ceph-port> up && ethtool <ceph-port> | grep -i 'link detected'
|
||||
```
|
||||
|
||||
This is the last revert point. A bad drive here means the Crucial goes back and
|
||||
the VM is recreated from tofu with nothing lost. Once Talos writes to the
|
||||
Kingston, PVE on that host is gone.
|
||||
|
||||
Relocating the boot drive is safe: `rpool` is referenced by NVMe EUI and the
|
||||
UEFI entries by GPT PARTUUID, neither of which encodes a slot.
|
||||
|
||||
## 4. Secure Boot
|
||||
|
||||
MS-01 ships with Secure Boot on, and Proxmox satisfies it with a
|
||||
Microsoft-signed shim. The Image Factory `metal-amd64.iso` is an unsigned UKI
|
||||
and the firmware rejects it outright. **Turn Secure Boot off in the BIOS.**
|
||||
|
||||
A signed variant exists per schematic, but it is signed with SideroLabs' key
|
||||
which firmware does not trust by default, and the installed system would have to
|
||||
use the secureboot installer image too. That is a separate project.
|
||||
|
||||
## 5. Install
|
||||
|
||||
Boot the stick. Maintenance mode DHCPs untagged on every link; on these hosts
|
||||
that lands on `192.168.8.x`, which is routable from the workstation. It DHCPs on
|
||||
*every* link, so one machine answers on several addresses at once — they are the
|
||||
same host, not several hosts. `get disks` on each confirms it.
|
||||
|
||||
Maintenance mode does serve `talosctl get`. `--insecure` is a flag on the
|
||||
subcommand rather than a global, so it must come *after* `get` — placed before
|
||||
it, cobra resolves to the legacy `talosctl disks` command and reports
|
||||
`unknown flag: --insecure`:
|
||||
|
||||
```sh
|
||||
talosctl -n <maintenance-ip> get disks --insecure
|
||||
talosctl -n <maintenance-ip> get links --insecure
|
||||
```
|
||||
|
||||
Verify serials and MACs here, on the machine itself, against the config about to
|
||||
be applied.
|
||||
|
||||
Build the config from the **live cluster's secrets**, not a fresh
|
||||
`talosctl gen config`, or the node forms a new cluster instead of joining:
|
||||
|
||||
```sh
|
||||
talosctl -n <surviving-node> get mc v1alpha1 -o jsonpath='{.spec}' > cp.yaml
|
||||
talosctl gen secrets --from-controlplane-config cp.yaml -o secrets.yaml
|
||||
talosctl gen config homelab https://k8s.renner.uno:6443 \
|
||||
--with-secrets secrets.yaml --output-types controlplane \
|
||||
--config-patch @talos/patches/cluster.yaml \
|
||||
--config-patch @<cilium additive half> \
|
||||
--config-patch @talos/patches/registry-mirror.yaml \
|
||||
--config-patch @talos/patches/k8s-N.yaml -o install.yaml
|
||||
```
|
||||
|
||||
Confirm `cluster.id` matches the running cluster before applying.
|
||||
|
||||
Strip the generated `HostnameConfig` document — it collides with
|
||||
`machine.network.hostname` from the per-node patch. `bootstrap.sh` does the same.
|
||||
|
||||
`cilium-cni.yaml` cannot be passed whole: its `$patch: delete` document fails
|
||||
against a base with no `inlineManifests` (issue #581). Use only the additive
|
||||
second document.
|
||||
|
||||
Validate:
|
||||
|
||||
```sh
|
||||
talosctl validate --config install.yaml --mode metal
|
||||
```
|
||||
|
||||
### Install minimal, complete on the running node
|
||||
|
||||
Do **not** apply the full per-node config in maintenance mode. A document that
|
||||
references an interface which does not exist yet hangs the install: the node
|
||||
sits at `STAGE: Installing` indefinitely with `KUBELET n/a`, `APISERVER n/a`,
|
||||
and the console scrolling
|
||||
|
||||
```
|
||||
k8s.NodeApplyController: error getting node: nodes "k8s-N" not found
|
||||
```
|
||||
|
||||
That message is a symptom of an install that never finished, not a Kubernetes or
|
||||
networking fault — the node holds a DHCP address and is reachable throughout. On
|
||||
saturn the offenders were the `ens19` VLAN document and the sysctl naming it,
|
||||
`net.ipv6.conf.ens19.disable_ipv6`. Three cold installs hung on it; the identical
|
||||
config applied cleanly once the node was up. Wiping disks makes no difference —
|
||||
it is not leftover state.
|
||||
|
||||
**Stage 1** — the smallest config that installs and joins, plus `EPHEMERAL`,
|
||||
which is one-shot (section 7) and so cannot be deferred to stage 2:
|
||||
|
||||
```
|
||||
machine.network.hostname, machine.install.diskSelector, certSANs, nodeLabels
|
||||
LinkConfig <uplink>
|
||||
VLANConfig mgmt0
|
||||
DHCPv4Config mgmt0
|
||||
VolumeConfig EPHEMERAL
|
||||
```
|
||||
|
||||
```sh
|
||||
talosctl apply-config --insecure -n <maintenance-ip> --file install-stage1.yaml
|
||||
```
|
||||
|
||||
**Stage 2** — once the node is `Ready` and `talosctl etcd members` lists it,
|
||||
apply the full config to the running node. `--mode=try` reverts on its own if the
|
||||
network config is wrong, so it cannot strand the node; re-apply with
|
||||
`--mode=auto` to persist.
|
||||
|
||||
```sh
|
||||
talosctl -e <node-ip> -n <node-ip> apply-config --mode=try --timeout=3m --file install.yaml
|
||||
talosctl -e <node-ip> -n <node-ip> apply-config --mode=auto --file install.yaml
|
||||
```
|
||||
|
||||
Both applied without a reboot on saturn. Confirm `ens19`, `mgmt0` and the Ceph
|
||||
link are up afterwards with `talosctl get links` and `get addresses`.
|
||||
|
||||
## 6. The network config will strand the node if it is wrong
|
||||
|
||||
Talos stops running default DHCP operators **for the entire machine** as soon as
|
||||
a single new-style network document exists — the gate is presence of a document,
|
||||
not success of resolution. If the configured path fails to materialise there is
|
||||
no fallback on any interface, and recovery needs a console and install media.
|
||||
|
||||
Concretely: do not use `LinkAliasConfig` as a `VLANConfig` parent without
|
||||
verifying it. Aliases *are* supported (confirmed in v1.12.6 source and by a
|
||||
maintainer), but a selector matching zero links is skipped **silently**, and
|
||||
`permanent_addr` comes from an ethtool ioctl with no fallback — if the driver
|
||||
does not answer, the match is against an empty string and everything downstream
|
||||
that references the alias does nothing.
|
||||
|
||||
Address links by predictable kernel name. To introduce aliases later, do it on a
|
||||
node that is already up, with `talosctl apply-config --mode=try --timeout=5m`,
|
||||
which rolls back on its own, and check `talosctl get linkaliasspec`.
|
||||
|
||||
Interface names derive from PCI topology and are stable — but not immune.
|
||||
Adding a third NVMe to jupiter renumbered the buses and renamed the unused
|
||||
second I226 from `enp89s0` to `enp90s0`.
|
||||
|
||||
## 7. EPHEMERAL placement is one-shot
|
||||
|
||||
```yaml
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
provisioning:
|
||||
diskSelector:
|
||||
match: disk.serial == '<crucial serial>'
|
||||
minSize: 50GiB
|
||||
grow: true
|
||||
```
|
||||
|
||||
Talos applies volume configuration **only when the volume does not yet exist**.
|
||||
Applied afterwards it is silently ignored, and relocating EPHEMERAL then needs a
|
||||
disk wipe and a fresh boot from install media. It must be in the first
|
||||
`apply-config`.
|
||||
|
||||
Match by serial. `disk.transport == 'nvme' && !system_disk` matches the PM983 as
|
||||
well, and losing that coin flip formats the OSD drive.
|
||||
|
||||
Verify afterwards:
|
||||
|
||||
```sh
|
||||
talosctl -n <node> get volumestatus EPHEMERAL -o yaml | grep -E 'location|phase'
|
||||
talosctl -n <node> get disks
|
||||
```
|
||||
|
||||
## 8. The mon will not come back on its own
|
||||
|
||||
**This is the step that cost the most time on jupiter.** The rebuilt node has a
|
||||
new MAC and therefore a new DHCP lease, but Ceph has its mon pinned to the old
|
||||
address in three places. The mon cannot bind an address the host no longer has,
|
||||
so it crashloops; the operator then tries to restart a *healthy* mon as its next
|
||||
step, Ceph refuses because that would drop quorum, and it retries every 60
|
||||
seconds forever without ever reaching the storage reconcile. Nothing else
|
||||
progresses behind it.
|
||||
|
||||
Fix in this order — the order matters, because doing it the other way round has
|
||||
the operator faithfully recreate the mon from the stale ConfigMap:
|
||||
|
||||
```sh
|
||||
# 1. ConfigMap first
|
||||
kubectl -n rook-ceph edit cm rook-ceph-mon-endpoints
|
||||
# remove the dead mon from `data` and from `mapping`
|
||||
|
||||
# 2. then the monmap
|
||||
ceph mon remove <id>
|
||||
|
||||
# 3. then the deployment — check it really carries the old address
|
||||
kubectl -n rook-ceph get deploy rook-ceph-mon-<id> -o yaml | grep public-addr
|
||||
kubectl -n rook-ceph delete deploy rook-ceph-mon-<id>
|
||||
```
|
||||
|
||||
Rook then creates a **new** mon with the next free ID (jupiter's `c` came back
|
||||
as `d`), on the correct address. It does not repair the old one.
|
||||
|
||||
Note mons use host ports under `network.provider: host`, so a replacement mon
|
||||
cannot schedule while the broken one's deployment still holds 3300/6789 — the
|
||||
canary sits `Pending` with "didn't have free ports" until the old deployment is
|
||||
deleted.
|
||||
|
||||
## 9. Re-add the OSD
|
||||
|
||||
Add the node back to `cephcluster.yaml` under `storage.nodes`, naming the PM983
|
||||
by `/dev/disk/by-id`:
|
||||
|
||||
```yaml
|
||||
- name: k8s-N
|
||||
devices:
|
||||
- fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_<serial>
|
||||
```
|
||||
|
||||
**Not** a bare device name. Talos enumerates these drives in a different order
|
||||
than Proxmox did — on jupiter `nvme0n1` is the Kingston *system* disk, and the
|
||||
PM983 is `nvme2n1`. Rook would probably decline a partitioned system disk, but
|
||||
that is luck rather than design.
|
||||
|
||||
Get the path from `talosctl -n <node> get disk <dev> -o yaml`, under `symlinks`.
|
||||
|
||||
## 10. Verify and restore
|
||||
|
||||
```sh
|
||||
kubectl get nodes # Ready
|
||||
talosctl -n <node> etcd members # 3 members
|
||||
ceph status # 3 mons in quorum, 3 osds
|
||||
ceph osd tree # OSD under the right zone/host
|
||||
ceph device ls # PM983 backing the new OSD
|
||||
```
|
||||
|
||||
Then undo the window changes: `cephfs-metadata` back to size 3, immich back to
|
||||
1 replica, and `ceph config set osd osd_mclock_profile balanced` if it was
|
||||
raised to `high_recovery_ops` to speed the evacuation.
|
||||
|
||||
The retired Crucial leaves a stale entry in `ceph device ls`. Remove it with
|
||||
`ceph config-key rm device/<devid>`; the mgr's devicehealth view may lag.
|
||||
|
||||
## Known-stale things that will bite
|
||||
|
||||
- `bootstrap.sh` cannot generate a config at all right now (issue #581), so a
|
||||
cold rebuild is broken independently of this work.
|
||||
- Manual Woodpecker pipelines are broken (issue #582), so anything gated on
|
||||
`PIPELINE == "..."` must be run another way.
|
||||
- Cilium's eBPF datapath drops VLAN-tagged traffic unless `bpf.vlanBypass` is
|
||||
set. **This happened** — every v30iot macvlan workload on a converted node
|
||||
stayed in sandbox creation until `vlanBypass: [30]` was added (issue #592).
|
||||
|
||||
Only `ens19` is affected, and the reason is worth knowing: Cilium attaches to
|
||||
devices that carry an address. `cilium-dbg status --verbose` lists `mgmt0`
|
||||
among its devices but not `ens19`, because `ens19` is deliberately
|
||||
address-less. Cilium therefore owns `mgmt0` and sees VLAN 40 frames already
|
||||
untagged by the 8021q layer, while VLAN 30 frames only ever meet the filter
|
||||
on the shared parent `enp2s0f0np0`, still tagged. Any future address-less
|
||||
VLAN carrying pod traffic will need the same treatment.
|
||||
|
||||
The symptom does not look like a VLAN problem. A tc-layer drop reaches an
|
||||
AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports `no buffer space
|
||||
available` and `network is down` rather than a DHCP timeout, and the pod sits
|
||||
in `ContainerCreating` behind `plugin type="macvlan" failed (add): error
|
||||
calling DHCP.Allocate: no more tries`. Confirm it with
|
||||
`cilium_drop_count_total{reason=~"VLAN.*"}` per node rather than by reading
|
||||
the CNI logs, and check `node_network_receive_packets_total{device="ens19"}`
|
||||
— it reads 0 on an affected node.
|
||||
@@ -79,30 +79,6 @@ which also pre-seeds the Talos image onto every Proxmox node. Wait until the
|
||||
node FQDNs (`k8s-{1,2,3}.srv.renner.uno`) resolve and the VMs are up before
|
||||
Step 2 — `bootstrap.sh` polls DNS but will not create the records.
|
||||
|
||||
**k8s-1 is not a VM and OpenTofu does not create it.** jupiter runs Talos on
|
||||
bare metal, so this step only provisions k8s-2 (saturn) and k8s-3 (neptune).
|
||||
Before Step 2, boot jupiter from the bare-metal image below so it comes up in
|
||||
maintenance mode and answers on `k8s-1.srv.renner.uno` — `bootstrap.sh` treats
|
||||
all three nodes identically from there, since `apply-config --insecure` does
|
||||
not care whether the target is a VM or bare metal.
|
||||
|
||||
```
|
||||
https://factory.talos.dev/image/a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64/v1.12.6/metal-amd64.iso
|
||||
```
|
||||
|
||||
That schematic is deliberately not the one `.woodpecker/deploy-infrastructure.yml`
|
||||
seeds onto the Proxmox nodes. It keeps the same `net.ifnames=1` kernel argument
|
||||
— without it the predictable interface names that `talos/patches/k8s-1.yaml`
|
||||
matches its VLAN parents against do not exist — and differs in three ways:
|
||||
|
||||
- drops `siderolabs/qemu-guest-agent`, which has nothing to talk to on bare metal
|
||||
- adds `siderolabs/intel-ucode`, since no hypervisor is supplying microcode now
|
||||
- adds `siderolabs/thunderbolt`, unused so far and inert until
|
||||
`machine.kernel.modules` requests `thunderbolt`/`thunderbolt_net`. It is
|
||||
present only so enabling the MS-01 Thunderbolt mesh later does not require a
|
||||
new image and a reinstall. Note its udev rule auto-authorises any Thunderbolt
|
||||
device attached to the machine once those modules are loaded.
|
||||
|
||||
## Step 2 — talos: form the cluster + get kubeconfig
|
||||
|
||||
```sh
|
||||
|
||||
@@ -121,9 +121,7 @@ spec:
|
||||
persistentVolumeClaim:
|
||||
claimName: audiobookshelf-books
|
||||
- name: le-ca
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -1326,121 +1326,7 @@ data:
|
||||
name: Caddy docs.renner.uno
|
||||
state: absent
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# wordpress-gate — www.renner.uno
|
||||
#
|
||||
# Unlike every other gate here, this one does NOT front a whole
|
||||
# host. www.renner.uno is a public family blog; only /wp-admin and
|
||||
# /wp-login.php are routed to this outpost (see
|
||||
# kubernetes/apps/wordpress/ingress.yaml). The front end never
|
||||
# touches it.
|
||||
#
|
||||
# The reverse carve-out — gate everything, skip_path_regex the
|
||||
# public parts, as paperless-gate does for /static/ — is not
|
||||
# possible here: the public set is every post URL, so the pattern
|
||||
# would need a negative lookahead and the outpost's RE2 has none.
|
||||
- id: wordpress-authors-group
|
||||
model: authentik_core.group
|
||||
identifiers:
|
||||
name: WordPress Authors
|
||||
state: present
|
||||
attrs:
|
||||
is_superuser: false
|
||||
# Git owns membership, as with the other gates: this group is the
|
||||
# explicit allowlist for who may publish on www.renner.uno.
|
||||
# Add/remove members HERE, not in the UI — blueprint apply is
|
||||
# authoritative for this M2M and reverts UI edits.
|
||||
#
|
||||
# s.renner is the Authentik account for the site's WordPress
|
||||
# Editor, which still carries the pre-2019 login `s.birzer` — the
|
||||
# two names do not match, and that is expected.
|
||||
users:
|
||||
- !Find [authentik_core.user, [username, k.renner]]
|
||||
- !Find [authentik_core.user, [username, s.renner]]
|
||||
|
||||
- id: wordpress-gate-provider
|
||||
model: authentik_providers_proxy.proxyprovider
|
||||
identifiers:
|
||||
name: wordpress-gate
|
||||
state: present
|
||||
attrs:
|
||||
authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
|
||||
invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
|
||||
mode: proxy
|
||||
internal_host: http://wordpress.wordpress.svc.cluster.local
|
||||
internal_host_ssl_validation: false
|
||||
external_host: https://www.renner.uno
|
||||
cookie_domain: renner.uno
|
||||
access_token_validity: hours=24
|
||||
refresh_token_validity: days=30
|
||||
intercept_header_auth: true
|
||||
# No skip_path_regex: wp-admin's assets are only ever fetched by an
|
||||
# already-authenticated session, and wp-admin has no manifest or
|
||||
# service worker — the credentials:'omit' PWA problem that forced
|
||||
# carve-outs on vscode-gate and retired music-assistant-gate does
|
||||
# not arise here.
|
||||
|
||||
- model: authentik_core.application
|
||||
identifiers:
|
||||
slug: wordpress-gate
|
||||
state: present
|
||||
attrs:
|
||||
name: WordPress (Gate)
|
||||
slug: wordpress-gate
|
||||
provider: !KeyOf wordpress-gate-provider
|
||||
meta_launch_url: https://www.renner.uno/wp-admin/
|
||||
meta_description: "Forward-auth gate in front of wp-admin on www.renner.uno"
|
||||
|
||||
# Only WordPress Authors pass the gate — see the group above for the
|
||||
# membership list. Both accounts that exist on the migrated site are
|
||||
# covered, so the gate cannot lock out the editor the site is
|
||||
# currently being kept alive for.
|
||||
- model: authentik_policies.policybinding
|
||||
identifiers:
|
||||
target: !Find [authentik_core.application, [slug, wordpress-gate]]
|
||||
group: !KeyOf wordpress-authors-group
|
||||
order: 0
|
||||
state: present
|
||||
attrs:
|
||||
enabled: true
|
||||
negate: false
|
||||
failure_result: false
|
||||
|
||||
- model: authentik_outposts.outpost
|
||||
identifiers:
|
||||
name: wordpress-gate
|
||||
state: present
|
||||
attrs:
|
||||
type: proxy
|
||||
service_connection: !KeyOf k8s-conn
|
||||
providers:
|
||||
- !KeyOf wordpress-gate-provider
|
||||
config:
|
||||
log_level: info
|
||||
authentik_host: "http://authentik-server.authentik.svc.cluster.local"
|
||||
authentik_host_insecure: false
|
||||
authentik_host_browser: "https://auth.renner.uno/"
|
||||
refresh_interval: "minutes=5"
|
||||
container_image: null
|
||||
object_naming_template: "ak-outpost-%(name)s"
|
||||
kubernetes_namespace: authentik
|
||||
kubernetes_replicas: 1
|
||||
kubernetes_service_type: ClusterIP
|
||||
kubernetes_disabled_components: ["ingress"]
|
||||
kubernetes_image_pull_secrets: []
|
||||
kubernetes_ingress_annotations: {}
|
||||
kubernetes_ingress_class_name: null
|
||||
kubernetes_ingress_secret_name: "authentik-outpost-tls"
|
||||
kubernetes_json_patches: null
|
||||
|
||||
# Passkey login for selected apps. Passkey is mandatory for accounts
|
||||
# that hold one and unavailable to accounts that do not, so both flows
|
||||
# branch on enrollment (policy passkey-user-has-webauthn): enrolled →
|
||||
# WebAuthn, not enrolled → password + MFA. Enrollment status is a
|
||||
# property of the identified account, so the weaker branch is never
|
||||
# something a requester can select.
|
||||
#
|
||||
# Two parts:
|
||||
# Passkey-only (WebAuthn) login for selected apps, in two parts:
|
||||
#
|
||||
# 1. A passwordless AUTHENTICATION flow (username → passkey, no
|
||||
# password prompt) bound per-provider, so fresh logins to these
|
||||
@@ -1449,13 +1335,7 @@ data:
|
||||
# 2. A step-up AUTHORIZATION flow — unlike the authentication flow,
|
||||
# this runs on EVERY interactive authorize, so an existing
|
||||
# password-only SSO session from another app cannot slip into
|
||||
# these apps without presenting a passkey. Two things narrow that:
|
||||
# the stage is skipped outright for accounts with no passkey (they
|
||||
# have no way to satisfy it), and last_auth_threshold reduces it to
|
||||
# "without presenting a passkey in this browser within the window"
|
||||
# — one touch then covers every app, including a later
|
||||
# password-only session for the same user. The cookie is keyed to
|
||||
# that user's device, so it never carries across users.
|
||||
# these apps without presenting a passkey.
|
||||
passkey-stepup.yaml: |
|
||||
version: 1
|
||||
metadata:
|
||||
@@ -1476,99 +1356,43 @@ data:
|
||||
# flow this replaces.
|
||||
authentication: require_authenticated
|
||||
|
||||
# ONE stage object, shared by the step-up authorization flow and
|
||||
# the passwordless authentication flow below.
|
||||
#
|
||||
# `last_auth_threshold` is enforced with a single browser cookie
|
||||
# (`authentik_mfa`) whose JWT is both signed with and carries the
|
||||
# stage's PK, so a validation stage only ever honours a cookie it
|
||||
# wrote itself. Two separate stage objects therefore each demand
|
||||
# their own touch, and — sharing the one cookie name — overwrite
|
||||
# each other, so neither window survives. Sharing this object is
|
||||
# what lets the login touch satisfy the per-app step-up.
|
||||
#
|
||||
# Declared here, ahead of every !KeyOf reference to it: the
|
||||
# blueprint importer resolves entries top-to-bottom.
|
||||
- id: passkey-webauthn-validation
|
||||
- id: passkey-stepup-webauthn-validation
|
||||
model: authentik_stages_authenticator_validate.authenticatorvalidatestage
|
||||
identifiers:
|
||||
name: passkey-webauthn-validation
|
||||
name: passkey-stepup-webauthn-validation
|
||||
state: present
|
||||
attrs:
|
||||
device_classes:
|
||||
- webauthn
|
||||
# `deny`, not `configure`: the authentication flow reaches this
|
||||
# stage with the user merely *identified* (a typed username)
|
||||
# and no password stage in front of it, so inline enrollment
|
||||
# would let anyone who knows a passkey-less username enroll
|
||||
# their own passkey and take over that account. The policy
|
||||
# binding means only accounts that already hold a passkey reach
|
||||
# this stage at all, so `deny` is defence in depth rather than
|
||||
# a path anyone walks — accounts without one branch to password
|
||||
# + MFA instead.
|
||||
not_configured_action: deny
|
||||
# Window within which an already-presented passkey satisfies a
|
||||
# later run of this stage. Non-zero for two reasons: (1) the
|
||||
# per-app step-up must not demand a second touch seconds after
|
||||
# the login touch; (2) NetBird's dashboard silently
|
||||
# re-authorizes via /silent-auth (prompt=none) on token expiry —
|
||||
# a stage that always demands interaction fails those and
|
||||
# forces a visible re-login every access-token lifetime.
|
||||
#
|
||||
# The cookie is browser-scoped, so inside the window a re-login
|
||||
# in that same browser is satisfied by the cookie alone and the
|
||||
# username is all that gets typed. Accepted for a single touch
|
||||
# per app.
|
||||
# `configure`, not `deny`: g.renner has Home Assistant access
|
||||
# but no enrolled WebAuthn device (live check 2026-07-07), so
|
||||
# a hard deny would lock her out with no self-service path.
|
||||
# `configure` injects the default WebAuthn enrollment stage
|
||||
# inline — first login without a passkey becomes "enroll one
|
||||
# now, then validate it". Still passkey-only: there is no way
|
||||
# through this stage without a WebAuthn credential.
|
||||
not_configured_action: configure
|
||||
configuration_stages:
|
||||
- !Find [authentik_stages_authenticator_webauthn.authenticatorwebauthnstage, [name, default-authenticator-webauthn-setup]]
|
||||
# Skip re-validation if a passkey was already presented within
|
||||
# the window. Non-zero for two reasons: (1) no double-touch
|
||||
# when the login itself used a passkey seconds earlier;
|
||||
# (2) NetBird's dashboard silently re-authorizes via
|
||||
# /silent-auth (prompt=none) on token expiry — a flow that
|
||||
# always demands interaction would fail those and force a
|
||||
# visible re-login every access-token lifetime.
|
||||
last_auth_threshold: hours=8
|
||||
webauthn_user_verification: preferred
|
||||
|
||||
# Gate on "this account has a passkey", so enrollment status picks
|
||||
# the branch. Enrolled accounts must use WebAuthn — at login and at
|
||||
# every authorize — while accounts with no passkey fall back to
|
||||
# password + MFA. An attacker cannot pick the weaker branch: the
|
||||
# user is already identified when this runs, so the branch follows
|
||||
# the victim's enrollment, not the requester's choice.
|
||||
#
|
||||
# request.user is the flow's pending_user, not the (anonymous)
|
||||
# requester: ReevaluateMarker builds the engine with
|
||||
# plan.context[PLAN_CONTEXT_PENDING_USER] (flows/markers.py). That
|
||||
# requires the bindings below to keep evaluate_on_plan false and
|
||||
# re_evaluate_policies true — on-plan evaluation happens before the
|
||||
# identification stage has resolved a user.
|
||||
- id: policy-has-passkey
|
||||
model: authentik_policies_expression.expressionpolicy
|
||||
identifiers:
|
||||
name: passkey-user-has-webauthn
|
||||
state: present
|
||||
attrs:
|
||||
expression: |
|
||||
return ak_user_has_authenticator(request.user, "webauthn")
|
||||
|
||||
- id: binding-stepup-webauthn
|
||||
model: authentik_flows.flowstagebinding
|
||||
- model: authentik_flows.flowstagebinding
|
||||
identifiers:
|
||||
target: !KeyOf passkey-stepup-flow
|
||||
stage: !KeyOf passkey-webauthn-validation
|
||||
stage: !KeyOf passkey-stepup-webauthn-validation
|
||||
order: 10
|
||||
state: present
|
||||
attrs:
|
||||
evaluate_on_plan: false
|
||||
re_evaluate_policies: true
|
||||
|
||||
- model: authentik_policies.policybinding
|
||||
identifiers:
|
||||
policy: !KeyOf policy-has-passkey
|
||||
target: !KeyOf binding-stepup-webauthn
|
||||
order: 0
|
||||
state: present
|
||||
attrs:
|
||||
enabled: true
|
||||
negate: false
|
||||
failure_result: false
|
||||
|
||||
# Passwordless authentication flow: identification (username or
|
||||
# email, NO password stage) → WebAuthn validation → login, with a
|
||||
# password + MFA branch for accounts that have no passkey.
|
||||
# email, NO password stage) → WebAuthn validation → login.
|
||||
- id: passkey-passwordless-flow
|
||||
model: authentik_flows.flow
|
||||
identifiers:
|
||||
@@ -1579,6 +1403,37 @@ data:
|
||||
title: "Sign in with your passkey"
|
||||
designation: authentication
|
||||
|
||||
# SEPARATE stage from the step-up one, and deliberately `deny`:
|
||||
# at this point in an authentication flow the user has only been
|
||||
# *identified* (typed a username), not authenticated — `configure`
|
||||
# here would let anyone who knows a passkey-less username enroll
|
||||
# their own passkey and take over that account. Inline enrollment
|
||||
# for passkey-less users happens exclusively in the authorization
|
||||
# step-up stage above, where require_authenticated guarantees a
|
||||
# fully authenticated session. A passkey-less user therefore
|
||||
# bootstraps via password login on auth.renner.uno, then gets the
|
||||
# enrollment prompt on first app access.
|
||||
#
|
||||
# Declared BEFORE the identification stage because the latter
|
||||
# references it via !KeyOf (webauthn_stage), and the blueprint
|
||||
# importer resolves entries top-to-bottom.
|
||||
- id: passkey-passwordless-webauthn-validation
|
||||
model: authentik_stages_authenticator_validate.authenticatorvalidatestage
|
||||
identifiers:
|
||||
name: passkey-passwordless-webauthn-validation
|
||||
state: present
|
||||
attrs:
|
||||
device_classes:
|
||||
- webauthn
|
||||
not_configured_action: deny
|
||||
# Small non-zero window so a passkey presented via the
|
||||
# identification stage's autofill (webauthn_stage below) is
|
||||
# not challenged a second time by this stage at order 20 in
|
||||
# the same flow run. The skip is session-scoped, so a fresh
|
||||
# (attacker) session never inherits it.
|
||||
last_auth_threshold: minutes=5
|
||||
webauthn_user_verification: preferred
|
||||
|
||||
# webauthn_stage enables passkey autofill (conditional WebAuthn,
|
||||
# authentik >= 2025.12): the browser offers enrolled passkeys in
|
||||
# the username field's autofill — one tap, no username typed.
|
||||
@@ -1594,7 +1449,7 @@ data:
|
||||
user_fields:
|
||||
- username
|
||||
- email
|
||||
webauthn_stage: !KeyOf passkey-webauthn-validation
|
||||
webauthn_stage: !KeyOf passkey-passwordless-webauthn-validation
|
||||
|
||||
- model: authentik_flows.flowstagebinding
|
||||
identifiers:
|
||||
@@ -1603,87 +1458,12 @@ data:
|
||||
order: 10
|
||||
state: present
|
||||
|
||||
# Order 20 and 21/22 are the two mutually exclusive branches.
|
||||
- id: binding-passwordless-webauthn
|
||||
model: authentik_flows.flowstagebinding
|
||||
- model: authentik_flows.flowstagebinding
|
||||
identifiers:
|
||||
target: !KeyOf passkey-passwordless-flow
|
||||
stage: !KeyOf passkey-webauthn-validation
|
||||
stage: !KeyOf passkey-passwordless-webauthn-validation
|
||||
order: 20
|
||||
state: present
|
||||
attrs:
|
||||
evaluate_on_plan: false
|
||||
re_evaluate_policies: true
|
||||
|
||||
- model: authentik_policies.policybinding
|
||||
identifiers:
|
||||
policy: !KeyOf policy-has-passkey
|
||||
target: !KeyOf binding-passwordless-webauthn
|
||||
order: 0
|
||||
state: present
|
||||
attrs:
|
||||
enabled: true
|
||||
negate: false
|
||||
failure_result: false
|
||||
|
||||
# The negated branch. failure_result: true is the safety net — the
|
||||
# UserLoginStage defaults a missing authentication backend to
|
||||
# BACKEND_INBUILT instead of refusing (user_login/stage.py), so a
|
||||
# flow that drops BOTH branches would log the user in on a typed
|
||||
# username alone. A PolicyException is resolved to failure_result
|
||||
# before negate is applied (policies/process.py), so a broken
|
||||
# expression pulls the password stage IN and degrades everyone to
|
||||
# password + MFA rather than to no factor at all.
|
||||
- id: binding-passwordless-password
|
||||
model: authentik_flows.flowstagebinding
|
||||
identifiers:
|
||||
target: !KeyOf passkey-passwordless-flow
|
||||
stage: !Find [authentik_stages_password.passwordstage, [name, default-authentication-password]]
|
||||
order: 21
|
||||
state: present
|
||||
attrs:
|
||||
evaluate_on_plan: false
|
||||
re_evaluate_policies: true
|
||||
|
||||
- model: authentik_policies.policybinding
|
||||
identifiers:
|
||||
policy: !KeyOf policy-has-passkey
|
||||
target: !KeyOf binding-passwordless-password
|
||||
order: 0
|
||||
state: present
|
||||
attrs:
|
||||
enabled: true
|
||||
negate: true
|
||||
failure_result: true
|
||||
|
||||
# Same MFA validation the default authentication flow runs at its
|
||||
# order 30, so the fallback is not a weaker way into these apps
|
||||
# than auth.renner.uno's own login already is. It carries
|
||||
# not_configured_action: configure, which is safe here and nowhere
|
||||
# else in this flow: the password stage above has established a
|
||||
# first factor by the time it runs, so it doubles as the enrollment
|
||||
# path for accounts holding no authenticator at all.
|
||||
- id: binding-passwordless-mfa
|
||||
model: authentik_flows.flowstagebinding
|
||||
identifiers:
|
||||
target: !KeyOf passkey-passwordless-flow
|
||||
stage: !Find [authentik_stages_authenticator_validate.authenticatorvalidatestage, [name, default-authentication-mfa-validation]]
|
||||
order: 22
|
||||
state: present
|
||||
attrs:
|
||||
evaluate_on_plan: false
|
||||
re_evaluate_policies: true
|
||||
|
||||
- model: authentik_policies.policybinding
|
||||
identifiers:
|
||||
policy: !KeyOf policy-has-passkey
|
||||
target: !KeyOf binding-passwordless-mfa
|
||||
order: 0
|
||||
state: present
|
||||
attrs:
|
||||
enabled: true
|
||||
negate: true
|
||||
failure_result: true
|
||||
|
||||
- model: authentik_flows.flowstagebinding
|
||||
identifiers:
|
||||
@@ -1797,19 +1577,3 @@ data:
|
||||
redirect_uris:
|
||||
- url: https://dash.renner.uno/login/generic_oauth
|
||||
matching_mode: strict
|
||||
|
||||
# Superseded by the single passkey-webauthn-validation stage above.
|
||||
# Their flowstagebindings are FK-CASCADEd away with them; the
|
||||
# identification stage's webauthn_stage FK is SET_NULL and is
|
||||
# re-pointed above, so no dangling reference survives. Declared
|
||||
# last: the importer applies entries top-to-bottom inside one
|
||||
# transaction, so the replacements exist before these go.
|
||||
- model: authentik_stages_authenticator_validate.authenticatorvalidatestage
|
||||
identifiers:
|
||||
name: passkey-stepup-webauthn-validation
|
||||
state: absent
|
||||
|
||||
- model: authentik_stages_authenticator_validate.authenticatorvalidatestage
|
||||
identifiers:
|
||||
name: passkey-passwordless-webauthn-validation
|
||||
state: absent
|
||||
|
||||
@@ -43,44 +43,6 @@ spec:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_connections: "400"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
#
|
||||
# checkpoint_timeout is the lever. The 5min default pushes every dirty
|
||||
# page to disk 288 times a day, so a small database whose same few pages
|
||||
# are re-dirtied constantly (sessions, tokens, job state) gets rewritten
|
||||
# over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
|
||||
# (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
|
||||
# day. At 15min those pages coalesce and are written a third as often.
|
||||
#
|
||||
# max_wal_size must rise with it, or checkpoints trigger on size instead
|
||||
# and the timeout change does nothing. It also bounds crash recovery to
|
||||
# at most 2GB of WAL replay.
|
||||
#
|
||||
# NOT changed: synchronous_commit stays at the default `on`. It is off for
|
||||
# ha-pg because a state recorder is telemetry; these hold credentials,
|
||||
# documents and job state, where losing the last few hundred ms of commits
|
||||
# on an unclean shutdown is not an acceptable trade.
|
||||
checkpoint_timeout: "15min"
|
||||
max_wal_size: "2GB"
|
||||
# zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
|
||||
# and slower than the lz4/zstd this build also accepts). Compresses the
|
||||
# full-page images that dominate WAL volume after each checkpoint.
|
||||
wal_compression: "zstd"
|
||||
# A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
|
||||
# on a timer -- the segment is then written and archived at its full size
|
||||
# no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
|
||||
# to 288 segments/day = 3.2 GB/day here, most of it padding.
|
||||
#
|
||||
# 15min, NOT the 30min used elsewhere in this rollout. This cluster holds
|
||||
# credentials, sessions and enrolment state, so its RPO is worth more than
|
||||
# the difference: 30min would save a further 0.3 GB/day, which does not
|
||||
# justify doubling the worst-case loss window on this data.
|
||||
#
|
||||
# It does carry a synchronous standby (instances: 2), so the realistic
|
||||
# exposure is losing both instances, not one -- but the conservative value
|
||||
# is cheap here and the aggressive one buys almost nothing.
|
||||
archive_timeout: "15min"
|
||||
# Async replication makes CNPG switchover slow because the standby has
|
||||
# to chase missing WAL via barman-cloud and replay queued records
|
||||
# before it can promote. With one synchronous standby, the standby is
|
||||
|
||||
@@ -14,7 +14,6 @@ resources:
|
||||
- zone-configurator-gate-egress.yaml
|
||||
- alertmanager-gate-egress.yaml
|
||||
- prometheus-gate-egress.yaml
|
||||
- wordpress-gate-egress.yaml
|
||||
- outpost-servicemonitors.yaml
|
||||
- ldap-outpost-cert.yaml
|
||||
- ldap-outpost-lb.yaml
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
# Allow the dedicated wordpress-gate outpost to egress into the wordpress
|
||||
# namespace and reach WordPress (:80). Without this, the authentik-namespace
|
||||
# baseline CNP has no egress rule to wordpress, so the gate outpost's
|
||||
# forwarded requests are dropped. Scoped to the gate outpost's pod label so no
|
||||
# other authentik component can reach it. Mirrors esphome-gate-egress.yaml.
|
||||
#
|
||||
# Note this gate only ever sees /wp-admin and /wp-login.php — the public site
|
||||
# is routed straight to the same Service by a separate Ingress, so this rule
|
||||
# is not the only path into the wordpress namespace, unlike the paperless case.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: wordpress-gate-egress
|
||||
namespace: authentik
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
goauthentik.io/outpost-name: wordpress-gate
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": wordpress
|
||||
app.kubernetes.io/name: wordpress
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "80"
|
||||
protocol: TCP
|
||||
@@ -29,52 +29,6 @@ spec:
|
||||
parameters:
|
||||
shared_buffers: "128MB"
|
||||
max_connections: "50"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
#
|
||||
# checkpoint_timeout is the lever. The 5min default pushes every dirty
|
||||
# page to disk 288 times a day, so a small database whose same few pages
|
||||
# are re-dirtied constantly (sessions, tokens, job state) gets rewritten
|
||||
# over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
|
||||
# (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
|
||||
# day. At 15min those pages coalesce and are written a third as often.
|
||||
#
|
||||
# max_wal_size must rise with it, or checkpoints trigger on size instead
|
||||
# and the timeout change does nothing. It also bounds crash recovery to
|
||||
# at most 2GB of WAL replay.
|
||||
#
|
||||
# NOT changed: synchronous_commit stays at the default `on`. It is off for
|
||||
# ha-pg because a state recorder is telemetry; these hold credentials,
|
||||
# documents and job state, where losing the last few hundred ms of commits
|
||||
# on an unclean shutdown is not an acceptable trade.
|
||||
checkpoint_timeout: "15min"
|
||||
max_wal_size: "2GB"
|
||||
# zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
|
||||
# and slower than the lz4/zstd this build also accepts). Compresses the
|
||||
# full-page images that dominate WAL volume after each checkpoint.
|
||||
wal_compression: "zstd"
|
||||
# THE dominant write source on this cluster, by three orders of magnitude.
|
||||
#
|
||||
# A WAL segment is a fixed 16 MiB file. archive_timeout forces a segment
|
||||
# switch on a timer so the archive never lags reality by more than that
|
||||
# interval -- and the switched-out segment is written and archived at its
|
||||
# full 16 MiB regardless of how few bytes of real WAL it contains.
|
||||
#
|
||||
# CNPG defaults this to 5min, i.e. up to 288 segments/day = 4.6 GB/day.
|
||||
# Measured here: this database is 21 MB and generates 4.5 MB/day of actual
|
||||
# WAL, while its RBD volume takes 211 MB/hour. Essentially every byte
|
||||
# written is padding -- roughly 1100x amplification.
|
||||
#
|
||||
# 30min cuts that to 48 segments/day. The cost is RPO: on total loss of
|
||||
# the cluster, up to 30min of Grafana state is unrecoverable rather than
|
||||
# 5min. Grafana's state here is dashboards, annotations and preferences,
|
||||
# and the write rate says it changes ~4.5 MB/day -- so that window is
|
||||
# nearly always empty anyway.
|
||||
#
|
||||
# Applied to gf-pg FIRST, alone, as a staged rollout: it is the lowest-risk
|
||||
# cluster and has the cleanest signal, since its real WAL is small enough
|
||||
# that any change in volume is unambiguously this parameter.
|
||||
archive_timeout: "30min"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
# Home Assistant recorder: SQLite → Postgres
|
||||
|
||||
## Why
|
||||
|
||||
The recorder's SQLite database is the single largest client writer in the
|
||||
cluster. Measured 2026-08-19 (cgroup `io.stat` on the home-assistant container,
|
||||
corroborated by kernel diskstats and Ceph pool counters):
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| HA writes to its RBD volume | **40 GB/day** |
|
||||
| share of all client writes | **~26%** |
|
||||
| actual row data underneath | **~5 GB/day** |
|
||||
|
||||
The 8× gap is the write *pattern*, not the volume:
|
||||
|
||||
```
|
||||
~30 state changes/sec (107k rows/hour, 2.57M/day)
|
||||
×2 SQLite WAL + checkpoint — every dirtied page written twice
|
||||
×4 ext4 journal + commit, because synchronous=FULL fsyncs every commit
|
||||
(measured: 819 KB of 4 KiB writes → 3.28 MB at the block device)
|
||||
= ~15 KB written per recorded state change
|
||||
```
|
||||
|
||||
Postgres replaces scattered fsync'd page rewrites with sequential batched WAL.
|
||||
This attacks the multiplier rather than trimming what gets recorded — excluding
|
||||
entities would hide the problem, not fix it.
|
||||
|
||||
**Context:** this is one of three write-reduction changes. The other two are
|
||||
merged and verified — `size 3→2` on the Ceph pools (#519) and
|
||||
`mon_compact_on_trim=false` (#520, which alone removed ~228 GB/day). See #434.
|
||||
|
||||
## What is at risk
|
||||
|
||||
`states` is disposable — `purge_keep_days: 7`, so it regenerates in a week.
|
||||
|
||||
**`statistics` is not.** As of 2026-08-19:
|
||||
|
||||
| table | rows | note |
|
||||
|---|---|---|
|
||||
| `statistics` | 10,793,712 | **spans 2024-03-13 → today, 2.4 years** |
|
||||
| `statistics_short_term` | 1,902,624 | 5-minute buckets |
|
||||
| `statistics_meta` | 1,448 | entity ↔ id mapping, required by both |
|
||||
| `states` | 18,289,190 | disposable, 7-day retention |
|
||||
| `events` | 483,989 | disposable |
|
||||
|
||||
**Home Assistant does not migrate data between database engines.** Repointing
|
||||
`db_url` at an empty Postgres silently starts from zero and the energy dashboard
|
||||
loses its entire history. The long-term statistics are irreplaceable — they are
|
||||
not in InfluxDB, which HA does not read for the energy dashboard.
|
||||
|
||||
## Phase 1 — deploy the cluster (this PR)
|
||||
|
||||
`ha-pg` comes up **empty** and nothing consumes it. HA stays on SQLite, so a
|
||||
failure here is inert and there is nothing to roll back.
|
||||
|
||||
Verify before going further:
|
||||
|
||||
```sh
|
||||
kubectl -n home-assistant get cluster ha-pg # Cluster in healthy state, 1/1
|
||||
kubectl -n home-assistant get backup # ha-pg-daily Completed (immediate: true)
|
||||
kubectl -n home-assistant logs ha-pg-1 -c postgres | grep -i archive # WAL archiving, no errors
|
||||
```
|
||||
|
||||
Confirm the backup actually landed in the bucket rather than trusting the CR
|
||||
status — the WordPress backup failed nightly for three days against a bucket
|
||||
that was declared but never created (see `opentofu/object_storage.tf`).
|
||||
|
||||
## Phase 2b — bulk statistics copy: DONE 2026-08-19
|
||||
|
||||
Completed with **HA running** — the SQLite file was read through a read-only
|
||||
mount, and SQLite's WAL mode gives readers a consistent snapshot, so no downtime
|
||||
was needed for the bulk copy.
|
||||
|
||||
```
|
||||
statistics_meta: sqlite=1449 pg=1449 OK
|
||||
statistics: sqlite=10799904 pg=10799904 OK (153.9s)
|
||||
statistics_short_term: sqlite=1969804 pg=1969804 OK ( 27.0s)
|
||||
|
||||
statistics span in Postgres: 2024-03-13 -> 2026-08-19
|
||||
entities with long-term history: 1449
|
||||
rows carrying sum (energy dashboard): 2627859
|
||||
database size: 2252 MB (PVC is 20Gi)
|
||||
```
|
||||
|
||||
`ANALYZE` was run on all three tables. That is not housekeeping: after a bulk
|
||||
`COPY` the planner has no statistics, and HA queries these tables constantly for
|
||||
history and energy views, so without it every view seq-scans 10.8M rows.
|
||||
|
||||
Two schema details that cost a failed first attempt, recorded so the top-up does
|
||||
not repeat them:
|
||||
|
||||
- `has_mean` / `has_sum` are INTEGER in SQLite and BOOLEAN in Postgres. Postgres
|
||||
will not coerce; convert explicitly and preserve NULL.
|
||||
- The legacy `created` / `start` / `last_reset` DATETIME columns are 100% NULL in
|
||||
both bulk tables (superseded by the `_ts` variants), which is what makes a
|
||||
tab-separated `COPY` safe — nothing in those tables needs escaping.
|
||||
|
||||
The schema itself was created with HA's own models
|
||||
(`Base.metadata.create_all`), **not** hand-written DDL, and `schema_changes` was
|
||||
left EMPTY on purpose. HA's `_initialize_database` detects a new database by
|
||||
looking for an index on `events.time_fired_ts`, then writes `SCHEMA_VERSION`
|
||||
itself. Pre-stamping that row would skip HA's own initialisation path.
|
||||
|
||||
### High-water marks for the cutover top-up
|
||||
|
||||
Rows written after the bulk copy are not yet in Postgres. Resume from:
|
||||
|
||||
```
|
||||
statistics max_id=13323904 max_start_ts=1787162400.0
|
||||
statistics_short_term max_id=159205762 max_start_ts=1787166000.0
|
||||
```
|
||||
|
||||
## Phase 2c — cutover (REMAINING)
|
||||
|
||||
Do not start until phase 1 is verified.
|
||||
|
||||
**Migrate statistics only.** Copying all 18M `states` rows is slow, needs a long
|
||||
HA outage, and buys a table that is discarded within 7 days anyway.
|
||||
|
||||
1. **Snapshot first.** Velero backup of the HA PVC, plus a `.backup` copy of
|
||||
`home-assistant_v2.db`. This is the rollback.
|
||||
2. **Stop the recorder** — scale `deploy/home-assistant` to 0. The database must
|
||||
be quiescent; copying a live SQLite file gives a torn read.
|
||||
3. **Create the schema.** Start HA once against the empty Postgres so it runs
|
||||
its own migrations. Never hand-create the tables — the schema version must
|
||||
match the HA version exactly or HA will refuse to start or silently migrate.
|
||||
4. **Copy** `statistics_meta` → `statistics` → `statistics_short_term`, in that
|
||||
order (the other two carry `metadata_id` foreign keys into the first).
|
||||
Sequences must be reset afterwards, or the first insert collides on a
|
||||
duplicate primary key:
|
||||
```sql
|
||||
SELECT setval('statistics_id_seq', (SELECT MAX(id) FROM statistics));
|
||||
SELECT setval('statistics_meta_id_seq', (SELECT MAX(id) FROM statistics_meta));
|
||||
SELECT setval('statistics_short_term_id_seq', (SELECT MAX(id) FROM statistics_short_term));
|
||||
```
|
||||
5. **Repoint** `recorder.db_url` in the `homeassistant` repo — that change lives
|
||||
there, not here. The `recorder_db_url` secret it references is already
|
||||
rendered into `secrets.yaml` by the `render-db-url` init container in
|
||||
`deployment.yaml` (phase 2a), so this order is mandatory: **the init
|
||||
container must be deployed first**, or HA refuses to start on a missing
|
||||
secret.
|
||||
|
||||
The init container percent-encodes the credentials and validates the
|
||||
resulting YAML rather than counting lines. That is not incidental — the CNPG
|
||||
password is random, and a naive `"postgresql://user:pass@host/db"` string
|
||||
makes `secrets.yaml` unparseable the moment the password contains a quote,
|
||||
which takes out *every* secret HA loads, not just this one. Verified against
|
||||
a deliberately hostile password before merge.
|
||||
```yaml
|
||||
recorder:
|
||||
db_url: !secret recorder_db_url # postgresql://homeassistant:<pw>@ha-pg-rw.home-assistant.svc.cluster.local/homeassistant
|
||||
```
|
||||
The password is in the CNPG-generated `ha-pg-app` Secret. It must not be
|
||||
committed — render it into `secrets.yaml` the same way the Z2M MQTT password
|
||||
is handled in `zigbee2mqtt-deployment.yaml`.
|
||||
6. **Verify** the energy dashboard renders history back to 2024, not just today.
|
||||
That is the check that proves the statistics migration worked; "HA starts and
|
||||
records" does not.
|
||||
|
||||
### Rollback
|
||||
|
||||
Revert `db_url` to the SQLite path and restore the `.backup` copy. Keep the
|
||||
SQLite file until the energy dashboard has been confirmed over a full day —
|
||||
deleting it early removes the only rollback that preserves the 2.4 years.
|
||||
|
||||
## After the migration
|
||||
|
||||
- The HA config PVC shrinks from 5.5 GB to a few hundred MB. Do not shrink the
|
||||
PVC itself: `ceph-block` allows expansion, not reduction.
|
||||
- Re-measure. `kubectl -n home-assistant exec deploy/home-assistant -c
|
||||
home-assistant -- cat /sys/fs/cgroup/io.stat` was 40 GB/day before. If it does
|
||||
not drop substantially, the migration did not achieve its purpose and that
|
||||
should be recorded rather than assumed.
|
||||
- `synchronous_commit: off` in `ha-pg-cluster.yaml` is the single biggest
|
||||
contributor to the reduction. It is safe for telemetry (Postgres stays crash
|
||||
consistent; at most a few hundred ms of commits are lost on unclean shutdown)
|
||||
but it is a deliberate trade and should not be "corrected" later without
|
||||
understanding why it is set.
|
||||
@@ -36,67 +36,9 @@ spec:
|
||||
# (sml2mqtt bridge owns the USB device on edge), so it no longer needs
|
||||
# the edge USB device and schedules on the general cluster pool. This
|
||||
# gets the heavy HA pod (8Gi) off the soon-to-be-tiny edge box.
|
||||
initContainers:
|
||||
# Puts CNPG's connection URI into /config/secrets.yaml as
|
||||
# `recorder_db_url`, for the recorder's move off SQLite
|
||||
# (MIGRATION-recorder-postgres.md).
|
||||
#
|
||||
# HA has no way to read a db_url from the environment — `!env_var` was
|
||||
# removed from its YAML loader — so `!secret` is the only mechanism, and
|
||||
# secrets.yaml is PVC state that config-sync deliberately does not
|
||||
# manage. Hence a small init container rather than a env: entry.
|
||||
#
|
||||
# The URI comes STRAIGHT from the ha-pg-app Secret. CNPG builds and
|
||||
# percent-encodes it, so there is nothing to escape here: the value is
|
||||
# restricted to [A-Za-z0-9:/@._%-], verified against the live secret.
|
||||
# An earlier version of this assembled the URI from user/password/dbname
|
||||
# and needed percent-encoding plus YAML round-trip validation to be
|
||||
# safe. That was solving a problem CNPG had already solved.
|
||||
#
|
||||
# Re-rendered on every pod start, so a CNPG password rotation takes
|
||||
# effect rather than silently leaving a stale URI behind.
|
||||
#
|
||||
# INERT until the recorder is repointed: nothing reads this key until
|
||||
# `db_url: !secret recorder_db_url` lands in configuration.yaml in the
|
||||
# homeassistant repo. That ordering is required — HA refuses to start
|
||||
# when a referenced secret is missing.
|
||||
- name: render-db-url
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
S=/config/secrets.yaml
|
||||
T=/config/.secrets.render.tmp
|
||||
[ -f "$S" ] || : > "$S"
|
||||
# Filter-then-append, never rewrite in place: this file holds
|
||||
# every other HA secret (influxdb_token and friends).
|
||||
grep -v '^recorder_db_url:' "$S" > "$T" || true
|
||||
printf 'recorder_db_url: "%s"\n' "$DB_URI" >> "$T"
|
||||
# Refuse to shrink the file. Cheap guard against the one failure
|
||||
# that would actually hurt: losing the other keys.
|
||||
[ "$(wc -l < "$T")" -ge "$(wc -l < "$S")" ] || {
|
||||
echo "FATAL: rewrite would shrink secrets.yaml; aborting" >&2; exit 1; }
|
||||
chmod 600 "$T"
|
||||
mv "$T" "$S"
|
||||
echo "recorder_db_url rendered"
|
||||
env:
|
||||
- name: DB_URI
|
||||
valueFrom:
|
||||
secretKeyRef: {name: ha-pg-app, key: fqdn-uri}
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
memory: 32Mi
|
||||
containers:
|
||||
- name: home-assistant
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.8.1 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
env:
|
||||
- name: TZ
|
||||
value: Europe/Berlin
|
||||
@@ -279,13 +221,9 @@ spec:
|
||||
# Writable emptyDir for the nginx entrypoint's envsubst output
|
||||
# (/etc/nginx/conf.d/default.conf rendered from the .template).
|
||||
- name: proxy-conf-d
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
# code-server's settings/extensions cache. Not on the config PVC
|
||||
# so installed extensions don't pollute HA's /config listing.
|
||||
# Trade-off: extensions reset across pod restarts.
|
||||
- name: vscode-state
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
---
|
||||
# Postgres backend for the Home Assistant recorder.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# The recorder's SQLite database is the single largest client writer in the
|
||||
# cluster. Measured 2026-08-19 (cgroup io.stat on the home-assistant container,
|
||||
# corroborated by kernel diskstats): 40 GB/day to its RBD volume, ~26% of all
|
||||
# client writes, against only ~5 GB/day of actual row data. The gap is the write
|
||||
# PATTERN, not the volume:
|
||||
#
|
||||
# ~30 state changes/sec (107k rows/hour, 2.57M/day)
|
||||
# x2 SQLite WAL + checkpoint — every dirtied page is written twice
|
||||
# x4 ext4 journal + commit, because synchronous=FULL fsyncs every commit
|
||||
# (measured: 819 KB of 4 KiB writes -> 3.28 MB at the block device)
|
||||
# = ~15 KB written per recorded state change
|
||||
#
|
||||
# Postgres replaces scattered fsync'd page rewrites with sequential, batched WAL.
|
||||
# That is the point of the move — it attacks the multiplier rather than trimming
|
||||
# what gets recorded, which would only hide the problem.
|
||||
#
|
||||
# STAGING
|
||||
# Phase 1 (this manifest): the cluster comes up EMPTY and HA keeps using SQLite.
|
||||
# Nothing depends on it, so a failure here is inert. Verify Ready, WAL archiving
|
||||
# to Hetzner, and a successful base backup before going further.
|
||||
# Phase 2: migrate the statistics tables and repoint recorder.db_url.
|
||||
# See MIGRATION-recorder-postgres.md for the sequence and the rollback.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ha-pg
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
# Sized for the migrated statistics plus 7 days of states at the current
|
||||
# rate: statistics 10.8M rows + statistics_short_term 1.9M (~1.8 GB with
|
||||
# indexes in Postgres) and ~18M states rows for the purge_keep_days:7 window
|
||||
# (~4 GB). 20Gi leaves room for autovacuum bloat and index churn.
|
||||
# ceph-block has allowVolumeExpansion: true, so this can grow in place.
|
||||
size: 20Gi
|
||||
plugins:
|
||||
- name: barman-cloud.cloudnative-pg.io
|
||||
isWALArchiver: true
|
||||
parameters:
|
||||
barmanObjectName: ha-pg-store
|
||||
bootstrap:
|
||||
initdb:
|
||||
# Explicit, not left to CNPG's `app` default: the generated ha-pg-app
|
||||
# Secret and the -app Service point at whatever is named here, and the
|
||||
# recorder db_url in phase 2 must match. Same trap that made the wp-pg
|
||||
# namespace migration point credentials at an empty database (#433).
|
||||
database: homeassistant
|
||||
owner: homeassistant
|
||||
postgresql:
|
||||
parameters:
|
||||
# 512MB, not the 256MB this shipped with. Measured after 15h on Postgres:
|
||||
# buffers_backend was 629,704 of 939,025 total page writes — 67% written
|
||||
# by backend processes rather than the checkpointer or bgwriter. That
|
||||
# ratio means the working set does not fit in shared_buffers, so backends
|
||||
# evict dirty pages themselves and the same pages are written out
|
||||
# repeatedly instead of coalescing in memory.
|
||||
#
|
||||
# The recorder touches a wide set of pages continuously (~30 state
|
||||
# changes/sec across states, state_attributes and five indexes), which is
|
||||
# exactly the workload that punishes a small buffer pool.
|
||||
#
|
||||
# Memory headroom checked before raising: the instance was using 383Mi of
|
||||
# a 1Gi limit with 256MB of that being shared_buffers, so the limit goes
|
||||
# to 1.5Gi to keep the same margin.
|
||||
shared_buffers: "512MB"
|
||||
max_connections: "50"
|
||||
# The rest of this block is tuned to MINIMISE WRITE VOLUME, which is the
|
||||
# entire reason for the migration. Defaults would give a correct database
|
||||
# that writes far more than it needs to on replicated Ceph storage.
|
||||
#
|
||||
# synchronous_commit off: the recorder is telemetry, not transactional
|
||||
# record-keeping. This does NOT risk corruption — Postgres stays crash
|
||||
# consistent; it only means the last few hundred ms of commits can be lost
|
||||
# on an unclean shutdown, which for state history is a non-event. In
|
||||
# exchange every commit stops forcing an fsync, which is precisely the
|
||||
# cost that makes SQLite expensive here.
|
||||
synchronous_commit: "off"
|
||||
# Compress full-page images in WAL. Recorder writes are many small row
|
||||
# inserts scattered across pages, so full-page writes after each
|
||||
# checkpoint dominate WAL volume; compression cuts that directly.
|
||||
#
|
||||
# Full-page images really are ~62% of WAL here: 61.9% measured as a
|
||||
# steady-state delta over one checkpoint cycle, against 62.4% lifetime.
|
||||
#
|
||||
# But switching pglz -> zstd changed NOTHING. Measured by
|
||||
# pg_current_wal_lsn advance over 20min windows: 4911 and 4976 MB/day
|
||||
# before, 5002 MB/day after.
|
||||
#
|
||||
# The reason is that `wal_compression: "on"` was never "no compression" --
|
||||
# it is pglz, which was already compressing these page images. zstd only
|
||||
# changes the ratio, and on recorder pages that margin is too small to
|
||||
# measure. The original framing of this change (that the clusters "were
|
||||
# not getting compression") was wrong; they were.
|
||||
#
|
||||
# Note the 62% figure is an upper bound: it is computed as
|
||||
# wal_fpi * 8192 / wal_bytes, which assumes uncompressed 8 KiB pages. With
|
||||
# compression active the true byte share is lower, which is consistent
|
||||
# with there being little headroom left for a better algorithm.
|
||||
#
|
||||
# Kept because zstd is not worse than pglz and costs nothing. It is simply
|
||||
# not a lever. The real one on this cluster was archive_timeout, below.
|
||||
#
|
||||
# zstd, not "on": the boolean spelling resolves to pglz, the original
|
||||
# 2016-era algorithm, which is both slower and weaker than the two
|
||||
# alternatives PostgreSQL 16 accepts here (lz4, zstd). zstd gives the
|
||||
# best ratio on 8 KiB page images, and the CPU cost lands on a cluster
|
||||
# that is IO-bound on QLC NVMe, not CPU-bound.
|
||||
wal_compression: "zstd"
|
||||
# A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
|
||||
# on a timer -- the segment is then written and archived at its full size
|
||||
# no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
|
||||
# to 288 segments/day = 4.6 GB/day that is largely padding.
|
||||
#
|
||||
# Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
|
||||
# cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
|
||||
# Two independent signals moving together is what confirms the volume was
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
# Fewer checkpoints => fewer full-page writes, since a page is only
|
||||
# fully-logged on its first modification after a checkpoint.
|
||||
#
|
||||
# That reasoning did not show up in WAL volume: 15min -> 30min moved the
|
||||
# WAL rate not at all (4911/4976 -> 5002 MB/day). The likely reason is
|
||||
# that this workload is append-mostly -- a page is dirtied while it fills
|
||||
# and then not touched again, so its full-page image is paid once
|
||||
# regardless of how often checkpoints run. Interval only helps pages that
|
||||
# are re-dirtied across many cycles, which is not what a recorder does.
|
||||
#
|
||||
# Kept, but NOT because WAL improved. checkpoint_timeout also governs
|
||||
# data-file writes at checkpoint, and those are unmeasured here: this
|
||||
# volume takes ~11 GB/day against 4.9 GB/day of WAL, so most of its
|
||||
# traffic is outside what the WAL measurement covers. Revisit with a
|
||||
# data-file measurement before treating this value as justified.
|
||||
#
|
||||
# Recovery cost is bounded by how much WAL a single interval produces,
|
||||
# not by the interval itself: at the measured rate 30min is well under
|
||||
# 200MB of replay, i.e. seconds. max_wal_size stays 2GB precisely so the
|
||||
# timeout stays the binding trigger -- if size were the binding one, the
|
||||
# interval change would do nothing.
|
||||
checkpoint_timeout: "30min"
|
||||
max_wal_size: "2GB"
|
||||
min_wal_size: "512MB"
|
||||
# full_page_writes is deliberately LEFT ON (the default). RBD can tear a
|
||||
# page across a 4 MiB object boundary, and turning this off would trade
|
||||
# write volume for the risk of an unrecoverable database. Not worth it.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1536Mi
|
||||
@@ -1,38 +0,0 @@
|
||||
---
|
||||
# Barman S3 credentials for ha-pg.
|
||||
#
|
||||
# FOLLOW-UP (#397): sourced from seed/paperless/hetzner-creds, which is not where
|
||||
# they belong. woodpecker/hetzner-creds-externalsecret.yaml has the identical
|
||||
# problem and the same note. The values are the same today, so this is correct
|
||||
# but coupled: rotating paperless's Hetzner credentials would silently change
|
||||
# the backup credentials for Home Assistant and Woodpecker too.
|
||||
#
|
||||
# The fix is one dedicated path per consumer (seed/home-assistant/hetzner-creds)
|
||||
# and a one-line change to the `key:` below. Creating it needs an OpenBao write:
|
||||
#
|
||||
# bao kv put homelab/seed/home-assistant/hetzner-creds \
|
||||
# @<(bao kv get -format=json homelab/seed/paperless/hetzner-creds | jq '.data.data')
|
||||
#
|
||||
# Note the mount is `homelab`, so the path is homelab/seed/... — #397 records a
|
||||
# 403 on `bao kv list seed`, which was a wrong-path error rather than a
|
||||
# permissions problem.
|
||||
#
|
||||
# The namespace allowlist on the openbao ClusterSecretStore
|
||||
# (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
|
||||
# namespace or ESO silently never syncs (#505). home-assistant is already
|
||||
# allowlisted — it has several ExternalSecrets in this directory already.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: hetzner-s3-creds
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
# Barman object store for ha-pg: base backups + continuous WAL archiving to
|
||||
# Hetzner Object Storage, same target bucket as the other CNPG clusters.
|
||||
#
|
||||
# Its own prefix. Two clusters archiving WAL to one destinationPath interleave
|
||||
# their streams and corrupt both — see the note in
|
||||
# kubernetes/apps/woodpecker/objectstore.yaml, which learned that the hard way.
|
||||
apiVersion: barmancloud.cnpg.io/v1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: ha-pg-store
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
configuration:
|
||||
destinationPath: s3://renner-uno-k8s-cnpg/ha-pg
|
||||
endpointURL: https://fsn1.your-objectstorage.com
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_SECRET_KEY
|
||||
region:
|
||||
name: hetzner-s3-creds
|
||||
key: REGION
|
||||
wal:
|
||||
compression: gzip
|
||||
data:
|
||||
compression: gzip
|
||||
# Matches the 30d retention the other CNPG clusters use and the
|
||||
# renner-uno-k8s-cnpg bucket's noncurrent-version expiry in
|
||||
# opentofu/object_storage.tf, so old S3 versions don't outlive the backups
|
||||
# that superseded them.
|
||||
retentionPolicy: "30d"
|
||||
@@ -1,23 +0,0 @@
|
||||
---
|
||||
# Daily base backups for ha-pg. Continuous WAL archiving runs separately
|
||||
# (ha-pg-cluster.yaml spec.plugins[].isWALArchiver); without periodic base
|
||||
# backups the prefix would have no recovery point of its own, and
|
||||
# ha-pg-store's retentionPolicy: 30d would have nothing to prune against, so the
|
||||
# WAL chain would grow without bound.
|
||||
#
|
||||
# 02:50 UTC, offset from the other CNPG clusters (02:30, wp-pg at 02:40) so they
|
||||
# do not all hit the Hetzner bucket at once.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: ScheduledBackup
|
||||
metadata:
|
||||
name: ha-pg-daily
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
schedule: "0 50 2 * * *"
|
||||
immediate: true
|
||||
backupOwnerReference: self
|
||||
cluster:
|
||||
name: ha-pg
|
||||
method: plugin
|
||||
pluginConfiguration:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
@@ -52,14 +52,6 @@ resources:
|
||||
# CNP ingress rule.
|
||||
- zigbee2mqtt-gate-service.yaml
|
||||
- zigbee2mqtt-ingress.yaml
|
||||
# Postgres backend for the recorder (MIGRATION-recorder-postgres.md).
|
||||
# PHASE 1: this cluster comes up EMPTY and nothing consumes it — HA is still
|
||||
# on SQLite. Deploying it is therefore inert; phase 2 migrates the statistics
|
||||
# tables and repoints recorder.db_url in the homeassistant repo.
|
||||
- ha-pg-hetzner-creds-externalsecret.yaml
|
||||
- ha-pg-objectstore.yaml
|
||||
- ha-pg-cluster.yaml
|
||||
- ha-pg-scheduled-backup.yaml
|
||||
configMapGenerator:
|
||||
# nginx config for the reverse-proxy sidecar. Hash suffix disabled so
|
||||
# the cm name stays stable; editing proxy.conf does not auto-roll the
|
||||
@@ -87,13 +79,5 @@ configMapGenerator:
|
||||
namespace: home-assistant
|
||||
files:
|
||||
- watch.sh=zigbee2mqtt-interview-watch.sh
|
||||
# Z2M discovery guard sidecar script (gitea issue #381). Hash suffix disabled
|
||||
# (name stays stable), so editing zigbee2mqtt-discovery-guard.sh does not
|
||||
# auto-roll the pod — restart manually with
|
||||
# `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`.
|
||||
- name: zigbee2mqtt-discovery-guard-script
|
||||
namespace: home-assistant
|
||||
files:
|
||||
- guard.sh=zigbee2mqtt-discovery-guard.sh
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
@@ -373,9 +373,8 @@ spec:
|
||||
# Broker user `zigbee2mqtt` — the Zigbee2MQTT bridge (ZHA replacement). Scoped
|
||||
# to its own `zigbee2mqtt/#` subtree plus `write homeassistant/#` (so it can
|
||||
# publish HA MQTT-discovery configs, exactly like the `shellies` user) and
|
||||
# `read homeassistant/status` (HA's birth/will topic; on `online` Z2M
|
||||
# republishes cached device states, NOT discovery configs — ACL and the full
|
||||
# reasoning in mosquitto-deployment.yaml). Own credential so its lifecycle is
|
||||
# `read homeassistant/status` (Z2M re-publishes discovery when HA restarts —
|
||||
# ACL in mosquitto-deployment.yaml). Own credential so its lifecycle is
|
||||
# independent of the others. The Z2M init container reads this password and
|
||||
# renders it into /app/data/secret.yaml.
|
||||
apiVersion: batch/v1
|
||||
|
||||
@@ -5,31 +5,13 @@ metadata:
|
||||
namespace: home-assistant
|
||||
data:
|
||||
mosquitto.conf: |
|
||||
# No on-disk persistence, and no PVC of any kind for this broker. The
|
||||
# broker keeps no durable state, so it starts even if Ceph is degraded and
|
||||
# reschedules anywhere in seconds — no RWO volume to fence/detach/reattach.
|
||||
#
|
||||
# Most clients re-populate their retained state on reconnect: evcc
|
||||
# republishes state, Shellies re-send their announce/online, HA re-publishes
|
||||
# its own discovery.
|
||||
#
|
||||
# Zigbee2MQTT is the EXCEPTION and does not self-heal. Its HA discovery
|
||||
# configs are published retained, and retention is the only mechanism it
|
||||
# relies on to survive a broker restart — verified against Z2M 2.12.1
|
||||
# source: discover() runs only from start(), the MQTT reconnect path
|
||||
# (lib/mqtt.ts onConnect) republishes bridge/state and nothing else, and the
|
||||
# homeassistant/status "online" handler (lib/extension/homeassistant.ts
|
||||
# onMQTTMessage) republishes bridge/state + cached device states but never
|
||||
# the discovery configs. There is no bridge/request endpoint that forces a
|
||||
# rediscovery. Upstream has declined to add one (Koenkk/zigbee2mqtt #27458,
|
||||
# #22977, #16950, all closed NOT_PLANNED).
|
||||
#
|
||||
# Consequence: a mosquitto restart that is not followed by a Zigbee2MQTT
|
||||
# restart leaves every Z2M entity unavailable in HA until Z2M is restarted
|
||||
# (`kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`). This is
|
||||
# a live operational hazard, tracked in gitea issue #381 — it caused a ~14 h
|
||||
# outage of every Z2M entity on 2026-07-23. Enabling persistence is NOT the
|
||||
# answer: this broker deliberately has no PVC (see above).
|
||||
# No on-disk persistence. Retained messages + sessions are re-populated by
|
||||
# clients on reconnect (evcc republishes state, Shellies re-send their
|
||||
# announce/online, HA re-publishes discovery), so the broker keeps no
|
||||
# durable state. This drops the Ceph PVC dependency (the broker starts even
|
||||
# if Ceph is degraded) and lets it reschedule anywhere in seconds — no RWO
|
||||
# volume to fence/detach/reattach. Trade-off: a broker restart briefly
|
||||
# empties retained state until clients reconnect (self-healing).
|
||||
persistence false
|
||||
log_dest stdout
|
||||
log_type error
|
||||
|
||||
@@ -25,13 +25,7 @@ spec:
|
||||
# a broker serving throughout an image bump / reschedule (clients on the old
|
||||
# pod just reconnect to the new one); maxSurge:1 permits the brief two-pod
|
||||
# overlap — two independent stateless brokers behind the LB, fine since there
|
||||
# is no shared state.
|
||||
#
|
||||
# NOTE: connectivity survives the roll, but RETAINED STATE DOES NOT — the
|
||||
# new pod starts with an empty retained store, so a roll is equivalent to a
|
||||
# broker restart for every client that depends on retention. Zigbee2MQTT
|
||||
# does not recover from that on its own; see mosquitto-config.yaml and
|
||||
# gitea issue #381.
|
||||
# is no shared state and discovery self-heals.
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
@@ -70,9 +64,7 @@ spec:
|
||||
# Pinned tag, auto-bumped within the 2.x line by Flux image-automation
|
||||
# (infrastructure/image-automation/mosquitto.yaml). The marker tells
|
||||
# Flux where to write; patch/minor updates land as git commits and
|
||||
# roll the broker. HA re-publishes its own discovery afterwards, but
|
||||
# Zigbee2MQTT does not — an image bump therefore needs the same
|
||||
# follow-up as any other broker restart (see mosquitto-config.yaml).
|
||||
# roll the broker (brief restart; discovery self-heals via HA).
|
||||
image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
|
||||
# The shim runs as root (image sets no USER) and prepares the
|
||||
# auth files for the broker, which drops to user mosquitto
|
||||
@@ -156,16 +148,8 @@ spec:
|
||||
# Owns its own `zigbee2mqtt/#` subtree; may WRITE `homeassistant/#`
|
||||
# to publish HA MQTT-discovery configs (write-only there, like the
|
||||
# shellies user, to deny discovery-tree recon from a compromised
|
||||
# bridge); and READs `homeassistant/status`, HA's birth/will topic.
|
||||
#
|
||||
# That read grant does NOT buy a discovery re-publish: on
|
||||
# `online`, Z2M 2.12.1 republishes bridge/state and cached device
|
||||
# states only (lib/extension/homeassistant.ts onMQTTMessage) —
|
||||
# discover() runs solely from start(). The grant is still correct,
|
||||
# because the cached-state republish is what makes HA show current
|
||||
# values right after an HA restart. Restoring discovery after a
|
||||
# BROKER restart is handled by the discovery-guard sidecar in
|
||||
# zigbee2mqtt-deployment.yaml.
|
||||
# bridge); and READs `homeassistant/status` so it re-publishes
|
||||
# discovery when HA restarts (birth/will).
|
||||
user zigbee2mqtt
|
||||
topic readwrite zigbee2mqtt/#
|
||||
topic write homeassistant/#
|
||||
@@ -313,9 +297,7 @@ spec:
|
||||
configMap:
|
||||
name: mosquitto-config
|
||||
- name: auth
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
- name: tls
|
||||
secret:
|
||||
secretName: mosquitto-tls
|
||||
|
||||
@@ -176,38 +176,3 @@ data:
|
||||
friendly_name: Treppenhaus Haustür Schalter 1
|
||||
'0x001788010b00d011':
|
||||
friendly_name: Treppenhaus Haustür Schalter 2
|
||||
# Ecowitt WS90 weather station. Joined after the ZHA-mirrored block above
|
||||
# was written, hence listed apart from it.
|
||||
#
|
||||
# This device also carries a non-default ZCL reporting configuration
|
||||
# (per-attribute min interval and reportable-change deadbands). That is
|
||||
# device-side state, not expressible here — Z2M has no settings field for
|
||||
# it — so it is documented, with restore commands, in
|
||||
# zigbee2mqtt-ws90-reporting.md. It is lost whenever Z2M re-runs
|
||||
# `configure` on the device, and a fresh PVC will not have it.
|
||||
#
|
||||
# The station transmits every 8.8s, as a burst of seven per-cluster ZCL
|
||||
# frames; Z2M publishes its full cumulative state once per frame. debounce
|
||||
# merges a burst into one publish carrying the last value per attribute,
|
||||
# and must stay below the transmit interval — a window that never sees
|
||||
# silence never flushes.
|
||||
#
|
||||
# Every few days the firmware zeroes its readings for ~8s, then skips a
|
||||
# transmit cycle and recovers on the next. The zeros propagate attribute
|
||||
# by attribute through a burst rather than arriving together, and they
|
||||
# start inside an otherwise normal cycle — the last valid reading and the
|
||||
# first zero are ~20ms apart.
|
||||
#
|
||||
# Merging cuts an episode from six published bad frames to one. It does
|
||||
# NOT make that frame recognisable: it carries whatever subset had zeroed
|
||||
# when the burst ended, which may be every field or only temperature, and
|
||||
# a frame reading {"temperature":0,"humidity":92,"pressure":96.9} is not
|
||||
# separable from a valid winter reading. Since the merge is last-wins and
|
||||
# the zeros arrive at the tail, it also discards the valid reading that
|
||||
# led the burst. This bounds the corruption; it does not filter it.
|
||||
#
|
||||
# Z2M reads this only at startup (requiresRestart), and this file is
|
||||
# SEED-ONLY per the header — the running instance takes it from its PVC.
|
||||
'0xc02cedfffe958f40':
|
||||
friendly_name: Wetterstation
|
||||
debounce: 2
|
||||
|
||||
@@ -158,7 +158,7 @@ spec:
|
||||
# Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
|
||||
# add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
|
||||
# follow-up if desired; pinned-and-manual is fine for the initial cutover.
|
||||
image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
|
||||
image: ghcr.io/koenkk/zigbee2mqtt:2.12.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: TZ
|
||||
@@ -272,64 +272,6 @@ spec:
|
||||
memory: 16Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
# Re-runs Zigbee2MQTT's HA discovery after the broker loses its retained
|
||||
# store. mosquitto runs `persistence false` with no PVC, so every broker
|
||||
# restart wipes the retained discovery configs, and nothing on the Z2M
|
||||
# side restores them — not reconnect, not HA's birth message, and there
|
||||
# is no rediscovery endpoint. Without this, a broker restart leaves every
|
||||
# Z2M entity unavailable in HA until someone restarts Z2M by hand (gitea
|
||||
# issue #381; ~14 h outage on 2026-07-23). The full reasoning, including
|
||||
# why bridge/groups is the sentinel and why the guard is tuned for
|
||||
# sensitivity, is in the script header.
|
||||
- name: discovery-guard
|
||||
# Same image and rationale as interview-watch above: reused purely for
|
||||
# its bundled mosquitto_sub/mosquitto_pub, nothing installed at
|
||||
# runtime. Verified present in this exact image: sh, mosquitto_sub
|
||||
# (incl. -C and -W), mosquitto_pub, date -u +%s, mkdir, cat.
|
||||
# Deliberately NOT carrying a Flux $imagepolicy marker, so it does not
|
||||
# drag the broker's image-automation policy along with it.
|
||||
image: eclipse-mosquitto:2.0.22
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [sh, /script/guard.sh]
|
||||
env:
|
||||
# Reuses the Z2M broker credential: its ACL is `readwrite
|
||||
# zigbee2mqtt/#`, which covers reading bridge/state/bridge/groups and
|
||||
# writing bridge/request/restart. No new broker user needed.
|
||||
- name: MQTT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mosquitto-zigbee2mqtt-credentials
|
||||
key: password
|
||||
# Must track zigbee2mqtt-config.yaml, same as interview-watch.
|
||||
- name: MQTT_HOST
|
||||
value: mosquitto
|
||||
- name: MQTT_USER
|
||||
value: zigbee2mqtt
|
||||
- name: BASE_TOPIC
|
||||
value: zigbee2mqtt
|
||||
- name: STATEDIR
|
||||
value: /state
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1883
|
||||
runAsGroup: 1883
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
volumeMounts:
|
||||
- name: discovery-guard-script
|
||||
mountPath: /script
|
||||
readOnly: true
|
||||
# Restart cooldown stamp. Its own emptyDir rather than the image's
|
||||
# /tmp, so it stays writable as non-root.
|
||||
- name: discovery-guard-state
|
||||
mountPath: /state
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
@@ -344,13 +286,4 @@ spec:
|
||||
configMap:
|
||||
name: zigbee2mqtt-interview-watch-script
|
||||
- name: interview-watch-state
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
- name: discovery-guard-script
|
||||
configMap:
|
||||
name: zigbee2mqtt-discovery-guard-script
|
||||
- name: discovery-guard-state
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Restores Home Assistant MQTT discovery after the broker loses its retained
|
||||
# store, which mosquitto does on every restart because it runs `persistence
|
||||
# false` and has no PVC (a deliberate, settled decision — see
|
||||
# mosquitto-config.yaml).
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# Zigbee2MQTT publishes its HA discovery configs retained and relies on broker
|
||||
# retention alone to survive a broker restart. Verified against Z2M 2.12.1
|
||||
# source, nothing on the Z2M side restores them:
|
||||
# - discover() runs only from HomeAssistant.start().
|
||||
# - lib/mqtt.ts onConnect() republishes bridge/state and resubscribes; the
|
||||
# republishRetainedTimer is armed only inside connect() and never re-armed
|
||||
# on reconnect.
|
||||
# - The homeassistant/status "online" handler republishes bridge/state plus
|
||||
# cached device states, never the discovery configs.
|
||||
# - There is no bridge/request endpoint that forces a rediscovery; upstream
|
||||
# declined to add one (Koenkk/zigbee2mqtt #27458, #22977, #16950).
|
||||
# So a broker restart not followed by a Z2M restart leaves every Z2M entity
|
||||
# unavailable in HA indefinitely. That cost ~14 h of dead entities on
|
||||
# 2026-07-23 (gitea issue #381).
|
||||
#
|
||||
# DETECTION — why bridge/groups
|
||||
# bridge/state and the four bridge/{info,devices,groups,definitions} topics are
|
||||
# all retained, but only bridge/state is republished on reconnect (mqtt.ts
|
||||
# onConnect). So "bridge/state present + one of the others absent" means: Z2M is
|
||||
# talking to this broker right now, but nothing has re-run start() since the
|
||||
# retained store was wiped. That is exactly the broken state.
|
||||
#
|
||||
# The absent-half partner must not be re-publishable by ordinary runtime
|
||||
# events, or it self-heals and the guard goes blind while HA is still broken:
|
||||
# bridge/devices — republished on announce/join/leave/interview. Worst.
|
||||
# bridge/definitions — republished on devicesChanged and OTA.
|
||||
# bridge/info — republished on devicesChanged, which reaches inbound
|
||||
# radio traffic via the deviceExposesChanged callback
|
||||
# handed to every fromZigbee converter (receive.ts).
|
||||
# bridge/health — self-heals on its own 10 min timer. Useless here.
|
||||
# bridge/groups — only re-published by deliberate admin actions: group
|
||||
# membership edits, group add/remove/rename, and scene
|
||||
# changes (which come solely from an explicit
|
||||
# <device>/set scene command, never an inbound report).
|
||||
# Hence bridge/groups. It is also safe with no groups defined: publishGroups()
|
||||
# always publishes, emitting `[]`, which is still a non-empty retained payload.
|
||||
#
|
||||
# DELIBERATELY NOT AND-ed with bridge/info being absent. That would cut false
|
||||
# positives but introduce false negatives, and the costs are wildly asymmetric:
|
||||
# a false positive is one ~60 s Z2M restart, a false negative is every Zigbee
|
||||
# entity dead until a human notices. This guard is tuned for sensitivity.
|
||||
#
|
||||
# Two known ways to get a spurious positive, both handled by CONFIRM_DELAY:
|
||||
# - Startup race. bridge/state is published at mqtt.connect() but
|
||||
# bridge/groups only at the end of BridgeExtension.start(), so the signature
|
||||
# is legitimately true for a few seconds during every normal Z2M boot.
|
||||
# - QoS 0. The bridge/* topics are published with no qos option, so they go
|
||||
# out at-most-once and a publish can in principle be dropped.
|
||||
#
|
||||
# ACTION
|
||||
# Publishes bridge/request/restart, which is a full in-process restart: index.js
|
||||
# restart() -> stop(true) tears down every extension, then start() builds a new
|
||||
# Controller and re-runs HomeAssistant.start(), hence discover(). It is NOT a
|
||||
# herdsman-only restart. discover() republishes only configs that are missing or
|
||||
# whose payload changed, which after a wipe is all of them.
|
||||
#
|
||||
# Credentials: reuses the existing zigbee2mqtt broker user, whose ACL is
|
||||
# `readwrite zigbee2mqtt/#` — enough to read bridge/state and bridge/groups and
|
||||
# to write bridge/request/restart. No new broker user, and no read grant on
|
||||
# homeassistant/# (checking for the discovery configs directly would have needed
|
||||
# one, which the ACL denies on purpose).
|
||||
set -eu
|
||||
|
||||
MQTT_HOST="${MQTT_HOST:-mosquitto}"
|
||||
MQTT_PORT="${MQTT_PORT:-1883}"
|
||||
MQTT_USER="${MQTT_USER:-zigbee2mqtt}"
|
||||
BASE_TOPIC="${BASE_TOPIC:-zigbee2mqtt}"
|
||||
# Poll cadence. Worst-case time-to-repair is CHECK_INTERVAL + CONFIRM_DELAY.
|
||||
CHECK_INTERVAL="${CHECK_INTERVAL:-300}"
|
||||
# Re-check gap before acting, to ride out a normal Z2M boot and a dropped QoS 0
|
||||
# publish. Must comfortably exceed a Z2M cold start.
|
||||
CONFIRM_DELAY="${CONFIRM_DELAY:-90}"
|
||||
# Floor between two restarts, so a Z2M that will not come back healthy is not
|
||||
# restarted in a loop.
|
||||
COOLDOWN="${COOLDOWN:-1800}"
|
||||
# How long to wait for a retained message to land before calling it absent.
|
||||
READ_TIMEOUT="${READ_TIMEOUT:-5}"
|
||||
# Hard ceiling on a single client invocation, as a backstop to the protocol-level
|
||||
# waits below. Every MQTT call here is bounded with `timeout`, because neither
|
||||
# client bounds a TCP connect that HANGS rather than being refused — and a stalled
|
||||
# connect is most likely precisely when this guard runs, around a broker restart.
|
||||
# Without it the loop would block forever and stop logging, which is exactly the
|
||||
# silent-failure mode the guard exists to prevent. Same reasoning and the same
|
||||
# PUB_TIMEOUT default as zigbee2mqtt-interview-watch.sh.
|
||||
#
|
||||
# NOT `mosquitto_pub -W`: that flag does not exist for mosquitto_pub (it is a
|
||||
# mosquitto_sub option) and makes every publish fail on a usage error.
|
||||
# mosquitto_sub keeps its -W, which bounds the wait for a message once CONNECTED;
|
||||
# the `timeout` wrapper is what covers the connect phase, so it is set above -W.
|
||||
PUB_TIMEOUT="${PUB_TIMEOUT:-10}"
|
||||
STATEDIR="${STATEDIR:-/state}"
|
||||
STAMP="$STATEDIR/last-restart"
|
||||
|
||||
log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] discovery-guard: $*"; }
|
||||
|
||||
# Echoes the retained payload of $1, or nothing if the topic holds none.
|
||||
# A retained message is delivered immediately on subscribe, so the wait is only
|
||||
# ever paid when the topic is genuinely empty.
|
||||
retained() {
|
||||
timeout "$((READ_TIMEOUT + 5))" mosquitto_sub -h "$MQTT_HOST" -p "$MQTT_PORT" \
|
||||
-u "$MQTT_USER" -P "$MQTT_PASSWORD" \
|
||||
-t "$1" -C 1 -W "$READ_TIMEOUT" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# True when Z2M is connected to this broker but has not re-run start() since the
|
||||
# retained store was last wiped.
|
||||
discovery_lost() {
|
||||
case "$(retained "$BASE_TOPIC/bridge/state")" in
|
||||
*online*) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
[ -z "$(retained "$BASE_TOPIC/bridge/groups")" ]
|
||||
}
|
||||
|
||||
in_cooldown() {
|
||||
[ -f "$STAMP" ] || return 1
|
||||
last=$(cat "$STAMP" 2>/dev/null || echo 0)
|
||||
now=$(date -u +%s)
|
||||
[ $((now - last)) -lt "$COOLDOWN" ]
|
||||
}
|
||||
|
||||
mkdir -p "$STATEDIR"
|
||||
log "watching $BASE_TOPIC/bridge/groups on $MQTT_HOST:$MQTT_PORT" \
|
||||
"(interval ${CHECK_INTERVAL}s, confirm ${CONFIRM_DELAY}s, cooldown ${COOLDOWN}s)"
|
||||
|
||||
while true; do
|
||||
sleep "$CHECK_INTERVAL"
|
||||
|
||||
discovery_lost || continue
|
||||
|
||||
log "bridge/groups missing while Z2M is online — retained store looks wiped;" \
|
||||
"re-checking in ${CONFIRM_DELAY}s before acting"
|
||||
sleep "$CONFIRM_DELAY"
|
||||
|
||||
if ! discovery_lost; then
|
||||
log "recovered on its own, no action taken"
|
||||
continue
|
||||
fi
|
||||
|
||||
if in_cooldown; then
|
||||
log "confirmed, but a restart was requested less than ${COOLDOWN}s ago —" \
|
||||
"skipping. Z2M may be failing to come up; check its logs."
|
||||
continue
|
||||
fi
|
||||
|
||||
log "confirmed — requesting a Zigbee2MQTT restart to re-run discovery"
|
||||
if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" \
|
||||
-u "$MQTT_USER" -P "$MQTT_PASSWORD" \
|
||||
-t "$BASE_TOPIC/bridge/request/restart" -m '' 2>/dev/null; then
|
||||
date -u +%s > "$STAMP"
|
||||
log "restart requested"
|
||||
else
|
||||
log "failed to publish the restart request; will retry next cycle"
|
||||
fi
|
||||
done
|
||||
@@ -1,149 +0,0 @@
|
||||
# WS90 Zigbee reporting configuration
|
||||
|
||||
Records the ZCL attribute-reporting configuration applied to the Wetterstation
|
||||
(Ecowitt WS90 powered by Shelly, IEEE `0xc02cedfffe958f40`) and how to restore
|
||||
it, because **this configuration lives on the device and in Z2M's
|
||||
`database.db`, not in git**. Zigbee2MQTT has no settings-schema field for
|
||||
reporting configuration, so there is nothing declarative to reconcile — Flux
|
||||
cannot restore it and a fresh PVC will not have it.
|
||||
|
||||
## Applied configuration
|
||||
|
||||
Endpoint 1, all intervals in seconds. `change` is in the attribute's raw ZCL
|
||||
units; the physical deadband is `change ÷ scale`.
|
||||
|
||||
| attribute | cluster | min | max | change | deadband | converter default |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| temperature | `msTemperatureMeasurement` (1026) | 5 | 3600 | 10 | 0.1 °C | 10 / 3600 / 100 |
|
||||
| humidity | `msRelativeHumidity` (1029) | 5 | 3600 | 50 | 0.5 % | 10 / 3600 / 100 |
|
||||
| pressure | `msPressureMeasurement` (1027) | 5 | 3600 | 1 | 1 hPa | 10 / 3600 / 50 |
|
||||
| illuminance | `msIlluminanceMeasurement` (1024) | 5 | 3600 | 5 | 5 lx | 10 / 3600 / 5 |
|
||||
| windSpeed | `shellyWS90Wind` (0xfc01/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| windDirection | `shellyWS90Wind` (0xfc01/4) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| gustSpeed | `shellyWS90Wind` (0xfc01/7) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| uvIndex | `shellyWS90UV` (0xfc02/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| rainStatus | `shellyWS90Rain` (0xfc03/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| precipitation | `shellyWS90Rain` (0xfc03/1) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
|
||||
`genPowerCfg.batteryVoltage` (cluster 1, attribute 33) is deliberately left at
|
||||
its hourly default.
|
||||
|
||||
The three `shellyWS90*` clusters are manufacturer-specific and require
|
||||
`"options": {"manufacturerCode": 5264}` on every reconfiguration request.
|
||||
|
||||
## Why min is 5 and not 10
|
||||
|
||||
The station transmits every 8.8 s. `min` is a hold-off: a change-triggered
|
||||
report is held until at least `min` seconds after the previous one. With
|
||||
`min: 10` every report was therefore held past a transmit cycle and landed on
|
||||
the next, so attributes reported at best every 17.6 s and drifted out of phase
|
||||
with each other — the published stream showed no 8.8 s periodicity at all,
|
||||
just a smear across 2–8 s.
|
||||
|
||||
At `min: 5` each attribute reports on its own transmit. The MQTT stream then
|
||||
carries one publish per transmit cycle, measured at a mean gap of 8.69 s.
|
||||
Total publish volume *fell* (~631/h to ~445/h) because each publish now
|
||||
carries a whole cycle instead of a fragment of one.
|
||||
|
||||
Any value below 8.8 s achieves this. 5 leaves margin without approaching the
|
||||
cycle time.
|
||||
|
||||
## Why the deadbands differ from the converter defaults
|
||||
|
||||
The defaults come from the shared `modernExtend` helpers in
|
||||
zigbee-herdsman-converters (`m.temperature()`, `m.pressure()`, `m.humidity()`,
|
||||
`m.illuminance()`), which the WS90 definition in `shelly.ts` composes without
|
||||
overrides. They are anti-chatter values sized for battery-powered indoor
|
||||
sensors, where report volume costs coin-cell life and mesh airtime. This
|
||||
station is mains-and-solar and exists to resolve small changes, so the
|
||||
trade-off runs the other way.
|
||||
|
||||
Each value above is matched to the resolution the device actually reports:
|
||||
temperature and pressure arrive in 0.1-unit steps, humidity in whole percent.
|
||||
|
||||
Pressure is worth calling out. The converter default of `change: 50` with
|
||||
`scale: 10` is a 5 kPa (50 hPa) deadband, which real barometric pressure never
|
||||
traverses — so pressure only ever updated on the hourly `max` heartbeat, and
|
||||
`pressure_trend` and `weather_condition` were derived from that. The same
|
||||
library's older `reporting.ts` helper configures the identical ZCL attribute
|
||||
with `change: 5`, a 5 hPa deadband, so the two code paths disagree by 10× and
|
||||
the modernExtend value is the outlier.
|
||||
|
||||
## When this is lost
|
||||
|
||||
Anything that makes Z2M re-run `configure` on the device overwrites all of it
|
||||
with the converter defaults:
|
||||
|
||||
- re-pairing or re-joining the device
|
||||
- an explicit `zigbee2mqtt/bridge/request/device/configure`
|
||||
- a converter update that changes the definition and triggers reconfiguration
|
||||
|
||||
It also will not exist on a fresh PVC. Re-apply with the commands below.
|
||||
|
||||
## Re-applying
|
||||
|
||||
Published from the `discovery-guard` sidecar, which already holds the
|
||||
`zigbee2mqtt` broker credential (ACL `readwrite zigbee2mqtt/#`, which covers
|
||||
`bridge/request/#`) so no credential has to leave the cluster:
|
||||
|
||||
Ten separate requests, one per attribute — written out literally rather than
|
||||
generated by a shell helper, because the nested quoting through `kubectl exec
|
||||
-- sh -c` is where this goes wrong. Each line below is in the exact form that
|
||||
was executed and verified.
|
||||
|
||||
```sh
|
||||
POD=$(kubectl get pod -n home-assistant -l app.kubernetes.io/name=zigbee2mqtt -o jsonpath='{.items[0].metadata.name}')
|
||||
```
|
||||
|
||||
```sh
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msTemperatureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":10}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msRelativeHumidity\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":50}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msPressureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msIlluminanceMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":5}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windDirection\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"gustSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90UV\",\"attribute\":\"uvIndex\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"rainStatus\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"precipitation\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
```
|
||||
|
||||
These are idempotent — re-sending a request that matches the current
|
||||
configuration is a no-op on the device.
|
||||
|
||||
The device must be awake and reachable; each request performs a real
|
||||
`ConfigureReporting` exchange and fails if it is not.
|
||||
|
||||
## Verifying
|
||||
|
||||
Read back what the device actually accepted — Z2M records the negotiated
|
||||
result, so this reflects the device, not the request:
|
||||
|
||||
```sh
|
||||
kubectl exec -n home-assistant "$POD" -c zigbee2mqtt -- \
|
||||
grep 0xc02cedfffe958f40 /app/data/database.db |
|
||||
jq -r '.endpoints | to_entries[] | .value.configuredReportings[]?
|
||||
| "cluster=\(.cluster) attr=\(.attrId) min=\(.minRepIntval) max=\(.maxRepIntval) change=\(.repChange)"'
|
||||
```
|
||||
|
||||
All ten rows above should read `min=5 max=3600`. On the wire, the publish
|
||||
cadence on `zigbee2mqtt/Wetterstation` should sit at ~8.7 s and temperature
|
||||
should move in 0.1 °C steps; 1 °C steps mean the defaults are back.
|
||||
|
||||
## Making it durable upstream
|
||||
|
||||
The lasting fix is for the WS90 definition to pass explicit `reporting`
|
||||
overrides rather than inherit indoor-sensor defaults, which would fix every
|
||||
WS90 rather than this one. `m.temperature()` and the other helpers already
|
||||
accept a `reporting` argument. The pressure divergence between
|
||||
`modernExtend.ts` and `reporting.ts` is a separate upstream bug worth
|
||||
reporting on its own.
|
||||
@@ -17,33 +17,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: ical-proxy
|
||||
# Pinned to a commit build rather than :latest.
|
||||
#
|
||||
# :latest never rolls. The tag moves when a build publishes, but
|
||||
# nothing tells the Deployment about it: the pod template is
|
||||
# unchanged, so Kubernetes has no reason to replace anything, and
|
||||
# this image is deliberately NOT enrolled in Flux image automation
|
||||
# — see kubernetes/infrastructure/image-automation/README.md, which
|
||||
# lists ghcr.io/konairius/ical-proxy as having "only main, latest,
|
||||
# and main-<sha> commit builds; no release tags", so there is no
|
||||
# ImagePolicy watching it. The pods therefore sat on a 21-day-old
|
||||
# image while :latest moved underneath them, and the only way to
|
||||
# pick up a new build was a manual restart.
|
||||
#
|
||||
# A pinned tag makes the rollout a git change: Flux applies the new
|
||||
# pod template and Kubernetes rolls it, which is also what makes
|
||||
# what is RUNNING recoverable from this repo rather than "whatever
|
||||
# :latest meant the last time a pod happened to restart".
|
||||
#
|
||||
# main-99a912b is the merge of konairius/ical-proxy#12, which adds
|
||||
# the summary_match/summary_replace parameters this deployment
|
||||
# needs to strip "| Abfuhrkalender - Landkreis Amberg-Sulzbach"
|
||||
# from every waste collection.
|
||||
#
|
||||
# Bumping this is a manual, reviewed edit. The tag format is
|
||||
# main-<short-sha> (docker/metadata-action type=sha with the branch
|
||||
# as prefix).
|
||||
image: ghcr.io/konairius/ical-proxy:main-99a912b
|
||||
image: ghcr.io/konairius/ical-proxy:latest
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
|
||||
@@ -42,43 +42,7 @@ spec:
|
||||
parameters:
|
||||
shared_buffers: "512MB"
|
||||
max_wal_size: "2GB"
|
||||
# zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
|
||||
# and slower than the lz4/zstd this build also accepts). Compresses the
|
||||
# full-page images that dominate WAL volume after each checkpoint.
|
||||
wal_compression: "zstd"
|
||||
# A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
|
||||
# on a timer -- the segment is then written and archived at its full size
|
||||
# no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
|
||||
# to 288 segments/day = 4.6 GB/day that is largely padding.
|
||||
#
|
||||
# Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
|
||||
# cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
|
||||
# Two independent signals moving together is what confirms the volume was
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
#
|
||||
# checkpoint_timeout is the lever. The 5min default pushes every dirty
|
||||
# page to disk 288 times a day, so a small database whose same few pages
|
||||
# are re-dirtied constantly (sessions, tokens, job state) gets rewritten
|
||||
# over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
|
||||
# (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
|
||||
# day. At 15min those pages coalesce and are written a third as often.
|
||||
#
|
||||
# max_wal_size must rise with it, or checkpoints trigger on size instead
|
||||
# and the timeout change does nothing. It also bounds crash recovery to
|
||||
# at most 2GB of WAL replay.
|
||||
#
|
||||
# NOT changed: synchronous_commit stays at the default `on`. It is off for
|
||||
# ha-pg because a state recorder is telemetry; these hold credentials,
|
||||
# documents and job state, where losing the last few hundred ms of commits
|
||||
# on an unclean shutdown is not an acceptable trade.
|
||||
checkpoint_timeout: "15min"
|
||||
wal_compression: "on"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
|
||||
@@ -6,13 +6,6 @@ metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: immich-machine-learning
|
||||
spec:
|
||||
# Scale this and immich-server to 0 for the duration of any single-node
|
||||
# maintenance — cluster CPU requests exceed what two nodes can hold, and this
|
||||
# pair frees ~1000m against a shortfall of roughly 200m. Stopping them is the
|
||||
# right lever rather than lowering their requests: 500m reserved against a
|
||||
# 2m idle draw looks like waste, but inference bursts hard and the request is
|
||||
# honest about what it needs when someone uploads photos.
|
||||
# See RUNBOOK-baremetal-conversion.md.
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
|
||||
@@ -27,7 +27,7 @@ spec:
|
||||
# Pinned to 7.4-alpine to match the LXC 225 compose exactly.
|
||||
# Any redis major-version bump is a separate migration.
|
||||
- name: redis
|
||||
image: redis:7.4.11-alpine # {"$imagepolicy": "flux-system:redis-immich"}
|
||||
image: redis:7.4.10-alpine # {"$imagepolicy": "flux-system:redis-immich"}
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
|
||||
@@ -36,9 +36,6 @@ metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: immich-server
|
||||
spec:
|
||||
# Scaled to 0 alongside immich-machine-learning during single-node
|
||||
# maintenance; see the note there. The CNPG Postgres cluster stays running,
|
||||
# so recovery is just these two replica counts.
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
@@ -150,9 +147,7 @@ spec:
|
||||
- key: immich-config.json
|
||||
path: immich-config.json
|
||||
- name: le-ca
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -73,52 +73,3 @@ spec:
|
||||
interval: 30s
|
||||
podTargetLabels:
|
||||
- role
|
||||
---
|
||||
# wp-pg, restored into the `woodpecker` namespace (issue #433). It has never
|
||||
# been scraped: `apps` had no PodMonitor for it AND its baseline network policy
|
||||
# never allowed the monitoring namespace, so `cnpg_collector_up` has series for
|
||||
# the four clusters above but not this one. The namespace move fixes the policy
|
||||
# half; without this PodMonitor there is still no scrape target at all.
|
||||
#
|
||||
# Deliberately not adding one for the cluster still in `apps`: it is retired in
|
||||
# stage 3, and a PodMonitor there would only be reconciled to be deleted again.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: cnpg-wp-pg
|
||||
namespace: woodpecker
|
||||
labels:
|
||||
app.kubernetes.io/name: cloudnative-pg
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: wp-pg
|
||||
podMetricsEndpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
podTargetLabels:
|
||||
- role
|
||||
---
|
||||
# ha-pg — Postgres backend for the Home Assistant recorder
|
||||
# (kubernetes/apps/home-assistant/ha-pg-cluster.yaml). Added with the cluster
|
||||
# rather than after it, so the migration has metrics from the first moment
|
||||
# instead of discovering the gap later, which is how wp-pg's scrape hole was
|
||||
# found (#433).
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: cnpg-ha-pg
|
||||
namespace: home-assistant
|
||||
labels:
|
||||
app.kubernetes.io/name: cloudnative-pg
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: ha-pg
|
||||
podMetricsEndpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
podTargetLabels:
|
||||
- role
|
||||
|
||||
@@ -4,20 +4,13 @@ kind: Kustomization
|
||||
# come from the kube-prometheus-stack chart. Flux applies this Kustomization
|
||||
# only after `monitoring` is Ready (chart installed → CRDs present).
|
||||
resources:
|
||||
- prometheusrule-ceph.yaml
|
||||
- prometheusrule-cilium.yaml
|
||||
- prometheusrule-hetzner-backup.yaml
|
||||
- prometheusrule-loki.yaml
|
||||
- prometheusrule-mariadb-backup.yaml
|
||||
- prometheusrule-netbird-availability.yaml
|
||||
- prometheusrule-netbird-backup.yaml
|
||||
- prometheusrule-netbird-ingress.yaml
|
||||
- prometheusrule-velero.yaml
|
||||
- prometheusrule-woodpecker-rotation.yaml
|
||||
# CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
|
||||
# alerted if that stopped — HetznerBackupEndpointDown only probes the S3
|
||||
# endpoint, which proves reachability, not that anything was written.
|
||||
- prometheusrule-cnpg-backup.yaml
|
||||
- alloy-servicemonitor.yaml
|
||||
- cnpg-podmonitors.yaml
|
||||
- home-assistant-servicemonitor.yaml
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: ceph
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus resource selects on this label;
|
||||
# without it the rule is created but never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: ceph.rules
|
||||
rules:
|
||||
- alert: CephHealthWarn
|
||||
# Per-check rather than ceph_health_status, so one permanently-active
|
||||
# check cannot mask every other Ceph warning.
|
||||
#
|
||||
# AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a
|
||||
# kernel supporting aes256k over the CSI mount path (#527).
|
||||
#
|
||||
# POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes
|
||||
# `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked
|
||||
# maintenance and not just data risk — do not exclude it for a pool
|
||||
# that is unreplicated deliberately.
|
||||
#
|
||||
# 6h: Ceph warns for self-clearing conditions. Only a warning that
|
||||
# does not clear is worth paging on.
|
||||
#
|
||||
# BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT
|
||||
# excluded (#557). Ceph holds the check for
|
||||
# bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so
|
||||
# one multi-second stall outlives the 6h `for` and always pages,
|
||||
# with latency back to normal long before anyone looks. Measured:
|
||||
# both episodes in the 30d to 2026-08-23 ran ~26h and cleared
|
||||
# untouched, the first ~24h after the last stall.
|
||||
#
|
||||
# Both began inside the Velero restore-test sweep — a ~5h sustained
|
||||
# restore that outruns the SLC cache on these QLC NVMe. The sweep
|
||||
# runs monthly, 08:00-14:00Z on the 1st.
|
||||
#
|
||||
# Expected for the hardware, but not excludable:
|
||||
# this is also the earliest signal of a genuinely failing drive, and
|
||||
# all three are at 80-87% endurance used (see cephcluster.yaml).
|
||||
expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
|
||||
for: 6h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph health check {{ $labels.name }} active for 6h"
|
||||
description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the monthly Velero restore-test window (08:00-14:00Z on the 1st), which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
|
||||
|
||||
- alert: CephPoolLostRedundancy
|
||||
# Names the pool, which POOL_NO_REDUNDANCY does not.
|
||||
#
|
||||
# There is no pool-size metric, but bytes_used/stored IS the
|
||||
# replication factor. The >1e6 floor drops pools too small to measure
|
||||
# meaningfully, empty ones included.
|
||||
#
|
||||
# No pool is exempt.
|
||||
expr: |
|
||||
ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Ceph pool {{ $labels.pool_id }} is running unreplicated"
|
||||
description: "Pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning a single OSD loss destroys data. No pool is meant to be unreplicated — an unreplicated pool also blocks ALL OSD maintenance cluster-wide, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."
|
||||
|
||||
- alert: CephHealthError
|
||||
# HEALTH_ERR means data availability or redundancy is at risk.
|
||||
expr: ceph_health_status == 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Ceph is HEALTH_ERR"
|
||||
description: "The Ceph cluster is in HEALTH_ERR — availability or redundancy is at risk. Inspect immediately with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` and `ceph osd df tree`."
|
||||
|
||||
- alert: CephMonDiskLow
|
||||
# Named explicitly rather than left to CephHealthWarn, because the
|
||||
# mons sit on the node's /var alongside container storage: this
|
||||
# fires for a full node, not for Ceph growth. The mon's own data is
|
||||
# ~106 MB, so 30% of a 60 GB shared filesystem says nothing about
|
||||
# Ceph.
|
||||
# Image GC does not clear this — measured on k8s-2, the reclaimable
|
||||
# content store is a small fraction of /var/lib/containerd next to
|
||||
# ~20 GB of overlayfs snapshots held by running containers. Look at
|
||||
# what actually grew before reaching for a threshold.
|
||||
expr: |
|
||||
100 * node_filesystem_avail_bytes{mountpoint="/var"}
|
||||
/ node_filesystem_size_bytes{mountpoint="/var"} < 30
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.instance }} /var below 30% free — Ceph mon will warn"
|
||||
description: "/var on {{ $labels.instance }} is under 30% free, which is Ceph's mon_data_avail_warn threshold, so the mon on this node will report MON_DISK_LOW. Find the actual consumer with `talosctl -n <node> usage /var/lib --depth 1` and then `/var/lib/containerd --depth 1` — note that overlayfs snapshots belong to running containers and are not reclaimable by image GC."
|
||||
@@ -1,44 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: cilium
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus resource selects on this label;
|
||||
# without it the rule is created but never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: cilium.rules
|
||||
# Both alerts are map-agnostic on purpose. cilium_bpf_map_pressure is
|
||||
# emitted per map type — policy, conntrack, NAT, LB — distinguished by
|
||||
# the map_name label, and running out of any of them degrades the
|
||||
# datapath. The policy map is merely the one that has bitten us.
|
||||
# Cilium only exports the metric for maps above
|
||||
# policyMapPressureMetricsThreshold (chart default 0.1), so quiet maps
|
||||
# produce no series rather than a flood of near-zero ones.
|
||||
rules:
|
||||
- alert: CiliumBPFMapPressureHigh
|
||||
# cilium_bpf_map_pressure is used/capacity for the fullest map of
|
||||
# that name on the agent. A full map is not a slow path — it is a
|
||||
# hard failure: the agent cannot program the datapath, so endpoint
|
||||
# regeneration fails and every new pod on the node hangs in sandbox
|
||||
# creation until something frees space.
|
||||
expr: cilium_bpf_map_pressure > 0.8
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Cilium BPF map {{ $labels.map_name }} is {{ $value | humanizePercentage }} full on {{ $labels.node }}"
|
||||
description: "The BPF map {{ $labels.map_name }} on node {{ $labels.node }} has been over 80% full for 15 minutes. At 100% the Cilium agent can no longer program the datapath and new pods on this node fail sandbox creation. Raise the corresponding size in kubernetes/infrastructure/cilium/helmrelease.yaml (values.bpf) — the map is sized at creation, so the change needs an agent restart."
|
||||
|
||||
- alert: CiliumBPFMapFull
|
||||
# Pressure reaches (and can exceed) 1.0 when the agent wants more
|
||||
# entries than the map holds. Pods are already failing to start.
|
||||
expr: cilium_bpf_map_pressure >= 1
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Cilium BPF map {{ $labels.map_name }} is full on {{ $labels.node }}"
|
||||
description: "The BPF map {{ $labels.map_name }} on node {{ $labels.node }} is at or over capacity. Endpoint regeneration is failing with 'no space left on device' and pods scheduled to this node will hang in ContainerCreating. Check `{namespace=\"kube-system\",pod=~\"cilium-.+\"} |= \"no space left on device\"` in Loki for the affected endpoints."
|
||||
@@ -1,78 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: cnpg-backup
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus resource selects on this label;
|
||||
# without it the rule is created but never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: cnpg-backup.rules
|
||||
rules:
|
||||
# WHY THIS FILE EXISTS
|
||||
# Seven Postgres clusters back up to Hetzner and NOTHING alerted if that
|
||||
# stopped. The only related rule was HetznerBackupEndpointDown, which
|
||||
# probes the S3 endpoint — it proves the bucket is reachable, not that
|
||||
# anything was written to it. Same failure shape as the WordPress backup
|
||||
# that failed nightly for three days against a bucket that was declared
|
||||
# in tofu but never applied: the status surface looked fine because
|
||||
# nothing checked the thing that mattered.
|
||||
|
||||
- alert: CNPGWalArchiveBacklog
|
||||
# `ready` counts WAL segments sitting in pg_wal/archive_status waiting
|
||||
# to be archived. Steady state is 0 across every cluster (verified
|
||||
# 2026-08-19). A sustained backlog means the archiver is failing:
|
||||
# point-in-time recovery is silently broken from that moment, and
|
||||
# pg_wal keeps growing until it fills the PVC and takes Postgres down.
|
||||
#
|
||||
# This is the load-bearing alert of the two. It depends only on the
|
||||
# instance-level collector, so unlike the backup-age metrics below it
|
||||
# is unaffected by the missing Backup CRs (#522).
|
||||
#
|
||||
# Threshold: >5 segments sustained for 30m. Brief non-zero values are
|
||||
# normal under write bursts — a segment is 16 MiB, so 5 is roughly
|
||||
# 80 MiB of un-archived WAL, well below anything that threatens the
|
||||
# volume but far above idle noise.
|
||||
expr: cnpg_collector_pg_wal_archive_status{value="ready"} > 5
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "{{ $value }} WAL segments have been waiting to archive for over 30 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."
|
||||
|
||||
- alert: CNPGBackupFailed
|
||||
# cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0
|
||||
# when no failure is recorded. Guard on > 0 so the epoch-zero default
|
||||
# is not read as "failed in 1970".
|
||||
expr: |
|
||||
cnpg_collector_last_failed_backup_timestamp > 0
|
||||
and
|
||||
(time() - cnpg_collector_last_failed_backup_timestamp) < 86400
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "CNPG backup failed in the last 24h on {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "A base backup for {{ $labels.namespace }}/{{ $labels.pod }} failed within the last 24 hours. Verify what is actually in the bucket rather than trusting CR status — see the barman-cloud-backup-list recipe in kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md."
|
||||
|
||||
# DELIBERATELY ABSENT: a backup-age alert, which is the one you would
|
||||
# most want here.
|
||||
#
|
||||
# The natural expression is
|
||||
# (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600
|
||||
# and it CANNOT be used today: that metric reads exactly 0 on every
|
||||
# instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2,
|
||||
# pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any
|
||||
# successful backup anywhere, because the Backup CRs it derives that
|
||||
# from are being deleted within a minute of completing — see #522.
|
||||
#
|
||||
# Backups themselves DO work; that was verified independently by
|
||||
# listing the bucket with barman-cloud-backup-list and restoring a WAL
|
||||
# segment back out of S3. The gap is in the record, not the data.
|
||||
#
|
||||
# Adding the rule now would fire permanently for all seven clusters and
|
||||
# be silenced within a day, which is worse than not having it. Add it as
|
||||
# part of resolving #522, once the metric reflects reality.
|
||||
@@ -1,51 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: mariadb-backup
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus selects on this label.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: mariadb-backup.rules
|
||||
rules:
|
||||
- alert: WordpressMariadbBackupStale
|
||||
# The nightly 02:45 logical dump (kubernetes/apps/wordpress/mariadb.yaml)
|
||||
# is the only transaction-consistent copy of www.renner.uno's database
|
||||
# — Velero snapshots the PVC, which for a running DB is not a restore
|
||||
# point you would trust.
|
||||
#
|
||||
# KubeJobFailed already covers a failed Job, but only while the Job
|
||||
# object survives. On #550 it cleared when the failed Job was
|
||||
# garbage-collected, not when a backup succeeded — so a failure that
|
||||
# is GC'd before anyone looks leaves no signal at all.
|
||||
# lastSuccessfulTime lives on the CronJob and outlives Job GC.
|
||||
#
|
||||
# 26h = one missed cycle plus 2h slack, matching NetbirdVpsBackupStale.
|
||||
# It counts any Job owned by the CronJob, so a manual
|
||||
# `kubectl create job --from=cronjob/...` re-run clears it too.
|
||||
expr: |
|
||||
time() - kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"} > 26 * 3600
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "WordPress MariaDB backup has not succeeded in 26h+"
|
||||
description: "The wordpress-mariadb-backup CronJob's last successful run is over 26 hours old. Check the newest Job's pod logs in the wordpress namespace: `kubectl -n wordpress get jobs`. The 2026-08-19 failure was `The specified bucket does not exist.` against renner-uno-k8s-mariadb while the endpoint itself probed healthy, so a green HetznerBackupEndpointDown does not clear this."
|
||||
|
||||
- alert: WordpressMariadbBackupNeverRan
|
||||
# The staleness expression above goes blind when the metric is
|
||||
# absent, which is what happens once the operator recreates the
|
||||
# CronJob. Without this guard "never backed up" is quieter than
|
||||
# "backed up late".
|
||||
expr: |
|
||||
kube_cronjob_created{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
|
||||
unless
|
||||
kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
|
||||
for: 6h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "WordPress MariaDB backup has no recorded successful run"
|
||||
description: "The wordpress-mariadb-backup CronJob exists but has no lastSuccessfulTime: either it has not completed once since the Backup CR was recreated, or every run since has failed. Check `kubectl -n wordpress get jobs` and that the hetzner-s3-creds ExternalSecret is synced."
|
||||
@@ -12,37 +12,37 @@ spec:
|
||||
groups:
|
||||
- name: woodpecker-rotation.rules
|
||||
# Convergence safety net for the automated WOODPECKER_AGENT_SECRET rotation
|
||||
# (kubernetes/infrastructure/openbao/rotation/woodpecker.yaml). That rotation Job is
|
||||
# (kubernetes/apps/woodpecker/rotate-agent-secret.yaml). That rotation Job is
|
||||
# fire-and-forget: it writes OpenBao and exits WITHOUT verifying ESO re-synced
|
||||
# or that both consumers restarted healthy. These alerts close that gap. ESO
|
||||
# metrics are not scraped in this cluster, so everything here is built on
|
||||
# kube-state-metrics. The rotation has since landed: the CronJob lives in the
|
||||
# openbao namespace, not apps, which is where the last two rules look.
|
||||
# kube-state-metrics. The CronJob/Job series stay empty until the rotation
|
||||
# feature (branch feat/woodpecker-secret-auto-rotation) is merged — harmless.
|
||||
rules:
|
||||
- alert: WoodpeckerServerNotReady
|
||||
# 0 ready replicas well past the 10m ESO refresh + Reloader restart cycle.
|
||||
expr: kube_deployment_status_replicas_ready{namespace="woodpecker",deployment="woodpecker-server"} == 0
|
||||
expr: kube_statefulset_status_replicas_ready{namespace="apps",statefulset="woodpecker-server"} == 0
|
||||
for: 20m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Woodpecker CI server has no ready replica"
|
||||
description: >-
|
||||
Deployment woodpecker-server (ns woodpecker) has had 0 ready replicas
|
||||
for 20m; all CI is down (agents cannot reach
|
||||
woodpecker.woodpecker.svc:9000). If this followed a secret rotation, the
|
||||
server pod failed to return after its Reloader-triggered restart — check
|
||||
`kubectl -n woodpecker describe deploy woodpecker-server` and the
|
||||
woodpecker-secret ExternalSecret sync status.
|
||||
StatefulSet woodpecker-server (ns apps) has had 0 ready replicas for 20m;
|
||||
all CI is down (agents cannot reach woodpecker.apps.svc:9000). If this
|
||||
followed a secret rotation, the server pod failed to return after its
|
||||
Reloader-triggered restart — check `kubectl -n apps describe pod
|
||||
woodpecker-server-0` (PVC attach / crashloop) and the woodpecker-secret
|
||||
ExternalSecret sync status.
|
||||
- alert: WoodpeckerAgentNotReady
|
||||
expr: kube_deployment_status_replicas_available{namespace="woodpecker",deployment="woodpecker-agent"} == 0
|
||||
expr: kube_deployment_status_replicas_available{namespace="apps",deployment="woodpecker-agent"} == 0
|
||||
for: 20m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Woodpecker CI agent has no available replica"
|
||||
description: >-
|
||||
Deployment woodpecker-agent (ns woodpecker) has had 0 available replicas for
|
||||
Deployment woodpecker-agent (ns apps) has had 0 available replicas for
|
||||
20m; no pipelines will run. If this followed a secret rotation, the agent
|
||||
did not return after its Reloader restart, or its WOODPECKER_AGENT_SECRET
|
||||
no longer matches the server's — check the agent pod logs for a gRPC auth
|
||||
@@ -50,29 +50,29 @@ spec:
|
||||
- alert: WoodpeckerSecretRotationJobFailed
|
||||
# Named, clearer signal than the generic KubeJobFailed for the OpenBao-write
|
||||
# step. Fires while a failed rotation Job is retained (ttl 3d).
|
||||
expr: kube_job_status_failed{namespace="openbao",job_name=~"woodpecker-secret-rotate.*"} > 0
|
||||
expr: kube_job_status_failed{namespace="apps",job_name=~"woodpecker-secret-rotate.*"} > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Woodpecker agent-secret rotation Job failed"
|
||||
description: >-
|
||||
A woodpecker-secret-rotate Job (ns openbao) failed at the OpenBao-write step;
|
||||
A woodpecker-secret-rotate Job (ns apps) failed at the OpenBao-write step;
|
||||
the agent secret was NOT rotated. Most common cause: the one-time OpenBao
|
||||
`rotator` role/policy bootstrap was never applied (see
|
||||
kubernetes/infrastructure/openbao/policies/rotator.hcl). Check
|
||||
`kubectl -n openbao logs job/<name>`.
|
||||
`kubectl -n apps logs job/<name>`.
|
||||
- alert: WoodpeckerSecretRotationStale
|
||||
# Quarterly cadence (~91d) + grace. The series is ABSENT until the first
|
||||
# successful run, so this cannot false-fire before rotation has ever worked.
|
||||
expr: (time() - kube_cronjob_status_last_successful_time{namespace="openbao",cronjob="woodpecker-secret-rotate"}) > 100 * 24 * 3600
|
||||
expr: (time() - kube_cronjob_status_last_successful_time{namespace="apps",cronjob="woodpecker-secret-rotate"}) > 100 * 24 * 3600
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Woodpecker agent secret has not rotated in >100 days"
|
||||
description: >-
|
||||
The quarterly woodpecker-secret-rotate CronJob (ns openbao) has not completed
|
||||
The quarterly woodpecker-secret-rotate CronJob (ns apps) has not completed
|
||||
successfully in over 100 days — rotation has silently stopped (CronJob
|
||||
suspended/deleted, or every run failing). Investigate the CronJob and its
|
||||
recent Jobs.
|
||||
|
||||
@@ -402,52 +402,6 @@ spec:
|
||||
limits:
|
||||
memory: 256Mi
|
||||
|
||||
# Drop the apiserver's high-cardinality latency histograms.
|
||||
#
|
||||
# Measured 2026-08-20: Prometheus held 298,236 active series, of which
|
||||
# ~108,000 — over a third — were apiserver/etcd `_bucket` histograms. Each
|
||||
# histogram fans out to one series per bucket x verb x resource x scope, so
|
||||
# a single metric like apiserver_request_duration_seconds_bucket was 23,810
|
||||
# series on its own. For comparison, every Home Assistant metric combined is
|
||||
# ~4,100.
|
||||
#
|
||||
# That cardinality is the dominant driver of Prometheus write volume
|
||||
# (~11 GB/day of TSDB + WAL, doubled again by Ceph replication), and none of
|
||||
# it is observable value here: per-bucket apiserver latency quantiles are for
|
||||
# clusters with SLAs and dozens of nodes, not three.
|
||||
#
|
||||
# ONLY metrics with ZERO rule dependencies are dropped. Verified against the
|
||||
# live rule set before writing this — each metric checked individually for
|
||||
# references in alert and recording rules:
|
||||
#
|
||||
# apiserver_request_duration_seconds_bucket rules=0 23,810 series
|
||||
# apiserver_request_body_size_bytes_bucket rules=0 23,200
|
||||
# etcd_request_duration_seconds_bucket rules=0 15,840
|
||||
# apiserver_watch_list_duration_seconds_bucket rules=0 11,538
|
||||
# apiserver_watch_cache_read_wait_seconds_bucket rules=0 8,232
|
||||
# apiserver_response_sizes_bucket rules=0 7,592
|
||||
# apiserver_watch_events_sizes_bucket rules=0 5,220
|
||||
#
|
||||
# apiserver_request_sli_duration_seconds_bucket is DELIBERATELY KEPT: 23
|
||||
# recording rules depend on it, including the apiserver availability SLO and
|
||||
# the KubeAPIErrorBudgetBurn burn-rate series. Dropping it would silently
|
||||
# break that alerting, which is the failure mode this comment exists to
|
||||
# prevent if someone later "tidies up" this regex.
|
||||
#
|
||||
# Only the `_bucket` series are dropped. The matching `_count` and `_sum`
|
||||
# remain, so request rates and mean latencies still work — what is lost is
|
||||
# per-bucket quantiles.
|
||||
#
|
||||
# etcd_* appears here even though kubeEtcd is disabled below: these are the
|
||||
# apiserver's own etcd-client metrics, exposed on the apiserver endpoint, not
|
||||
# a separate etcd scrape.
|
||||
kubeApiServer:
|
||||
serviceMonitor:
|
||||
metricRelabelings:
|
||||
- action: drop
|
||||
sourceLabels: [__name__]
|
||||
regex: (apiserver_request_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|etcd_request_duration_seconds_bucket|apiserver_watch_list_duration_seconds_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket)
|
||||
|
||||
# Disable etcd/scheduler/controller-manager scraping (Talos doesn't expose them by default)
|
||||
kubeEtcd:
|
||||
enabled: false
|
||||
|
||||
@@ -64,9 +64,7 @@ spec:
|
||||
secret:
|
||||
secretName: pve-exporter-config
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -25,11 +25,8 @@ spec:
|
||||
# replaces the former hostNetwork attachment (see
|
||||
# network-attachment.yaml) so the NODE no longer holds an address
|
||||
# on the compromised IoT VLAN; MA gets its own leased IoT IP on
|
||||
# net1 instead. master: ens19 resolves on every k8s-{1,2,3}/edge
|
||||
# worker, though it is no longer the same kind of device everywhere:
|
||||
# on the bare-metal control-plane nodes it is a VLAN 30 sub-interface
|
||||
# of enp2s0f0np0 rather than a NIC, which is why the datapath needs
|
||||
# bpf.vlanBypass to carry it (issue #592).
|
||||
# net1 instead. master: ens19 is the v30iot NIC on every k8s-{1,2,3}/edge
|
||||
# worker.
|
||||
k8s.v1.cni.cncf.io/networks: music-assistant-iot
|
||||
spec:
|
||||
enableServiceLinks: false
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
# paperless-data: CephFS → RBD
|
||||
|
||||
## Why
|
||||
|
||||
`paperless-data` is `ReadWriteMany` on `ceph-filesystem`, but **only the paperless
|
||||
Deployment mounts it**. Verified 2026-08-20:
|
||||
|
||||
```
|
||||
$ pods mounting paperless-data
|
||||
paperless-68cb8b8744-7hjb8 <- one, and only one
|
||||
|
||||
paperless-data ReadWriteMany ceph-filesystem
|
||||
paperless-media ReadWriteMany ceph-filesystem <- genuinely shared (ftp-scanner)
|
||||
paperless-consume ReadWriteMany ceph-filesystem <- genuinely shared (ftp-scanner)
|
||||
```
|
||||
|
||||
Its contents are single-writer state — the 50 MB Whoosh index,
|
||||
`classification_model.pickle`, `celerybeat-schedule.db`, `log/`,
|
||||
`migration_lock`, `protonmail.pem`. 75 files. Nothing else reads or writes them.
|
||||
|
||||
It is on CephFS by default, not by requirement.
|
||||
|
||||
## What it costs
|
||||
|
||||
Every access goes through the MDS. Measured with **zero CI pipelines running**:
|
||||
|
||||
```
|
||||
paperless-data client request_load_avg 22 (all other sessions 0-1)
|
||||
MDS over 180s 4,052 client requests = 22.5/s
|
||||
2,718 journal events = 15/s
|
||||
2 dir commits <- almost none: not file writes
|
||||
```
|
||||
|
||||
So it is cap/inode churn from repeated access, not data being written. That is
|
||||
most of the constant **7-8 GB/day** written to `cephfs-metadata`, and since that
|
||||
pool is `size: 3`, roughly **21-24 GB/day** reaches the drives.
|
||||
|
||||
`cephfs-metadata` deliberately stays at size 3. Losing it destroys the entire
|
||||
filesystem rather than the objects on one OSD, and the 275 GiB in `cephfs-data0`
|
||||
depends on it. Cutting the writes at source is correct; cutting the replication
|
||||
is not.
|
||||
|
||||
## Sequence
|
||||
|
||||
Staged deliberately: the PVC lands first and empty, so merging it changes
|
||||
nothing. The copy and the repoint are separate.
|
||||
|
||||
### Phase 1 — provision (this PR)
|
||||
|
||||
`paperless-data-block` (RWO, `ceph-block`, 10Gi) is created and mounted by
|
||||
nothing. Inert.
|
||||
|
||||
Verify: `kubectl -n paperless get pvc paperless-data-block` → Bound.
|
||||
|
||||
### Phase 2 — copy
|
||||
|
||||
The Whoosh index must come across intact. Paperless *can* rebuild it
|
||||
(`document_index reindex`), but that reprocesses every document, so copying is
|
||||
both faster and less disruptive.
|
||||
|
||||
1. Scale paperless to 0. The source must be quiescent — `celerybeat-schedule.db`
|
||||
and the Whoosh index are both live-written, and copying them under an active
|
||||
writer gives a torn read.
|
||||
2. Run a one-off pod that mounts BOTH PVCs and copies:
|
||||
```
|
||||
cp -a /src/. /dst/
|
||||
```
|
||||
The RWO target pins the pod to one node; the RWX source will follow.
|
||||
3. Verify file count and index size match:
|
||||
```
|
||||
find /src -type f | wc -l == find /dst -type f | wc -l
|
||||
du -sh /src/index == du -sh /dst/index
|
||||
```
|
||||
|
||||
### Phase 3 — repoint
|
||||
|
||||
Change the Deployment's `data` volume to `claimName: paperless-data-block`,
|
||||
merge, let it roll.
|
||||
|
||||
Verify afterwards:
|
||||
- paperless starts, search returns results (proves the Whoosh index survived)
|
||||
- `ceph tell mds.cephfs-b session ls` no longer shows a busy `paperless-data`
|
||||
client
|
||||
- `rate(ceph_pool_wr_bytes{pool_id="3"})` drops from its 7-8 GB/day baseline
|
||||
|
||||
**Do not delete `paperless-data` until search has been confirmed working.** It is
|
||||
the rollback, and it is `prune: disabled` so Flux will not remove it.
|
||||
|
||||
## Expected effect
|
||||
|
||||
~7-8 GB/day off `cephfs-metadata`, so ~21-24 GB/day off the drives once
|
||||
replication is counted. Against ~432 GB/day cluster-wide that is roughly 5% —
|
||||
worth doing, but it is a correctness fix (a single-writer volume does not belong
|
||||
on a shared filesystem) more than a large win.
|
||||
|
||||
It also removes 22 req/sec of MDS load, which benefits the workloads that
|
||||
genuinely need CephFS — Woodpecker CI workspaces share the same MDS.
|
||||
@@ -34,42 +34,6 @@ spec:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_connections: "200"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
#
|
||||
# checkpoint_timeout is the lever. The 5min default pushes every dirty
|
||||
# page to disk 288 times a day, so a small database whose same few pages
|
||||
# are re-dirtied constantly (sessions, tokens, job state) gets rewritten
|
||||
# over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
|
||||
# (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
|
||||
# day. At 15min those pages coalesce and are written a third as often.
|
||||
#
|
||||
# max_wal_size must rise with it, or checkpoints trigger on size instead
|
||||
# and the timeout change does nothing. It also bounds crash recovery to
|
||||
# at most 2GB of WAL replay.
|
||||
#
|
||||
# NOT changed: synchronous_commit stays at the default `on`. It is off for
|
||||
# ha-pg because a state recorder is telemetry; these hold credentials,
|
||||
# documents and job state, where losing the last few hundred ms of commits
|
||||
# on an unclean shutdown is not an acceptable trade.
|
||||
checkpoint_timeout: "15min"
|
||||
max_wal_size: "2GB"
|
||||
# zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
|
||||
# and slower than the lz4/zstd this build also accepts). Compresses the
|
||||
# full-page images that dominate WAL volume after each checkpoint.
|
||||
wal_compression: "zstd"
|
||||
# A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
|
||||
# on a timer -- the segment is then written and archived at its full size
|
||||
# no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
|
||||
# to 288 segments/day = 4.1 GB/day that is largely padding.
|
||||
#
|
||||
# Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
|
||||
# cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if both instances
|
||||
# are lost. The documents themselves live on the media PVC, not in
|
||||
# Postgres -- this database holds their metadata.
|
||||
archive_timeout: "30min"
|
||||
# See ak-pg cluster.yaml for the rationale on synchronous replication.
|
||||
synchronous:
|
||||
method: any
|
||||
|
||||
@@ -22,7 +22,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: gotenberg
|
||||
image: gotenberg/gotenberg:8.36.0 # {"$imagepolicy": "flux-system:gotenberg"}
|
||||
image: gotenberg/gotenberg:8.35.0 # {"$imagepolicy": "flux-system:gotenberg"}
|
||||
args:
|
||||
- "gotenberg"
|
||||
- "--chromium-disable-javascript=true"
|
||||
|
||||
@@ -208,25 +208,9 @@ spec:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumes:
|
||||
# paperless-data-block (RWO, ceph-block), NOT paperless-data
|
||||
# (RWX, ceph-filesystem). Only this Deployment ever mounted the volume,
|
||||
# and its contents are single-writer state — the Whoosh index,
|
||||
# classification model, celerybeat schedule, logs. Being on CephFS meant
|
||||
# every access went through the MDS: 22 requests/sec sustained with zero
|
||||
# CI running, ~15 journal events/sec against 2 directory commits per
|
||||
# 180s, i.e. cap/inode churn rather than file writes. That was most of
|
||||
# the constant 7-8 GB/day on cephfs-metadata, and that pool is size 3.
|
||||
#
|
||||
# `consume` and `media` below STAY on CephFS: the ftp-scanner writes to
|
||||
# those from another node, so they genuinely need RWX.
|
||||
#
|
||||
# paperless-data is retained (prune: disabled) as the rollback. Do not
|
||||
# delete it until search has been confirmed working — that is what
|
||||
# proves the Whoosh index survived the copy, not merely that paperless
|
||||
# starts. Migration record: MIGRATION-data-to-block.md.
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-data-block
|
||||
claimName: paperless-data
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-media
|
||||
@@ -234,9 +218,7 @@ spec:
|
||||
persistentVolumeClaim:
|
||||
claimName: paperless-consume
|
||||
- name: bridge-cert
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -18,45 +18,6 @@ spec:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
# Replacement for paperless-data, on RBD instead of CephFS.
|
||||
#
|
||||
# WHY: paperless-data is declared ReadWriteMany on ceph-filesystem, but only ONE
|
||||
# pod ever mounts it — the paperless Deployment itself. Its contents are all
|
||||
# single-writer state: the 50 MB Whoosh index, classification_model.pickle,
|
||||
# celerybeat-schedule.db, logs, migration_lock. Nothing shares it. Only `consume`
|
||||
# and `media` genuinely need RWX, because the FTP scanner writes to those from a
|
||||
# different node.
|
||||
#
|
||||
# COST OF LEAVING IT: every access goes through the MDS. Measured 2026-08-20 with
|
||||
# zero CI running, this volume's client was doing 22 MDS requests/sec sustained,
|
||||
# and the MDS logged ~15 journal events/sec against only 2 directory commits per
|
||||
# 180s — i.e. cap and inode churn from repeated access, not file writes. That is
|
||||
# most of the constant 7-8 GB/day written to cephfs-metadata, which is a size 3
|
||||
# pool, so ~21-24 GB/day actually hits the drives.
|
||||
#
|
||||
# cephfs-metadata deliberately STAYS at size 3 (see storageclass.yaml): losing
|
||||
# the metadata pool destroys the whole filesystem, not just the objects on one
|
||||
# OSD, so the 275 GiB in cephfs-data0 rides on it. Reducing the writes at source
|
||||
# is the right fix; reducing the replication is not.
|
||||
#
|
||||
# MIGRATION: this PVC is created EMPTY and nothing mounts it yet. The data copy
|
||||
# and the deployment repoint are a separate, staged step — see
|
||||
# MIGRATION-data-to-block.md. Merging this alone is inert.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: paperless-data-block
|
||||
namespace: paperless
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:8.10.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
image: redis:8.10.0-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
|
||||
@@ -32,44 +32,7 @@ spec:
|
||||
parameters:
|
||||
shared_buffers: "128MB"
|
||||
max_connections: "50"
|
||||
# zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
|
||||
# and slower than the lz4/zstd this build also accepts). Compresses the
|
||||
# full-page images that dominate WAL volume after each checkpoint.
|
||||
wal_compression: "zstd"
|
||||
# A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
|
||||
# on a timer -- the segment is then written and archived at its full size
|
||||
# no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
|
||||
# to 288 segments/day = 4.6 GB/day that is largely padding.
|
||||
#
|
||||
# Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
|
||||
# cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
|
||||
# Two independent signals moving together is what confirms the volume was
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
#
|
||||
# checkpoint_timeout is the lever. The 5min default pushes every dirty
|
||||
# page to disk 288 times a day, so a small database whose same few pages
|
||||
# are re-dirtied constantly (sessions, tokens, job state) gets rewritten
|
||||
# over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
|
||||
# (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
|
||||
# day. At 15min those pages coalesce and are written a third as often.
|
||||
#
|
||||
# max_wal_size must rise with it, or checkpoints trigger on size instead
|
||||
# and the timeout change does nothing. It also bounds crash recovery to
|
||||
# at most 2GB of WAL replay.
|
||||
#
|
||||
# NOT changed: synchronous_commit stays at the default `on`. It is off for
|
||||
# ha-pg because a state recorder is telemetry; these hold credentials,
|
||||
# documents and job state, where losing the last few hundred ms of commits
|
||||
# on an unclean shutdown is not an acceptable trade.
|
||||
checkpoint_timeout: "15min"
|
||||
max_wal_size: "2GB"
|
||||
wal_compression: "on"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
|
||||
@@ -29,6 +29,11 @@ data:
|
||||
# The app re-reads schedules.json every 60 s, so edits here take
|
||||
# effect without a restart once the init container has rewritten it.
|
||||
#
|
||||
# timestamp stays false: the dashboard carries a clock CARD, which
|
||||
# the browser draws at capture time — so the time in the image is
|
||||
# already the time the screenshot was taken, and the renderer's own
|
||||
# overlay would just print it twice.
|
||||
#
|
||||
# lang is "de", not null. It calls Home Assistant's own
|
||||
# _selectLanguage() for the screenshot session, so the rendered page
|
||||
# is localised by HA rather than by hand. With it left null the
|
||||
@@ -362,22 +367,6 @@ spec:
|
||||
name: trmnl-renderer-schedule
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
# RAM-backed, NOT disk. Without `medium: Memory` an emptyDir lands
|
||||
# on the node's filesystem, and this one was the single largest
|
||||
# writer in the cluster: 249 GB over 7.3 days = ~34 GB/day onto
|
||||
# /dev/vda5, more than every Ceph OSD and every database combined
|
||||
# (measured 2026-08-20 from the container's cgroup io.stat).
|
||||
#
|
||||
# The cause is cadence, not volume. schedules.template.json renders
|
||||
# every 60 s, so Chrome is launched 1440 times a day and each launch
|
||||
# creates, fills and deletes a /tmp/puppeteer_dev_chrome_profile-*
|
||||
# directory. At rest /tmp holds under 8 MB; it is pure churn for
|
||||
# data that never outlives the render.
|
||||
#
|
||||
# tmpfs pages count against the container's memory limit, which is
|
||||
# why the cap stays at 512Mi: measured usage is 129Mi against a 2Gi
|
||||
# limit, so even the full cap leaves ample headroom.
|
||||
medium: Memory
|
||||
sizeLimit: 512Mi
|
||||
---
|
||||
apiVersion: v1
|
||||
|
||||
@@ -18,7 +18,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: whisper
|
||||
image: rhasspy/wyoming-whisper:3.6.0 # {"$imagepolicy": "flux-system:wyoming-whisper"}
|
||||
image: rhasspy/wyoming-whisper:3.5.0 # {"$imagepolicy": "flux-system:wyoming-whisper"}
|
||||
args:
|
||||
- --model=base-int8
|
||||
- --language=de
|
||||
|
||||
@@ -1,388 +0,0 @@
|
||||
# Woodpecker: `apps` → `woodpecker` namespace migration
|
||||
|
||||
Moves the Woodpecker service — server, agent and the `wp-pg` CNPG backend — out
|
||||
of the shared `apps` namespace into one of its own. Distinct from
|
||||
`MIGRATION.md`, which covers the earlier SQLite → CNPG move.
|
||||
|
||||
## Why
|
||||
|
||||
`apps` holds gitea, gatus, external-dns, ical-proxy and woodpecker together,
|
||||
and its `default-deny-ingress` uses `endpointSelector: {}`. Every allow rule it
|
||||
carries therefore applies to *every* workload in the namespace.
|
||||
|
||||
That is not academic. Issue #433 — `wp-pg` stuck at `Ready=False` with
|
||||
`Instance Status Extraction Error` for three weeks — is caused by the CNPG
|
||||
operator in `database` being unable to reach `https://<pod-ip>:8000/pg/status`.
|
||||
The only namespace-level fix is `fromEndpoints: namespace=database`, which
|
||||
would expose gitea, gatus, external-dns and ical-proxy to the whole `database`
|
||||
namespace to solve a problem affecting one pod on one port.
|
||||
|
||||
`cnpg_collector_up` has series for authentik, immich, monitoring and paperless
|
||||
but not for `wp-pg`, for the same reason: `apps` never allowed the monitoring
|
||||
namespace either.
|
||||
|
||||
A namespace per service keeps each grant on the workload that needs it.
|
||||
|
||||
**`woodpecker-ci` is not the target.** It enforces `pod-security: privileged`
|
||||
and exists solely for ephemeral pipeline step pods running repo-supplied code.
|
||||
Putting the database there would let any build step reach it through the
|
||||
intra-namespace allow.
|
||||
|
||||
## Staging
|
||||
|
||||
Rollback is trivial through stage 2. The `apps` cluster is untouched until
|
||||
stage 3, so reverting means pointing the DSN back and scaling up.
|
||||
|
||||
| Stage | Contents | CI impact |
|
||||
|---|---|---|
|
||||
| **1** (this PR) | `woodpecker` namespace, its network policies, ExternalSecret, ObjectStores, and `wp-pg` restored from barman | none — parallel, nothing points at it |
|
||||
| **runbook** | verify the restore holds real data | none |
|
||||
| **2** | move server + agent, flip DSN, RBAC subject, rotator policies, docs | **downtime** |
|
||||
| **3** | delete the `apps` originals, add the prune guard to the survivor | none |
|
||||
|
||||
## Stage 1 design notes
|
||||
|
||||
**Two ObjectStores, deliberately.** `wp-pg-store-source` points at
|
||||
`s3://renner-uno-k8s-cnpg/wp-pg` — the prefix the live cluster archives to —
|
||||
and is read-only. `wp-pg-store` points at a **new** prefix,
|
||||
`s3://renner-uno-k8s-cnpg/wp-pg-woodpecker`, for ongoing archiving.
|
||||
|
||||
Both clusters writing one `destinationPath` would interleave their WAL streams
|
||||
and corrupt both. Splitting them also leaves the pre-migration backup intact
|
||||
and independently restorable.
|
||||
|
||||
**`serverName: wp-pg`** on the `externalCluster` names the cluster that
|
||||
*produced* the backup, not the one being created.
|
||||
|
||||
**The namespace is declared in `infrastructure/namespaces/namespaces.yaml`,**
|
||||
not alongside these workloads. Its CiliumNetworkPolicies ship with the
|
||||
`infrastructure-crds` KS (`infrastructure/post-crds` includes
|
||||
`../network-policies/`), which `dependsOn` `infrastructure` — so the namespace
|
||||
exists before any policy targets it. Creating it in the `woodpecker` workload
|
||||
KS would invert that order, since that KS runs after both.
|
||||
|
||||
**Ingress from `woodpecker-ci` is deliberately absent in stage 1.** Step pods do
|
||||
call the server back on `:8000`, but the CNPG instance manager serves its status
|
||||
API on `:8000` too, and the baseline policy uses `endpointSelector: {}`. A
|
||||
`toPorts` block narrows the port, not the destination workload — so putting that
|
||||
rule on the baseline would hand arbitrary repo-supplied build steps a route to
|
||||
wp-pg's instance manager. It lands in stage 2 as its own policy selecting the
|
||||
server pod.
|
||||
|
||||
**No `prune: disabled` guard on the restored cluster.** The original carries one
|
||||
because it holds the only copy of the data. This one is reconstructable by
|
||||
re-running the restore, and a prune guard on a migration artifact is how orphans
|
||||
survive an abandoned migration. It gains the guard in stage 3.
|
||||
|
||||
**Barman credentials still come from `seed/paperless/hetzner-creds`.** Issue
|
||||
#397 wants woodpecker to own `seed/woodpecker/hetzner-creds`, and this would
|
||||
have been the natural moment — but creating that path needs OpenBao write
|
||||
access that was not available (`bao kv list seed` → 403). Repointing at a path
|
||||
that may not exist would leave the restored cluster unable to archive WAL,
|
||||
which is the thing this stage exists to prove. #397 stays open; the swap is one
|
||||
line once the path exists.
|
||||
|
||||
## Expect the `woodpecker` Kustomization to go not-Ready during the restore
|
||||
|
||||
`kubernetes/clusters/homelab/woodpecker.yaml` sets `wait: true` and a
|
||||
`healthCheckExprs` entry matching `postgresql.cnpg.io/v1 Cluster` **with no
|
||||
name or namespace qualifier**. Flux evaluates it against every Cluster in that
|
||||
Kustomization's inventory — which, after this stage, is both the live
|
||||
`apps/wp-pg` and the restoring `woodpecker/wp-pg`.
|
||||
|
||||
So while the restore runs, the whole `woodpecker` KS — the one that owns
|
||||
production Woodpecker — reports not-Ready, and a reconcile timeout in that
|
||||
window will trip the `flux-failures` Alert. **That is expected and is not
|
||||
Woodpecker breaking.** Nothing `dependsOn: woodpecker`, so it does not cascade.
|
||||
|
||||
If the restore is going to sit for a long time, or the noise is unwelcome, the
|
||||
options are to suspend the KS while it completes, or to split the migration
|
||||
resources into their own `wait: false` Kustomization. For a ~660 MB database the
|
||||
window should be minutes.
|
||||
|
||||
## Verifying stage 1
|
||||
|
||||
Restore is asynchronous. Watch it reach a healthy state:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get cluster wp-pg -w
|
||||
kubectl -n woodpecker get pods
|
||||
```
|
||||
|
||||
Expect `Cluster in healthy state`. If it sticks in bootstrap, the recovery job's
|
||||
logs carry the reason:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker logs -l cnpg.io/jobRole=full-recovery --tail=100
|
||||
```
|
||||
|
||||
**First check the generated app credentials target the right database.** CNPG
|
||||
defaults `bootstrap.recovery.database` to `app`; the restored data uses
|
||||
`woodpecker`. If this reads anything but `woodpecker`, the cluster has
|
||||
provisioned an empty `app` database and the real data is unreachable through
|
||||
the `-app` Secret and Service:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get secret wp-pg-app -o jsonpath='{.data.dbname}' | base64 -d; echo
|
||||
kubectl -n woodpecker get secret wp-pg-app -o jsonpath='{.data.username}' | base64 -d; echo
|
||||
```
|
||||
|
||||
**Then confirm the restore actually holds Woodpecker's data** — a cluster that
|
||||
comes up healthy but empty is the failure mode worth catching:
|
||||
|
||||
```sh
|
||||
PGPASS=$(kubectl -n woodpecker get secret wp-pg-app -o jsonpath='{.data.password}' | base64 -d)
|
||||
kubectl -n woodpecker exec -it wp-pg-1 -- env PGPASSWORD="$PGPASS" \
|
||||
psql -U woodpecker -d woodpecker -c '\dt'
|
||||
kubectl -n woodpecker exec -it wp-pg-1 -- env PGPASSWORD="$PGPASS" \
|
||||
psql -U woodpecker -d woodpecker -c 'select count(*) from repos;'
|
||||
```
|
||||
|
||||
Row counts should match the live cluster:
|
||||
|
||||
```sh
|
||||
PGPASS_OLD=$(kubectl -n apps get secret wp-pg-app -o jsonpath='{.data.password}' | base64 -d)
|
||||
kubectl -n apps exec -it wp-pg-1 -- env PGPASSWORD="$PGPASS_OLD" \
|
||||
psql -U woodpecker -d woodpecker -c 'select count(*) from repos;'
|
||||
```
|
||||
|
||||
They will not match exactly if builds ran after the backup the restore used.
|
||||
|
||||
**And stage 2 does not close that gap.** An earlier draft of this document
|
||||
claimed it quiesced the server first; it does not — Flux simply prunes the
|
||||
`apps` workloads and creates the `woodpecker` ones, so anything written to the
|
||||
old database between the restore and the cutover stays only in the old database.
|
||||
|
||||
Measured on the live cutover: the restore captured through pipeline `9488`,
|
||||
four more (`9491`, `9492`, `9494`, `9496`) landed in `apps/wp-pg` before the
|
||||
swap, and the new cluster resumed at `9521`. Four pipeline records and their
|
||||
log entries exist only in the old cluster. No ID collision, no functional
|
||||
breakage — Woodpecker's own manifest calls this database "small and
|
||||
non-critical (state is largely reconstructable)" — but it is real history loss
|
||||
and should be a decision, not a surprise.
|
||||
|
||||
If that gap matters for a future migration, quiesce before cutting over:
|
||||
|
||||
```sh
|
||||
flux suspend kustomization woodpecker
|
||||
kubectl -n <old-ns> scale deploy/woodpecker-server deploy/woodpecker-agent --replicas=0
|
||||
# re-run the restore so it captures the final state, then resume
|
||||
```
|
||||
|
||||
The old cluster keeps its `prune: disabled` guard until stage 3, so the missing
|
||||
rows remain recoverable until then.
|
||||
|
||||
Confirm the two fixes this move exists for:
|
||||
|
||||
```sh
|
||||
# #433: operator can now reach the instance
|
||||
kubectl -n woodpecker get cluster wp-pg -o jsonpath='{.status.conditions}'
|
||||
# metrics now scraped — needs BOTH the PodMonitor (cnpg-podmonitors.yaml) and
|
||||
# the monitoring ingress allow; wp-pg had neither in `apps`, which is why it is
|
||||
# absent from cnpg_collector_up while ak-pg/im-pg/gf-pg/pprl-pg are present
|
||||
kubectl get --raw '/api/v1/namespaces/monitoring/services/kube-prometheus-stack-prometheus:9090/proxy/api/v1/query?query=cnpg_collector_up{namespace="woodpecker"}'
|
||||
```
|
||||
|
||||
Confirm the scrape target is actually registered, not just that the metric is
|
||||
missing for some other reason:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get podmonitor cnpg-wp-pg
|
||||
```
|
||||
|
||||
**Check the policy semantics by hand.** `kubectl --dry-run=server` validates
|
||||
manifest shape, not Cilium behaviour, so it cannot catch a rule that is
|
||||
syntactically fine but scoped wider than intended.
|
||||
|
||||
`cilium-dbg policy trace` does **not** exist in Cilium 1.20 — it was removed,
|
||||
and `cilium-dbg policy` now offers only `get`, `selectors`, `subject-selectors`
|
||||
and `wait`. Read the realized BPF policy map instead, which is better evidence
|
||||
anyway: it is the datapath as programmed, not a simulation of it, and it
|
||||
carries live byte/packet counters.
|
||||
|
||||
Find the agent on wp-pg's node and the endpoint id:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get pod wp-pg-1 -o jsonpath='{.spec.nodeName}'
|
||||
kubectl -n kube-system get pod -l k8s-app=cilium \
|
||||
--field-selector spec.nodeName=<node> --no-headers -o custom-columns=N:.metadata.name
|
||||
|
||||
kubectl -n kube-system exec <cilium-pod> -c cilium-agent -- \
|
||||
cilium-dbg endpoint list -o json \
|
||||
| jq -r '.[] | select((.status.labels."security-relevant"[]?|tostring)
|
||||
| test("cnpg.io/cluster=wp-pg")) | "\(.id) \(.status.networking.addressing[0].ipv4)"'
|
||||
```
|
||||
|
||||
Then, on that endpoint id — who may reach `:8000`:
|
||||
|
||||
```sh
|
||||
kubectl -n kube-system exec <cilium-pod> -c cilium-agent -- \
|
||||
cilium-dbg bpf policy get <endpoint-id> | awk '$4 ~ /8000/ || NR<=1'
|
||||
```
|
||||
|
||||
Expect exactly one row, the CNPG operator, with non-zero counters once it has
|
||||
polled:
|
||||
|
||||
```
|
||||
Allow Ingress k8s:app.kubernetes.io/instance=cloudnative-pg 8000/TCP 11756 58
|
||||
```
|
||||
|
||||
And confirm CI step pods reach **ICMP only** — never a TCP port:
|
||||
|
||||
```sh
|
||||
kubectl -n kube-system exec <cilium-pod> -c cilium-agent -- \
|
||||
cilium-dbg bpf policy get <endpoint-id> | grep woodpecker-ci | awk '{print $4}' | sort -u
|
||||
```
|
||||
|
||||
Every line must be an ICMP/ICMPv6 type. A bare port number here means the
|
||||
namespace baseline has grown a rule it should not have.
|
||||
|
||||
Note the operator's allow is also confirmed indirectly and more strongly by
|
||||
`Ready=True` on the Cluster: that condition only goes true once the operator has
|
||||
successfully polled this exact endpoint and port.
|
||||
|
||||
And that archiving goes to the new prefix, leaving the old one alone:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get cluster wp-pg -o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")]}'
|
||||
```
|
||||
|
||||
## Stage 2 — the cutover
|
||||
|
||||
Moves server, agent, service, ingress, pdb, the agent ServiceAccount, the
|
||||
`woodpecker-secret` ExternalSecret and the rotator policy into `woodpecker`,
|
||||
points the server's DSN at `wp-pg-rw.woodpecker.svc`, repoints the agent's gRPC
|
||||
at `woodpecker.woodpecker.svc:9000`, moves the openbao rotators' egress, and
|
||||
adds the `woodpecker-ci` → server `:8000` policy that stage 1 deliberately
|
||||
deferred — scoped to the server pod, not the namespace baseline.
|
||||
|
||||
**This is the downtime window.** Flux prunes the `apps` copies and creates the
|
||||
`woodpecker` ones; `ci.renner.uno` is down in between, and in-flight pipelines
|
||||
die with their agent.
|
||||
|
||||
**The outage outlasts the pods becoming Ready, because of TLS.** Certificates
|
||||
are namespace-scoped, so moving the Ingress does not move `woodpecker-tls` —
|
||||
cert-manager issues a fresh one in the new namespace. Until it completes,
|
||||
Traefik serves a default certificate and clients fail before they get an HTTP
|
||||
status at all:
|
||||
|
||||
```
|
||||
curl: (60) SSL: no alternative certificate subject name matches target hostname 'ci.renner.uno'
|
||||
```
|
||||
|
||||
That is expected, not a routing or policy fault. Watch it clear with:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get certificate woodpecker-tls -w
|
||||
```
|
||||
|
||||
`letsencrypt-cluster` solves **DNS-01** via Cloudflare, so there is no in-cluster
|
||||
ACME solver pod and no HTTP challenge to admit through the network policies —
|
||||
worth knowing before scoping any ingress rule, since an HTTP-01 issuer would
|
||||
need the solver reachable too.
|
||||
|
||||
The four CNPG files (`cluster.yaml`, `objectstore.yaml`, `scheduled-backup.yaml`,
|
||||
`hetzner-creds-externalsecret.yaml`) still describe the **original** apps-namespace
|
||||
cluster and are deliberately untouched — they are the rollback. Stage 3 retires
|
||||
them.
|
||||
|
||||
### Verifying stage 2
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get pods
|
||||
kubectl -n apps get pods -l app.kubernetes.io/name=woodpecker # expect none
|
||||
```
|
||||
|
||||
The server must reach the restored database, not the old one:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker logs deploy/woodpecker-server --tail=40 | grep -i "database\|migrat\|error"
|
||||
kubectl -n woodpecker exec deploy/woodpecker-server -- env | grep DATASOURCE
|
||||
```
|
||||
|
||||
Then confirm the UI and a real pipeline, which is the only check that exercises
|
||||
server → agent → step pod → server end to end:
|
||||
|
||||
```sh
|
||||
curl -sS -o /dev/null -w '%{http_code}\n' https://ci.renner.uno/
|
||||
woodpecker pipeline create --branch master --var PIPELINE=e2e-test k.renner/homelab
|
||||
```
|
||||
|
||||
Step pods must still reach the server on `:8000` while still being unable to
|
||||
reach wp-pg on any TCP port — re-run the `bpf policy get` checks above against
|
||||
both the server's and wp-pg's endpoint ids.
|
||||
|
||||
Check the rotators can still reach the server, since their egress moved:
|
||||
|
||||
```sh
|
||||
kubectl -n openbao get cronjob
|
||||
kubectl -n openbao logs -l app.kubernetes.io/component=secret-rotator --tail=20
|
||||
```
|
||||
|
||||
### Rolling back stage 2
|
||||
|
||||
Revert the PR. Flux recreates the `apps` workloads and the DSN points back at
|
||||
`wp-pg-rw.apps.svc`, which never stopped running. Data written to the new
|
||||
cluster during the window is lost, so roll back promptly or not at all — if
|
||||
pipelines have run against `woodpecker/wp-pg`, prefer fixing forward.
|
||||
|
||||
## Stage 3 — retiring the old cluster
|
||||
|
||||
Deletes the four `apps` manifests, promotes the `migration-*` set onto their
|
||||
names, and gives the surviving Cluster the `prune: disabled` data guard it was
|
||||
deliberately denied while a rollback still existed.
|
||||
|
||||
**Merging stage 3 does not delete anything.** The old resources carry
|
||||
`kustomize.toolkit.fluxcd.io/prune: disabled`, so dropping them from git only
|
||||
stops Flux managing them — they keep running, and keep occupying their PVC.
|
||||
That is the point: the PR is reversible, the teardown is a separate, deliberate
|
||||
act.
|
||||
|
||||
Before deleting anything, confirm the new cluster is genuinely standing on its
|
||||
own — that it has taken at least one base backup to its own prefix, so recovery
|
||||
no longer depends on the retired cluster's data:
|
||||
|
||||
```sh
|
||||
kubectl -n woodpecker get backup.postgresql.cnpg.io
|
||||
kubectl -n woodpecker get cluster wp-pg -o jsonpath='{.status.conditions}'
|
||||
```
|
||||
|
||||
`LastBackupSucceeded` must be `True`, not merely `BackupStarted`.
|
||||
|
||||
Then retire the old set:
|
||||
|
||||
```sh
|
||||
kubectl -n apps delete scheduledbackup.postgresql.cnpg.io wp-pg-daily
|
||||
kubectl -n apps delete cluster.postgresql.cnpg.io wp-pg # deletes its PVCs
|
||||
kubectl -n apps delete objectstore.barmancloud.cnpg.io wp-pg-store
|
||||
kubectl -n apps delete externalsecret hetzner-s3-creds
|
||||
```
|
||||
|
||||
Delete the ScheduledBackup first: leaving it while the Cluster goes away
|
||||
produces failing backup jobs against a cluster that no longer exists.
|
||||
|
||||
**What this destroys and what it does not.** It destroys the old cluster's PVCs
|
||||
— including the four pipeline records that only ever existed there (see the
|
||||
stage-1 verification section). It does **not** touch
|
||||
`s3://renner-uno-k8s-cnpg/wp-pg`: that prefix keeps its backups until their own
|
||||
retention expires, and `wp-pg-store-source` is retained precisely so it stays
|
||||
restorable. That is the real safety net, not the running cluster.
|
||||
|
||||
### After the teardown
|
||||
|
||||
The `apps` namespace should have no woodpecker-related objects left:
|
||||
|
||||
```sh
|
||||
kubectl -n apps get cluster,objectstore,scheduledbackup,externalsecret 2>/dev/null | grep -i "wp-pg\|hetzner" || echo "clean"
|
||||
```
|
||||
|
||||
## Rolling back stage 1
|
||||
|
||||
Delete the four resources and the namespace. Nothing references them, and the
|
||||
`apps` cluster is untouched:
|
||||
|
||||
```sh
|
||||
kubectl delete ns woodpecker
|
||||
```
|
||||
|
||||
Then revert this PR so Flux stops recreating them.
|
||||
@@ -2,7 +2,7 @@ apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: woodpecker-agent
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: agent
|
||||
@@ -12,7 +12,7 @@ metadata:
|
||||
secret.reloader.stakater.com/reload: woodpecker-secret
|
||||
spec:
|
||||
# 1: the in-cluster server owns ci.renner.uno now, so this agent registers with
|
||||
# it (woodpecker.woodpecker.svc:9000) and runs pipelines on the kubernetes backend,
|
||||
# it (woodpecker.apps.svc:9000) and runs pipelines on the kubernetes backend,
|
||||
# replacing the retired docker agents (LXC 910/911).
|
||||
replicas: 1
|
||||
selector:
|
||||
@@ -34,10 +34,10 @@ spec:
|
||||
# Image automation, range 3.x (see image-automation/woodpecker.yaml).
|
||||
# MUST stay version-matched with woodpecker-server (gRPC parity) — both
|
||||
# track the same range/filter so they bump together on each release.
|
||||
image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
image: woodpeckerci/woodpecker-agent:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
env:
|
||||
- name: WOODPECKER_SERVER
|
||||
value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
|
||||
value: woodpecker.apps.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
|
||||
- name: WOODPECKER_MAX_WORKFLOWS
|
||||
value: "2"
|
||||
# --- kubernetes backend: pipeline steps run as native pods (retires
|
||||
@@ -50,39 +50,12 @@ spec:
|
||||
# clone + subsequent steps share the workspace PVC and the k8s backend
|
||||
# can schedule them on different nodes -> an RWO volume then fails to
|
||||
# mount on the second node. CephFS (RWX) lets the workspace follow
|
||||
# the pipeline across nodes. Reclaim MUST be Delete: workspaces are
|
||||
# throwaway, and a Retain class leaked one Released PV per finished
|
||||
# pipeline (issue #262 follow-up).
|
||||
#
|
||||
# ceph-filesystem-ci, not ceph-filesystem-ephemeral: a dedicated
|
||||
# CephFS whose pools are BOTH size 1 (infrastructure/
|
||||
# rook-ceph-cluster/cephfs-ci.yaml). The shared filesystem
|
||||
# replicates data0 2x and metadata 3x, which is the correct trade
|
||||
# for real data and pure waste for a git clone that is deleted
|
||||
# minutes later.
|
||||
#
|
||||
# Measured 2026-08-21, polling both pools every 15s (a 5-minute
|
||||
# interval misses an entire pipeline -- PVC created, used and
|
||||
# deleted inside ~2 minutes):
|
||||
# CI active 1.5min: data0 143.2 MB, metadata 17.2 MB
|
||||
# CI idle 16.0min: data0 0.0 MB, metadata 11.0 MB
|
||||
# data0 write volume is 100% CI. Per pipeline this moves the drive
|
||||
# cost from 71.6x2 + 8.2x3 = ~168 MB to ~80 MB.
|
||||
#
|
||||
# ceph-filesystem-ephemeral is NOT repointed, and the reason is a
|
||||
# technical constraint rather than a safety one. Its other consumer
|
||||
# is velero/restore-test.sh, whose probe PVCs are snapshot CLONES:
|
||||
# `ceph fs subvolume snapshot clone` operates within one filesystem,
|
||||
# so a snapshot taken in `cephfs` cannot be cloned into `cephfs-ci`.
|
||||
#
|
||||
# Those probes are just as throwaway as a CI workspace -- the
|
||||
# authoritative copy is the backup on Hetzner, and a lost probe just
|
||||
# means re-running the test -- so they are equally undeserving of
|
||||
# replication. Landing them unreplicated would mean a second size-1
|
||||
# data pool inside `cephfs`, and restore-test runs MONTHLY against
|
||||
# CI's ~96 pipelines/day, so it is not worth the complexity.
|
||||
# the pipeline across nodes. The -ephemeral variant (reclaim Delete,
|
||||
# rook-ceph-cluster/storageclass.yaml) is REQUIRED here: workspaces
|
||||
# are throwaway, and the Retain class leaked one Released PV per
|
||||
# finished pipeline (issue #262 follow-up).
|
||||
- name: WOODPECKER_BACKEND_K8S_STORAGE_CLASS
|
||||
value: ceph-filesystem-ci
|
||||
value: ceph-filesystem-ephemeral
|
||||
- name: WOODPECKER_BACKEND_K8S_STORAGE_RWX
|
||||
value: "true"
|
||||
# MUST be a JSON object literal — the backend json.Unmarshal's this
|
||||
|
||||
@@ -1,27 +1,19 @@
|
||||
---
|
||||
# Woodpecker's Postgres backend. Serves the server at wp-pg-rw.woodpecker.svc.
|
||||
#
|
||||
# Originally created in `apps` with bootstrap.initdb; moved here by restoring
|
||||
# from barman into the new namespace rather than by editing a namespace field,
|
||||
# because PVCs are namespace-scoped and a Cluster cannot be relocated in place
|
||||
# (issue #433, MIGRATION-namespace.md).
|
||||
#
|
||||
# The bootstrap.recovery stanza and its externalCluster are retained rather than
|
||||
# reverted to initdb. CNPG honours `bootstrap` only when PGDATA is absent, so on
|
||||
# this live cluster it is inert — and keeping it means a cold rebuild of an
|
||||
# empty cluster restores from backup instead of coming up empty, which is the
|
||||
# gap kubernetes/RECOVERY.md exists to close for the other CNPG clusters.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: wp-pg
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
annotations:
|
||||
# Data guard, earned at stage 3 now that this is the only copy: prevents a
|
||||
# Flux prune from deleting the Cluster and with it the data PVCs. The
|
||||
# migration deliberately withheld this while a rollback still existed.
|
||||
# CRITICAL data-guard: prune-disabled so the seed-woodpecker -> dedicated
|
||||
# woodpecker KS ownership handoff can never prune this Cluster (which would
|
||||
# delete its data PVCs). Landed here first, via seed-woodpecker, so it's
|
||||
# live before the move. Kept permanently as a standing guard.
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
# Single instance: the Woodpecker CI DB is small and non-critical (state is
|
||||
# largely reconstructable), so HA isn't warranted — but WAL archiving to
|
||||
# object storage still gives point-in-time backups. Mirrors grafana-db's
|
||||
# single-instance shape, with barman enabled like immich/paperless.
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
@@ -33,67 +25,13 @@ spec:
|
||||
parameters:
|
||||
barmanObjectName: wp-pg-store
|
||||
bootstrap:
|
||||
recovery:
|
||||
source: wp-pg-apps
|
||||
# Must mirror the initdb values of the cluster this restores from
|
||||
# (apps/wp-pg: database woodpecker, owner woodpecker). CNPG defaults
|
||||
# `database` to `app` and `owner` to whatever `database` is, and those
|
||||
# names do not exist in the physically restored data — it would provision
|
||||
# an empty `app` database, point the generated wp-pg-app Secret and the
|
||||
# -app Service at it, and leave the real data unreachable through them.
|
||||
# Same pattern as every recovery example in kubernetes/RECOVERY.md.
|
||||
initdb:
|
||||
database: woodpecker
|
||||
owner: woodpecker
|
||||
externalClusters:
|
||||
- name: wp-pg-apps
|
||||
plugin:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
parameters:
|
||||
barmanObjectName: wp-pg-store-source
|
||||
# The cluster that produced the backup, not this one.
|
||||
serverName: wp-pg
|
||||
postgresql:
|
||||
parameters:
|
||||
shared_buffers: "128MB"
|
||||
max_connections: "50"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
#
|
||||
# checkpoint_timeout is the lever. The 5min default pushes every dirty
|
||||
# page to disk 288 times a day, so a small database whose same few pages
|
||||
# are re-dirtied constantly (sessions, tokens, job state) gets rewritten
|
||||
# over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
|
||||
# (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
|
||||
# day. At 15min those pages coalesce and are written a third as often.
|
||||
#
|
||||
# max_wal_size must rise with it, or checkpoints trigger on size instead
|
||||
# and the timeout change does nothing. It also bounds crash recovery to
|
||||
# at most 2GB of WAL replay.
|
||||
#
|
||||
# NOT changed: synchronous_commit stays at the default `on`. It is off for
|
||||
# ha-pg because a state recorder is telemetry; these hold credentials,
|
||||
# documents and job state, where losing the last few hundred ms of commits
|
||||
# on an unclean shutdown is not an acceptable trade.
|
||||
checkpoint_timeout: "15min"
|
||||
max_wal_size: "2GB"
|
||||
# zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
|
||||
# and slower than the lz4/zstd this build also accepts). Compresses the
|
||||
# full-page images that dominate WAL volume after each checkpoint.
|
||||
wal_compression: "zstd"
|
||||
# A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
|
||||
# on a timer -- the segment is then written and archived at its full size
|
||||
# no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
|
||||
# to 288 segments/day = 4.6 GB/day that is largely padding.
|
||||
#
|
||||
# Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
|
||||
# cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
|
||||
# Two independent signals moving together is what confirms the volume was
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
|
||||
@@ -2,7 +2,7 @@ apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: woodpecker-secret
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
spec:
|
||||
# 10m (was 1h) so an automated rotation (rotate-agent-secret.yaml writes a new
|
||||
# value to OpenBao) propagates into the k8s Secret within ~10m without a manual
|
||||
|
||||
@@ -1,20 +1,11 @@
|
||||
---
|
||||
# Barman S3 credentials for wp-pg.
|
||||
#
|
||||
# Sourced from seed/paperless/hetzner-creds, which is not where they belong.
|
||||
# Issue #397 wants woodpecker to own seed/woodpecker/hetzner-creds; creating
|
||||
# that path needs OpenBao write access (`bao kv list seed` → 403 for the
|
||||
# identities available here), so it stays open. The values are identical, so it
|
||||
# is a one-line swap of the `key:` below once the path exists.
|
||||
#
|
||||
# The namespace allowlist on the openbao ClusterSecretStore
|
||||
# (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
|
||||
# namespace, or ESO silently never syncs — see #505.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: hetzner-s3-creds
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->dedicated-KS handoff (barman S3 creds).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
@@ -23,5 +14,10 @@ spec:
|
||||
target:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
# Reuses the paperless copy of the shared Hetzner S3 credentials (identical
|
||||
# ACCESS_KEY_ID / ACCESS_SECRET_KEY / REGION) to avoid seeding a new OpenBao
|
||||
# path for this migration. FOLLOW-UP: give woodpecker its own
|
||||
# seed/woodpecker/hetzner-creds path once OpenBao can be written, and repoint
|
||||
# this here (the value is the same, so it's a drop-in swap).
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
|
||||
@@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: woodpecker
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
annotations:
|
||||
|
||||
@@ -21,20 +21,11 @@ resources:
|
||||
# (kubernetes/infrastructure/openbao/rotation/woodpecker.yaml) — intra-ns
|
||||
# OpenBao access, no NetworkPolicy needed. server.yaml/agent.yaml keep their
|
||||
# Reloader annotations; externalsecret.yaml keeps the 10m refreshInterval.
|
||||
# --- namespace migration, stages 1-2 (issue #433) ---
|
||||
# Stage 2 has moved the workloads above (server, agent, service, ingress, pdb,
|
||||
# rbac, secrets, policies) into the `woodpecker` namespace and pointed the
|
||||
# server's DSN at wp-pg-rw.woodpecker.svc. Stage 3 retired the `apps`
|
||||
# originals; these four now describe the surviving cluster. History and the
|
||||
# reasoning behind the three stages: MIGRATION-namespace.md.
|
||||
#
|
||||
# The `woodpecker` Namespace itself is declared in infrastructure/namespaces,
|
||||
# not here: it must exist before the `infrastructure-crds` KS applies the
|
||||
# namespace's CiliumNetworkPolicies, and this KS runs after both of those.
|
||||
#
|
||||
# This KS depends on infrastructure-cnpg-barman for the barman plugin behind
|
||||
# the ObjectStore and ScheduledBackup.
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
- objectstore.yaml
|
||||
# wp-pg CNPG Postgres backend (moved here from the retired seed-woodpecker KS;
|
||||
# each object carries prune: disabled so ownership transferred without a
|
||||
# teardown). The dedicated `woodpecker` KS depends on infrastructure-cnpg-barman
|
||||
# for the barman plugin behind the ObjectStore/ScheduledBackup.
|
||||
- cluster.yaml
|
||||
- objectstore.yaml
|
||||
- scheduled-backup.yaml
|
||||
- hetzner-creds-externalsecret.yaml
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
# Secret rotation: the review-bot and claude-oauth rotators (openbao ns) reach
|
||||
# the Woodpecker SERVER to upsert org-level CI secrets (review_bot_token,
|
||||
# claude_code_oauth_token). They use the in-cluster Service
|
||||
# (woodpecker.woodpecker.svc:8000) — the public ci.renner.uno name hairpins off the
|
||||
# ingress LB from a pod. This namespace's baseline
|
||||
# (network-policies/woodpecker-ns.yaml) admits only same-namespace, monitoring,
|
||||
# host/remote-node and ICMP, so without this the rotator's connect to :8000 is
|
||||
# (woodpecker.apps.svc:8000) — the public ci.renner.uno name hairpins off the
|
||||
# ingress LB from a pod. The apps default-deny-ingress only allows same-ns /
|
||||
# host / a few named sources, so without this the rotator's connect to :8000 is
|
||||
# dropped. Scoped to the woodpecker SERVER pod :8000 and the two rotator labels;
|
||||
# the egress halves are rotator-review-bot-egress + rotator-claude-oauth-egress
|
||||
# in network-policies/openbao-ns.yaml.
|
||||
#
|
||||
# Cilium unions allow rules, so this composes with the baseline and with the
|
||||
# sibling policies below (traefik, CI steps) rather than replacing them. Note
|
||||
# allow-ingress.yaml does NOT apply here — it is hard-scoped to `namespace:
|
||||
# apps`, which is why traefik needs its own rule in this file now.
|
||||
# This is additive to the apps-ns baseline allows (allow-ingress.yaml selects all
|
||||
# apps pods), so the server's existing ingress — Traefik, agents — is unaffected.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: woodpecker-allow-rotators
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
@@ -37,25 +34,3 @@ spec:
|
||||
- ports:
|
||||
- port: "8000"
|
||||
protocol: TCP
|
||||
---
|
||||
# Pipeline step pods call the server back on :8000 to stream logs and report
|
||||
# status. Deferred from stage 1 on purpose, and scoped to the SERVER pod rather
|
||||
# than granted on the namespace baseline.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: woodpecker-allow-ci-steps
|
||||
namespace: woodpecker
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: server
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": woodpecker-ci
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8000"
|
||||
protocol: TCP
|
||||
|
||||
@@ -1,20 +1,18 @@
|
||||
---
|
||||
# RECOVERY SOURCE — read-only. The prefix the retired `apps/wp-pg` archived to.
|
||||
#
|
||||
# Kept after the migration on purpose. Nothing writes to it, and the backups
|
||||
# under it age out on their own retention, but while they last this is a second,
|
||||
# independent restore point that predates the move — worth having until the new
|
||||
# prefix has a meaningful backup history of its own.
|
||||
#
|
||||
# serverName on the consuming externalCluster must be `wp-pg`, the name of the
|
||||
# cluster that produced these backups.
|
||||
apiVersion: barmancloud.cnpg.io/v1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: wp-pg-store-source
|
||||
namespace: woodpecker
|
||||
name: wp-pg-store
|
||||
namespace: apps
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->dedicated-KS handoff (backup target).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
configuration:
|
||||
# Same offsite Hetzner Object Storage bucket as the other CNPG clusters,
|
||||
# under a per-cluster prefix. See apps/paperless/objectstore.yaml for the
|
||||
# SSE / addressing rationale (Hetzner is SSE-C only; transit is TLS,
|
||||
# bucket private).
|
||||
destinationPath: s3://renner-uno-k8s-cnpg/wp-pg
|
||||
endpointURL: https://fsn1.your-objectstorage.com
|
||||
s3Credentials:
|
||||
@@ -31,36 +29,4 @@ spec:
|
||||
compression: gzip
|
||||
data:
|
||||
compression: gzip
|
||||
# No retentionPolicy: this store is a read source during the migration, and
|
||||
# the old cluster in `apps` owns the lifecycle of the data underneath it.
|
||||
---
|
||||
# ARCHIVING TARGET — a DIFFERENT prefix from the source, deliberately.
|
||||
#
|
||||
# Two clusters archiving WAL to one destinationPath interleave their streams and
|
||||
# corrupt both. That mattered during the parallel-run window; it still matters
|
||||
# now, because the source prefix above is retained and must stay frozen for the
|
||||
# independent restore point it provides.
|
||||
apiVersion: barmancloud.cnpg.io/v1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: wp-pg-store
|
||||
namespace: woodpecker
|
||||
spec:
|
||||
configuration:
|
||||
destinationPath: s3://renner-uno-k8s-cnpg/wp-pg-woodpecker
|
||||
endpointURL: https://fsn1.your-objectstorage.com
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_SECRET_KEY
|
||||
region:
|
||||
name: hetzner-s3-creds
|
||||
key: REGION
|
||||
wal:
|
||||
compression: gzip
|
||||
data:
|
||||
compression: gzip
|
||||
retentionPolicy: "30d"
|
||||
|
||||
@@ -15,7 +15,7 @@ apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: woodpecker-server
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: server
|
||||
|
||||
@@ -17,7 +17,7 @@ apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: woodpecker-agent
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: agent
|
||||
@@ -60,7 +60,4 @@ roleRef:
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: woodpecker-agent
|
||||
# The agent's SA lives in `woodpecker`; this Role/RoleBinding stays in
|
||||
# `woodpecker-ci`, because that is the namespace it grants pod
|
||||
# create/delete in. Cross-namespace subject, deliberately.
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
|
||||
@@ -1,19 +1,18 @@
|
||||
---
|
||||
# Daily base backups onto the wp-pg-woodpecker prefix. Continuous WAL archiving
|
||||
# runs separately (cluster.yaml spec.plugins[].isWALArchiver); without periodic
|
||||
# base backups the prefix would have no recovery point of its own and
|
||||
# wp-pg-store's retentionPolicy: 30d would have nothing to prune against, so the
|
||||
# WAL chain would grow without bound.
|
||||
#
|
||||
# 02:40 UTC, offset from the other CNPG clusters' 02:30 so they do not all hit
|
||||
# the Hetzner bucket together.
|
||||
# Daily base backup at 02:35 UTC (offset from the other clusters' 02:30 to
|
||||
# avoid a thundering herd against the Hetzner bucket). WAL archiving runs
|
||||
# continuously (see cluster.yaml spec.plugins[].isWALArchiver). immediate:true
|
||||
# takes the first base backup as soon as this CR applies.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: ScheduledBackup
|
||||
metadata:
|
||||
name: wp-pg-daily
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
annotations:
|
||||
# Prune-disabled — a standing guard kept from the seed->dedicated-KS handoff (backup schedule).
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
schedule: "0 40 2 * * *"
|
||||
schedule: "0 35 2 * * *"
|
||||
immediate: true
|
||||
backupOwnerReference: self
|
||||
cluster:
|
||||
|
||||
@@ -14,7 +14,7 @@ apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: woodpecker-secret
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
type: Opaque
|
||||
|
||||
@@ -2,7 +2,7 @@ apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: woodpecker-server
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: server
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
# minor + patch auto-update within 3.x; only the 4.0 major is held for
|
||||
# a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
|
||||
# -bump hazard is gone). Stays in lockstep with the agent.
|
||||
image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
image: woodpeckerci/woodpecker-server:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
env:
|
||||
- name: WOODPECKER_HOST
|
||||
value: https://ci.renner.uno
|
||||
@@ -84,7 +84,7 @@ spec:
|
||||
- name: WOODPECKER_DATABASE_DRIVER
|
||||
value: postgres
|
||||
- name: WOODPECKER_DATABASE_DATASOURCE
|
||||
value: "host=wp-pg-rw.woodpecker.svc.cluster.local port=5432 user=$(WP_DB_USER) password=$(WP_DB_PASS) dbname=woodpecker sslmode=require"
|
||||
value: "host=wp-pg-rw.apps.svc.cluster.local port=5432 user=$(WP_DB_USER) password=$(WP_DB_PASS) dbname=woodpecker sslmode=require"
|
||||
# AGENT_SECRET + GITEA_CLIENT + GITEA_SECRET come from the SOPS secret.
|
||||
envFrom:
|
||||
- secretRef:
|
||||
|
||||
@@ -2,13 +2,13 @@ apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: woodpecker
|
||||
namespace: woodpecker
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
spec:
|
||||
# Flipped ExternalName -> ClusterIP: ci.renner.uno (via the Ingress, by name)
|
||||
# and the in-cluster agent both target this Service now. http:8000 serves the
|
||||
# UI/API; grpc:9000 is how the agent (woodpecker.woodpecker.svc:9000) reaches the
|
||||
# UI/API; grpc:9000 is how the agent (woodpecker.apps.svc:9000) reaches the
|
||||
# server. Selects the server pod only (not the agent).
|
||||
type: ClusterIP
|
||||
selector:
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: wordpress
|
||||
namespace: wordpress
|
||||
annotations:
|
||||
secret.reloader.stakater.com/reload: wordpress-secrets
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
# RWO volume, single writer — the rolling default would deadlock on the
|
||||
# new pod waiting for a volume the old pod still holds.
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wordpress
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: wordpress
|
||||
spec:
|
||||
securityContext:
|
||||
# wp-content must be writable by Apache (www-data, uid/gid 33) for
|
||||
# media uploads and plugin installs.
|
||||
fsGroup: 33
|
||||
containers:
|
||||
- name: wordpress
|
||||
image: wordpress:7.0.4-php8.3-apache
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 80
|
||||
env:
|
||||
# The operator names the base Service after the MariaDB resource;
|
||||
# the CRD's separate primaryService/secondaryService fields are the
|
||||
# suffixed variants for replicated setups, which this is not
|
||||
# (replicas: 1). Not verified against a running cluster — if it is
|
||||
# wrong the pod fails loudly on connect at startup rather than
|
||||
# degrading quietly. Confirm with `kubectl get svc -n wordpress`
|
||||
# on first deploy.
|
||||
- name: WORDPRESS_DB_HOST
|
||||
value: wordpress-mariadb.wordpress.svc.cluster.local:3306
|
||||
- name: WORDPRESS_DB_NAME
|
||||
value: wordpress
|
||||
- name: WORDPRESS_DB_USER
|
||||
value: wordpress
|
||||
- name: WORDPRESS_DB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
# Generated by the operator from the MariaDB CR — never
|
||||
# stored in Git or OpenBao.
|
||||
name: wordpress-mariadb-app
|
||||
key: password
|
||||
- name: WORDPRESS_CONFIG_EXTRA
|
||||
value: |
|
||||
define('WP_HOME', 'https://www.renner.uno');
|
||||
define('WP_SITEURL', 'https://www.renner.uno');
|
||||
define('FORCE_SSL_ADMIN', true);
|
||||
/* Traefik terminates TLS; without this WordPress sees plain
|
||||
HTTP and builds http:// URLs into every page and redirect. */
|
||||
if (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https') {
|
||||
$_SERVER['HTTPS'] = 'on';
|
||||
}
|
||||
/* Core lives in the image, not on the volume — the updater
|
||||
would only fail against a read-only docroot. Plugin and
|
||||
theme installs are unaffected. */
|
||||
define('WP_AUTO_UPDATE_CORE', false);
|
||||
/* No add_filter() here. WORDPRESS_CONFIG_EXTRA is eval'd inside
|
||||
wp-config.php, long before wp-settings.php loads the plugin
|
||||
API, so any add_filter call is a fatal error on every request
|
||||
— which is exactly what happened on first deploy. xmlrpc is
|
||||
handled at the ingress instead, by routing /xmlrpc.php
|
||||
through the Authentik gate. */
|
||||
envFrom:
|
||||
# The eight WordPress salts. Rotating them in OpenBao invalidates
|
||||
# every session — which is the intended lever, not a side effect.
|
||||
- secretRef:
|
||||
name: wordpress-secrets
|
||||
volumeMounts:
|
||||
- name: content
|
||||
mountPath: /var/www/html/wp-content
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 768Mi
|
||||
volumes:
|
||||
- name: content
|
||||
persistentVolumeClaim:
|
||||
claimName: wordpress-content
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: wordpress
|
||||
namespace: wordpress
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: wordpress
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: http
|
||||
@@ -1,46 +0,0 @@
|
||||
# The eight WordPress salts (AUTH_KEY, SECURE_AUTH_KEY, LOGGED_IN_KEY,
|
||||
# NONCE_KEY, AUTH_SALT, SECURE_AUTH_SALT, LOGGED_IN_SALT, NONCE_SALT).
|
||||
# Expected keys in OpenBao at homelab/apps/wordpress, named exactly as the
|
||||
# image's env vars: WORDPRESS_AUTH_KEY, WORDPRESS_SECURE_AUTH_KEY, ...
|
||||
#
|
||||
# Deliberately NOT carried over from the old wp-config.php: the Hetzner host
|
||||
# is being decommissioned and its salts have been sitting on a VM whose
|
||||
# neighbour site was compromised this month. New salts cost one logout.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wordpress-secrets
|
||||
namespace: wordpress
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: wordpress-secrets
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/wordpress
|
||||
---
|
||||
# S3 credentials for the MariaDB Backup.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: hetzner-s3-creds
|
||||
namespace: wordpress
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: hetzner-s3-creds
|
||||
dataFrom:
|
||||
# Reuses the paperless copy of the shared Hetzner S3 credentials (identical
|
||||
# ACCESS_KEY_ID / ACCESS_SECRET_KEY / REGION) to avoid seeding a new OpenBao
|
||||
# path for this migration — same approach woodpecker took. FOLLOW-UP: give
|
||||
# wordpress its own seed/wordpress/hetzner-creds path once OpenBao can be
|
||||
# written, and repoint this here (the value is the same, so it's a drop-in
|
||||
# swap).
|
||||
- extract:
|
||||
key: seed/paperless/hetzner-creds
|
||||
@@ -1,15 +0,0 @@
|
||||
# A plain Ingress cannot point at a Service in another namespace, and the
|
||||
# Authentik outposts all live in `authentik`. Same workaround as
|
||||
# zone-configurator and paperless: mirror the outpost Service in as an
|
||||
# ExternalName.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ak-outpost-wordpress-gate
|
||||
namespace: wordpress
|
||||
spec:
|
||||
type: ExternalName
|
||||
externalName: ak-outpost-wordpress-gate.authentik.svc.cluster.local
|
||||
ports:
|
||||
- port: 9000
|
||||
targetPort: 9000
|
||||
@@ -1,110 +0,0 @@
|
||||
# Two routers on one host. The site itself must stay readable by anyone --
|
||||
# it is a family blog -- while the editing surface sits behind Authentik.
|
||||
#
|
||||
# Why not one gated Ingress with skip_path_regex, the way paperless-gate
|
||||
# carves out /static/: that works when the PUBLIC paths are enumerable. Here
|
||||
# it is the reverse -- every post URL is public and only two paths are gated
|
||||
# -- which would need a negative lookahead. The outpost is Go, and RE2 has no
|
||||
# lookahead. So the split happens at the router instead.
|
||||
#
|
||||
# Traefik orders routers by rule length, so PathPrefix(`/wp-admin`) is
|
||||
# evaluated before PathPrefix(`/`); no explicit priority is needed.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wordpress
|
||||
namespace: wordpress
|
||||
labels:
|
||||
dns.renner.uno/public: "true"
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
rules:
|
||||
- host: www.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: wordpress
|
||||
port:
|
||||
number: 80
|
||||
tls:
|
||||
- secretName: wordpress-tls
|
||||
hosts:
|
||||
- www.renner.uno
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wordpress-admin
|
||||
namespace: wordpress
|
||||
labels:
|
||||
# Same label as the public Ingress on purpose. external-dns-unifi runs
|
||||
# with --label-filter=dns.renner.uno/public!=true, so an unlabelled
|
||||
# Ingress on this host would quietly gain a UDM A-record pointing LAN
|
||||
# clients straight at the Traefik VIP -- a second, unintended path to
|
||||
# the same host that skips the public route entirely.
|
||||
dns.renner.uno/public: "true"
|
||||
# No cert-manager annotation and no tls block: the public Ingress above
|
||||
# already owns the certificate for this host. Declaring it twice would
|
||||
# create two Certificates racing for the same wordpress-tls Secret.
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
rules:
|
||||
- host: www.renner.uno
|
||||
http:
|
||||
paths:
|
||||
# The outpost's own OAuth leg. Without this the gate is a dead end,
|
||||
# not a gate: hitting /wp-admin redirects to
|
||||
# /outpost.goauthentik.io/start on THIS host, which otherwise falls
|
||||
# through to the public `/` router and lands on WordPress's 404
|
||||
# page — verified exactly that way on first deploy. paperless needs
|
||||
# the same route and has it as a separate Ingress on auth.renner.uno
|
||||
# (apps/authentik/paperless-gate-ingress.yaml); here the flow stays
|
||||
# on www.renner.uno, so it belongs in this rule set.
|
||||
#
|
||||
# It is not an authentication hole: these endpoints ARE the login
|
||||
# flow, and the outpost serves them itself.
|
||||
- path: /outpost.goauthentik.io
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: ak-outpost-wordpress-gate
|
||||
port:
|
||||
number: 9000
|
||||
# Covers admin-ajax.php too. That is only safe because the gift
|
||||
# registry plugin -- the sole thing on this site that ever emitted
|
||||
# an ajaxurl -- is being dropped in the migration. Verified against
|
||||
# the live site: 121 admin-ajax hits in the current log window, not
|
||||
# one of them a 200.
|
||||
- path: /wp-admin
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: ak-outpost-wordpress-gate
|
||||
port:
|
||||
number: 9000
|
||||
- path: /wp-login.php
|
||||
pathType: Exact
|
||||
backend:
|
||||
service:
|
||||
name: ak-outpost-wordpress-gate
|
||||
port:
|
||||
number: 9000
|
||||
# xmlrpc.php would otherwise be served by the public `/` prefix
|
||||
# router — an earlier version of this file claimed it was "blocked
|
||||
# at the ingress" while nothing blocked it at all. There is no deny
|
||||
# verb in an Ingress, so it goes through the gate instead: anonymous
|
||||
# callers get Authentik, not the XML-RPC endpoint. Nothing on this
|
||||
# site uses it (no Jetpack, no mobile app, no pingbacks worth
|
||||
# keeping), so no legitimate caller is affected.
|
||||
- path: /xmlrpc.php
|
||||
pathType: Exact
|
||||
backend:
|
||||
service:
|
||||
name: ak-outpost-wordpress-gate
|
||||
port:
|
||||
number: 9000
|
||||
@@ -1,9 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- externalsecret.yaml
|
||||
- mariadb.yaml
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- gate-service.yaml
|
||||
- ingress.yaml
|
||||
@@ -1,86 +0,0 @@
|
||||
# The database for www.renner.uno. Named `wordpress-mariadb`, not `wp-*`:
|
||||
# `wp-pg` in the woodpecker namespace is Woodpecker's Postgres, and reusing
|
||||
# the prefix for an unrelated engine invites exactly the wrong guess.
|
||||
#
|
||||
# Mirrors the CNPG convention: the operator lives in `database`, the database
|
||||
# resource lives with its consumer (ak-pg in authentik, im-pg in immich).
|
||||
apiVersion: k8s.mariadb.com/v1alpha1
|
||||
kind: MariaDB
|
||||
metadata:
|
||||
name: wordpress-mariadb
|
||||
namespace: wordpress
|
||||
annotations:
|
||||
# Data guard — a Kustomization rebuild must never take the database with it.
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
# generate: true lets the operator create both Secrets on first reconcile.
|
||||
# Neither password is ever typed by a human or parked in OpenBao, so there
|
||||
# is nothing to rotate by hand and nothing to leak through Git.
|
||||
rootPasswordSecretKeyRef:
|
||||
name: wordpress-mariadb-root
|
||||
key: password
|
||||
generate: true
|
||||
username: wordpress
|
||||
passwordSecretKeyRef:
|
||||
name: wordpress-mariadb-app
|
||||
key: password
|
||||
generate: true
|
||||
database: wordpress
|
||||
port: 3306
|
||||
replicas: 1
|
||||
|
||||
storage:
|
||||
size: 5Gi
|
||||
storageClassName: ceph-block
|
||||
# The source database is 527 KB. 5Gi is not a sizing estimate, it is the
|
||||
# smallest allocation that leaves room for binlogs and a decade of posts.
|
||||
|
||||
myCnf: |
|
||||
[mariadb]
|
||||
bind-address=*
|
||||
default_storage_engine=InnoDB
|
||||
innodb_buffer_pool_size=256M
|
||||
# WordPress writes utf8mb4 throughout; the source dump carries
|
||||
# utf8mb4_unicode_520_ci on every table that survives the migration.
|
||||
character_set_server=utf8mb4
|
||||
collation_server=utf8mb4_unicode_520_ci
|
||||
max_allowed_packet=64M
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
---
|
||||
# Logical backup to Hetzner Object Storage. Velero only captures PVC data,
|
||||
# which for a running database is crash-consistent, not transaction-consistent
|
||||
# — that is a snapshot, not a backup you would want to restore from.
|
||||
apiVersion: k8s.mariadb.com/v1alpha1
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: wordpress-mariadb-backup
|
||||
namespace: wordpress
|
||||
spec:
|
||||
mariaDbRef:
|
||||
name: wordpress-mariadb
|
||||
schedule:
|
||||
# 02:45, ahead of the Velero window that starts at 03:15.
|
||||
cron: "45 2 * * *"
|
||||
suspend: false
|
||||
maxRetention: 720h # 30d, matching the CNPG ObjectStore retentionPolicy
|
||||
compression: gzip
|
||||
storage:
|
||||
s3:
|
||||
bucket: renner-uno-k8s-mariadb
|
||||
prefix: wordpress
|
||||
endpoint: fsn1.your-objectstorage.com
|
||||
region: fsn1
|
||||
accessKeyIdSecretKeyRef:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKeySecretKeyRef:
|
||||
name: hetzner-s3-creds
|
||||
key: ACCESS_SECRET_KEY
|
||||
tls:
|
||||
enabled: true
|
||||
@@ -1,35 +0,0 @@
|
||||
# Only wp-content is persisted, not the whole docroot. WordPress core comes
|
||||
# from the pinned image, so upgrading the site means bumping a tag in Git
|
||||
# rather than clicking "update" in wp-admin — which is the entire point of
|
||||
# moving off the Hetzner VM, where core was a mutable pile of PHP on disk.
|
||||
#
|
||||
# Consequence to be aware of: WordPress's own core updater will fail against
|
||||
# a read-only core. Plugin and theme installs still work, because those land
|
||||
# in wp-content.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: wordpress-content
|
||||
namespace: wordpress
|
||||
labels:
|
||||
# The Velero Schedule selects ON this label rather than excluding the
|
||||
# database PVC by one the mariadb-operator is merely assumed to set. If
|
||||
# that assumption were wrong the exclusion would fail silently and Velero
|
||||
# would snapshot the live database volume; selecting a label this repo
|
||||
# owns cannot fail that way. Same direction as openbao-daily, which picks
|
||||
# its snapshot PVC by app.kubernetes.io/component.
|
||||
app.kubernetes.io/name: wordpress
|
||||
app.kubernetes.io/component: content
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
# RWO is sufficient: a single replica with strategy Recreate. ceph-block is
|
||||
# the cluster default; named explicitly so a default change can't silently
|
||||
# move the media library onto another class.
|
||||
storageClassName: ceph-block
|
||||
resources:
|
||||
requests:
|
||||
# The migrated wp-content is 29 MB compressed, 13 MB of it uploads.
|
||||
storage: 10Gi
|
||||
@@ -48,10 +48,6 @@ spec:
|
||||
kind: HelmRelease
|
||||
name: cloudnative-pg # DB operator — seed jobs create databases
|
||||
namespace: database
|
||||
- apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
name: mariadb-operator # the MySQL half — WordPress can't use CNPG
|
||||
namespace: database
|
||||
# decryption block dropped 2026-07-13: no *.sops.yaml left under this
|
||||
# path — secrets come from OpenBao via ESO (MIGRATION-sops-openbao.md).
|
||||
---
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: wordpress
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 15m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/apps/wordpress
|
||||
prune: true
|
||||
dependsOn:
|
||||
# infrastructure gates on the mariadb-operator HelmRelease being Ready,
|
||||
# so the MariaDB CRD exists and has a controller before this applies.
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
# kstatus has no built-in rule for k8s.mariadb.com resources, so without
|
||||
# this the Kustomization would report Ready the moment the manifests are
|
||||
# accepted — before the database is actually serving. Same reasoning as
|
||||
# the CephCluster expressions in infrastructure.yaml.
|
||||
healthCheckExprs:
|
||||
- apiVersion: k8s.mariadb.com/v1alpha1
|
||||
kind: MariaDB
|
||||
failed: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'False' && c.reason == 'Failed')"
|
||||
current: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')"
|
||||
@@ -268,7 +268,7 @@ EOF
|
||||
|
||||
# 120x15s = 30 min, not the old 60x5s = 5 min. node-agent has no
|
||||
# loadConcurrency override, so Velero's default of one data path per node
|
||||
# applies: while the restore-test sweep holds the slot, even this 1 GiB
|
||||
# applies: while the Sunday restore-test sweep holds the slot, even this 1 GiB
|
||||
# DataUpload sits in phase Prepared for well over 5 minutes. The old 300 s wait
|
||||
# then let the teardown below delete the source namespace and the Backup
|
||||
# mid-upload ("Cancel du under phase Prepared because it is being deleted"),
|
||||
|
||||
@@ -17,11 +17,7 @@ spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: cilium
|
||||
# Patch-only range: Flux reconciles every 30m and would otherwise take a
|
||||
# CNI minor or major unattended. Widening this is a deliberate, reviewed
|
||||
# act — a minor bump changes datapath behaviour and rolls 2 of 3 agents
|
||||
# at once.
|
||||
version: ">=1.20.0 <1.21.0"
|
||||
version: ">=1.19.0"
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: cilium
|
||||
@@ -33,34 +29,6 @@ spec:
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
# Roll the agent DaemonSet when cilium-config changes. Cilium reads
|
||||
# datapath options once, at agent start, and the chart puts no config
|
||||
# checksum on the pod template — so a changed ConfigMap leaves every agent
|
||||
# running the previous datapath until something restarts it. Nothing did.
|
||||
#
|
||||
# This is not hypothetical. vlan-bpf-bypass reached cilium-config and every
|
||||
# v30iot macvlan workload stayed broken until the agents were deleted by
|
||||
# hand, one node at a time (issue #592). policyMapMax has the same
|
||||
# exposure, and its comment below says "takes effect on agent restart" —
|
||||
# documentation standing in for automation.
|
||||
#
|
||||
# The trade: any cilium-config change now rolls the CNI cluster-wide,
|
||||
# node by node. That is disruptive-ish, but it is what a datapath change
|
||||
# already required — the difference is that it now happens on its own
|
||||
# instead of being forgotten. Reloader is already deployed for exactly
|
||||
# this pattern; named per-ConfigMap rather than blanket, as elsewhere.
|
||||
annotations:
|
||||
configmap.reloader.stakater.com/reload: cilium-config
|
||||
# Which pod labels feed security-identity computation. Patterns are
|
||||
# start-anchored regexes; a leading "!" excludes while keeping the default
|
||||
# set. Woodpecker stamps every step pod with woodpecker-ci.org/task-uuid
|
||||
# and step=<name>, so without this each build step is its own identity —
|
||||
# and every identity costs one entry per allowed (direction, proto, port)
|
||||
# in every endpoint's policy map, cluster-wide. Nothing selects on these
|
||||
# labels in policy, so they buy identity churn and nothing else.
|
||||
# Keep in lockstep with kubernetes/talos/patches/cilium-cni.yaml and
|
||||
# kubernetes/talos/cilium.yaml.
|
||||
labels: '!woodpecker-ci\.org/.* !step$'
|
||||
ipv6:
|
||||
enabled: true
|
||||
ipam:
|
||||
@@ -78,49 +46,6 @@ spec:
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
bpf:
|
||||
# Size of each endpoint's own cilium_policy_v3_<id> map, holding one
|
||||
# entry per (identity, direction, protocol, port) the endpoint is
|
||||
# allowed. When it fills, endpoint regeneration fails with "no space
|
||||
# left on device", the CNI ADD call never returns, and the pod is stuck
|
||||
# in sandbox creation — so this must stay comfortably above the busiest
|
||||
# endpoint's entry count, tracked by cilium_bpf_map_pressure.
|
||||
# Takes effect on agent restart; the map is sized at creation.
|
||||
# Bootstrap installs Cilium from the inline manifest in
|
||||
# kubernetes/talos/patches/cilium-cni.yaml, so bpf-policy-map-max there
|
||||
# must move with this value or a cold rebuild starts with the old size.
|
||||
# kubernetes/talos/cilium.yaml carries a third copy of the same key.
|
||||
# Nothing enforces any of this; grep bpf-policy-map-max before changing it.
|
||||
policyMapMax: 65536
|
||||
# VLAN ids the eBPF datapath is allowed to pass. Without this it drops
|
||||
# VLAN-tagged frames, accounted as "VLAN traffic disallowed by VLAN
|
||||
# filter" in cilium_drop_count_total.
|
||||
#
|
||||
# 30 is v30iot, the macvlan parent for the shelly-manager, matter-server,
|
||||
# otbr and music-assistant attachments. On bare metal it is a VLAN
|
||||
# sub-interface of enp2s0f0np0; inside the former Proxmox guests it was
|
||||
# an untagged virtio NIC, which is why this was not needed before.
|
||||
#
|
||||
# VLAN 40 (mgmt0) is deliberately NOT listed, and the asymmetry is the
|
||||
# interesting part. Cilium attaches to devices that carry an address —
|
||||
# `cilium-dbg status --verbose` lists enp87s0, enp2s0f0np0, enp2s0f1np1
|
||||
# and mgmt0, but NOT ens19. mgmt0 holds the node IP, so Cilium owns it as
|
||||
# a device and sees frames the 8021q layer has already untagged; nothing
|
||||
# for a VLAN filter to match. ens19 is deliberately address-less (see
|
||||
# talos/patches/k8s-1.yaml — host addresses on that L2 make node replies
|
||||
# bypass the UDM), so Cilium never attaches to it and VLAN 30 frames only
|
||||
# meet the filter on the shared parent, still tagged.
|
||||
#
|
||||
# So the fix belongs on 30 alone. Listing 40 would change nothing, and
|
||||
# would touch the datapath carrying etcd and the API for no reason.
|
||||
#
|
||||
# A tc-layer drop surfaces to an AF_PACKET sender as ENOBUFS, so the
|
||||
# symptom is the CNI dhcp plugin reporting "no buffer space available"
|
||||
# rather than a DHCP timeout. See issue #592.
|
||||
#
|
||||
# Same three-copy problem as policyMapMax above: vlan-bpf-bypass in
|
||||
# kubernetes/talos/cilium.yaml and kubernetes/talos/patches/cilium-cni.yaml
|
||||
# must move with this list. Takes effect on agent restart.
|
||||
vlanBypass: [30]
|
||||
# Let pods reach the cluster's own LoadBalancer external IPs; Cilium
|
||||
# redirects the socket directly to the backend pod instead of sending
|
||||
# traffic out to the VIP and back (which fails with L2 announcements).
|
||||
|
||||
@@ -917,27 +917,11 @@ subjects:
|
||||
name: plugin-barman-cloud
|
||||
namespace: database
|
||||
---
|
||||
# stringData, NOT base64 `data`. An image reference is not a secret, and
|
||||
# encoding it here had two real costs: no reviewer sees a version change in a
|
||||
# diff, and no $imagepolicy marker can be attached to a base64 blob. The result
|
||||
# was that automation advanced the controller to v0.14.0 while this sidecar sat
|
||||
# at v0.12.0 — a two-minor skew in the component that performs every base backup
|
||||
# and WAL archive, invisible until someone read the pod spec (#535).
|
||||
#
|
||||
# The Secret KIND is kept because the controller reads it via secretKeyRef
|
||||
# (deployment env SIDECAR_IMAGE); changing it to a ConfigMap would need a matching
|
||||
# edit to that reference and diverge further from upstream. stringData is
|
||||
# accepted identically by the API server and serialised into `data` on write.
|
||||
#
|
||||
# The name carries a kustomize hash suffix from upstream's secretGenerator. It is
|
||||
# static here, so editing the value does NOT change the name and the deployment's
|
||||
# secretKeyRef stays valid.
|
||||
#
|
||||
# IF YOU RE-IMPORT THE UPSTREAM MANIFEST: upstream ships this as base64 `data`
|
||||
# with no marker. Re-apply this block, or the skew silently returns.
|
||||
apiVersion: v1
|
||||
stringData:
|
||||
SIDECAR_IMAGE: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar:v0.14.0 # {"$imagepolicy": "flux-system:cnpg-barman-plugin-sidecar"}
|
||||
data:
|
||||
SIDECAR_IMAGE: |
|
||||
Z2hjci5pby9jbG91ZG5hdGl2ZS1wZy9wbHVnaW4tYmFybWFuLWNsb3VkLXNpZGVjYXI6dj
|
||||
AuMTIuMA==
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: plugin-barman-cloud-kg5c2795f7
|
||||
|
||||
@@ -87,16 +87,6 @@ spec:
|
||||
|
||||
lapi:
|
||||
replicas: 1
|
||||
# The chart defaults request 500m. Measured steady-state draw is 2m, and
|
||||
# the request is what the scheduler reserves, so the default parks half a
|
||||
# core of unusable capacity per LAPI. That is affordable on three nodes
|
||||
# and is not on two: with one node out for maintenance the remaining pair
|
||||
# sit at 100% of requested CPU while actually using ~22%, and nothing new
|
||||
# can be placed. The limit stays at the chart default so a log flood can
|
||||
# still burst; only the reservation shrinks.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
podAnnotations:
|
||||
# Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
|
||||
# env-var secret, so a restart is required to pick up a rotated key). See
|
||||
@@ -132,12 +122,6 @@ spec:
|
||||
key: key
|
||||
|
||||
agent:
|
||||
# Same reasoning as lapi.resources above, multiplied by three: this is a
|
||||
# DaemonSet, so the chart's 500m default reserves half a core on every
|
||||
# node to run a process that measures 1-2m.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
# The agent parses the ingress controller's access logs (read from
|
||||
# the pod's stdout) to detect attacks. Repointed from the retired
|
||||
# ingress-nginx (decommissioned 2026-05-17) to Traefik. The
|
||||
|
||||
@@ -28,40 +28,3 @@ spec:
|
||||
policy:
|
||||
semver:
|
||||
range: "0.x"
|
||||
---
|
||||
# The SIDECAR image, tracked separately from the controller above.
|
||||
#
|
||||
# These are two DIFFERENT images that must move together:
|
||||
# plugin-barman-cloud the controller, in the database namespace
|
||||
# plugin-barman-cloud-sidecar injected into every Postgres pod
|
||||
#
|
||||
# Only the controller was tracked, so automation walked it from v0.12.0 to
|
||||
# v0.14.0 while the sidecar stayed at v0.12.0 — a two-minor skew in the
|
||||
# component that performs every base backup and WAL archive (#535). The skew was
|
||||
# invisible because the sidecar tag was base64-encoded inside a Secret, where no
|
||||
# $imagepolicy marker can reach it and no reviewer would see it in a diff.
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageRepository
|
||||
metadata:
|
||||
name: cnpg-barman-plugin-sidecar
|
||||
namespace: flux-system
|
||||
spec:
|
||||
image: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar
|
||||
interval: 1h
|
||||
---
|
||||
# Same filter and range as the controller policy deliberately: the two images
|
||||
# are released together and must not be allowed to drift apart again.
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImagePolicy
|
||||
metadata:
|
||||
name: cnpg-barman-plugin-sidecar
|
||||
namespace: flux-system
|
||||
spec:
|
||||
imageRepositoryRef:
|
||||
name: cnpg-barman-plugin-sidecar
|
||||
filterTags:
|
||||
pattern: '^v(?P<ver>\d+\.\d+\.\d+)$'
|
||||
extract: '$ver'
|
||||
policy:
|
||||
semver:
|
||||
range: "0.x"
|
||||
|
||||
@@ -15,7 +15,6 @@ resources:
|
||||
- rook-ceph/
|
||||
- ceph-csi-drivers/
|
||||
- cloudnative-pg/
|
||||
- mariadb-operator/
|
||||
- netbird-ingress/
|
||||
- netbird-backup/
|
||||
- coredns-config/
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# MariaDB operator, for the one workload in this cluster that cannot use
|
||||
# CloudNativePG: WordPress. WordPress speaks MySQL natively — the PostgreSQL
|
||||
# route would mean pg4wp, an SQL-rewriting shim whose own README lists
|
||||
# WordPress 6.5.3 as its newest tested version while renner.uno runs 7.0.4.
|
||||
#
|
||||
# The charts are OCI artifacts (ghcr.io), so this follows the OCIRepository
|
||||
# pattern from infrastructure/spegel rather than the HelmRepository pattern
|
||||
# used for CNPG.
|
||||
#
|
||||
# CRDs ship as a SEPARATE chart that must be installed first — upstream is
|
||||
# explicit about the ordering, hence the dependsOn below rather than a single
|
||||
# release with crds: CreateReplace.
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: OCIRepository
|
||||
metadata:
|
||||
name: mariadb-operator-crds
|
||||
namespace: database
|
||||
spec:
|
||||
interval: 24h
|
||||
url: oci://ghcr.io/mariadb-operator/charts/mariadb-operator-crds
|
||||
ref:
|
||||
# Calendar versioning: the major is the year. Upstream ships an upgrade
|
||||
# guide with each minor (Helm repo deprecations, registry moves), so the
|
||||
# range is capped at the year boundary instead of the usual open ">=".
|
||||
semver: ">=26.6.0 <27.0.0"
|
||||
---
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: mariadb-operator-crds
|
||||
namespace: database
|
||||
spec:
|
||||
interval: 30m
|
||||
chartRef:
|
||||
kind: OCIRepository
|
||||
name: mariadb-operator-crds
|
||||
namespace: database
|
||||
install:
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
remediation:
|
||||
retries: 3
|
||||
---
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: OCIRepository
|
||||
metadata:
|
||||
name: mariadb-operator
|
||||
namespace: database
|
||||
spec:
|
||||
interval: 24h
|
||||
url: oci://ghcr.io/mariadb-operator/charts/mariadb-operator
|
||||
ref:
|
||||
semver: ">=26.6.0 <27.0.0"
|
||||
---
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: mariadb-operator
|
||||
namespace: database
|
||||
spec:
|
||||
interval: 30m
|
||||
dependsOn:
|
||||
- name: mariadb-operator-crds
|
||||
namespace: database
|
||||
chartRef:
|
||||
kind: OCIRepository
|
||||
name: mariadb-operator
|
||||
namespace: database
|
||||
install:
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
# The operator watches all namespaces; the MariaDB CR itself lives with
|
||||
# the app in kubernetes/apps/wordpress/, mirroring how CNPG Clusters sit
|
||||
# in their consumer's namespace (ak-pg in authentik, im-pg in immich).
|
||||
metrics:
|
||||
enabled: false
|
||||
@@ -1,4 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- helmrelease.yaml
|
||||
@@ -151,35 +151,3 @@ apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: trmnl
|
||||
---
|
||||
# WordPress for www.renner.uno, migrated off the Hetzner VM `wordpress-hochzeit`.
|
||||
# No privileged label needed: the cluster enforces `baseline`, which permits the
|
||||
# official Apache image's root master process binding :80. It does trip the
|
||||
# `restricted` audit/warn level — that is expected and not an error.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: wordpress
|
||||
---
|
||||
# The Woodpecker service itself — server, agent and its wp-pg Postgres backend,
|
||||
# moving out of the shared `apps` namespace (issue #433). Distinct from
|
||||
# `woodpecker-ci`, which enforces pod-security: privileged and holds only
|
||||
# ephemeral pipeline step pods.
|
||||
#
|
||||
# `apps` uses an endpointSelector:{} default-deny, so every allow it carries
|
||||
# applies to gitea, gatus, external-dns and ical-proxy alike — granting the
|
||||
# CNPG operator access to wp-pg there would have opened all of `apps` to all of
|
||||
# `database`. A namespace per service keeps that grant on the one workload.
|
||||
#
|
||||
# Declared here rather than alongside the workloads so it is created by the
|
||||
# `infrastructure` KS. The namespace's CiliumNetworkPolicies ship with the
|
||||
# `infrastructure-crds` KS (infrastructure/post-crds includes
|
||||
# ../network-policies/), which itself dependsOn `infrastructure` — so the
|
||||
# namespace exists before any policy targets it. Creating it in the `woodpecker`
|
||||
# workload KS would invert that order, since that KS runs after both.
|
||||
#
|
||||
# No pod-security labels: the cluster default applies.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: woodpecker
|
||||
|
||||
@@ -156,6 +156,4 @@ spec:
|
||||
# each pod enrolls its own peer and a restart re-enrolls cleanly
|
||||
# (the old peer is reaped).
|
||||
- name: state
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
emptyDir: {}
|
||||
|
||||
@@ -31,20 +31,43 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
@@ -80,17 +103,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -38,20 +38,43 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
@@ -80,17 +103,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -36,17 +36,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -32,17 +32,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -27,20 +27,43 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
@@ -79,17 +102,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -28,20 +28,43 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
@@ -80,17 +103,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -157,21 +157,44 @@ spec:
|
||||
icmps:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 } # Echo Reply
|
||||
- { type: 3, family: IPv4 } # Destination Unreachable — PMTUD
|
||||
- { type: 3, family: IPv4 } # Destination Unreachable
|
||||
- { type: 4, family: IPv4 } # Source Quench
|
||||
- { type: 5, family: IPv4 } # Redirect
|
||||
- { type: 8, family: IPv4 } # Echo Request
|
||||
- { type: 9, family: IPv4 } # Router Advertisement
|
||||
- { type: 10, family: IPv4 } # Router Solicitation
|
||||
- { type: 11, family: IPv4 } # Time Exceeded
|
||||
- { type: 12, family: IPv4 } # Parameter Problem
|
||||
- { type: 13, family: IPv4 } # Timestamp
|
||||
- { type: 14, family: IPv4 } # Timestamp Reply
|
||||
- { type: 15, family: IPv4 } # Information Request
|
||||
- { type: 16, family: IPv4 } # Information Reply
|
||||
- { type: 17, family: IPv4 } # Address Mask Request
|
||||
- { type: 18, family: IPv4 } # Address Mask Reply
|
||||
- { type: 30, family: IPv4 } # Traceroute
|
||||
- { type: 1, family: IPv6 } # Destination Unreachable
|
||||
- { type: 2, family: IPv6 } # Packet Too Big — PMTUD, mandatory
|
||||
- { type: 2, family: IPv6 } # Packet Too Big
|
||||
- { type: 3, family: IPv6 } # Time Exceeded
|
||||
- { type: 4, family: IPv6 } # Parameter Problem
|
||||
- { type: 128, family: IPv6 } # Echo Request
|
||||
- { type: 129, family: IPv6 } # Echo Reply
|
||||
- { type: 130, family: IPv6 } # Multicast Listener Query
|
||||
- { type: 131, family: IPv6 } # Multicast Listener Report
|
||||
- { type: 132, family: IPv6 } # Multicast Listener Done
|
||||
- { type: 133, family: IPv6 } # Router Solicitation
|
||||
- { type: 134, family: IPv6 } # Router Advertisement
|
||||
- { type: 135, family: IPv6 } # Neighbor Solicitation
|
||||
- { type: 136, family: IPv6 } # Neighbor Advertisement
|
||||
- { type: 137, family: IPv6 } # Redirect
|
||||
- { type: 138, family: IPv6 } # Router Renumbering
|
||||
- { type: 141, family: IPv6 } # Inverse ND Solicitation
|
||||
- { type: 142, family: IPv6 } # Inverse ND Advertisement
|
||||
- { type: 143, family: IPv6 } # MLDv2 Listener Report
|
||||
- { type: 148, family: IPv6 } # Certification Path Solicitation
|
||||
- { type: 149, family: IPv6 } # Certification Path Advertisement
|
||||
- { type: 151, family: IPv6 } # Multicast Router Advertisement
|
||||
- { type: 152, family: IPv6 } # Multicast Router Solicitation
|
||||
- { type: 153, family: IPv6 } # Multicast Router Termination
|
||||
egress:
|
||||
# Cluster-internal callers HA initiates connections to: mosquitto,
|
||||
# whisper, matter-server, etc.
|
||||
@@ -230,20 +253,43 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
---
|
||||
# Scoped world ingress — replaces the former namespace-wide `fromEntities:
|
||||
# world` allow (see the note in the default-deny-ingress policy above). IoT-VLAN
|
||||
@@ -358,67 +404,3 @@ spec:
|
||||
- ports:
|
||||
- port: "8123"
|
||||
protocol: TCP
|
||||
---
|
||||
# ha-pg (CNPG) — instance-manager access for the CloudNativePG operator.
|
||||
#
|
||||
# DEDICATED policy with its own endpointSelector, NOT another rule on the
|
||||
# namespace-wide default-deny above. That policy has endpointSelector:{}, so any
|
||||
# rule added there applies to EVERY pod in the namespace — and :8000 is served
|
||||
# both by the CNPG instance manager and by other workloads, so a namespace-wide
|
||||
# :8000 grant would expose far more than intended. Same reasoning as
|
||||
# wp-pg-allow-cnpg-operator in woodpecker-ns.yaml, which was corrected during
|
||||
# review for exactly this (#433).
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: ha-pg-allow-cnpg-operator
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: ha-pg
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": database
|
||||
app.kubernetes.io/name: cloudnative-pg
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8000"
|
||||
protocol: TCP
|
||||
# ICMP to the ha-pg pods. REQUIRED because the L4 rules in this policy
|
||||
# select those endpoints, and Cilium drops ICMP to an endpoint selected by
|
||||
# any layer 4 policy unless the traffic is CT-related. Without this, PMTUD
|
||||
# Destination Unreachable / Packet Too Big toward ha-pg are dropped
|
||||
# silently, and the symptom is stalled transfers rather than a policy
|
||||
# denial — the failure mode documented in woodpecker-ns.yaml.
|
||||
- fromEntities:
|
||||
- cluster
|
||||
icmps:
|
||||
- fields:
|
||||
- { type: 3, family: IPv4 } # Destination Unreachable — PMTUD
|
||||
- { type: 1, family: IPv6 } # Destination Unreachable
|
||||
- { type: 2, family: IPv6 } # Packet Too Big — PMTUD, mandatory
|
||||
---
|
||||
# ha-pg — Postgres access for the Home Assistant recorder.
|
||||
#
|
||||
# Scoped to the HA pod on both ends: only home-assistant reaches ha-pg, and only
|
||||
# on 5432. Nothing else in this namespace has any business in the recorder
|
||||
# database.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: ha-pg-allow-home-assistant
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: ha-pg
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
app.kubernetes.io/name: home-assistant
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "5432"
|
||||
protocol: TCP
|
||||
|
||||
@@ -42,17 +42,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -42,17 +42,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -21,7 +21,5 @@ resources:
|
||||
- openbao-ns.yaml
|
||||
- external-secrets-ns.yaml
|
||||
- woodpecker-ci-ns.yaml
|
||||
- woodpecker-ns.yaml
|
||||
- trmnl-ns.yaml
|
||||
- wordpress-ns.yaml
|
||||
- allow-cilium-health.yaml
|
||||
|
||||
@@ -67,17 +67,40 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
|
||||
@@ -141,9 +141,9 @@ spec:
|
||||
---
|
||||
# Egress for the review-bot rotator. Its CronJob has two steps: the init step
|
||||
# mints review-bot's PAT against gitea (apps :3000), the main step upserts the
|
||||
# Woodpecker org secret against the woodpecker server (woodpecker :8000). Both run
|
||||
# Woodpecker org secret against the woodpecker server (apps :8000). Both run
|
||||
# under the same pod labels, so one egress policy covers both hops. Complements
|
||||
# gitea-allow-review-bot-rotator (apps) + woodpecker-allow-rotators (woodpecker).
|
||||
# gitea-allow-review-bot-rotator + woodpecker-allow-rotators (apps side).
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
@@ -165,16 +165,16 @@ spec:
|
||||
protocol: TCP
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": woodpecker
|
||||
"k8s:io.kubernetes.pod.namespace": apps
|
||||
app.kubernetes.io/name: woodpecker
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8000"
|
||||
protocol: TCP
|
||||
---
|
||||
# Egress for the claude-oauth sync rotator: woodpecker server only (woodpecker :8000).
|
||||
# Egress for the claude-oauth sync rotator: woodpecker server only (apps :8000).
|
||||
# It never touches gitea (the OAuth value is seeded into OpenBao by hand, not
|
||||
# minted). Complements woodpecker-allow-rotators (woodpecker side).
|
||||
# minted). Complements woodpecker-allow-rotators (apps side).
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
@@ -188,7 +188,7 @@ spec:
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": woodpecker
|
||||
"k8s:io.kubernetes.pod.namespace": apps
|
||||
app.kubernetes.io/name: woodpecker
|
||||
toPorts:
|
||||
- ports:
|
||||
|
||||
@@ -37,20 +37,43 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
@@ -86,20 +109,43 @@ spec:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 4, family: IPv4 }
|
||||
- { type: 5, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 9, family: IPv4 }
|
||||
- { type: 10, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 13, family: IPv4 }
|
||||
- { type: 14, family: IPv4 }
|
||||
- { type: 15, family: IPv4 }
|
||||
- { type: 16, family: IPv4 }
|
||||
- { type: 17, family: IPv4 }
|
||||
- { type: 18, family: IPv4 }
|
||||
- { type: 30, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 130, family: IPv6 }
|
||||
- { type: 131, family: IPv6 }
|
||||
- { type: 132, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 137, family: IPv6 }
|
||||
- { type: 138, family: IPv6 }
|
||||
- { type: 141, family: IPv6 }
|
||||
- { type: 142, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
- { type: 148, family: IPv6 }
|
||||
- { type: 149, family: IPv6 }
|
||||
- { type: 151, family: IPv6 }
|
||||
- { type: 152, family: IPv6 }
|
||||
- { type: 153, family: IPv6 }
|
||||
---
|
||||
# Allow FTP access (from scanners on the LAN) to the ftp-scanner pod.
|
||||
apiVersion: cilium.io/v2
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user