Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  claude-botandClaude Opus 5
|
401d23ac48 |
@@ -159,61 +159,3 @@ steps:
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-jq:latest
|
||||
|
||||
# Runtime image, not CI tooling: upstream containernetworking/plugins binaries
|
||||
# baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
|
||||
# node-side fetch from GitHub that had NO checksum verification and sat on the
|
||||
# critical path of every node rebuild. The download now happens here, once,
|
||||
# where the sha256 is checked — see the Containerfile for the full rationale.
|
||||
- name: build-cni-plugins
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
directory: ci/images/cni-plugins
|
||||
depends_on: []
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/cni-plugins:latest
|
||||
|
||||
# Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
|
||||
# needing s_client/x509. Named for what it is; first consumer is the paperless
|
||||
# fetch-bridge-cert init container. Replaces an `apk add openssl` that ran on
|
||||
# every paperless pod start. The openssl CLI is genuinely absent from the alpine
|
||||
# base, so this one needs an image rather than a busybox substitute.
|
||||
- name: build-alpine-openssl
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
directory: ci/images/alpine-openssl
|
||||
depends_on: []
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-openssl:latest
|
||||
|
||||
@@ -1,254 +0,0 @@
|
||||
# Validate the GitOps tree and the pipeline definitions on PRs that touch them.
|
||||
#
|
||||
# Nothing covered kubernetes/** before this. deploy-infrastructure.yml only
|
||||
# gates opentofu/**, and ci-scripts-test.yml only runs scripts/ci/ unit tests —
|
||||
# so a broken manifest reached the cluster and was caught by Flux failing to
|
||||
# reconcile, or by a human noticing. Three separate defects on 2026-07-30 would
|
||||
# have been caught here instead:
|
||||
#
|
||||
# * an unquoted YAML scalar containing ": " in .woodpecker/build-ci-images.yml,
|
||||
# which made EVERY pipeline in the repo fail to compile (#443). `yq` parsed
|
||||
# it happily — it is valid YAML, just a map instead of a string — so
|
||||
# "it parses" was mistaken for "it is valid". Only woodpecker lint catches it.
|
||||
# * a rotator pinned to an image digest that no longer existed (#458).
|
||||
# * manifests whose kustomize wiring was only ever checked by hand.
|
||||
#
|
||||
# Deliberately CHEAP and offline: no cluster, no registry, no secrets. Everything
|
||||
# here is a static check that either passes or fails on the checkout alone, so it
|
||||
# can gate PRs without becoming a bottleneck or a flake source.
|
||||
when:
|
||||
# Path-filtered on purpose, unlike ci-scripts-test.yml. That pipeline is one
|
||||
# step on one already-cached image, so running it everywhere is free; this one
|
||||
# pulls three images and builds every kustomization in the tree, which is real
|
||||
# cost to put on a docs-only PR. The filter covers everything the steps actually read:
|
||||
# kubernetes/** for the manifests and scripts, .woodpecker/** for the lint (and
|
||||
# so an edit to this file re-runs its own checks).
|
||||
- event: pull_request
|
||||
path:
|
||||
- "kubernetes/**"
|
||||
- ".woodpecker/**"
|
||||
|
||||
# ADDING A THIRD DISCOVERY CHECK: kustomize-build and shellcheck both hand-roll
|
||||
# the same shape — `find`, a floor guard against empty discovery, then a
|
||||
# file-fed `while read`. Two copies is deliberate: the two traps being commented
|
||||
# around (BusyBox find lacking `-printf`, and a piped loop running in a subshell
|
||||
# where `fail` cannot propagate) are different, and both are worth spelling out
|
||||
# where they bite. A third instance is the point to stop copying and factor it
|
||||
# into a shared script under `scripts/ci/` — which is already covered by
|
||||
# ci-scripts-test.yml, so it would gain unit tests in the process.
|
||||
#
|
||||
# Every step carries `depends_on: []` so all four run in PARALLEL and every
|
||||
# failure shows in one run. Woodpecker's default is sequential, which would stop
|
||||
# at the first failure and make a contributor fix-and-repush to discover the
|
||||
# next — and these four are independent, cheap, offline checks with no ordering
|
||||
# between them. Keep it on every step: `depends_on` is per-step, so dropping it
|
||||
# from one silently re-serialises that step behind the others.
|
||||
steps:
|
||||
# Every kustomization must build. This is the check that catches a resource
|
||||
# entry pointing at a file that does not exist, a bad patch target, or YAML
|
||||
# that is fine in isolation but broken once overlaid — the class of error that
|
||||
# otherwise surfaces as a Flux reconcile failure after merge.
|
||||
#
|
||||
# Verified green across every kustomization in the tree when this was added,
|
||||
# so this gates new breakage rather than pre-existing debt.
|
||||
#
|
||||
# ci-k8s TAG SPLIT: this file pins a newer dated tag than e2e.yml, e2e-test.yml,
|
||||
# alert-monitor.yml and issue-fix.yml, which still sit on an older one. That is
|
||||
# staleness in those four, not a deliberate split here — everything should be on
|
||||
# the newest dated tag. They are not bumped here because none of them are
|
||||
# triggered by this PR, so it would ship untested changes under cover of a green
|
||||
# run that never touched them. Tracked in #461.
|
||||
- name: kustomize-build
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-07-30
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
# NOT `find -printf '%h\n'`: that is a GNU findutils extension and this
|
||||
# image ships BusyBox find (v1.37.0), which rejects it outright with
|
||||
# "find: unrecognized: -printf". Because the result feeds a command
|
||||
# substitution used as a word list, the failure is SILENT — empty output,
|
||||
# zero loop iterations, fail=0, step exits green having validated
|
||||
# nothing. That is exactly what this pipeline shipped in its first
|
||||
# revision, and CI reported it as passing.
|
||||
dirs=$(find kubernetes -name kustomization.yaml | sed 's|/kustomization.yaml$||' | sort)
|
||||
|
||||
# Assert the discovery actually found something. Without this, any future
|
||||
# breakage of the find/sed above degrades back into a green no-op rather
|
||||
# than a failure — the specific trap that made the first revision of this
|
||||
# step useless while looking healthy.
|
||||
count=$(printf '%s\n' "$dirs" | grep -c . || true)
|
||||
if [ "$count" -lt 50 ]; then
|
||||
echo "FAIL: found only $count kustomizations; floor is 50."
|
||||
echo " If the tree legitimately shrank, lower this floor in"
|
||||
echo " .woodpecker/ci-manifests-test.yml — do not delete it."
|
||||
echo " Otherwise discovery is broken — refusing to report success."
|
||||
exit 1
|
||||
fi
|
||||
echo "discovered $count kustomizations"
|
||||
|
||||
# `while read` fed from a FILE, not a pipe: a pipe would run the loop in
|
||||
# a subshell where `fail` cannot propagate, so a failing build would exit
|
||||
# 0 — the same silent-pass shape this step already shipped once. Reading
|
||||
# a file keeps the loop in the current shell, and read -r handles paths
|
||||
# containing spaces without IFS games.
|
||||
printf '%s\n' "$dirs" > /tmp/dirs
|
||||
fail=0
|
||||
while IFS= read -r d; do
|
||||
[ -n "$d" ] || continue
|
||||
if kubectl kustomize "$d" >/dev/null 2>/tmp/err; then
|
||||
echo "ok $d"
|
||||
else
|
||||
echo "FAIL $d"
|
||||
sed 's/^/ /' /tmp/err
|
||||
fail=1
|
||||
fi
|
||||
done < /tmp/dirs
|
||||
exit $fail
|
||||
|
||||
# Lint the pipeline definitions themselves. This is the one that would have
|
||||
# prevented the repo-wide CI outage: `woodpecker lint` exits 1 on a config that
|
||||
# fails to compile, and exits 0 on mere warnings — so it fails on real breakage
|
||||
# without blocking on pre-existing noise. Both behaviours were verified against
|
||||
# this repo (rc=1 reproducing #443, rc=0 on the tree as it stands).
|
||||
#
|
||||
# An earlier revision of this comment also asserted that several deploy-*.yml
|
||||
# files emit "Additional property timeout is not allowed" warnings. That was
|
||||
# observed under the v3.9.0 CLI this step used to pin, and it is NOT known to
|
||||
# still hold on v3.16.0 — review raised the likelihood that the warning was
|
||||
# schema drift in a 7-minor-old CLI rather than real debt in those files, which
|
||||
# is one of the reasons this pin moved to latest stable. It cannot be confirmed
|
||||
# from a green run either, because the loop below sends lint output to /tmp/out
|
||||
# and prints it only on failure, so warnings are invisible when the step passes.
|
||||
# The claim is dropped rather than restated: what this step guarantees is the
|
||||
# exit-code contract above, which is what the gate actually rests on.
|
||||
# Latest stable, per the standing rule that any software uses the latest
|
||||
# stable version. Matching the deployed woodpecker-server (v3.16.0) is a
|
||||
# CONSEQUENCE of that, not a separate requirement — the server is bumped by
|
||||
# image-automation within the 3.x range, so both land on the same release.
|
||||
#
|
||||
# It matters here because the CLI compiles the config against its own schema:
|
||||
# one behind the server can pass a config the server then rejects, or reject
|
||||
# one it accepts, and the gate stops predicting what the server will do —
|
||||
# which is its only purpose.
|
||||
#
|
||||
# `-alpine` is required, not cosmetic: the plain vX.Y.Z CLI image ships no
|
||||
# shell, and the loop below needs `for`, `sed` and `echo` around the binary.
|
||||
# A step using the plain tag fails to start rather than failing a check.
|
||||
#
|
||||
# DRIFT RISK — accepted, and NOT fixable here. The server auto-bumps via
|
||||
# image-automation; this pin cannot, so the two will drift apart.
|
||||
# Image-automation can only patch YAML that is a Kubernetes manifest —
|
||||
# fluxcd/flux2#2826: "Flux can only patch Kubernetes manifests. This means the
|
||||
# YAML manifest must have `apiVersion` and `kind` fields." A Woodpecker
|
||||
# pipeline file has neither, so an `$imagepolicy` marker here is silently
|
||||
# ignored. That is a property of the FILE, not of `update.path`: widening the
|
||||
# automation's path to `./` would buy nothing, and would additionally drag the
|
||||
# whole ansible/ and esphome/ trees into the walk. Do not try it — it looks
|
||||
# like the fix and is not.
|
||||
#
|
||||
# So this bump is manual, and drift is caught by the update-audit agent
|
||||
# (ci/agents/update-audit/prompt.md, item 5) which is told to flag this image
|
||||
# whenever it trails the deployed server. If you bump the server, bump here.
|
||||
- name: woodpecker-lint
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: woodpeckerci/woodpecker-cli:v3.16.0-alpine
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
fail=0
|
||||
for f in .woodpecker/*.yml; do
|
||||
if /bin/woodpecker-cli lint "$f" >/tmp/out 2>&1; then
|
||||
echo "ok $f"
|
||||
else
|
||||
echo "FAIL $f"
|
||||
sed 's/^/ /' /tmp/out
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
exit $fail
|
||||
|
||||
# Standalone shell scripts under kubernetes/ (sidecar and bootstrap scripts
|
||||
# mounted via configMapGenerator). Severity is pinned to `error` on purpose:
|
||||
# style and info findings would fail the build on existing scripts for no
|
||||
# safety gain. All scripts in the tree are clean at this level today.
|
||||
#
|
||||
# NOTE this does NOT cover shell embedded in manifests (init containers, probe
|
||||
# exec blocks) — shellcheck cannot see inside YAML. That embedded shell is
|
||||
# where several real bugs lived this year, so it remains a gap worth closing
|
||||
# separately rather than one this step quietly implies is covered.
|
||||
- name: shellcheck
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: koalaman/shellcheck-alpine:v0.11.0
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
files=$(find kubernetes -name '*.sh' | sort)
|
||||
# Same no-op guard as kustomize-build: if discovery ever returns nothing,
|
||||
# fail loudly instead of passing having checked no files.
|
||||
#
|
||||
# This floor is 1 while kustomize-build's is 50, and the asymmetry is
|
||||
# deliberate rather than an oversight. That floor is loose relative to the
|
||||
# tree it guards, so it catches both total failure and gross partial loss
|
||||
# while tolerating normal churn. The shell-script set is small enough that
|
||||
# any floor tight enough to catch PARTIAL loss would sit inside normal
|
||||
# churn — deleting one script would fail CI spuriously, and the fix a
|
||||
# hurried person reaches for is lowering the floor, which is how a guard
|
||||
# quietly becomes decoration. A floor that has to be edited whenever the
|
||||
# tree changes is not a guard, it is a snapshot with an exit code.
|
||||
#
|
||||
# So this catches the failure that actually happened here (discovery
|
||||
# silently returning nothing, step green having checked no files) and not
|
||||
# partial loss. Worth revisiting only if kubernetes/ grows a lot more
|
||||
# shell, at which point a loose proportionate floor becomes possible.
|
||||
count=$(printf '%s\n' "$files" | grep -c . || true)
|
||||
if [ "$count" -lt 1 ]; then
|
||||
echo "FAIL: no shell scripts found under kubernetes/ — discovery is broken."
|
||||
exit 1
|
||||
fi
|
||||
echo "discovered $count shell scripts"
|
||||
|
||||
# Same file-fed while-read reasoning as kustomize-build above.
|
||||
printf '%s\n' "$files" > /tmp/files
|
||||
fail=0
|
||||
while IFS= read -r f; do
|
||||
[ -n "$f" ] || continue
|
||||
if shellcheck -S error "$f"; then
|
||||
echo "ok $f"
|
||||
else
|
||||
echo "FAIL $f"
|
||||
fail=1
|
||||
fi
|
||||
done < /tmp/files
|
||||
exit $fail
|
||||
|
||||
# Guard the convention that cost a day to diagnose: registry-local images are
|
||||
# pinned by DATED TAG, never by digest. Digests here are garbage-collected once
|
||||
# :latest and the dated tags move past them — and that happens on edits to
|
||||
# .woodpecker/build-ci-images.yml, a shared file, so an unrelated image addition
|
||||
# silently orphans the manifest a rotator was pinned to. 11 rotators were dead
|
||||
# this way and would have failed on their first scheduled run months later.
|
||||
#
|
||||
# Scoped to the rotation directory, which is where the convention is documented
|
||||
# and where the failure actually bit.
|
||||
#
|
||||
# The `grep -v kustomization.yaml` is load-bearing, not tidying: that file
|
||||
# documents this very rule, so its comment contains the literal string
|
||||
# `image:.*@sha256` and therefore MATCHES the pattern. Anchoring on `image:`
|
||||
# does not prevent that — verified by running the bare grep, which returns
|
||||
# kustomization.yaml. Only the filename filter excludes it. Drop the filter and
|
||||
# this step fails permanently on its own documentation.
|
||||
- name: no-digest-pins
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-07-30
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
hits=$(grep -l 'image:.*@sha256' kubernetes/infrastructure/openbao/rotation/*.yaml 2>/dev/null \
|
||||
| grep -v kustomization.yaml || true)
|
||||
if [ -n "$hits" ]; then
|
||||
echo "FAIL: digest-pinned images found; use a dated tag (see rotation/kustomization.yaml):"
|
||||
echo "$hits" | sed 's/^/ /'
|
||||
exit 1
|
||||
fi
|
||||
echo "ok no digest pins under openbao/rotation/"
|
||||
@@ -18,26 +18,6 @@ chrony_upstream_pools:
|
||||
# Local stratum if disconnected from upstream
|
||||
chrony_local_stratum: 10
|
||||
|
||||
# Step (rather than slew) the clock when the offset exceeds the threshold, but
|
||||
# only for the first N updates after chronyd starts. Without a makestep
|
||||
# directive chronyd only ever slews, so a large offset is never closed in
|
||||
# practice.
|
||||
#
|
||||
# edge-1, 2026-08-04: the CMOS battery is dead, so after a power loss the RTC
|
||||
# came up reading 2018-08-20. The kernel set the clock from it, systemd floored
|
||||
# that forward to its own built-in epoch (2026-06-26), and it ended up at
|
||||
# ~2026-07-19 -- about 16 days behind. chronyd then synced to ptbtime1, reported
|
||||
# "System time: 1376235.375 seconds slow of NTP time", and corrected none of it.
|
||||
# With host time before the cluster certificate's NotBefore, every outbound TLS
|
||||
# handshake failed x509 validation, taking out the MQTT bridge (heating
|
||||
# telemetry) and the Loki log shipper at once. The 16 days is incidental -- it
|
||||
# is just where the software flooring landed; the fault is that nothing steps.
|
||||
#
|
||||
# Limiting to the first few updates keeps steps confined to boot, so a
|
||||
# long-running server is never jumped out from under its workloads.
|
||||
chrony_makestep_threshold: 1.0
|
||||
chrony_makestep_limit: 3
|
||||
|
||||
# Hardware timestamping. Fine on the PVE nodes' Intel NICs; the Wyse 3040
|
||||
# (edge-1) has a Realtek NIC that does not support it, and chronyd logs an
|
||||
# error per interface when asked for it. Overridden to false in
|
||||
|
||||
@@ -14,11 +14,6 @@ pool {{ pool }}
|
||||
# Record the rate at which the system clock gains/losses time
|
||||
driftfile /var/lib/chrony/chrony.drift
|
||||
|
||||
# Step the clock instead of slewing it when the offset is large, limited to the
|
||||
# first few updates after start so only boot-time corrections jump. Rationale
|
||||
# and the incident behind it: chrony_makestep_* in the role's defaults/main.yml.
|
||||
makestep {{ chrony_makestep_threshold }} {{ chrony_makestep_limit }}
|
||||
|
||||
# Allow NTP client access from local networks
|
||||
{% for network in chrony_allowed_networks %}
|
||||
allow {{ network }}
|
||||
|
||||
@@ -59,22 +59,6 @@ For each, find the **current pin in the repo** (Read/Glob/Grep), then look up th
|
||||
5. **Images deliberately excluded from automation** — listed in the
|
||||
image-automation `README.md` (e.g. vsftpd, otbr, ical-proxy) and any image with
|
||||
no `# {"$imagepolicy": ...}` marker. Check whether a newer usable tag exists.
|
||||
This includes **CI step images in `.woodpecker/*.yml`**, which are all manual:
|
||||
image-automation cannot patch those files at all (they are not Kubernetes
|
||||
manifests — no `apiVersion`/`kind` — so markers there are ignored), and no
|
||||
amount of `update.path` widening changes that. Grep `.woodpecker/` for
|
||||
`image:` and check each pin.
|
||||
|
||||
Give `woodpeckerci/woodpecker-cli` in `.woodpecker/ci-manifests-test.yml`
|
||||
particular attention: the woodpecker **server** IS auto-bumped (`3.x`), the
|
||||
CLI is not, so they silently drift apart. The CLI lints pipeline configs
|
||||
against its own schema, so one behind the server can pass a config the server
|
||||
then rejects — the gate stops predicting server behaviour. Report the CLI as
|
||||
needing an update whenever it trails the server version, not only when it
|
||||
trails the newest CLI release. Its tag carries an `-alpine` suffix (the plain
|
||||
tag is scratch-based with no shell and would break the step) — compare the
|
||||
`vX.Y.Z` part, and ignore the `next-*` / `pull_*` development tags, which are
|
||||
the newest by date on that repo and are not releases.
|
||||
6. **Ansible roles & OpenTofu providers** — `ansible/requirements.yml` (Galaxy
|
||||
roles/collections) and `opentofu/providers.tf` + `opentofu/.terraform.lock.hcl`.
|
||||
Report any with a newer published version.
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
|
||||
|
||||
# Alpine plus the openssl CLI. Nothing application-specific — named for what it
|
||||
# is, so anything needing `openssl s_client` / `x509` from a shell script can use
|
||||
# it.
|
||||
#
|
||||
# It exists because installing tools at container start is not acceptable here:
|
||||
# `apk add` makes every start depend on network and package-index availability,
|
||||
# and the resolved version can drift between restarts of an otherwise pinned
|
||||
# image.
|
||||
#
|
||||
# Note the openssl CLI is genuinely absent from the alpine base — verified, the
|
||||
# base carries the libraries but not the command — so unlike the multus CNI
|
||||
# installer (where busybox wget/tar already covered the need) this one really
|
||||
# does require an image rather than just dropping the apk add.
|
||||
#
|
||||
# First consumer: the paperless fetch-bridge-cert init container, which captures
|
||||
# the Proton Mail Bridge's self-signed IMAP certificate via STARTTLS and pins it
|
||||
# for paperless to trust. busybox has no s_client equivalent, so openssl is not
|
||||
# substitutable there.
|
||||
RUN apk add --no-cache openssl
|
||||
@@ -1,68 +0,0 @@
|
||||
# Upstream containernetworking/plugins binaries, baked in at build time.
|
||||
#
|
||||
# WHY THIS IMAGE EXISTS
|
||||
# The multus cni-plugins-installer DaemonSet used to fetch this tarball from
|
||||
# GitHub on the node itself, at node-provisioning time, and extract it straight
|
||||
# into /opt/cni/bin. Two problems with that:
|
||||
#
|
||||
# 1. Security. The download had NO integrity verification — wget, then tar,
|
||||
# into a directory whose contents are executed as root in the CNI path on
|
||||
# every node. A tampered or MITM'd artifact would have got privileged
|
||||
# execution cluster-wide, with nothing to detect it.
|
||||
# 2. Node rebuild. It put a live dependency on github.com (and previously the
|
||||
# Alpine package index too) on the critical path of bringing a node up, so
|
||||
# a GitHub outage or slow transfer stalled or broke CNI installation
|
||||
# exactly when a node was being rebuilt.
|
||||
#
|
||||
# Baking the binaries in fixes both: the fetch happens once, here, where the
|
||||
# checksum IS verified, and the installer becomes a local file copy with no
|
||||
# network at all.
|
||||
#
|
||||
# The version and digest are pinned literals, so bumping them is a reviewable
|
||||
# commit rather than a silent change in what runs on every node. The digest was
|
||||
# cross-checked against upstream's own published
|
||||
# cni-plugins-linux-amd64-<version>.tgz.sha256 asset rather than merely recorded
|
||||
# from a local download — a download can be tampered with; the point of the
|
||||
# check is to compare against what upstream states.
|
||||
#
|
||||
# amd64 only, matching the ARCH the installer already hardcoded and the cluster's
|
||||
# node architecture. Add a second stage if a different arch is ever needed.
|
||||
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d AS fetch
|
||||
|
||||
ARG CNI_PLUGINS_VERSION=v1.5.1
|
||||
ARG CNI_PLUGINS_SHA256=77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b
|
||||
|
||||
# busybox wget and tar are already in the base; nothing is installed here
|
||||
# either. sha256sum -c fails the BUILD if the artifact does not match, which is
|
||||
# the whole point — a bad download must never reach a node.
|
||||
RUN set -eu; \
|
||||
F="cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz"; \
|
||||
wget -q -O "/tmp/${F}" \
|
||||
"https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${F}"; \
|
||||
echo "${CNI_PLUGINS_SHA256} /tmp/${F}" | sha256sum -c -; \
|
||||
mkdir -p /plugins; \
|
||||
tar -xzf "/tmp/${F}" -C /plugins; \
|
||||
rm -f "/tmp/${F}"; \
|
||||
# Drop the non-executable payload: the tarball ships LICENSE and README.md,
|
||||
# which the old installer extracted into /opt/cni/bin alongside the binaries.
|
||||
rm -f /plugins/LICENSE /plugins/README.md; \
|
||||
# Fail loudly here rather than shipping an image that silently installs
|
||||
# nothing, if upstream ever restructures the archive.
|
||||
#
|
||||
# This list must cover every plugin the repo actually depends on, not just
|
||||
# the ones the installer happens to log. `dhcp` is included because the
|
||||
# sibling cni-dhcp-daemon DaemonSet runs that binary and documents that it
|
||||
# ships from here (see kubernetes/infrastructure/multus/cni-dhcp-daemon.yaml)
|
||||
# — without it in the check, upstream dropping `dhcp` while keeping the
|
||||
# others would build green and only surface later as a broken DHCP daemon on
|
||||
# whichever node rolled first.
|
||||
for p in macvlan static host-local dhcp; do \
|
||||
[ -x "/plugins/$p" ] || { echo "FATAL: /plugins/$p missing or not executable" >&2; exit 1; }; \
|
||||
done
|
||||
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
|
||||
|
||||
# Alpine rather than scratch: the installer needs a shell to copy the binaries
|
||||
# onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
|
||||
COPY --from=fetch /plugins/ /opt/cni-plugins/
|
||||
@@ -1,179 +0,0 @@
|
||||
# Proposal: enrol the mosquitto broker credentials in Class-A rotation
|
||||
|
||||
**Status:** proposal, nothing implemented.
|
||||
**Scope:** the 8 MQTT broker users and the credentials behind them.
|
||||
|
||||
## Problem
|
||||
|
||||
Every self-generated secret in this cluster rotates automatically except these.
|
||||
`kubernetes/infrastructure/openbao/rotation/` holds 13 manifests defining 15
|
||||
rotator CronJobs (`home-assistant.yaml` alone defines three). Five are **Class-A**
|
||||
— authentik, crowdsec, paperless, shelly-manager and woodpecker — meaning
|
||||
"self-generated random secret", which is exactly what a broker password is.
|
||||
The mosquitto credentials are the only Class-A-shaped secrets not enrolled.
|
||||
|
||||
They are currently minted once by seven near-identical bootstrap Jobs
|
||||
(`mosquitto-bootstrap.yaml`, 508 lines — the ServiceAccount, Role and RoleBinding
|
||||
are inside that same file) which generate a password, hand-build Secret JSON with
|
||||
`jq` and POST it to the Kubernetes API with `curl`. Once created they never
|
||||
change again.
|
||||
|
||||
Three consequences:
|
||||
|
||||
1. **They never rotate.** Not on a schedule, not on compromise, not ever.
|
||||
2. **They exist only in the Secret object** — not in OpenBao, so not covered by
|
||||
the OpenBao snapshot CronJob, and unrecoverable if lost.
|
||||
3. **The Jobs cannot be modified.** A Job's `spec.template` is immutable, so
|
||||
changing even the container image makes the whole `home-assistant`
|
||||
Kustomization fail to reconcile with `field is immutable`. Any edit requires
|
||||
deleting or renaming the Jobs.
|
||||
|
||||
## Why the obvious fixes do not work
|
||||
|
||||
Recorded because each was tried and rejected on evidence, and the reasons are
|
||||
not obvious from the code.
|
||||
|
||||
| Approach | Why it fails |
|
||||
| --- | --- |
|
||||
| ExternalSecrets reading hand-seeded OpenBao paths | Depends on content copied in by hand. Git cannot recreate it; a fresh OpenBao leaves the paths empty and ESO then overwrites live broker credentials with nothing. |
|
||||
| ESO `Password` generator | **Overwrites pre-existing Secrets.** Verified: a Secret holding `KNOWN-PREEXISTING-VALUE` was replaced with a freshly generated password on first sync. Adopting it rotates the 7 Job-minted credentials at once, uncoordinated (`evcc` is already ESO-extract-backed and so unaffected). |
|
||||
| Keep the Jobs, fix only the image | Blocked by `spec.template` immutability. Renaming the Jobs works but is a hack that recurs on every future edit. |
|
||||
|
||||
The common thread: each treats the credential as something to *place*, when the
|
||||
real requirement is a credential that can be *replaced*.
|
||||
|
||||
## How rotation actually has to work here
|
||||
|
||||
The broker rebuilds its password file from scratch on **every pod start**
|
||||
(`mosquitto-deployment.yaml`): `rm -f /mosquitto/auth/passwd` followed by
|
||||
`mosquitto_passwd -c -b …` for each of the 8 users, sourced from env vars backed
|
||||
by the 8 Secrets.
|
||||
|
||||
Two things follow, and they shape the whole design:
|
||||
|
||||
- **A Secret change has no effect until the broker restarts.** There is no
|
||||
Reloader annotation on the mosquitto Deployment today, unlike the consumers of
|
||||
the existing Class-A rotators.
|
||||
- **There is no make-before-break.** A mosquitto user has exactly one password.
|
||||
The LLAT rotators can mint a second token and revoke the first afterwards; a
|
||||
broker user cannot hold old and new simultaneously. Unless a transitional
|
||||
second user is created, broker and client must cut over together.
|
||||
|
||||
### Who holds each credential
|
||||
|
||||
This is the part that determines cutover cost, and it is not visible from the
|
||||
manifests alone.
|
||||
|
||||
| Broker user | Held by | Cutover |
|
||||
| --- | --- | --- |
|
||||
| `health` | the broker's own liveness/readiness probes | in-cluster, same pod restart |
|
||||
| `zigbee2mqtt` | zigbee2mqtt Deployment (`secretKeyRef`) | in-cluster, pod restart |
|
||||
| `evcc` | evcc app; already OpenBao-backed via ESO | in-cluster, pod restart |
|
||||
| `sml2mqtt` | edge host, from `vault_edge_sml2mqtt_mqtt_password` | re-run the edge-host play |
|
||||
| `open3e` | edge host, from `vault_edge_open3e_mqtt_password` | re-run the edge-host play |
|
||||
| `homeassistant` | HA MQTT integration — a config entry created in the UI | needs an API/WebSocket update, or a manual re-entry |
|
||||
| `shellies` | Shelly device firmware | pushed per-device; `shelly-manager` is deployed and is the natural mechanism |
|
||||
| `admin` | a workstation client, held by a human | read the new value from OpenBao |
|
||||
|
||||
Note `sml2mqtt` and `open3e` currently keep **the same password in two unlinked
|
||||
places** — the Kubernetes Secret for the broker, and `ansible-vault` for the
|
||||
client — synchronised by hand. That duplication, not any client limitation, is
|
||||
what makes rotation feel impossible today.
|
||||
|
||||
## Proposed design
|
||||
|
||||
Follow the existing Class-A pattern exactly (`rotation/paperless.yaml` is the
|
||||
closest model): *generate random bytes → merge-patch into OpenBao → ESO syncs the
|
||||
Secret → restart the consumer*.
|
||||
|
||||
1. **One rotator CronJob per broker user**, running the existing `secret-rotator`
|
||||
binary in random mode, writing `apps/home-assistant/mosquitto-<user>-credentials`
|
||||
in OpenBao. Reuses the established policy/role/SA pattern; no new machinery.
|
||||
2. **ExternalSecrets** project each OpenBao entry to the existing Secret name and
|
||||
key, so `mosquitto-deployment.yaml` and the in-cluster consumers are unchanged.
|
||||
Key names stay as they are (`homeassistant-password` on one, `password` on the
|
||||
rest) — consumers reference them, so normalising is a separate change.
|
||||
3. **Add a Reloader annotation to the mosquitto Deployment** so a Secret change
|
||||
restarts the broker, instead of silently taking effect at some unrelated future
|
||||
restart.
|
||||
4. **Move the edge host off `ansible-vault` for these two values**, reading the
|
||||
same OpenBao entry instead, so there is one source of truth per credential.
|
||||
5. **Rotate one user at a time.** Each rotation then affects exactly one client,
|
||||
not all eight.
|
||||
|
||||
### The cutover window
|
||||
|
||||
With no make-before-break, each rotation has a window where broker and client
|
||||
disagree. Two options:
|
||||
|
||||
- **Accept a brief window** (recommended for most users). Order: rotator writes
|
||||
OpenBao → client picks up the new value → broker restarts. MQTT clients
|
||||
reconnect automatically; the window is seconds. Because rotation is per-user,
|
||||
only that one client is affected.
|
||||
|
||||
One failure mode to expect rather than be alarmed by: the broker rolls with
|
||||
`maxUnavailable: 0` / `maxSurge: 1`, so mid-rollout **two broker pods with
|
||||
different passwd files sit behind the LB**. A client reconnecting during that
|
||||
window can land on the old pod and fail auth until the rollout completes. It is
|
||||
self-healing and lasts seconds to a minute, but it will look alarming in logs
|
||||
during the first `health` / `zigbee2mqtt` rotation, so it is worth knowing in
|
||||
advance.
|
||||
- **Transitional dual user** (`<user>` and `<user>-next` in the passwd file and
|
||||
ACL) for any credential where even a brief outage is unacceptable. More moving
|
||||
parts; probably only worth it for `homeassistant`, which carries everything.
|
||||
|
||||
## Suggested phasing
|
||||
|
||||
Ordered so the risky work happens after the pattern is proven, and so each phase
|
||||
is independently valuable.
|
||||
|
||||
1. **`health` and `zigbee2mqtt` first.** Both are entirely in-cluster and pick up
|
||||
a new password on pod restart with no external coordination. Proves rotator →
|
||||
OpenBao → ESO → Reloader end to end with the smallest blast radius. `health`
|
||||
is the safest possible start: only the broker's own probes consume it.
|
||||
2. **`evcc`.** Already OpenBao-backed via ESO; only the rotator is missing.
|
||||
3. **`sml2mqtt` and `open3e`**, together with switching the edge host to read from
|
||||
OpenBao. This is where the duplicate-source-of-truth problem gets fixed.
|
||||
4. **`shellies`**, via shelly-manager pushing to devices. Needs investigation:
|
||||
shelly-manager has its own Class-A rotator today, but that rotates
|
||||
shelly-manager's own secret and does not touch device MQTT credentials.
|
||||
5. **`homeassistant`** last. Highest blast radius, and the config entry is UI-created,
|
||||
so it needs an API/WebSocket update path — the HA LLAT rotators
|
||||
(`rotation/home-assistant.yaml`) already talk to HA over WebSocket and are the
|
||||
obvious precedent.
|
||||
6. **`admin`** whenever convenient; a human reads the new value from OpenBao.
|
||||
|
||||
Only once a user is enrolled does its bootstrap Job become dead code. Delete the
|
||||
Jobs incrementally as their users migrate, rather than in one step — that also
|
||||
avoids the immutability problem entirely, since the Jobs are deleted, never
|
||||
patched.
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Does `secret-rotator` random mode fit unchanged?** The existing rotators write
|
||||
a field into an OpenBao entry; here the entry holds a single password field, so
|
||||
it should, but this needs confirming against the binary rather than assumed.
|
||||
- **Can shelly-manager push MQTT credentials to devices**, and does it cover both
|
||||
Gen1 (`/settings/mqtt`) and Gen2+ (`Mqtt.SetConfig`) firmwares? If not, the
|
||||
`shellies` user may need a different mechanism or stay manual longest.
|
||||
- **How is the HA MQTT config entry updated programmatically?** Reusing the
|
||||
WebSocket approach from the LLAT rotator is plausible but unverified.
|
||||
- **Rotation cadence.** The existing rotators' schedules should be the reference
|
||||
rather than inventing a new one.
|
||||
- **ESO refresh latency.** Rotator-writes-OpenBao → Secret-updates →
|
||||
Reloader-restarts is delayed by up to one `refreshInterval`; the `evcc`
|
||||
ExternalSecret uses `1h`. That does not widen the outage window — broker and
|
||||
client still move together — but it does stretch how long a rotation takes to
|
||||
land, which matters for verifying one completed and for the cold-rebuild
|
||||
ordering question above. Worth deciding whether these ExternalSecrets want a
|
||||
shorter interval than 1h, or whether the rotator should trigger the sync
|
||||
directly.
|
||||
- **Bootstrap on a cold rebuild.** On a fresh cluster the rotator must create the
|
||||
OpenBao entry before ESO can sync it; check this against
|
||||
`kubernetes/RUNBOOK-cold-rebuild.md` so broker bring-up is not blocked.
|
||||
|
||||
## What this does not change
|
||||
|
||||
- Secret names, key names and every `secretKeyRef` in `mosquitto-deployment.yaml`.
|
||||
- The ACL model and the per-user topic scoping.
|
||||
- The `evcc` ExternalSecret, which already follows the target pattern.
|
||||
@@ -13,9 +13,6 @@ resources:
|
||||
- prometheus-token-externalsecret.yaml
|
||||
- config-sync-token-externalsecret.yaml
|
||||
- mosquitto-bootstrap.yaml
|
||||
# health broker credential: OpenBao-backed + Class-A rotated (first of the
|
||||
# eight users to migrate off mosquitto-bootstrap.yaml).
|
||||
- mosquitto-health-credentials-externalsecret.yaml
|
||||
- mosquitto-config.yaml
|
||||
- mosquitto-certificate.yaml
|
||||
- mosquitto-service.yaml
|
||||
|
||||
@@ -370,6 +370,74 @@ spec:
|
||||
limits:
|
||||
memory: 64Mi
|
||||
---
|
||||
# Broker user `health`: a dedicated, least-privilege credential used ONLY by
|
||||
# mosquitto's own liveness/readiness probes (mosquitto-deployment.yaml). Its
|
||||
# ACL is `write health/#` and nothing else — it cannot read any topic or
|
||||
# write outside the throwaway probe subtree, so a leak is worthless (nothing
|
||||
# subscribes to health/#). Kept on its own Secret, like every other user, so
|
||||
# the probe never has to borrow the admin credential.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: mosquitto-health-credentials-bootstrap
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
backoffLimit: 4
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: mosquitto-bootstrap
|
||||
spec:
|
||||
serviceAccountName: mosquitto-bootstrap
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: bootstrap
|
||||
image: alpine:3.21
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache curl jq >/dev/null
|
||||
|
||||
NS=home-assistant
|
||||
NAME=mosquitto-health-credentials
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
API=https://kubernetes.default.svc
|
||||
|
||||
if curl -fsSL --cacert "$CA" -H "Authorization: Bearer $TOKEN" \
|
||||
"$API/api/v1/namespaces/$NS/secrets/$NAME" >/dev/null 2>&1; then
|
||||
echo "$NAME already exists; nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PW=$(tr -dc 'A-Za-z0-9' </dev/urandom | head -c 32)
|
||||
PW_B64=$(printf '%s' "$PW" | base64 | tr -d '\n')
|
||||
|
||||
body=$(jq -nc \
|
||||
--arg name "$NAME" \
|
||||
--arg ns "$NS" \
|
||||
--arg pw "$PW_B64" \
|
||||
'{apiVersion: "v1",
|
||||
kind: "Secret",
|
||||
metadata: {name: $name, namespace: $ns},
|
||||
type: "Opaque",
|
||||
data: {password: $pw}}')
|
||||
|
||||
curl -fsSL --cacert "$CA" -H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" -X POST \
|
||||
--data "$body" \
|
||||
"$API/api/v1/namespaces/$NS/secrets" >/dev/null
|
||||
|
||||
echo "generated $NAME"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
---
|
||||
# Broker user `zigbee2mqtt` — the Zigbee2MQTT bridge (ZHA replacement). Scoped
|
||||
# to its own `zigbee2mqtt/#` subtree plus `write homeassistant/#` (so it can
|
||||
# publish HA MQTT-discovery configs, exactly like the `shellies` user) and
|
||||
|
||||
@@ -5,18 +5,6 @@ metadata:
|
||||
namespace: home-assistant
|
||||
labels:
|
||||
app.kubernetes.io/name: mosquitto
|
||||
annotations:
|
||||
# REQUIRED for credential rotation to take effect at all. The startup shim
|
||||
# rebuilds /mosquitto/auth/passwd from these Secrets on every pod start, so
|
||||
# a rotated Secret is inert until the pod restarts — without this the new
|
||||
# password would sit unused until some unrelated restart, while the old one
|
||||
# stayed valid indefinitely.
|
||||
#
|
||||
# Listed per-Secret rather than using the blanket `reloader.stakater.com/
|
||||
# auto: "true"`, so that enrolling each remaining broker user in rotation is
|
||||
# a visible, reviewable edit to this line rather than something that starts
|
||||
# happening implicitly.
|
||||
secret.reloader.stakater.com/reload: mosquitto-health-credentials
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
@@ -189,9 +177,7 @@ spec:
|
||||
secretKeyRef:
|
||||
name: mosquitto-evcc-credentials
|
||||
key: password
|
||||
# Probe-only credential. Now OpenBao-backed and Class-A rotated (see
|
||||
# mosquitto-health-credentials-externalsecret.yaml); its bootstrap Job
|
||||
# is gone.
|
||||
# Probe-only credential (mosquitto-health-credentials-bootstrap).
|
||||
# Present here for the same reason as the others — the startup
|
||||
# shim hashes it into the passwd file — and reused by the exec
|
||||
# probes below, which run inside this container.
|
||||
@@ -244,7 +230,7 @@ spec:
|
||||
#
|
||||
# Uses the dedicated least-privilege `health` user (ACL: readwrite
|
||||
# health/# only — see the passwd/acl build above and the
|
||||
# mosquitto-health-credentials ExternalSecret). Its password is in
|
||||
# mosquitto-health-credentials-bootstrap Job). Its password is in
|
||||
# this container's env (HEALTH_PASSWORD) because the startup shim
|
||||
# hashes every user into the passwd file; the exec probes run
|
||||
# inside this container and read it directly. A leak of this
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
# Projects the mosquitto `health` broker credential from OpenBao into the Secret
|
||||
# the broker already consumes. FIRST of the eight broker users to move off the
|
||||
# bootstrap Jobs and onto Class-A rotation (see
|
||||
# PROPOSAL-mosquitto-credential-rotation.md (PR #453 — not yet merged; this
|
||||
# file lands with it)); the rotator lives at
|
||||
# kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml.
|
||||
#
|
||||
# Secret name and key are UNCHANGED (`password`), so mosquitto-deployment.yaml's
|
||||
# secretKeyRef and the exec probes that read $HEALTH_PASSWORD need no edit.
|
||||
#
|
||||
# refreshInterval is 10m rather than the 1h used by mosquitto-evcc-credentials:
|
||||
# a rotation only lands once ESO re-reads, so 1h would mean up to an hour between
|
||||
# the rotator running and the broker actually using the new password, with the
|
||||
# old one still valid in the meantime. 10m keeps that window short without
|
||||
# polling OpenBao pointlessly hard.
|
||||
#
|
||||
# ⚠️ OWNERSHIP: creationPolicy is implicitly Owner, so ESO sets an ownerReference
|
||||
# on the Secret. Deleting this ExternalSecret therefore DELETES the live Secret
|
||||
# and with it the broker's probe credential — the bootstrap Job it replaces had
|
||||
# no such coupling. To detach without destroying, set creationPolicy: Orphan
|
||||
# before removing it.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: mosquitto-health-credentials
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
refreshInterval: 10m
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: mosquitto-health-credentials
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/home-assistant/mosquitto-health-credentials
|
||||
@@ -52,7 +52,7 @@ spec:
|
||||
# stdlib only; Flux keeps the tag in lockstep via the shared image
|
||||
# policy) — no extra dependency.
|
||||
- name: reconcile-publish-ip
|
||||
image: ghcr.io/music-assistant/server:2.9.10 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.9.9 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
@@ -62,7 +62,7 @@ spec:
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: music-assistant
|
||||
image: ghcr.io/music-assistant/server:2.9.10 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.9.9 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
env:
|
||||
- name: LOG_LEVEL
|
||||
value: info
|
||||
|
||||
@@ -88,17 +88,13 @@ spec:
|
||||
# trust without disabling verification.
|
||||
initContainers:
|
||||
- name: fetch-bridge-cert
|
||||
# alpine + the openssl CLI, built by .woodpecker/build-ci-images.yml
|
||||
# from ci/images/alpine-openssl/. Replaces `apk add --no-cache openssl`
|
||||
# on every paperless pod start: tools belong in the image, not
|
||||
# installed at runtime. Dated tag so a change is a reviewable diff
|
||||
# here, matching the ci-ansible / alpine-jq convention.
|
||||
image: gitea.renner.uno/gitea_admin/alpine-openssl:2026-07-30
|
||||
image: alpine:3.21
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
apk add --no-cache openssl
|
||||
# Wait until the bridge presents a cert via STARTTLS.
|
||||
# Re-fetched on every paperless restart so a bridge cert
|
||||
# rotation only requires a paperless rollout to pick up.
|
||||
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:8.10.0-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
image: redis:8.8.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
|
||||
@@ -34,7 +34,7 @@ spec:
|
||||
# Image automation, range 3.x (see image-automation/woodpecker.yaml).
|
||||
# MUST stay version-matched with woodpecker-server (gRPC parity) — both
|
||||
# track the same range/filter so they bump together on each release.
|
||||
image: woodpeckerci/woodpecker-agent:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
image: woodpeckerci/woodpecker-agent:v3.16.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
env:
|
||||
- name: WOODPECKER_SERVER
|
||||
value: woodpecker.apps.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
|
||||
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
# minor + patch auto-update within 3.x; only the 4.0 major is held for
|
||||
# a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
|
||||
# -bump hazard is gone). Stays in lockstep with the agent.
|
||||
image: woodpeckerci/woodpecker-server:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
image: woodpeckerci/woodpecker-server:v3.16.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
env:
|
||||
- name: WOODPECKER_HOST
|
||||
value: https://ci.renner.uno
|
||||
|
||||
@@ -36,15 +36,6 @@ spec:
|
||||
kubeProxyReplacement: true
|
||||
socketLB:
|
||||
enabled: true
|
||||
prometheus:
|
||||
# The agent exported no metrics at all, so BPF map utilisation was
|
||||
# invisible: when per-endpoint policy maps (cilium_policy_v3_*) run out
|
||||
# of space the agent only says "no space left on device" in its log and
|
||||
# pod sandbox creation fails (issue #452). cilium_bpf_map_pressure makes
|
||||
# that observable — and alertable — before it stalls a rollout.
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
bpf:
|
||||
# Let pods reach the cluster's own LoadBalancer external IPs; Cilium
|
||||
# redirects the socket directly to the backend pod instead of sending
|
||||
|
||||
@@ -55,31 +55,6 @@ existing `flux-system` GitRepository whose Secret was created by
|
||||
| mmwave-configurator | `2.x` | Plain 3-component filter (rejects `addon-`/`-armv7`/`-arm64`/`-amd64` variants) |
|
||||
| cnpg-barman-plugin | `0.x` | v-prefix filter; marker lives in upstream-imported manifest (see below) |
|
||||
|
||||
## Not enrolled (structurally impossible)
|
||||
|
||||
- `woodpeckerci/woodpecker-cli` — the lint image in
|
||||
`.woodpecker/ci-manifests-test.yml`. It publishes a perfectly good
|
||||
version stream (`v3.16.0-alpine`), so unlike the three below this is
|
||||
**not** a tag-format problem. It cannot be enrolled because
|
||||
**image-automation can only patch YAML that is a Kubernetes manifest** —
|
||||
a maintainer states it plainly in fluxcd/flux2#2826: *"Flux can only
|
||||
patch Kubernetes manifests. This means the YAML manifest must have
|
||||
`apiVersion` and `kind` fields."* A Woodpecker pipeline file is plain
|
||||
YAML with neither, so the marker would be silently ignored.
|
||||
|
||||
Note this is **not** fixable by widening `update.path`. The path only
|
||||
controls which directory is walked; objects without `apiVersion`/`kind`
|
||||
are skipped regardless. Widening the path to `./` buys nothing here and
|
||||
additionally pulls the whole `ansible/` and `esphome/` trees into the
|
||||
walk, so don't.
|
||||
|
||||
Consequence: the CLI is a **manual** pin while the server and agent
|
||||
auto-bump, so it *will* drift. That is a known, accepted gap, covered
|
||||
by the update-audit agent (item 5) rather than by automation. When
|
||||
bumping the server major/minor, bump this too — it must stay on the
|
||||
latest stable like everything else, and a CLI behind the server can
|
||||
pass a config the server then rejects.
|
||||
|
||||
## Not enrolled (deliberate)
|
||||
|
||||
These three images publish no usable version stream — there is nothing
|
||||
|
||||
@@ -38,57 +38,54 @@ spec:
|
||||
effect: NoExecute
|
||||
initContainers:
|
||||
- name: install
|
||||
# Plugin binaries are baked into this image, checksum-verified at build
|
||||
# time (ci/images/cni-plugins/Containerfile). This replaces fetching the
|
||||
# release tarball from GitHub on the node, which had NO integrity check
|
||||
# at all and put github.com on the critical path of every node rebuild.
|
||||
# Dated tag so a version change is a reviewable diff here, matching the
|
||||
# ci-ansible / alpine-jq convention.
|
||||
image: gitea.renner.uno/gitea_admin/cni-plugins:2026-07-30
|
||||
image: alpine:3.21
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
SRC=/opt/cni-plugins
|
||||
DST=/host/opt/cni/bin
|
||||
|
||||
# No network, no download, no "already installed?" guard. The old
|
||||
# guard existed to avoid re-fetching; a local copy is cheap, so
|
||||
# copying unconditionally is better — the node then always matches
|
||||
# the version pinned in this manifest instead of keeping whatever
|
||||
# it happened to install first.
|
||||
for src in "$SRC"/*; do
|
||||
name=${src##*/}
|
||||
# Copy to a temp name in the SAME directory and rename over the
|
||||
# target. A plain `cp` onto a plugin the kubelet is currently
|
||||
# executing can fail with ETXTBSY; rename(2) never does, and is
|
||||
# atomic, so a concurrent CNI invocation sees either the old or
|
||||
# the new binary and never a partial one.
|
||||
cp "$src" "$DST/.$name.tmp"
|
||||
chmod 0755 "$DST/.$name.tmp"
|
||||
mv -f "$DST/.$name.tmp" "$DST/$name"
|
||||
done
|
||||
|
||||
echo "Installed from $(basename "$SRC") image:"
|
||||
ls -la "$DST/macvlan" "$DST/static" "$DST/host-local"
|
||||
echo "Version: $("$DST/macvlan" 2>&1 | head -1 || true)"
|
||||
# Kept, but now as a REPAIR rather than a workaround. The original
|
||||
# cause is gone: extracting the tarball on the node applied the
|
||||
# archive's `./` entry uid/gid (1001/127 — the github-actions runner
|
||||
# that built the release) to the destination directory, and `cp`
|
||||
# into an existing directory does not touch its ownership, so that
|
||||
# can no longer happen.
|
||||
#
|
||||
# It still runs because nodes provisioned by the old installer may
|
||||
# ALREADY carry the wrong ownership, and this is what fixes them.
|
||||
# Why it matters: Cilium's apply-sysctl-overwrites init container
|
||||
# runs as root without DAC_OVERRIDE, so with /opt/cni/bin not owned
|
||||
# by root it fails with "Permission denied" copying cilium-sysctlfix
|
||||
# into /hostbin. The old cluster only survived because the original
|
||||
# Cilium pods initialised before the chown and were never re-rolled;
|
||||
# a rolling daemonset replacement on rebuild is what exposed it.
|
||||
# Safe to drop once every node has been through this version.
|
||||
VERSION=v1.5.1
|
||||
ARCH=amd64
|
||||
URL="https://github.com/containernetworking/plugins/releases/download/${VERSION}/cni-plugins-linux-${ARCH}-${VERSION}.tgz"
|
||||
cd /host/opt/cni/bin
|
||||
if [ -x ./macvlan ] && [ -x ./static ] && [ -x ./host-local ]; then
|
||||
installed=$(./macvlan 2>&1 | head -1 || true)
|
||||
echo "CNI plugins already installed (${installed})"
|
||||
else
|
||||
echo "Fetching ${URL}"
|
||||
# busybox wget + tar, NOT `apk add --no-cache curl tar`.
|
||||
# Installing tools at container start is not acceptable here: it
|
||||
# makes the install path depend on package-index availability at
|
||||
# exactly the moment a node is being (re)built, and this is a
|
||||
# DaemonSet — it runs on every node.
|
||||
#
|
||||
# Nothing needed to be added in the first place. Verified in
|
||||
# alpine:3.21: `tar` is already busybox (v1.37.0) and handles
|
||||
# -xzf, and busybox `wget` follows GitHub's release redirect to
|
||||
# objects.githubusercontent.com over TLS using the base image's
|
||||
# /etc/ssl/certs/ca-certificates.crt — fetched the real 48MB
|
||||
# tarball and extracted an executable macvlan from it.
|
||||
# Only `curl` is genuinely absent from the base, and wget covers
|
||||
# what it was used for here.
|
||||
wget -q -O /tmp/plugins.tgz "${URL}"
|
||||
# Extract everything: the tarball entries are stored with a
|
||||
# `./` prefix, so selective extraction by bare name fails.
|
||||
# All standard plugins are tiny (~5-10MB total) and harmless
|
||||
# to keep alongside Cilium's CNI binaries.
|
||||
tar -xzf /tmp/plugins.tgz -C .
|
||||
echo "Installed: $(ls -la macvlan static host-local)"
|
||||
fi
|
||||
# Always restore root ownership of /opt/cni/bin. The cni-plugins
|
||||
# tarball includes a `./` directory entry whose uid/gid (1001/127
|
||||
# — the github-actions runner that built the release) gets
|
||||
# applied to the destination during extract. After that, Cilium's
|
||||
# apply-sysctl-overwrites init container fails with "Permission
|
||||
# denied" trying to `cp` cilium-sysctlfix into /hostbin, because
|
||||
# the init runs root without DAC_OVERRIDE and the directory is
|
||||
# no longer owned by root. The previous cluster only worked
|
||||
# because the original Cilium pods initialised before the chown
|
||||
# happened and were never re-rolled; on rebuild, the rolling
|
||||
# daemonset replacement bites.
|
||||
chown 0:0 /host/opt/cni/bin
|
||||
securityContext:
|
||||
privileged: true
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
# OpenBao policy: rotator-mosquitto-health
|
||||
#
|
||||
# Least-privilege policy for the mosquitto `health` broker-credential rotation
|
||||
# CronJob (kubernetes auth role `rotator-mosquitto-health`). PER-SECRET
|
||||
# isolation, same template as rotator-paperless: one policy + one role per
|
||||
# rotated secret, so a compromised rotation Job cannot touch any other secret.
|
||||
#
|
||||
# Scoped to the `health` credential ALONE, deliberately not to a wildcard over
|
||||
# apps/home-assistant/mosquitto-*. The broker has eight users and they are being
|
||||
# enrolled one at a time; a wildcard would silently grant the first rotator write
|
||||
# access to the other seven before they are ready, which is exactly the blast
|
||||
# radius this per-secret pattern exists to avoid.
|
||||
#
|
||||
# Plan and phasing: kubernetes/apps/home-assistant/
|
||||
# PROPOSAL-mosquitto-credential-rotation.md — added by PR #453, which is not yet
|
||||
# merged, so that path does not resolve until it lands.
|
||||
#
|
||||
# NOT applied by Flux — OpenBao policies/roles are not Kubernetes resources
|
||||
# (same convention as eso-read.hcl and every other rotator policy). Apply ONCE
|
||||
# with an admin token:
|
||||
#
|
||||
# bao policy write rotator-mosquitto-health \
|
||||
# kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl
|
||||
#
|
||||
# bao write auth/kubernetes/role/rotator-mosquitto-health \
|
||||
# bound_service_account_names=mosquitto-health-secret-rotator \
|
||||
# bound_service_account_namespaces=openbao \
|
||||
# policies=rotator-mosquitto-health ttl=15m
|
||||
#
|
||||
# Do NOT set token_no_default_policy=true — the `bao kv patch` kv-v2 preflight
|
||||
# needs the default policy (see rotator.hcl). The rotator writes via patch, so
|
||||
# hardening the role that way would break rotation rather than tighten it.
|
||||
#
|
||||
# BOOTSTRAP: `random` mode calls assert_fields_exist BEFORE patching, so it
|
||||
# ROTATES an existing field and never creates one. The path must already hold a
|
||||
# `password` field or the Job fails safe without writing. Seed it once:
|
||||
#
|
||||
# bao kv put homelab/apps/home-assistant/mosquitto-health-credentials \
|
||||
# password="$(kubectl -n home-assistant get secret mosquitto-health-credentials \
|
||||
# -o jsonpath='{.data.password}' | base64 -d)"
|
||||
#
|
||||
# (Reads are already covered by eso-read.hcl's `homelab/data/*`, so the
|
||||
# ExternalSecret needs no policy change.)
|
||||
path "homelab/data/apps/home-assistant/mosquitto-health-credentials" {
|
||||
capabilities = ["read", "patch"]
|
||||
}
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: sync-woodpecker
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:828bf4863286411bfd7ef9fdff617586588050c814743bec601a9222692162e4
|
||||
args: ["woodpecker-secret"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -61,7 +61,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:978fe53cad83a8d55319f6201ecd171775a101b21aaa21b7ec3c7b14304c6a42
|
||||
args: ["cloudflare-roll"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -56,7 +56,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:978fe53cad83a8d55319f6201ecd171775a101b21aaa21b7ec3c7b14304c6a42
|
||||
args: ["cloudflare-roll"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -84,7 +84,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:b41cc3645f144179dddaa5ab1d32282c57c257432a613c15a6e15634fd0c4b81
|
||||
args: ["gitea-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["ha-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
@@ -130,7 +130,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["ha-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
@@ -192,7 +192,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["ha-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -4,46 +4,6 @@ kind: Kustomization
|
||||
# (intra-namespace OpenBao access → no NetworkPolicy needed). One file per rotated
|
||||
# secret. The matching OpenBao policy/role for each lives in ../policies/ and is
|
||||
# bootstrapped manually (not GitOps-managed).
|
||||
#
|
||||
# IMAGE PINNING: every rotator here uses a DATED TAG of secret-rotator
|
||||
# (`:YYYY-MM-DD`), never a digest and never `:latest`. This is a registry-local
|
||||
# image, so it does not need digest pinning, and a dated tag makes "what actually
|
||||
# runs" a reviewable diff.
|
||||
#
|
||||
# Digests were tried and actively broke: untagged manifests are garbage-collected
|
||||
# once :latest and the dated tags move past them, so a digest pinned today stops
|
||||
# resolving after a few rebuilds. That decay is driven by edits to
|
||||
# .woodpecker/build-ci-images.yml — a shared file — so adding an UNRELATED image
|
||||
# republishes secret-rotator and orphans the manifest every rotator here was
|
||||
# pinned to. 11 of these were pinned to digests that no longer existed and would
|
||||
# have ImagePullBackOff'd on their first scheduled run (issue #458); because they
|
||||
# are quarterly, that would have looked like nothing happening rather than like a
|
||||
# failure.
|
||||
#
|
||||
# Keep them all on one tag: it is a single binary containing every mode
|
||||
# (random | ha-token | cloudflare-roll | netbird-key | proxmox-token |
|
||||
# gitea-token | woodpecker-secret), so a split serves no purpose and just makes
|
||||
# the next bump partial.
|
||||
#
|
||||
# ADDING A ROTATOR: copy an existing file, and check you did not inherit a
|
||||
# digest pin from an older example — nothing enforces this automatically, so it
|
||||
# is easy to reintroduce silently and the failure only shows up months later on
|
||||
# the first scheduled run:
|
||||
#
|
||||
# grep -l 'image:.*@sha256' \
|
||||
# kubernetes/infrastructure/openbao/rotation/*.yaml | grep -v kustomization.yaml
|
||||
#
|
||||
# (want: no output. The trailing filter is not padding — this file documents the
|
||||
# pattern, so it necessarily matches its own grep. Verified both directions:
|
||||
# silent across this directory as it stands, and it still reports a file that
|
||||
# actually carries an `image: …@sha256:` pin.)
|
||||
#
|
||||
# BUMPING the tag: verify the new one exists BEFORE committing — a tag that was
|
||||
# never built fails exactly like the digests did, and just as quietly:
|
||||
#
|
||||
# kubectl run tagcheck --rm -i --restart=Never \
|
||||
# --image=gitea.renner.uno/gitea_admin/secret-rotator:<DATE> \
|
||||
# --command -- /secret-rotator # prints the usage/mode list if the tag resolves
|
||||
resources:
|
||||
- paperless.yaml
|
||||
- woodpecker.yaml
|
||||
@@ -58,4 +18,3 @@ resources:
|
||||
- home-assistant.yaml
|
||||
- review-bot.yaml
|
||||
- claude-oauth.yaml
|
||||
- mosquitto-health.yaml
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
---
|
||||
# Class-A automated rotation of the mosquitto `health` broker credential — a
|
||||
# self-generated random secret.
|
||||
#
|
||||
# FIRST of the eight mosquitto broker users to be enrolled. The credentials were
|
||||
# previously minted once by bootstrap Jobs and then never changed at all: no
|
||||
# schedule, no compromise response, and they existed only in the Secret object,
|
||||
# outside the OpenBao snapshot. See kubernetes/apps/home-assistant/
|
||||
# PROPOSAL-mosquitto-credential-rotation.md (PR #453 — not yet merged; this
|
||||
# file lands with it) for the full plan and phasing.
|
||||
#
|
||||
# `health` deliberately goes first: it is the broker's OWN liveness/readiness
|
||||
# probe credential, consumed by exec probes inside the mosquitto container via
|
||||
# the same Secret that seeds the passwd file. Nothing outside the pod holds it,
|
||||
# so a rotation needs no coordination with any client — the smallest possible
|
||||
# blast radius for proving rotator -> OpenBao -> ESO -> Reloader end to end.
|
||||
#
|
||||
# Runs the secret-rotator binary (random mode): generate 32 random bytes,
|
||||
# standard-base64 encode, and merge-patch into OpenBao. assert_fields_exist runs
|
||||
# FIRST, so a misconfigured path fails safe (no rotation) rather than adding a
|
||||
# stray field — which also means this ROTATES an existing value and never
|
||||
# bootstraps one. Seeding is a one-time out-of-band step, documented in
|
||||
# policies/rotator-mosquitto-health.hcl.
|
||||
#
|
||||
# Base64 output is safe for this credential specifically: the probes pass it as
|
||||
# `-P "$HEALTH_PASSWORD"` to mosquitto_pub/sub inside the container, so the `+`,
|
||||
# `/` and `=` characters are never parsed by a URL or a config file. That is NOT
|
||||
# automatically true of the other seven users (sml2mqtt and open3e templates on
|
||||
# the edge host, Shelly device firmware, the HA config entry), so the charset
|
||||
# must be re-checked per user before enrolling each one — the old bootstrap Jobs
|
||||
# generated alphanumeric-only passwords with `tr -dc 'A-Za-z0-9'`.
|
||||
#
|
||||
# openbao-ns pattern: the Job runs in the openbao namespace, reaching the
|
||||
# OpenBao API intra-namespace — no NetworkPolicy needed. It writes OpenBao only;
|
||||
# ESO syncs mosquitto-health-credentials (ns home-assistant) and Reloader
|
||||
# restarts the broker so the passwd file is rebuilt from the new value.
|
||||
#
|
||||
# Rotation is not instant: it completes only after the ExternalSecret's
|
||||
# refreshInterval elapses and the broker rolls. During that roll two broker pods
|
||||
# with different passwd files sit behind the LB (maxUnavailable: 0/maxSurge: 1),
|
||||
# so a probe against the outgoing pod can fail auth briefly. Self-healing.
|
||||
#
|
||||
# ON-DEMAND: kubectl -n openbao create job --from=cronjob/mosquitto-health-secret-rotate mosquitto-health-secret-rotate-now
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: mosquitto-health-secret-rotator
|
||||
namespace: openbao
|
||||
labels:
|
||||
app.kubernetes.io/name: mosquitto
|
||||
app.kubernetes.io/component: secret-rotator
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: mosquitto-health-secret-rotate
|
||||
namespace: openbao
|
||||
labels:
|
||||
app.kubernetes.io/name: mosquitto
|
||||
app.kubernetes.io/component: secret-rotator
|
||||
spec:
|
||||
# Quarterly, offset one month from paperless (which runs Jan/Apr/Jul/Oct) so
|
||||
# unrelated rotations do not land in the same window and complicate diagnosis.
|
||||
schedule: "0 4 1 2,5,8,11 *"
|
||||
timeZone: Europe/Berlin
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 2
|
||||
failedJobsHistoryLimit: 5
|
||||
startingDeadlineSeconds: 3600
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 300
|
||||
ttlSecondsAfterFinished: 259200 # 3 d
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: mosquitto
|
||||
app.kubernetes.io/component: secret-rotator
|
||||
spec:
|
||||
serviceAccountName: mosquitto-health-secret-rotator
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
# Dated tag, per the convention documented in kustomization.yaml —
|
||||
# every rotator here now uses the same one. Verified for this tag:
|
||||
# it pulls, `random` is among its modes, and it carries the
|
||||
# RANDOM_BYTES>=16 floor documented below (RANDOM_BYTES=8 exits
|
||||
# "refusing to generate a weak secret").
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: BAO_ADDR
|
||||
value: http://openbao.openbao.svc.cluster.local:8200
|
||||
- name: BAO_ROLE
|
||||
value: rotator-mosquitto-health
|
||||
- name: BAO_PATH
|
||||
value: homelab/apps/home-assistant/mosquitto-health-credentials
|
||||
# Matches the key the ExternalSecret projects and the key the
|
||||
# Secret already carries, so mosquitto-deployment.yaml's
|
||||
# secretKeyRef is unchanged.
|
||||
- name: BAO_FIELDS
|
||||
value: "password"
|
||||
# 32 bytes -> 44 base64 chars. The binary refuses anything below
|
||||
# 16 bytes outright.
|
||||
- name: RANDOM_BYTES
|
||||
value: "32"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
memory: 48Mi
|
||||
@@ -59,7 +59,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:49bfddd8eeb5fd871cbeb05bc3b4e2fcaaab7d9e24cc3007caae6084b4634c18
|
||||
args: ["netbird-key"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:97c22bcff324a93d2bb7bf3adb2894b405ed1a8035d02ad1ff6baced3813cefe
|
||||
args: ["proxmox-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -81,7 +81,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: rotate-mint
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:828bf4863286411bfd7ef9fdff617586588050c814743bec601a9222692162e4
|
||||
args: ["gitea-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
@@ -128,7 +128,7 @@ spec:
|
||||
memory: 48Mi
|
||||
containers:
|
||||
- name: sync-woodpecker
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:828bf4863286411bfd7ef9fdff617586588050c814743bec601a9222692162e4
|
||||
args: ["woodpecker-secret"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
Reference in New Issue
Block a user