Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 claude-bot
|
803c741362 | ||
|
claude-bot
|
aff01d5a32 | ||
|
claude-bot
|
80c3051dc7 | ||
|
claude-bot
|
bbbc42c79f |
+91
-129
@@ -14,6 +14,12 @@
|
||||
# Flux-deployed CronJobs, so it is tracked separately in #330 rather than
|
||||
# bundled here. Note this pipeline runs no LLM agent, so the exfil path that
|
||||
# motivated the other cutovers does not apply to it.
|
||||
#
|
||||
# The build steps run the plain kaniko executor, which has no shell: they pass
|
||||
# flags via `entrypoint` and read credentials from DOCKER_CONFIG. Registry auth
|
||||
# is written once by the registry-auth step below. Dated tags are literals: bump
|
||||
# them in the commit that changes an image. The build timestamp itself lives in
|
||||
# the image at /etc/image-build-date, written by a RUN in the Containerfile.
|
||||
when:
|
||||
- event: push
|
||||
path:
|
||||
@@ -24,10 +30,22 @@ when:
|
||||
evaluate: 'PIPELINE == "build-ci-images"'
|
||||
|
||||
steps:
|
||||
- name: build-ansible
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
directory: ci/images/ansible
|
||||
- name: registry-auth
|
||||
image: alpine:3.24.1
|
||||
depends_on: []
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p .docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > .docker/config.json
|
||||
|
||||
- name: build-ansible
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: ci/images/ansible
|
||||
depends_on: [registry-auth]
|
||||
# Only rebuild when THIS image's source (or the pipeline file) changed, or
|
||||
# on a manual run. Without this every step ran on any matching push, so a
|
||||
# secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
|
||||
@@ -37,68 +55,47 @@ steps:
|
||||
- event: push
|
||||
path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-ansible:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-ansible:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/ci-ansible:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/ci-ansible:latest
|
||||
|
||||
- name: build-k8s
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: ci/images/k8s
|
||||
depends_on: []
|
||||
depends_on: [registry-auth]
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-k8s:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-k8s:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/ci-k8s:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/ci-k8s:latest
|
||||
|
||||
- name: build-tofu
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: ci/images/tofu
|
||||
depends_on: []
|
||||
depends_on: [registry-auth]
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-tofu:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-tofu:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/ci-tofu:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/ci-tofu:latest
|
||||
|
||||
# NOTE: ci-claude is deliberately NOT built here. It tracks upstream Claude
|
||||
# Code releases on its own cadence and is built + version-tagged by
|
||||
@@ -108,28 +105,21 @@ steps:
|
||||
# Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
|
||||
# used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
|
||||
- name: build-secret-rotator
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: secret-rotator
|
||||
depends_on: []
|
||||
depends_on: [registry-auth]
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/secret-rotator:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/secret-rotator:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/secret-rotator:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/secret-rotator:latest
|
||||
|
||||
# Runtime image, not CI tooling: a general-purpose alpine + jq base for
|
||||
# workloads that need to read JSON from a shell script. Named for what it IS,
|
||||
@@ -137,28 +127,21 @@ steps:
|
||||
# container, but nothing about it is zigbee-specific. Exists so tools come from
|
||||
# the image rather than an `apk add` on every container start.
|
||||
- name: build-alpine-jq
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: ci/images/alpine-jq
|
||||
depends_on: []
|
||||
depends_on: [registry-auth]
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-jq:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/alpine-jq:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/alpine-jq:latest
|
||||
|
||||
# Runtime image, not CI tooling: upstream containernetworking/plugins binaries
|
||||
# baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
|
||||
@@ -166,28 +149,21 @@ steps:
|
||||
# critical path of every node rebuild. The download now happens here, once,
|
||||
# where the sha256 is checked — see the Containerfile for the full rationale.
|
||||
- name: build-cni-plugins
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: ci/images/cni-plugins
|
||||
depends_on: []
|
||||
depends_on: [registry-auth]
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/cni-plugins:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/cni-plugins:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/cni-plugins:latest
|
||||
|
||||
# Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
|
||||
# needing s_client/x509. Named for what it is; first consumer is the paperless
|
||||
@@ -195,28 +171,21 @@ steps:
|
||||
# every paperless pod start. The openssl CLI is genuinely absent from the alpine
|
||||
# base, so this one needs an image rather than a busybox substitute.
|
||||
- name: build-alpine-openssl
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: ci/images/alpine-openssl
|
||||
depends_on: []
|
||||
depends_on: [registry-auth]
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-openssl:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/alpine-openssl:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/alpine-openssl:latest
|
||||
|
||||
# Heating CAN readers: open3e (Vitocal/VX3) + E3onCAN (E380 meter), one image
|
||||
# run as two services on edge-1. Replaces pip-into-venv on the host, where an
|
||||
@@ -224,25 +193,18 @@ steps:
|
||||
# Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
|
||||
# builds the same way and lands in the same registry so the audit sees it.
|
||||
- name: build-heating-can
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.24.0
|
||||
directory: ci/images/heating-can
|
||||
depends_on: []
|
||||
depends_on: [registry-auth]
|
||||
when:
|
||||
- event: manual
|
||||
- event: push
|
||||
path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/heating-can:latest
|
||||
DOCKER_CONFIG: "${CI_WORKSPACE}/.docker"
|
||||
entrypoint:
|
||||
- /kaniko/executor
|
||||
- --context=.
|
||||
- --dockerfile=Containerfile
|
||||
- --destination=gitea.renner.uno/gitea_admin/heating-can:2026-08-23
|
||||
- --destination=gitea.renner.uno/gitea_admin/heating-can:latest
|
||||
|
||||
@@ -13,7 +13,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: validate-secrets
|
||||
image: alpine:3.24.1
|
||||
image: alpine:3.21
|
||||
environment:
|
||||
TF_VAR_proxmox_api_url:
|
||||
from_secret: proxmox_api_url
|
||||
@@ -129,20 +129,13 @@ steps:
|
||||
- echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
|
||||
- export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
|
||||
- |
|
||||
# Ensure Talos image exists on all Proxmox nodes via API.
|
||||
#
|
||||
# Only nodes still running Proxmox belong in this list. jupiter and
|
||||
# saturn run Talos on bare metal now, so the storage API returns
|
||||
# nothing for them and the download-wait loop below would burn its full
|
||||
# 60x10s per node before giving up. Bare-metal nodes boot the
|
||||
# metal-amd64 image from their own installed disk instead, and the
|
||||
# whole step goes away once neptune is converted too.
|
||||
# Ensure Talos image exists on all Proxmox nodes via API
|
||||
TALOS_VERSION="v1.12.6"
|
||||
TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
|
||||
TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
|
||||
TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
|
||||
API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
|
||||
for node in neptune; do
|
||||
for node in jupiter saturn neptune; do
|
||||
EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
|
||||
-H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
|
||||
if [ "$EXISTS" -gt 0 ]; then
|
||||
|
||||
@@ -22,3 +22,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
|
||||
# unanchored extraction therefore yields a device link key, and herdsman then
|
||||
# forms a NEW Zigbee network and drops every device.
|
||||
RUN apk add --no-cache jq
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Last layer, so it only invalidates
|
||||
# itself.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -19,3 +19,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
|
||||
# for paperless to trust. busybox has no s_client equivalent, so openssl is not
|
||||
# substitutable there.
|
||||
RUN apk add --no-cache openssl
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Last layer, so it only invalidates
|
||||
# itself.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -24,3 +24,8 @@ RUN pip3 install --break-system-packages --no-cache-dir \
|
||||
RUN ansible-galaxy collection install \
|
||||
community.proxmox \
|
||||
community.docker
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Last layer, so it only invalidates
|
||||
# itself.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -44,3 +44,9 @@ RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \
|
||||
/usr \
|
||||
/root \
|
||||
/tmp
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# version was built, this says when. Deliberately the LAST layer and its own
|
||||
# layer, so the byte-stable npm layer above still dedups in the registry — that
|
||||
# dedup is what #351 was for, and a few dozen bytes here does not affect it.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -66,3 +66,8 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
|
||||
# Alpine rather than scratch: the installer needs a shell to copy the binaries
|
||||
# onto the host. busybox covers cp/mv/chown/ls, so nothing is added.
|
||||
COPY --from=fetch /plugins/ /opt/cni-plugins/
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Last layer, so it only invalidates
|
||||
# itself.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -120,3 +120,8 @@ USER open3e
|
||||
WORKDIR /var/lib/open3e
|
||||
|
||||
ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Last layer, so it only invalidates
|
||||
# itself.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -42,3 +42,8 @@ RUN curl -sfL -o /tmp/velero.tar.gz \
|
||||
&& tar -xz -C /tmp -f /tmp/velero.tar.gz \
|
||||
&& install "/tmp/velero-${VELERO_VERSION}-linux-amd64/velero" /usr/local/bin/velero \
|
||||
&& rm -rf "/tmp/velero-${VELERO_VERSION}-linux-amd64" /tmp/velero.tar.gz
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Last layer, so it only invalidates
|
||||
# itself.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -31,3 +31,8 @@ RUN pip3 install --break-system-packages --no-cache-dir \
|
||||
RUN ansible-galaxy collection install \
|
||||
community.proxmox \
|
||||
community.docker
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Last layer, so it only invalidates
|
||||
# itself.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /etc/image-build-date
|
||||
|
||||
@@ -167,7 +167,7 @@ permanent recovery.
|
||||
|
||||
## CNPG restore-test (verify backups are restorable — non-destructive)
|
||||
|
||||
The monthly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
|
||||
The weekly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
|
||||
PVC restores but **explicitly does not touch CNPG** (barman-cloud is a separate
|
||||
code path). Run this drill periodically to prove the CNPG backups are actually
|
||||
restorable, **without touching the live clusters**. It restores into a throwaway
|
||||
|
||||
@@ -1,340 +0,0 @@
|
||||
# Runbook — converting a control-plane node from Proxmox VM to bare metal
|
||||
|
||||
Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
|
||||
metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
|
||||
after doing jupiter on 2026-08-25; saturn and neptune remain.
|
||||
|
||||
Everything here that reads like an over-specific warning is something that
|
||||
actually happened. The conversion itself is straightforward — the failures were
|
||||
all in state that outlived the VM.
|
||||
|
||||
## What the node ends up as
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| System disk | Kingston OM8PGP41024Q-A0, selected by serial |
|
||||
| EPHEMERAL (`/var`) | the retired Crucial P3, selected by serial |
|
||||
| Ceph OSD | Samsung PM983, selected by `/dev/disk/by-id` |
|
||||
| Management | VLAN 40 tagged on X710 port 0 |
|
||||
| v30iot | VLAN 30 on the same port, named `ens19` |
|
||||
| Ceph cluster net | X710 port 1, `10.99.0.x/29`, MTU 9000 |
|
||||
|
||||
## Before you start
|
||||
|
||||
- [ ] All three OSDs up, all mons in quorum, PGs `active+clean`. Do not start
|
||||
from a degraded cluster; the window itself removes one of three.
|
||||
- [ ] Cluster fits on two nodes. Check `kubectl describe node` allocation on the
|
||||
two survivors. Take immich down (`kubernetes/apps/immich`, both
|
||||
Deployments to `replicas: 0`) — it frees ~1000m against a shortfall of
|
||||
roughly 200m and takes its image pulls off the remaining nodes.
|
||||
- [ ] `cephfs-metadata` to size 2 / min_size 1 for the duration. At size 3 on
|
||||
two hosts every PG is undersized with min_size 2, so one further OSD loss
|
||||
blocks IO on every CephFS mount cluster-wide.
|
||||
- [ ] Talos metal ISO on a USB stick. Schematic
|
||||
`a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64`
|
||||
(`net.ifnames=1`, intel-ucode, thunderbolt; no qemu-guest-agent).
|
||||
Verify by comparing partition offsets and sizes against the ISO with
|
||||
`fdisk -l` — blkid labels lie after a rewrite.
|
||||
- [ ] Physical access. Secure Boot must be turned off (see below), and that is
|
||||
the one step nothing can do remotely.
|
||||
|
||||
## 1. Evacuate the OSD
|
||||
|
||||
```sh
|
||||
ceph osd out <id>
|
||||
```
|
||||
|
||||
Wait until no PGs are `backfilling` or `backfill_wait`. Do **not** wait for
|
||||
`ceph osd safe-to-destroy` — pools at size 3 keep the OSD in their acting set
|
||||
because three copies cannot be placed on two hosts, so it never returns clean.
|
||||
|
||||
Then remove it. The node must already be out of `cephcluster.yaml`'s
|
||||
`storage.nodes` list, or the operator rebuilds the OSD within one reconcile and
|
||||
starts backfilling onto the drive you are about to pull:
|
||||
|
||||
```sh
|
||||
kubectl -n rook-ceph scale deployment rook-ceph-osd-<id> --replicas=0
|
||||
ceph osd purge <id> --yes-i-really-mean-it
|
||||
kubectl -n rook-ceph delete deployment rook-ceph-osd-<id>
|
||||
```
|
||||
|
||||
## 2. Drain, and leave the cluster
|
||||
|
||||
```sh
|
||||
kubectl drain <node> --ignore-daemonsets --delete-emptydir-data --timeout=300s
|
||||
```
|
||||
|
||||
Single-replica workloads with `minAvailable: 1` PDBs will refuse eviction — that
|
||||
is deliberate, they exist to surface the pod rather than let it be yanked. On
|
||||
this cluster that is `influxdb-influxdb2-0`, `woodpecker-server` and the
|
||||
CNPG primary. Move each one deliberately with `kubectl delete pod`; they
|
||||
reschedule onto the survivors and their RBD volumes follow.
|
||||
|
||||
```sh
|
||||
talosctl -n <node> reset --graceful --wipe-mode system-disk
|
||||
kubectl delete node <node>
|
||||
```
|
||||
|
||||
`--wipe-mode system-disk` and not the default `all`: the OSD drive is being
|
||||
pulled anyway and wiping a terabyte of it is wasted time. `--graceful` leaves
|
||||
etcd cleanly. Delete the Node object — it carries the `unschedulable` flag from
|
||||
the drain, which the rebuilt node would otherwise inherit.
|
||||
|
||||
## 2a. Destroy the VM in OpenTofu — while the host is still a PVE node
|
||||
|
||||
Remove `opentofu/vm_15N_k8s_N.tf`, then apply. This **must** happen before the
|
||||
host leaves the Proxmox cluster: afterwards the provider cannot reach the node,
|
||||
the plan errors, and the only escape is `tofu state rm`, which is off the table.
|
||||
|
||||
Point `proxmox_api_url` at a host that will *survive* the conversion, not the one
|
||||
being converted.
|
||||
|
||||
Manual triggers are currently broken by the Woodpecker v3.18.0 regression
|
||||
(issue #582). Until that is resolved, run it locally:
|
||||
|
||||
```sh
|
||||
AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… AWS_REGION=fsn1 \
|
||||
tofu -chdir=opentofu init -backend-config=<file>
|
||||
… plan -target=module.k8s_N -var-file=<file> -out=<plan>
|
||||
… apply <plan>
|
||||
```
|
||||
|
||||
The `AWS_*` environment variables are required *in addition to*
|
||||
`-backend-config`, or the S3 backend fails `HeadObject` with 403 even with
|
||||
credentials that can read the object fine via the aws CLI.
|
||||
|
||||
## 2b. Proxmox quorum — matters from the second conversion onward
|
||||
|
||||
Two nodes remain after jupiter, with `expected_votes: 2` and quorum 2. Removing
|
||||
one leaves the last node inquorate and `/etc/pve` read-only, which blocks the
|
||||
final VM destroy. On the surviving node:
|
||||
|
||||
```sh
|
||||
pvecm expected 1
|
||||
```
|
||||
|
||||
This is the official recovery. Proxmox documents it in the storage replication
|
||||
chapter (`/usr/share/pve-docs/chapter-pvesr.html`), verbatim:
|
||||
|
||||
> If you have no quorum, we strongly advise to fix this first and make the node
|
||||
> operable again. Only if this is not possible at the moment, you may use the
|
||||
> following command to enforce quorum on the current node: `# pvecm expected 1`
|
||||
|
||||
The caveat there is about restoring the missing node instead. Here the missing
|
||||
node is deliberately gone for good, so enforcing quorum is the correct branch
|
||||
rather than a workaround.
|
||||
|
||||
Run it only once the node is actually alone — attempting it while both nodes are
|
||||
up fails with `CS_ERR_INVALID_PARAM`, since corosync will not set expected votes
|
||||
below the members it can currently see. Which is exactly why it becomes valid
|
||||
when one remains.
|
||||
|
||||
Runtime only; it does not survive a reboot, so re-run it if the survivor
|
||||
restarts before its own conversion.
|
||||
|
||||
## 3. Hardware
|
||||
|
||||
Power the host off, then:
|
||||
|
||||
1. Kingston boot drive out of slot 1, into slot 2.
|
||||
2. Crucial P3 out of slot 2 — it becomes EPHEMERAL, so put it in slot 3 rather
|
||||
than setting it aside.
|
||||
3. Slot 1 switch to **U.2**. Set it before the drive goes in; there is nothing
|
||||
to record afterwards, the installed drive type implies the position.
|
||||
4. PM983 onto the included 7mm M.2-to-U.2 adapter, into slot 1.
|
||||
|
||||
Boot the host's existing PVE once before touching Talos and check the drive:
|
||||
|
||||
```sh
|
||||
nvme smart-log /dev/nvmeXn1 | grep -iE 'percentage_used|media_errors|critical'
|
||||
nvme id-ns /dev/nvmeXn1 -H | grep -i 'in use' # want 512b data, 0 metadata
|
||||
wipefs /dev/nvmeYn1 # Crucial: clear the old bluestore sig
|
||||
ip link set <ceph-port> up && ethtool <ceph-port> | grep -i 'link detected'
|
||||
```
|
||||
|
||||
This is the last revert point. A bad drive here means the Crucial goes back and
|
||||
the VM is recreated from tofu with nothing lost. Once Talos writes to the
|
||||
Kingston, PVE on that host is gone.
|
||||
|
||||
Relocating the boot drive is safe: `rpool` is referenced by NVMe EUI and the
|
||||
UEFI entries by GPT PARTUUID, neither of which encodes a slot.
|
||||
|
||||
## 4. Secure Boot
|
||||
|
||||
MS-01 ships with Secure Boot on, and Proxmox satisfies it with a
|
||||
Microsoft-signed shim. The Image Factory `metal-amd64.iso` is an unsigned UKI
|
||||
and the firmware rejects it outright. **Turn Secure Boot off in the BIOS.**
|
||||
|
||||
A signed variant exists per schematic, but it is signed with SideroLabs' key
|
||||
which firmware does not trust by default, and the installed system would have to
|
||||
use the secureboot installer image too. That is a separate project.
|
||||
|
||||
## 5. Install
|
||||
|
||||
Boot the stick. Maintenance mode DHCPs untagged on every link; on these hosts
|
||||
that lands on `192.168.8.x`, which is routable from the workstation.
|
||||
|
||||
Maintenance mode is effectively write-only — `talosctl get` has no `--insecure`
|
||||
and `version` returns "API is not implemented in maintenance mode". You cannot
|
||||
inspect disks or links before applying. Verify serials and MACs from the host's
|
||||
PVE beforehand, in step 3.
|
||||
|
||||
Build the config from the **live cluster's secrets**, not a fresh
|
||||
`talosctl gen config`, or the node forms a new cluster instead of joining:
|
||||
|
||||
```sh
|
||||
talosctl -n <surviving-node> get mc v1alpha1 -o jsonpath='{.spec}' > cp.yaml
|
||||
talosctl gen secrets --from-controlplane-config cp.yaml -o secrets.yaml
|
||||
talosctl gen config homelab https://k8s.renner.uno:6443 \
|
||||
--with-secrets secrets.yaml --output-types controlplane \
|
||||
--config-patch @talos/patches/cluster.yaml \
|
||||
--config-patch @<cilium additive half> \
|
||||
--config-patch @talos/patches/registry-mirror.yaml \
|
||||
--config-patch @talos/patches/k8s-N.yaml -o install.yaml
|
||||
```
|
||||
|
||||
Confirm `cluster.id` matches the running cluster before applying.
|
||||
|
||||
Strip the generated `HostnameConfig` document — it collides with
|
||||
`machine.network.hostname` from the per-node patch. `bootstrap.sh` does the same.
|
||||
|
||||
`cilium-cni.yaml` cannot be passed whole: its `$patch: delete` document fails
|
||||
against a base with no `inlineManifests` (issue #581). Use only the additive
|
||||
second document.
|
||||
|
||||
Validate, then apply:
|
||||
|
||||
```sh
|
||||
talosctl validate --config install.yaml --mode metal
|
||||
talosctl apply-config --insecure -n <maintenance-ip> --file install.yaml
|
||||
```
|
||||
|
||||
## 6. The network config will strand the node if it is wrong
|
||||
|
||||
Talos stops running default DHCP operators **for the entire machine** as soon as
|
||||
a single new-style network document exists — the gate is presence of a document,
|
||||
not success of resolution. If the configured path fails to materialise there is
|
||||
no fallback on any interface, and recovery needs a console and install media.
|
||||
|
||||
Concretely: do not use `LinkAliasConfig` as a `VLANConfig` parent without
|
||||
verifying it. Aliases *are* supported (confirmed in v1.12.6 source and by a
|
||||
maintainer), but a selector matching zero links is skipped **silently**, and
|
||||
`permanent_addr` comes from an ethtool ioctl with no fallback — if the driver
|
||||
does not answer, the match is against an empty string and everything downstream
|
||||
that references the alias does nothing.
|
||||
|
||||
Address links by predictable kernel name. To introduce aliases later, do it on a
|
||||
node that is already up, with `talosctl apply-config --mode=try --timeout=5m`,
|
||||
which rolls back on its own, and check `talosctl get linkaliasspec`.
|
||||
|
||||
Interface names derive from PCI topology and are stable — but not immune.
|
||||
Adding a third NVMe to jupiter renumbered the buses and renamed the unused
|
||||
second I226 from `enp89s0` to `enp90s0`.
|
||||
|
||||
## 7. EPHEMERAL placement is one-shot
|
||||
|
||||
```yaml
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
provisioning:
|
||||
diskSelector:
|
||||
match: disk.serial == '<crucial serial>'
|
||||
minSize: 50GiB
|
||||
grow: true
|
||||
```
|
||||
|
||||
Talos applies volume configuration **only when the volume does not yet exist**.
|
||||
Applied afterwards it is silently ignored, and relocating EPHEMERAL then needs a
|
||||
disk wipe and a fresh boot from install media. It must be in the first
|
||||
`apply-config`.
|
||||
|
||||
Match by serial. `disk.transport == 'nvme' && !system_disk` matches the PM983 as
|
||||
well, and losing that coin flip formats the OSD drive.
|
||||
|
||||
Verify afterwards:
|
||||
|
||||
```sh
|
||||
talosctl -n <node> get volumestatus EPHEMERAL -o yaml | grep -E 'location|phase'
|
||||
talosctl -n <node> get disks
|
||||
```
|
||||
|
||||
## 8. The mon will not come back on its own
|
||||
|
||||
**This is the step that cost the most time on jupiter.** The rebuilt node has a
|
||||
new MAC and therefore a new DHCP lease, but Ceph has its mon pinned to the old
|
||||
address in three places. The mon cannot bind an address the host no longer has,
|
||||
so it crashloops; the operator then tries to restart a *healthy* mon as its next
|
||||
step, Ceph refuses because that would drop quorum, and it retries every 60
|
||||
seconds forever without ever reaching the storage reconcile. Nothing else
|
||||
progresses behind it.
|
||||
|
||||
Fix in this order — the order matters, because doing it the other way round has
|
||||
the operator faithfully recreate the mon from the stale ConfigMap:
|
||||
|
||||
```sh
|
||||
# 1. ConfigMap first
|
||||
kubectl -n rook-ceph edit cm rook-ceph-mon-endpoints
|
||||
# remove the dead mon from `data` and from `mapping`
|
||||
|
||||
# 2. then the monmap
|
||||
ceph mon remove <id>
|
||||
|
||||
# 3. then the deployment — check it really carries the old address
|
||||
kubectl -n rook-ceph get deploy rook-ceph-mon-<id> -o yaml | grep public-addr
|
||||
kubectl -n rook-ceph delete deploy rook-ceph-mon-<id>
|
||||
```
|
||||
|
||||
Rook then creates a **new** mon with the next free ID (jupiter's `c` came back
|
||||
as `d`), on the correct address. It does not repair the old one.
|
||||
|
||||
Note mons use host ports under `network.provider: host`, so a replacement mon
|
||||
cannot schedule while the broken one's deployment still holds 3300/6789 — the
|
||||
canary sits `Pending` with "didn't have free ports" until the old deployment is
|
||||
deleted.
|
||||
|
||||
## 9. Re-add the OSD
|
||||
|
||||
Add the node back to `cephcluster.yaml` under `storage.nodes`, naming the PM983
|
||||
by `/dev/disk/by-id`:
|
||||
|
||||
```yaml
|
||||
- name: k8s-N
|
||||
devices:
|
||||
- fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_<serial>
|
||||
```
|
||||
|
||||
**Not** a bare device name. Talos enumerates these drives in a different order
|
||||
than Proxmox did — on jupiter `nvme0n1` is the Kingston *system* disk, and the
|
||||
PM983 is `nvme2n1`. Rook would probably decline a partitioned system disk, but
|
||||
that is luck rather than design.
|
||||
|
||||
Get the path from `talosctl -n <node> get disk <dev> -o yaml`, under `symlinks`.
|
||||
|
||||
## 10. Verify and restore
|
||||
|
||||
```sh
|
||||
kubectl get nodes # Ready
|
||||
talosctl -n <node> etcd members # 3 members
|
||||
ceph status # 3 mons in quorum, 3 osds
|
||||
ceph osd tree # OSD under the right zone/host
|
||||
ceph device ls # PM983 backing the new OSD
|
||||
```
|
||||
|
||||
Then undo the window changes: `cephfs-metadata` back to size 3, immich back to
|
||||
1 replica, and `ceph config set osd osd_mclock_profile balanced` if it was
|
||||
raised to `high_recovery_ops` to speed the evacuation.
|
||||
|
||||
The retired Crucial leaves a stale entry in `ceph device ls`. Remove it with
|
||||
`ceph config-key rm device/<devid>`; the mgr's devicehealth view may lag.
|
||||
|
||||
## Known-stale things that will bite
|
||||
|
||||
- `bootstrap.sh` cannot generate a config at all right now (issue #581), so a
|
||||
cold rebuild is broken independently of this work.
|
||||
- Manual Woodpecker pipelines are broken (issue #582), so anything gated on
|
||||
`PIPELINE == "..."` must be run another way.
|
||||
- Cilium's eBPF datapath silently drops VLAN-tagged traffic unless
|
||||
`bpf.vlanBypass` is set. Both `mgmt0` and `ens19` are VLAN interfaces on a
|
||||
converted node. Not observed on jupiter, but it is the first place to look if
|
||||
the v30iot macvlan workloads misbehave there.
|
||||
@@ -79,30 +79,6 @@ which also pre-seeds the Talos image onto every Proxmox node. Wait until the
|
||||
node FQDNs (`k8s-{1,2,3}.srv.renner.uno`) resolve and the VMs are up before
|
||||
Step 2 — `bootstrap.sh` polls DNS but will not create the records.
|
||||
|
||||
**k8s-1 is not a VM and OpenTofu does not create it.** jupiter runs Talos on
|
||||
bare metal, so this step only provisions k8s-2 (saturn) and k8s-3 (neptune).
|
||||
Before Step 2, boot jupiter from the bare-metal image below so it comes up in
|
||||
maintenance mode and answers on `k8s-1.srv.renner.uno` — `bootstrap.sh` treats
|
||||
all three nodes identically from there, since `apply-config --insecure` does
|
||||
not care whether the target is a VM or bare metal.
|
||||
|
||||
```
|
||||
https://factory.talos.dev/image/a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64/v1.12.6/metal-amd64.iso
|
||||
```
|
||||
|
||||
That schematic is deliberately not the one `.woodpecker/deploy-infrastructure.yml`
|
||||
seeds onto the Proxmox nodes. It keeps the same `net.ifnames=1` kernel argument
|
||||
— without it the predictable interface names that `talos/patches/k8s-1.yaml`
|
||||
matches its VLAN parents against do not exist — and differs in three ways:
|
||||
|
||||
- drops `siderolabs/qemu-guest-agent`, which has nothing to talk to on bare metal
|
||||
- adds `siderolabs/intel-ucode`, since no hypervisor is supplying microcode now
|
||||
- adds `siderolabs/thunderbolt`, unused so far and inert until
|
||||
`machine.kernel.modules` requests `thunderbolt`/`thunderbolt_net`. It is
|
||||
present only so enabling the MS-01 Thunderbolt mesh later does not require a
|
||||
new image and a reinstall. Note its udev rule auto-authorises any Thunderbolt
|
||||
device attached to the machine once those modules are loaded.
|
||||
|
||||
## Step 2 — talos: form the cluster + get kubeconfig
|
||||
|
||||
```sh
|
||||
|
||||
@@ -84,7 +84,7 @@ spec:
|
||||
env:
|
||||
- name: DB_URI
|
||||
valueFrom:
|
||||
secretKeyRef: {name: ha-pg-app, key: fqdn-uri}
|
||||
secretKeyRef: { name: ha-pg-app, key: fqdn-uri }
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
@@ -96,7 +96,7 @@ spec:
|
||||
memory: 32Mi
|
||||
containers:
|
||||
- name: home-assistant
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.8.2 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
env:
|
||||
- name: TZ
|
||||
value: Europe/Berlin
|
||||
|
||||
@@ -176,38 +176,3 @@ data:
|
||||
friendly_name: Treppenhaus Haustür Schalter 1
|
||||
'0x001788010b00d011':
|
||||
friendly_name: Treppenhaus Haustür Schalter 2
|
||||
# Ecowitt WS90 weather station. Joined after the ZHA-mirrored block above
|
||||
# was written, hence listed apart from it.
|
||||
#
|
||||
# This device also carries a non-default ZCL reporting configuration
|
||||
# (per-attribute min interval and reportable-change deadbands). That is
|
||||
# device-side state, not expressible here — Z2M has no settings field for
|
||||
# it — so it is documented, with restore commands, in
|
||||
# zigbee2mqtt-ws90-reporting.md. It is lost whenever Z2M re-runs
|
||||
# `configure` on the device, and a fresh PVC will not have it.
|
||||
#
|
||||
# The station transmits every 8.8s, as a burst of seven per-cluster ZCL
|
||||
# frames; Z2M publishes its full cumulative state once per frame. debounce
|
||||
# merges a burst into one publish carrying the last value per attribute,
|
||||
# and must stay below the transmit interval — a window that never sees
|
||||
# silence never flushes.
|
||||
#
|
||||
# Every few days the firmware zeroes its readings for ~8s, then skips a
|
||||
# transmit cycle and recovers on the next. The zeros propagate attribute
|
||||
# by attribute through a burst rather than arriving together, and they
|
||||
# start inside an otherwise normal cycle — the last valid reading and the
|
||||
# first zero are ~20ms apart.
|
||||
#
|
||||
# Merging cuts an episode from six published bad frames to one. It does
|
||||
# NOT make that frame recognisable: it carries whatever subset had zeroed
|
||||
# when the burst ended, which may be every field or only temperature, and
|
||||
# a frame reading {"temperature":0,"humidity":92,"pressure":96.9} is not
|
||||
# separable from a valid winter reading. Since the merge is last-wins and
|
||||
# the zeros arrive at the tail, it also discards the valid reading that
|
||||
# led the burst. This bounds the corruption; it does not filter it.
|
||||
#
|
||||
# Z2M reads this only at startup (requiresRestart), and this file is
|
||||
# SEED-ONLY per the header — the running instance takes it from its PVC.
|
||||
'0xc02cedfffe958f40':
|
||||
friendly_name: Wetterstation
|
||||
debounce: 2
|
||||
|
||||
@@ -158,7 +158,7 @@ spec:
|
||||
# Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
|
||||
# add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
|
||||
# follow-up if desired; pinned-and-manual is fine for the initial cutover.
|
||||
image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
|
||||
image: ghcr.io/koenkk/zigbee2mqtt:2.12.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: TZ
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
# WS90 Zigbee reporting configuration
|
||||
|
||||
Records the ZCL attribute-reporting configuration applied to the Wetterstation
|
||||
(Ecowitt WS90 powered by Shelly, IEEE `0xc02cedfffe958f40`) and how to restore
|
||||
it, because **this configuration lives on the device and in Z2M's
|
||||
`database.db`, not in git**. Zigbee2MQTT has no settings-schema field for
|
||||
reporting configuration, so there is nothing declarative to reconcile — Flux
|
||||
cannot restore it and a fresh PVC will not have it.
|
||||
|
||||
## Applied configuration
|
||||
|
||||
Endpoint 1, all intervals in seconds. `change` is in the attribute's raw ZCL
|
||||
units; the physical deadband is `change ÷ scale`.
|
||||
|
||||
| attribute | cluster | min | max | change | deadband | converter default |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| temperature | `msTemperatureMeasurement` (1026) | 5 | 3600 | 10 | 0.1 °C | 10 / 3600 / 100 |
|
||||
| humidity | `msRelativeHumidity` (1029) | 5 | 3600 | 50 | 0.5 % | 10 / 3600 / 100 |
|
||||
| pressure | `msPressureMeasurement` (1027) | 5 | 3600 | 1 | 1 hPa | 10 / 3600 / 50 |
|
||||
| illuminance | `msIlluminanceMeasurement` (1024) | 5 | 3600 | 5 | 5 lx | 10 / 3600 / 5 |
|
||||
| windSpeed | `shellyWS90Wind` (0xfc01/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| windDirection | `shellyWS90Wind` (0xfc01/4) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| gustSpeed | `shellyWS90Wind` (0xfc01/7) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| uvIndex | `shellyWS90UV` (0xfc02/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| rainStatus | `shellyWS90Rain` (0xfc03/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
| precipitation | `shellyWS90Rain` (0xfc03/1) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
|
||||
|
||||
`genPowerCfg.batteryVoltage` (cluster 1, attribute 33) is deliberately left at
|
||||
its hourly default.
|
||||
|
||||
The three `shellyWS90*` clusters are manufacturer-specific and require
|
||||
`"options": {"manufacturerCode": 5264}` on every reconfiguration request.
|
||||
|
||||
## Why min is 5 and not 10
|
||||
|
||||
The station transmits every 8.8 s. `min` is a hold-off: a change-triggered
|
||||
report is held until at least `min` seconds after the previous one. With
|
||||
`min: 10` every report was therefore held past a transmit cycle and landed on
|
||||
the next, so attributes reported at best every 17.6 s and drifted out of phase
|
||||
with each other — the published stream showed no 8.8 s periodicity at all,
|
||||
just a smear across 2–8 s.
|
||||
|
||||
At `min: 5` each attribute reports on its own transmit. The MQTT stream then
|
||||
carries one publish per transmit cycle, measured at a mean gap of 8.69 s.
|
||||
Total publish volume *fell* (~631/h to ~445/h) because each publish now
|
||||
carries a whole cycle instead of a fragment of one.
|
||||
|
||||
Any value below 8.8 s achieves this. 5 leaves margin without approaching the
|
||||
cycle time.
|
||||
|
||||
## Why the deadbands differ from the converter defaults
|
||||
|
||||
The defaults come from the shared `modernExtend` helpers in
|
||||
zigbee-herdsman-converters (`m.temperature()`, `m.pressure()`, `m.humidity()`,
|
||||
`m.illuminance()`), which the WS90 definition in `shelly.ts` composes without
|
||||
overrides. They are anti-chatter values sized for battery-powered indoor
|
||||
sensors, where report volume costs coin-cell life and mesh airtime. This
|
||||
station is mains-and-solar and exists to resolve small changes, so the
|
||||
trade-off runs the other way.
|
||||
|
||||
Each value above is matched to the resolution the device actually reports:
|
||||
temperature and pressure arrive in 0.1-unit steps, humidity in whole percent.
|
||||
|
||||
Pressure is worth calling out. The converter default of `change: 50` with
|
||||
`scale: 10` is a 5 kPa (50 hPa) deadband, which real barometric pressure never
|
||||
traverses — so pressure only ever updated on the hourly `max` heartbeat, and
|
||||
`pressure_trend` and `weather_condition` were derived from that. The same
|
||||
library's older `reporting.ts` helper configures the identical ZCL attribute
|
||||
with `change: 5`, a 5 hPa deadband, so the two code paths disagree by 10× and
|
||||
the modernExtend value is the outlier.
|
||||
|
||||
## When this is lost
|
||||
|
||||
Anything that makes Z2M re-run `configure` on the device overwrites all of it
|
||||
with the converter defaults:
|
||||
|
||||
- re-pairing or re-joining the device
|
||||
- an explicit `zigbee2mqtt/bridge/request/device/configure`
|
||||
- a converter update that changes the definition and triggers reconfiguration
|
||||
|
||||
It also will not exist on a fresh PVC. Re-apply with the commands below.
|
||||
|
||||
## Re-applying
|
||||
|
||||
Published from the `discovery-guard` sidecar, which already holds the
|
||||
`zigbee2mqtt` broker credential (ACL `readwrite zigbee2mqtt/#`, which covers
|
||||
`bridge/request/#`) so no credential has to leave the cluster:
|
||||
|
||||
Ten separate requests, one per attribute — written out literally rather than
|
||||
generated by a shell helper, because the nested quoting through `kubectl exec
|
||||
-- sh -c` is where this goes wrong. Each line below is in the exact form that
|
||||
was executed and verified.
|
||||
|
||||
```sh
|
||||
POD=$(kubectl get pod -n home-assistant -l app.kubernetes.io/name=zigbee2mqtt -o jsonpath='{.items[0].metadata.name}')
|
||||
```
|
||||
|
||||
```sh
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msTemperatureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":10}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msRelativeHumidity\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":50}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msPressureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msIlluminanceMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":5}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windDirection\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"gustSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90UV\",\"attribute\":\"uvIndex\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"rainStatus\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
|
||||
kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"precipitation\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
|
||||
```
|
||||
|
||||
These are idempotent — re-sending a request that matches the current
|
||||
configuration is a no-op on the device.
|
||||
|
||||
The device must be awake and reachable; each request performs a real
|
||||
`ConfigureReporting` exchange and fails if it is not.
|
||||
|
||||
## Verifying
|
||||
|
||||
Read back what the device actually accepted — Z2M records the negotiated
|
||||
result, so this reflects the device, not the request:
|
||||
|
||||
```sh
|
||||
kubectl exec -n home-assistant "$POD" -c zigbee2mqtt -- \
|
||||
grep 0xc02cedfffe958f40 /app/data/database.db |
|
||||
jq -r '.endpoints | to_entries[] | .value.configuredReportings[]?
|
||||
| "cluster=\(.cluster) attr=\(.attrId) min=\(.minRepIntval) max=\(.maxRepIntval) change=\(.repChange)"'
|
||||
```
|
||||
|
||||
All ten rows above should read `min=5 max=3600`. On the wire, the publish
|
||||
cadence on `zigbee2mqtt/Wetterstation` should sit at ~8.7 s and temperature
|
||||
should move in 0.1 °C steps; 1 °C steps mean the defaults are back.
|
||||
|
||||
## Making it durable upstream
|
||||
|
||||
The lasting fix is for the WS90 definition to pass explicit `reporting`
|
||||
overrides rather than inherit indoor-sensor defaults, which would fix every
|
||||
WS90 rather than this one. `m.temperature()` and the other helpers already
|
||||
accept a `reporting` argument. The pressure divergence between
|
||||
`modernExtend.ts` and `reporting.ts` is a separate upstream bug worth
|
||||
reporting on its own.
|
||||
@@ -6,13 +6,6 @@ metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: immich-machine-learning
|
||||
spec:
|
||||
# Scale this and immich-server to 0 for the duration of any single-node
|
||||
# maintenance — cluster CPU requests exceed what two nodes can hold, and this
|
||||
# pair frees ~1000m against a shortfall of roughly 200m. Stopping them is the
|
||||
# right lever rather than lowering their requests: 500m reserved against a
|
||||
# 2m idle draw looks like waste, but inference bursts hard and the request is
|
||||
# honest about what it needs when someone uploads photos.
|
||||
# See RUNBOOK-baremetal-conversion.md.
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
|
||||
@@ -36,9 +36,6 @@ metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: immich-server
|
||||
spec:
|
||||
# Scaled to 0 alongside immich-machine-learning during single-node
|
||||
# maintenance; see the note there. The CNPG Postgres cluster stays running,
|
||||
# so recovery is just these two replica counts.
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
|
||||
@@ -8,7 +8,6 @@ resources:
|
||||
- prometheusrule-cilium.yaml
|
||||
- prometheusrule-hetzner-backup.yaml
|
||||
- prometheusrule-loki.yaml
|
||||
- prometheusrule-mariadb-backup.yaml
|
||||
- prometheusrule-netbird-availability.yaml
|
||||
- prometheusrule-netbird-backup.yaml
|
||||
- prometheusrule-netbird-ingress.yaml
|
||||
|
||||
@@ -25,29 +25,13 @@ spec:
|
||||
#
|
||||
# 6h: Ceph warns for self-clearing conditions. Only a warning that
|
||||
# does not clear is worth paging on.
|
||||
#
|
||||
# BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT
|
||||
# excluded (#557). Ceph holds the check for
|
||||
# bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so
|
||||
# one multi-second stall outlives the 6h `for` and always pages,
|
||||
# with latency back to normal long before anyone looks. Measured:
|
||||
# both episodes in the 30d to 2026-08-23 ran ~26h and cleared
|
||||
# untouched, the first ~24h after the last stall.
|
||||
#
|
||||
# Both began inside the Velero restore-test sweep — a ~5h sustained
|
||||
# restore that outruns the SLC cache on these QLC NVMe. The sweep
|
||||
# runs monthly, 08:00-14:00Z on the 1st.
|
||||
#
|
||||
# Expected for the hardware, but not excludable:
|
||||
# this is also the earliest signal of a genuinely failing drive, and
|
||||
# all three are at 80-87% endurance used (see cephcluster.yaml).
|
||||
expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
|
||||
for: 6h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph health check {{ $labels.name }} active for 6h"
|
||||
description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the monthly Velero restore-test window (08:00-14:00Z on the 1st), which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
|
||||
description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
|
||||
|
||||
- alert: CephPoolLostRedundancy
|
||||
# Names the pool, which POOL_NO_REDUNDANCY does not.
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: mariadb-backup
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus selects on this label.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: mariadb-backup.rules
|
||||
rules:
|
||||
- alert: WordpressMariadbBackupStale
|
||||
# The nightly 02:45 logical dump (kubernetes/apps/wordpress/mariadb.yaml)
|
||||
# is the only transaction-consistent copy of www.renner.uno's database
|
||||
# — Velero snapshots the PVC, which for a running DB is not a restore
|
||||
# point you would trust.
|
||||
#
|
||||
# KubeJobFailed already covers a failed Job, but only while the Job
|
||||
# object survives. On #550 it cleared when the failed Job was
|
||||
# garbage-collected, not when a backup succeeded — so a failure that
|
||||
# is GC'd before anyone looks leaves no signal at all.
|
||||
# lastSuccessfulTime lives on the CronJob and outlives Job GC.
|
||||
#
|
||||
# 26h = one missed cycle plus 2h slack, matching NetbirdVpsBackupStale.
|
||||
# It counts any Job owned by the CronJob, so a manual
|
||||
# `kubectl create job --from=cronjob/...` re-run clears it too.
|
||||
expr: |
|
||||
time() - kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"} > 26 * 3600
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "WordPress MariaDB backup has not succeeded in 26h+"
|
||||
description: "The wordpress-mariadb-backup CronJob's last successful run is over 26 hours old. Check the newest Job's pod logs in the wordpress namespace: `kubectl -n wordpress get jobs`. The 2026-08-19 failure was `The specified bucket does not exist.` against renner-uno-k8s-mariadb while the endpoint itself probed healthy, so a green HetznerBackupEndpointDown does not clear this."
|
||||
|
||||
- alert: WordpressMariadbBackupNeverRan
|
||||
# The staleness expression above goes blind when the metric is
|
||||
# absent, which is what happens once the operator recreates the
|
||||
# CronJob. Without this guard "never backed up" is quieter than
|
||||
# "backed up late".
|
||||
expr: |
|
||||
kube_cronjob_created{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
|
||||
unless
|
||||
kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
|
||||
for: 6h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "WordPress MariaDB backup has no recorded successful run"
|
||||
description: "The wordpress-mariadb-backup CronJob exists but has no lastSuccessfulTime: either it has not completed once since the Backup CR was recreated, or every run since has failed. Check `kubectl -n wordpress get jobs` and that the hetzner-s3-creds ExternalSecret is synced."
|
||||
@@ -34,7 +34,7 @@ spec:
|
||||
# Image automation, range 3.x (see image-automation/woodpecker.yaml).
|
||||
# MUST stay version-matched with woodpecker-server (gRPC parity) — both
|
||||
# track the same range/filter so they bump together on each release.
|
||||
image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
image: woodpeckerci/woodpecker-agent:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
env:
|
||||
- name: WOODPECKER_SERVER
|
||||
value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
|
||||
@@ -79,7 +79,7 @@ spec:
|
||||
# authoritative copy is the backup on Hetzner, and a lost probe just
|
||||
# means re-running the test -- so they are equally undeserving of
|
||||
# replication. Landing them unreplicated would mean a second size-1
|
||||
# data pool inside `cephfs`, and restore-test runs MONTHLY against
|
||||
# data pool inside `cephfs`, and restore-test runs WEEKLY against
|
||||
# CI's ~96 pipelines/day, so it is not worth the complexity.
|
||||
- name: WOODPECKER_BACKEND_K8S_STORAGE_CLASS
|
||||
value: ceph-filesystem-ci
|
||||
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
# minor + patch auto-update within 3.x; only the 4.0 major is held for
|
||||
# a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
|
||||
# -bump hazard is gone). Stays in lockstep with the agent.
|
||||
image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
image: woodpeckerci/woodpecker-server:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
env:
|
||||
- name: WOODPECKER_HOST
|
||||
value: https://ci.renner.uno
|
||||
|
||||
@@ -268,7 +268,7 @@ EOF
|
||||
|
||||
# 120x15s = 30 min, not the old 60x5s = 5 min. node-agent has no
|
||||
# loadConcurrency override, so Velero's default of one data path per node
|
||||
# applies: while the restore-test sweep holds the slot, even this 1 GiB
|
||||
# applies: while the Sunday restore-test sweep holds the slot, even this 1 GiB
|
||||
# DataUpload sits in phase Prepared for well over 5 minutes. The old 300 s wait
|
||||
# then let the teardown below delete the source namespace and the Backup
|
||||
# mid-upload ("Cancel du under phase Prepared because it is being deleted"),
|
||||
|
||||
@@ -87,16 +87,6 @@ spec:
|
||||
|
||||
lapi:
|
||||
replicas: 1
|
||||
# The chart defaults request 500m. Measured steady-state draw is 2m, and
|
||||
# the request is what the scheduler reserves, so the default parks half a
|
||||
# core of unusable capacity per LAPI. That is affordable on three nodes
|
||||
# and is not on two: with one node out for maintenance the remaining pair
|
||||
# sit at 100% of requested CPU while actually using ~22%, and nothing new
|
||||
# can be placed. The limit stays at the chart default so a log flood can
|
||||
# still burst; only the reservation shrinks.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
podAnnotations:
|
||||
# Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
|
||||
# env-var secret, so a restart is required to pick up a rotated key). See
|
||||
@@ -132,12 +122,6 @@ spec:
|
||||
key: key
|
||||
|
||||
agent:
|
||||
# Same reasoning as lapi.resources above, multiplied by three: this is a
|
||||
# DaemonSet, so the chart's 500m default reserves half a core on every
|
||||
# node to run a process that measures 1-2m.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
# The agent parses the ingress controller's access logs (read from
|
||||
# the pod's stdout) to detect attacks. Repointed from the retired
|
||||
# ingress-nginx (decommissioned 2026-05-17) to Traefik. The
|
||||
|
||||
@@ -51,15 +51,7 @@ spec:
|
||||
startingDeadlineSeconds: 3600
|
||||
jobTemplate:
|
||||
spec:
|
||||
# 4 (not the sibling rotators' 1-2) because this is the only DAILY
|
||||
# rotator — every other CronJob here is quarterly, so it takes ~90x as
|
||||
# many runs and is correspondingly more exposed to per-run transients.
|
||||
# backoffLimit: 1 gave two attempts ~11 s apart, and on 2026-08-19 a
|
||||
# sub-minute resolver transient took both out (Job
|
||||
# claude-oauth-secret-sync-29785100, BackoffLimitExceeded), costing a
|
||||
# day's sync. 4 spreads the attempts over ~150 s (10/20/40/80 s backoff),
|
||||
# still well inside activeDeadlineSeconds.
|
||||
backoffLimit: 4
|
||||
backoffLimit: 1
|
||||
activeDeadlineSeconds: 300
|
||||
ttlSecondsAfterFinished: 259200 # 3 d
|
||||
template:
|
||||
|
||||
@@ -160,41 +160,11 @@ spec:
|
||||
- "10.2.40.0/24"
|
||||
cluster:
|
||||
- "10.99.0.0/29"
|
||||
# Nodes are listed explicitly rather than useAllNodes, so that a node can be
|
||||
# taken out of the storage fleet by removing it here. Under useAllNodes the
|
||||
# operator re-provisions an OSD on any node whose device still carries a
|
||||
# BlueStore signature, which makes purging an OSD ahead of pulling its drive
|
||||
# impossible: the OSD is recreated within a reconcile and starts backfilling
|
||||
# onto the drive being replaced.
|
||||
#
|
||||
# nvme0n1 on k8s-2/k8s-3 is the NVMe drive PCI-passed-through from saturn and
|
||||
# neptune, and is the only block device those guests can see, so a bare device
|
||||
# name is unambiguous there.
|
||||
#
|
||||
# k8s-1 is bare metal and sees three NVMe drives, so it names its OSD by
|
||||
# /dev/disk/by-id path instead. A bare name would be actively wrong: Talos
|
||||
# enumerates this machine's drives in a different order than Proxmox did, and
|
||||
# nvme0n1 here is the Kingston SYSTEM disk. Rook would probably decline it for
|
||||
# carrying partitions, but that is luck rather than intent. The by-id path is
|
||||
# derived from model and serial and does not depend on enumeration at all.
|
||||
#
|
||||
# nvme0n1 50026B7382B48B86 Kingston — Talos system disk
|
||||
# nvme1n1 2332E86B749A Crucial — Talos EPHEMERAL (/var)
|
||||
# nvme2n1 S437NX0M400426 PM983 — this OSD
|
||||
storage:
|
||||
useAllNodes: false
|
||||
useAllNodes: true
|
||||
useAllDevices: false
|
||||
nodes:
|
||||
# k8s-2 is absent while saturn's OSD drive is swapped and the node is
|
||||
# converted to bare metal. RE-ADD IT once the PM983 is in and the node has
|
||||
# rejoined, naming the device by /dev/disk/by-id as k8s-1 does — bare
|
||||
# metal sees three NVMe drives and the bare name is not the OSD.
|
||||
- name: k8s-3
|
||||
devices:
|
||||
- name: nvme0n1
|
||||
- name: k8s-1
|
||||
devices:
|
||||
- fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M400426
|
||||
devices:
|
||||
- name: nvme0n1 # NVMe drive PCI-passed-through from each PVE node
|
||||
config:
|
||||
osdsPerDevice: "1"
|
||||
# Storage layer must never be evicted by kubelet under node pressure
|
||||
|
||||
@@ -87,31 +87,15 @@ metadata:
|
||||
name: cephfs
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
# TEMPORARY — RESTORE TO size 3 / no min_size OVERRIDE ONCE SATURN AND
|
||||
# NEPTUNE HAVE BOTH BEEN CONVERTED TO BARE METAL. It stays at 2/1 across all
|
||||
# three conversions rather than being toggled per node: each change
|
||||
# re-replicates the pool, and the window reopens with the next node anyway.
|
||||
#
|
||||
# The standing trade for this pool is size 3: it holds ~253 MiB against the
|
||||
# data pool's 280 GiB, so it contributes nothing measurable to drive wear —
|
||||
# the entire reason data0 dropped to 2 — while losing it would destroy the
|
||||
# whole filesystem, not just the objects on the lost OSD. Paying 3x on a
|
||||
# quarter of a gigabyte to keep that intact is the right trade in steady
|
||||
# state.
|
||||
#
|
||||
# It does not hold while one of three hosts is out. size 3 on two remaining
|
||||
# hosts leaves every PG undersized at two copies with min_size 2, so a single
|
||||
# further OSD loss drops the pool below min_size and blocks IO on every
|
||||
# CephFS mount cluster-wide. At 2/1 the same two copies are active+clean
|
||||
# instead, and a further loss degrades rather than blocks — availability
|
||||
# bought at the cost of accepting writes onto a single copy while degraded.
|
||||
#
|
||||
# See RUNBOOK-baremetal-conversion.md.
|
||||
# metadataPool STAYS at size 3 while the data pool drops to 2. It holds
|
||||
# 467 MiB against the data pool's 275 GiB, so it contributes nothing
|
||||
# measurable to drive wear — the entire reason for the size reduction — while
|
||||
# losing it would destroy the whole filesystem, not just the objects on the
|
||||
# lost OSD. Paying 3x on half a gigabyte to keep that intact is the right
|
||||
# trade; the wear saving comes from data0.
|
||||
metadataPool:
|
||||
replicated:
|
||||
size: 2
|
||||
parameters:
|
||||
min_size: "1"
|
||||
size: 3
|
||||
dataPools:
|
||||
- name: data0
|
||||
# size 2 / min_size 1 for the same reasons as replicapool above — see
|
||||
|
||||
@@ -1,30 +1,19 @@
|
||||
---
|
||||
# Monthly Velero restore-test CronJob — 08:00 UTC on the 1st.
|
||||
# Weekly Velero restore-test CronJob.
|
||||
#
|
||||
# The sweep restores every discovered schedule, immich's ~180 GiB library
|
||||
# included. At `size: 2` that is ~360 GiB raw, ~120 GiB per OSD, against a
|
||||
# ~65 GB/day weekday baseline on three QLC NVMe at 81-87% endurance used —
|
||||
# ~47% of each drive's weekly write budget. Monthly keeps every restore path
|
||||
# exercised and cuts that share ~4x.
|
||||
#
|
||||
# 08:00 — *after* that day's own daily backups have finished (paperless
|
||||
# 03:15 through apps 07:15, each normally ~1 min). The previous 02:00 slot
|
||||
# was chosen as "well before the next day's backups", which missed that the
|
||||
# daily schedules fire on the same day: a sweep starting at 02:00 and
|
||||
# running ~5 h sat on top of the entire 03:15-07:15 backup window and
|
||||
# Sunday 08:00 UTC — *after* Sunday's own daily backups have finished
|
||||
# (paperless 03:15 through apps 07:15, each normally ~1 min). The previous
|
||||
# 02:00 slot was chosen as "well before Monday's backups", which missed
|
||||
# that the daily schedules also fire on Sunday: a sweep starting at 02:00
|
||||
# and running ~5 h sat on top of the entire 03:15-07:15 backup window and
|
||||
# starved it of node-agent DataUpload concurrency. Observed effect on
|
||||
# 2026-07-19 and 2026-07-26 — audiobookshelf-daily took 2 h 18 m and
|
||||
# 2 h 33 m respectively against a ~1 min baseline, with the books
|
||||
# DataUpload not starting until 06:18. Both are Sundays, the only day the
|
||||
# sweep ran then; no other day shows it.
|
||||
# DataUpload not starting until 06:18. Both are Sundays; no other day
|
||||
# shows it.
|
||||
#
|
||||
# 08:00 + the 6 h activeDeadlineSeconds cap still lands well clear of the
|
||||
# next day's 03:15.
|
||||
#
|
||||
# Day-of-month rather than a fixed weekday: Kubernetes parses cron with
|
||||
# robfig/cron's standard parser, which has no `#` nth-weekday operator, and
|
||||
# restricting both day fields (`0 8 1-7 * 0`) ORs them — every Sunday plus
|
||||
# the 1st through 7th.
|
||||
# 08:00 + the 6 h activeDeadlineSeconds cap still lands well clear of
|
||||
# Monday's 03:15.
|
||||
#
|
||||
# Concurrency policy Forbid: the immich restore (~180 GiB, millions of
|
||||
# small files) alone can run multiple hours, so the full sweep is sized
|
||||
@@ -40,7 +29,7 @@ metadata:
|
||||
name: restore-test
|
||||
namespace: velero
|
||||
spec:
|
||||
schedule: "0 8 1 * *"
|
||||
schedule: "0 8 * * 0"
|
||||
timeZone: Etc/UTC
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 2
|
||||
@@ -50,7 +39,7 @@ spec:
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 21600 # 6 h hard cap (immich alone can take ~3 h)
|
||||
ttlSecondsAfterFinished: 3456000 # 40 d — enough to inspect after the next monthly run
|
||||
ttlSecondsAfterFinished: 1209600 # 14 d — enough to inspect after the next weekly run
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: restore-test
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
---
|
||||
# RBAC for the monthly restore-test CronJob.
|
||||
# RBAC for the weekly restore-test CronJob.
|
||||
#
|
||||
# Permissions required by restore-test.sh:
|
||||
# - velero.io: read schedules/backups/BSL, create+delete restores
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Monthly verification that Velero PVC backups are actually restorable.
|
||||
# Weekly verification that Velero PVC backups are actually restorable.
|
||||
#
|
||||
# What this exercises (per service):
|
||||
# 1. Velero Restore reaches phase=Completed against yesterday's
|
||||
@@ -53,7 +53,7 @@ check() {
|
||||
# and ceph-filesystem StorageClasses ship with reclaimPolicy: Retain so that
|
||||
# production PVC deletions can be undone manually. The restore-test has the
|
||||
# opposite need: every PVC it creates is throwaway and MUST not survive the
|
||||
# iteration. Without this, each run leaks O(100 GiB) of CephFS
|
||||
# iteration. Without this, each weekly run leaks O(100 GiB) of CephFS
|
||||
# subvolumes and the cluster eventually goes OSD_FULL (incident 2026-05-17).
|
||||
#
|
||||
# Why this still exists after ceph-*-ephemeral landed. Three categories of
|
||||
|
||||
@@ -1,129 +1,62 @@
|
||||
# k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
|
||||
# still Proxmox VMs. Every machine-specific selector below differs from theirs
|
||||
# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
|
||||
# four physical NICs and all three NVMe drives are visible to Talos at once.
|
||||
#
|
||||
# Networking uses the v1.12 multi-document config rather than the deprecated
|
||||
# machine.network.interfaces. That is deliberate: the docs do not define what
|
||||
# happens when the same link is configured by both schemas, so this file keeps
|
||||
# every link in one schema instead of straddling them.
|
||||
machine:
|
||||
network:
|
||||
hostname: k8s-1
|
||||
# Shared control-plane VIP — etcd-elected leader hosts 10.2.40.60 as a
|
||||
# secondary IP. Exposed via UniFi DNS as k8s.renner.uno.
|
||||
#
|
||||
# Two virtio NICs (mgmt + v30iot). `driver: virtio_net` would match
|
||||
# both, so each is pinned by its PCI bus path:
|
||||
# - 0000:06:12.0 — original mgmt NIC (the only virtio before)
|
||||
# - 0000:06:13.0 — extra_network_bridges = ["v30iot"] in tofu
|
||||
# Bus paths are consistent across k8s-{1,2,3}; verified live 2026-05-13.
|
||||
# The X710 selector stays driver-based (i40e is unambiguous) and
|
||||
# tolerates kernel renaming.
|
||||
interfaces:
|
||||
- deviceSelector:
|
||||
busPath: "0000:06:12.0"
|
||||
dhcp: true
|
||||
vip:
|
||||
ip: 10.2.40.60
|
||||
# v30iot: LINK ONLY — no host addressing. The NIC is the macvlan
|
||||
# parent for the v30iot pod attachments (matter/otbr/shelly-manager),
|
||||
# whose children do their own DHCP on the L2; host addresses were
|
||||
# never needed for that. Host DHCP (formerly dhcp + routeMetric 4096)
|
||||
# installed a connected 10.2.30.0/24 route, so node replies to
|
||||
# IoT-subnet clients bypassed the UDM and broke every UDM-DNATed
|
||||
# flow (hairpin NAT: un-translated SYN-ACK -> client kernel RST,
|
||||
# verified live 2026-07-23). The purpose the old comment claimed —
|
||||
# Cilium L2 announce for an IoT LB pool — does not exist (no
|
||||
# L2AnnouncementPolicy, no such pool). v6 RA addressing on this NIC
|
||||
# is disabled via machine.sysctls for the same reason.
|
||||
- deviceSelector:
|
||||
busPath: "0000:06:13.0"
|
||||
dhcp: false
|
||||
# Dedicated Ceph cluster network on the X710 SFP+ port (PCI passthrough,
|
||||
# mapping x710-port1 in opentofu/vm_150_k8s_1.tf). Separate /29 on VLAN 49,
|
||||
# MTU 9000, no default route — used by Rook for OSD↔OSD replication via
|
||||
# CephCluster.spec.network.addressRanges.cluster.
|
||||
- deviceSelector:
|
||||
driver: i40e
|
||||
addresses:
|
||||
- 10.99.0.1/29
|
||||
mtu: 9000
|
||||
sysctls:
|
||||
# ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It exists as
|
||||
# the macvlan parent for the v30iot pod attachments
|
||||
# (matter/otbr/shelly-manager/music-assistant), whose children do their own
|
||||
# DHCP on that L2. Host addresses were never needed and are actively
|
||||
# harmful: a connected 10.2.30.0/24 route makes node replies to IoT-subnet
|
||||
# clients bypass the UDM, which breaks every UDM-DNATed flow (hairpin NAT:
|
||||
# un-translated SYN-ACK -> client kernel RST, verified live 2026-07-23).
|
||||
# This disables v6 entirely on that link, including link-local.
|
||||
# v30iot NIC carries no host addressing (see interfaces above); this
|
||||
# disables v6 entirely (incl. link-local) so the host is unreachable on that L2.
|
||||
#
|
||||
# Keyed on the STABLE predictable name `ens19` (busPath 0000:06:13.0 =
|
||||
# v30iot). Predictable naming is re-enabled via the Image Factory
|
||||
# schematic (net.ifnames=1; see .woodpecker/deploy-infrastructure.yml),
|
||||
# so `ens19` is topology-derived and does NOT shuffle across reboots —
|
||||
# unlike the former `eth0/1/2` (driver-probe order) which renamed on a
|
||||
# k8s-3 reboot and silently mis-targeted this sysctl onto mgmt. The four
|
||||
# v30iot macvlan NADs key on the same `ens19`.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are ordinary block devices to Talos here: the Kingston
|
||||
# OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
|
||||
# Ceph OSD, and the retired Crucial P3 that carries EPHEMERAL (see the
|
||||
# VolumeConfig below). Every one of them is selected by serial. Nothing in
|
||||
# this file may use a broad selector: `disk.transport == 'nvme'` matches all
|
||||
# three, and the failure mode is Talos taking the OSD drive.
|
||||
diskSelector:
|
||||
serial: 50026B7382B48B86
|
||||
disk: /dev/vda
|
||||
certSANs:
|
||||
- k8s-1.srv.renner.uno
|
||||
nodeLabels:
|
||||
topology.kubernetes.io/zone: jupiter
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
|
||||
#
|
||||
# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
|
||||
# the PM983 replaced it. /var is exactly the right role for a disk in that
|
||||
# state: containerd's image and snapshot store, pod logs and the Ceph mon store
|
||||
# are all reconstructible, so the drive is consumable here in a way it is not as
|
||||
# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
|
||||
# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
|
||||
# they absorb another node's pods.
|
||||
#
|
||||
# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; a config applied after EPHEMERAL
|
||||
# has been created elsewhere is silently ignored, and relocating it then
|
||||
# requires wiping the disk and booting install media again.
|
||||
#
|
||||
# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
|
||||
# would match the PM983 as well, and losing that coin flip means Talos formats
|
||||
# the OSD drive.
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
provisioning:
|
||||
diskSelector:
|
||||
match: disk.serial == '2332E86B749A'
|
||||
minSize: 50GiB
|
||||
grow: true
|
||||
---
|
||||
# X710 port 0 carries the switch trunk. It takes no addressing itself — the
|
||||
# port's untagged VLAN is the 192.168.8.0/24 network, which this node has no
|
||||
# business being on — so management and v30iot both ride it as tagged VLANs.
|
||||
#
|
||||
# Links are addressed by predictable kernel name. Those derive from PCI
|
||||
# topology, not probe order — though not immune: adding the third NVMe
|
||||
# renumbered the buses and moved the unused second I226 from enp89s0 to
|
||||
# enp90s0. Neither name used here was affected.
|
||||
#
|
||||
# LinkAliasConfig would be the better way to pin these, and is worth
|
||||
# reintroducing — but only once it is proven. Aliasing both X710 ports by
|
||||
# permanent MAC and using the aliases as VLAN parents produced a node with no
|
||||
# address on any interface: the aliases were never created, so every document
|
||||
# referencing them did nothing, including the static address on port 1. Verify
|
||||
# any alias-based revision with `talosctl apply-config --mode=try`, which rolls
|
||||
# back on its own, before making it permanent.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
|
||||
# network. The k8s-1 VM reached 10.2.40.174 over this same physical port via
|
||||
# Proxmox's v40srv VNet (vmbr0.40), so the switch trunks VLAN 40 here.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: mgmt0
|
||||
vlanID: 40
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
kind: DHCPv4Config
|
||||
name: mgmt0
|
||||
---
|
||||
# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
|
||||
# k8s.renner.uno.
|
||||
apiVersion: v1alpha1
|
||||
kind: Layer2VIPConfig
|
||||
name: 10.2.40.60
|
||||
link: mgmt0
|
||||
---
|
||||
# Named ens19 on purpose. The four v30iot macvlan NADs hard-code
|
||||
# `master: ens19`, and k8s-2/k8s-3 still present the VLAN as a virtio NIC of
|
||||
# that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
|
||||
# serve both VM and bare-metal nodes for as long as the cluster is mixed.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ens19
|
||||
vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
|
||||
# untagged on the wire. Used by Rook for OSD<->OSD replication via
|
||||
# CephCluster.spec.network.addressRanges.cluster. Not routed off its own
|
||||
# segment, and deliberately has no default route.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
up: true
|
||||
mtu: 9000
|
||||
addresses:
|
||||
- address: 10.99.0.1/29
|
||||
|
||||
@@ -1,99 +1,31 @@
|
||||
# k8s-2 runs Talos directly on saturn's MS-01 hardware. Structurally identical
|
||||
# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
|
||||
# serials, MACs and addresses differ. k8s-3 is still a Proxmox VM and keeps the
|
||||
# VM-era layout until neptune is converted.
|
||||
machine:
|
||||
network:
|
||||
hostname: k8s-2
|
||||
# Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
|
||||
interfaces:
|
||||
- deviceSelector:
|
||||
busPath: "0000:06:12.0"
|
||||
dhcp: true
|
||||
vip:
|
||||
ip: 10.2.40.60
|
||||
# v30iot: link only, no host addressing — see k8s-1.yaml.
|
||||
- deviceSelector:
|
||||
busPath: "0000:06:13.0"
|
||||
dhcp: false
|
||||
# See k8s-1.yaml for context on the dedicated Ceph cluster network.
|
||||
- deviceSelector:
|
||||
driver: i40e
|
||||
addresses:
|
||||
- 10.99.0.2/29
|
||||
mtu: 9000
|
||||
sysctls:
|
||||
# ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It is the
|
||||
# macvlan parent for the v30iot pod attachments; host addresses on that L2
|
||||
# make node replies bypass the UDM and break every UDM-DNATed flow.
|
||||
# v30iot NIC carries no host addressing — see k8s-1.yaml.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are visible on bare metal. Select the installer target
|
||||
# by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
|
||||
# order the kernel enumerates them in — Talos and Proxmox do not agree on
|
||||
# that order.
|
||||
diskSelector:
|
||||
serial: 50026B7382DA5F6E
|
||||
disk: /dev/vda
|
||||
certSANs:
|
||||
- k8s-2.srv.renner.uno
|
||||
nodeLabels:
|
||||
topology.kubernetes.io/zone: saturn
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
|
||||
# drive is at 81% of its rated endurance and is finished as an OSD, but
|
||||
# everything EPHEMERAL holds is reconstructible, so it is consumable in this
|
||||
# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had.
|
||||
#
|
||||
# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; applied later it is silently
|
||||
# ignored and relocating needs a disk wipe and a boot from install media.
|
||||
#
|
||||
# Matched by serial deliberately: a broad selector such as
|
||||
# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
provisioning:
|
||||
diskSelector:
|
||||
match: disk.serial == '2332E86B74BC'
|
||||
minSize: 50GiB
|
||||
grow: true
|
||||
---
|
||||
# X710 port 0 carries the switch trunk and takes no addressing itself — the
|
||||
# port's untagged VLAN is not one this node belongs on. Management and v30iot
|
||||
# both ride it as tagged VLANs.
|
||||
#
|
||||
# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are
|
||||
# valid VLAN parents, but a selector matching zero links is skipped silently
|
||||
# and Talos disables default DHCP machine-wide as soon as any new-style network
|
||||
# document exists — so a silent miss leaves the node with no address on any
|
||||
# interface. See RUNBOOK-baremetal-conversion.md.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Management, VLAN 40. Node IP, etcd, the API and Ceph's public network.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: mgmt0
|
||||
vlanID: 40
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
kind: DHCPv4Config
|
||||
name: mgmt0
|
||||
---
|
||||
# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
|
||||
# k8s.renner.uno.
|
||||
apiVersion: v1alpha1
|
||||
kind: Layer2VIPConfig
|
||||
name: 10.2.40.60
|
||||
link: mgmt0
|
||||
---
|
||||
# Named ens19 on purpose: the four v30iot macvlan NADs hard-code
|
||||
# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that
|
||||
# name. VLANConfig.name is free-form, so one set of NADs serves both.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ens19
|
||||
vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
|
||||
# untagged on the wire. OSD<->OSD replication only. Not routed off its own
|
||||
# segment and deliberately has no default route.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
up: true
|
||||
mtu: 9000
|
||||
addresses:
|
||||
- address: 10.99.0.2/29
|
||||
|
||||
@@ -47,13 +47,8 @@ variable "tags" {
|
||||
}
|
||||
|
||||
variable "template_name" {
|
||||
type = string
|
||||
# No default clone source. The ubuntu-24.04-template it used to name was
|
||||
# deleted with saturn's conversion to bare metal, and Proxmox is being
|
||||
# retired, so a default pointing at a template that does not exist would
|
||||
# only fail at apply time. Set it explicitly if a clone source is ever
|
||||
# wanted again.
|
||||
default = null
|
||||
type = string
|
||||
default = "ubuntu-24-04-template"
|
||||
}
|
||||
|
||||
variable "full_clone" {
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
module "k8s_1" {
|
||||
source = "./modules/proxmox_vm"
|
||||
|
||||
name = "k8s-1.srv.renner.uno"
|
||||
target_node = "jupiter"
|
||||
vmid = 150
|
||||
memory = 49152
|
||||
cores = 8
|
||||
|
||||
template_name = null
|
||||
cloud_init = false
|
||||
full_clone = false
|
||||
bios = "ovmf"
|
||||
machine = "q35"
|
||||
boot = "order=virtio0"
|
||||
agent_enabled = true
|
||||
|
||||
# Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
|
||||
# VLAN from this node (previously only k8s-edge had v30iot, so any
|
||||
# IoT-pool LB lease landing here was unreachable from devices).
|
||||
extra_network_bridges = ["v30iot"]
|
||||
|
||||
disks = [
|
||||
{
|
||||
storage = "local-zfs"
|
||||
size = "64G"
|
||||
format = "raw"
|
||||
slot = "virtio0"
|
||||
file_id = local.talos_image
|
||||
}
|
||||
]
|
||||
|
||||
hostpci = [
|
||||
{
|
||||
mapping = "osd-ssd"
|
||||
},
|
||||
{
|
||||
mapping = "x710-port1"
|
||||
}
|
||||
]
|
||||
|
||||
tags = "kubernetes;talos;controlplane"
|
||||
ssh_public_keys = local.common_ssh_keys
|
||||
|
||||
vm_state = "running"
|
||||
|
||||
# Rebuild pipeline issues `-replace`; skip graceful shutdown so a
|
||||
# non-responsive Talos guest can't hang the destroy for 30 minutes.
|
||||
stop_on_destroy = true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
module "k8s_2" {
|
||||
source = "./modules/proxmox_vm"
|
||||
|
||||
name = "k8s-2.srv.renner.uno"
|
||||
target_node = "saturn"
|
||||
vmid = 151
|
||||
memory = 49152
|
||||
cores = 8
|
||||
|
||||
template_name = null
|
||||
cloud_init = false
|
||||
full_clone = false
|
||||
bios = "ovmf"
|
||||
machine = "q35"
|
||||
boot = "order=virtio0"
|
||||
agent_enabled = true
|
||||
|
||||
# Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
|
||||
# VLAN from this node (previously only k8s-edge had v30iot, so any
|
||||
# IoT-pool LB lease landing here was unreachable from devices).
|
||||
extra_network_bridges = ["v30iot"]
|
||||
|
||||
disks = [
|
||||
{
|
||||
storage = "local-zfs"
|
||||
size = "64G"
|
||||
format = "raw"
|
||||
slot = "virtio0"
|
||||
file_id = local.talos_image
|
||||
}
|
||||
]
|
||||
|
||||
hostpci = [
|
||||
{
|
||||
mapping = "osd-ssd"
|
||||
},
|
||||
{
|
||||
mapping = "x710-port1"
|
||||
}
|
||||
]
|
||||
|
||||
tags = "kubernetes;talos;controlplane"
|
||||
ssh_public_keys = local.common_ssh_keys
|
||||
|
||||
vm_state = "running"
|
||||
|
||||
# Rebuild pipeline issues `-replace`; skip graceful shutdown so a
|
||||
# non-responsive Talos guest can't hang the destroy for 30 minutes.
|
||||
stop_on_destroy = true
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
resource "proxmox_virtual_environment_vm" "ubuntu_24_04_template" {
|
||||
name = "ubuntu-24.04-template"
|
||||
node_name = "saturn"
|
||||
vm_id = 605
|
||||
|
||||
started = false
|
||||
template = true
|
||||
on_boot = false
|
||||
|
||||
keyboard_layout = "en-us"
|
||||
|
||||
bios = "seabios"
|
||||
|
||||
agent {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = 2
|
||||
type = "x86-64-v3"
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = 2048
|
||||
}
|
||||
|
||||
initialization {
|
||||
datastore_id = "local-zfs"
|
||||
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
|
||||
user_account {
|
||||
username = "ladmin"
|
||||
keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
disk {
|
||||
datastore_id = "local-zfs"
|
||||
interface = "scsi0"
|
||||
size = 32
|
||||
file_format = "raw"
|
||||
iothread = false
|
||||
}
|
||||
|
||||
network_device {
|
||||
bridge = "v40srv"
|
||||
model = "virtio"
|
||||
}
|
||||
|
||||
serial_device {}
|
||||
|
||||
scsi_hardware = "virtio-scsi-pci"
|
||||
|
||||
lifecycle {
|
||||
ignore_changes = [
|
||||
node_name,
|
||||
started,
|
||||
initialization,
|
||||
disk,
|
||||
cpu,
|
||||
memory
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,14 @@ COPY src ./src
|
||||
RUN cargo build --release --target x86_64-unknown-linux-musl \
|
||||
&& strip target/x86_64-unknown-linux-musl/release/secret-rotator
|
||||
|
||||
# Build timestamp, recorded in the image rather than the tag: the tag says which
|
||||
# source revision was built, this says when. Written here because the final
|
||||
# image is scratch and has no shell to run `date` in.
|
||||
RUN date -u +%Y-%m-%dT%H:%M:%SZ > /build/image-build-date
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /build/target/x86_64-unknown-linux-musl/release/secret-rotator /secret-rotator
|
||||
COPY --from=build /build/image-build-date /etc/image-build-date
|
||||
|
||||
# Matches runAsUser on the rotation Jobs. There is no /etc/passwd in a scratch
|
||||
# image, so this is a bare uid by necessity — which is also all Kubernetes needs.
|
||||
|
||||
Reference in New Issue
Block a user