Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  claude-botandClaude Opus 5
|
e30d833624 |
@@ -42,7 +42,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: fetch-alerts
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
+12
-188
@@ -19,13 +19,12 @@
|
||||
# the admin account could (#330).
|
||||
#
|
||||
# The token is still OWNED by gitea_admin, because Gitea scopes packages to
|
||||
# a user or org namespace and most of these images live under `gitea_admin/*`.
|
||||
# The rebuild-critical ones additionally publish to `konairius/` — an org
|
||||
# created for the ghcr mirror path, with gitea_admin as a member so this same
|
||||
# token can push there. The full gitea_admin/* -> org migration is a
|
||||
# 38-reference change tracked separately in #330 rather than bundled here.
|
||||
# Note this pipeline runs no LLM agent, so the exfil path that motivated the
|
||||
# other cutovers does not apply to it.
|
||||
# a user or org namespace and these images live under `gitea_admin/*` — a
|
||||
# different account cannot publish there. Moving them to a dedicated org is
|
||||
# the remaining cleanup; it is a 38-reference migration across pipelines and
|
||||
# Flux-deployed CronJobs, so it is tracked separately in #330 rather than
|
||||
# bundled here. Note this pipeline runs no LLM agent, so the exfil path that
|
||||
# motivated the other cutovers does not apply to it.
|
||||
when:
|
||||
- event: push
|
||||
path:
|
||||
@@ -88,30 +87,6 @@ steps:
|
||||
--destination gitea.renner.uno/gitea_admin/ci-k8s:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-k8s:latest
|
||||
|
||||
- name: build-vmtest
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/vmtest
|
||||
depends_on: []
|
||||
when:
|
||||
- event: deployment
|
||||
- event: push
|
||||
path: ["ci/images/vmtest/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-vmtest:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/ci-vmtest:latest
|
||||
|
||||
- name: build-tofu
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/tofu
|
||||
@@ -172,28 +147,6 @@ steps:
|
||||
# not for who uses it — its first consumer is the zigbee2mqtt seed-data init
|
||||
# container, but nothing about it is zigbee-specific. Exists so tools come from
|
||||
# the image rather than an `apk add` on every container start.
|
||||
#
|
||||
# Published THREE times, and the third one is not a typo. Its consumers — the
|
||||
# zigbee2mqtt seed-data init container and the trmnl renderer — start while
|
||||
# the cluster is still being rebuilt, and at that moment gitea.renner.uno does
|
||||
# not exist, because Gitea runs inside the cluster being built. A cold-rebuild
|
||||
# rehearsal caught both in ImagePullBackOff.
|
||||
#
|
||||
# So there is a copy on ghcr.io. But the manifests must NOT point at it:
|
||||
# normal operation pulls from the local registry, and an external reference
|
||||
# would make every steady-state pull leave the network. The recovery source
|
||||
# and the everyday source are different things, exactly as they are for Flux's
|
||||
# git source.
|
||||
#
|
||||
# Talos resolves that with machine.registries.mirrors, which takes an ordered
|
||||
# endpoint list — gitea first, ghcr second (bootstrap.sh). Containerd keeps
|
||||
# the repository path when it falls through, so the two registries have to
|
||||
# agree on it: `gitea_admin/alpine-jq` cannot be served by ghcr, where the
|
||||
# namespace is fixed to the account name. Hence the second Gitea destination
|
||||
# under konairius/, which the manifests reference and which matches the ghcr
|
||||
# path exactly.
|
||||
#
|
||||
# The gitea_admin/ copy stays until nothing references it any more.
|
||||
- name: build-alpine-jq
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/alpine-jq
|
||||
@@ -206,47 +159,23 @@ steps:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
GHCR_USER: konairius
|
||||
GHCR_TOKEN:
|
||||
from_secret: ghcr_mirror_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-jq:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-jq:latest \
|
||||
--destination gitea.renner.uno/konairius/alpine-jq:$TAG \
|
||||
--destination gitea.renner.uno/konairius/alpine-jq:latest \
|
||||
--destination ghcr.io/konairius/alpine-jq:$TAG \
|
||||
--destination ghcr.io/konairius/alpine-jq:latest
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-jq:latest
|
||||
|
||||
# Runtime image, not CI tooling: upstream containernetworking/plugins binaries
|
||||
# baked in, consumed by the multus cni-plugins-installer DaemonSet. Replaces a
|
||||
# node-side fetch from GitHub that had NO checksum verification and sat on the
|
||||
# critical path of every node rebuild. The download now happens here, once,
|
||||
# where the sha256 is checked — see the Containerfile for the full rationale.
|
||||
#
|
||||
# This one ALSO publishes to ghcr.io. A cold-rebuild rehearsal proved why: the
|
||||
# DaemonSet could not pull from gitea.renner.uno, because Gitea runs inside the
|
||||
# very cluster being rebuilt. Without the plugins there is no macvlan, so
|
||||
# Multus cannot attach ceph-san, so Rook's network canary never starts, so the
|
||||
# CephCluster sits at "Configuring Ceph Mons" — and OpenBao waits forever for a
|
||||
# PVC that will never bind. Moving the download here fixed the integrity gap
|
||||
# but quietly swapped github.com on the critical path for something strictly
|
||||
# worse: a registry that by definition does not exist during a recovery. The
|
||||
# GHCR copy is public, so a bare node can pull it with no credential and no
|
||||
# Gitea.
|
||||
#
|
||||
# The test that proved this also proved the rule is not specific to this
|
||||
# image. alpine-jq and alpine-openssl mirror for the same reason — see the
|
||||
# note on build-alpine-jq. Anything an app needs while the cluster is still
|
||||
# being built cannot live only in that cluster's own registry.
|
||||
- name: build-cni-plugins
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/cni-plugins
|
||||
@@ -259,36 +188,23 @@ steps:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
GHCR_USER: konairius
|
||||
GHCR_TOKEN:
|
||||
from_secret: ghcr_mirror_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/cni-plugins:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/cni-plugins:latest \
|
||||
--destination gitea.renner.uno/konairius/cni-plugins:$TAG \
|
||||
--destination gitea.renner.uno/konairius/cni-plugins:latest \
|
||||
--destination ghcr.io/konairius/cni-plugins:$TAG \
|
||||
--destination ghcr.io/konairius/cni-plugins:latest
|
||||
--destination gitea.renner.uno/gitea_admin/cni-plugins:latest
|
||||
|
||||
# Runtime image, not CI tooling: alpine + the openssl CLI, for shell scripts
|
||||
# needing s_client/x509. Named for what it is; first consumer is the paperless
|
||||
# fetch-bridge-cert init container. Replaces an `apk add openssl` that ran on
|
||||
# every paperless pod start. The openssl CLI is genuinely absent from the alpine
|
||||
# base, so this one needs an image rather than a busybox substitute.
|
||||
#
|
||||
# Mirrored to ghcr.io for the reason given on build-alpine-jq: it is an init
|
||||
# container, so paperless does not start without it, and during a rebuild
|
||||
# gitea.renner.uno is not there to serve it. A cold-rebuild rehearsal left
|
||||
# paperless in Init:ErrImagePull on exactly this image.
|
||||
- name: build-alpine-openssl
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/alpine-openssl
|
||||
@@ -301,25 +217,17 @@ steps:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
GHCR_USER: konairius
|
||||
GHCR_TOKEN:
|
||||
from_secret: ghcr_mirror_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- GH_AUTH=$(printf '%s:%s' "$GHCR_USER" "$GHCR_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"},"ghcr.io":{"auth":"%s"}}}' "$AUTH" "$GH_AUTH" > /kaniko/.docker/config.json
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-openssl:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-openssl:latest \
|
||||
--destination gitea.renner.uno/konairius/alpine-openssl:$TAG \
|
||||
--destination gitea.renner.uno/konairius/alpine-openssl:latest \
|
||||
--destination ghcr.io/konairius/alpine-openssl:$TAG \
|
||||
--destination ghcr.io/konairius/alpine-openssl:latest
|
||||
--destination gitea.renner.uno/gitea_admin/alpine-openssl:latest
|
||||
|
||||
# Heating CAN readers: open3e (Vitocal/VX3) + E3onCAN (E380 meter), one image
|
||||
# run as two services on edge-1. Replaces pip-into-venv on the host, where an
|
||||
@@ -349,87 +257,3 @@ steps:
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/heating-can:latest
|
||||
|
||||
# netbird-proxy-services: the controller that reconciles NetBird reverse-proxy
|
||||
# services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
|
||||
# Runtime artifact like heating-can; same registry so the audit tracks it.
|
||||
- name: build-netbird-proxy-services
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/netbird-proxy-services
|
||||
depends_on: []
|
||||
when:
|
||||
- event: deployment
|
||||
- event: push
|
||||
path: ["ci/images/netbird-proxy-services/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/netbird-proxy-services:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/netbird-proxy-services:latest
|
||||
|
||||
# wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea and
|
||||
# queried by ~/Notes/bin/wiki_rank.py. Runtime artifact like netbird-proxy-services.
|
||||
- name: build-wiki-index
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/wiki-index
|
||||
depends_on: []
|
||||
when:
|
||||
- event: deployment
|
||||
- event: push
|
||||
path: ["ci/images/wiki-index/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
# Also a per-commit tag: several builds a day (branch pushes and the
|
||||
# merge) all rewrite the dated tag, and the deployment pin raced them
|
||||
# on 2026-09-20. The commit tag is immutable and needs no digest lookup.
|
||||
- SHORT=$(printf '%s' "$CI_COMMIT_SHA" | cut -c1-8)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/wiki-index:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/wiki-index:$TAG-$SHORT \
|
||||
--destination gitea.renner.uno/gitea_admin/wiki-index:latest
|
||||
|
||||
# bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
|
||||
# artifact like heating-can; same registry so the audit tracks it.
|
||||
- name: build-bird
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/bird
|
||||
depends_on: []
|
||||
when:
|
||||
- event: deployment
|
||||
- event: push
|
||||
path: ["ci/images/bird/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/bird:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/bird:latest
|
||||
|
||||
@@ -60,7 +60,7 @@ steps:
|
||||
# run that never touched them. Tracked in #461.
|
||||
- name: kustomize-build
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
@@ -158,7 +158,7 @@ steps:
|
||||
# whenever it trails the deployed server. If you bump the server, bump here.
|
||||
- name: woodpecker-lint
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
|
||||
image: woodpeckerci/woodpecker-cli:v3.17.0-alpine
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
@@ -246,7 +246,7 @@ steps:
|
||||
# this step fails permanently on its own documentation.
|
||||
- name: no-digest-pins
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
@@ -7,21 +7,7 @@ when:
|
||||
# install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
|
||||
# so changes to it must redeploy this host too.
|
||||
- "ansible/roles/netbird-proxy-watchdog/**"
|
||||
# install-netbird-proxy.yml also applies the edge bouncer.
|
||||
- "ansible/roles/crowdsec-edge-bouncer/**"
|
||||
- "ansible/plays/install-netbird-proxy.yml"
|
||||
# The variables that steer those roles, not just the roles themselves.
|
||||
# Without these two, a change that only touches inventory never deploys
|
||||
# and nothing says so: arming the bouncer lives in host_vars, and its
|
||||
# LAPI key lives in group_vars, so both would sit in git looking applied
|
||||
# while the hosts kept running the previous configuration.
|
||||
#
|
||||
# pve_hosts.yml is deliberately NOT listed. It is the single inventory
|
||||
# file for the whole fleet, so listing it would redeploy the proxies for
|
||||
# any unrelated host edit. The cost is that a change to [netbird_proxy]
|
||||
# group membership needs a manual deployment run.
|
||||
- "ansible/inventory/host_vars/proxy-*.yml"
|
||||
- "ansible/inventory/group_vars/netbird_proxy/**"
|
||||
- ".woodpecker/deploy-netbird-proxy.yml"
|
||||
- event: deployment
|
||||
evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
|
||||
|
||||
@@ -4,7 +4,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: e2e
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ steps:
|
||||
- sh scripts/ci/test-model-selection.sh
|
||||
|
||||
- name: e2e-test
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
@@ -49,7 +49,7 @@ steps:
|
||||
# (the issue-pick logic needs jq, which only the ci-claude image has);
|
||||
# minting a 1h read-only token per run is cheap. Degrades to
|
||||
# snapshot-only if the ci-diagnostics ServiceAccount isn't reconciled.
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
# Nightly wiki maintenance for the Notes vault (k.renner/notes), replacing the
|
||||
# workstation's wiki-maintenance.timer: distill the raw/ backlog into wiki/,
|
||||
# then audit wiki/. Each agent is its own `claude` run, not a subagent of an
|
||||
# orchestrator, so nothing depends on background tasks in print mode (the
|
||||
# workstation job died on their 600 s ceiling). The pipeline commits and pushes
|
||||
# as Claude Bot; the push webhook then refreshes the wiki-index service.
|
||||
#
|
||||
# The agents and their retrieval scripts address the vault as ~/Notes, so each
|
||||
# agent step links $HOME/Notes to the checkout instead of rewriting them.
|
||||
when:
|
||||
- event: cron
|
||||
cron: wiki-maintenance
|
||||
# Hand-started runs: `woodpecker pipeline create k.renner/homelab --branch master`.
|
||||
# The typesafe_api_key secret is scoped to cron and manual, not deployment.
|
||||
- event: manual
|
||||
|
||||
steps:
|
||||
- name: checkout-notes
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
|
||||
pull: true
|
||||
environment:
|
||||
WIKI_BOT_TOKEN:
|
||||
from_secret: wiki_bot_token
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
# Token only on the wire, never in .git/config. Secrets set from files carry
|
||||
# a trailing newline (run 10220: "could not read Username"); strip CR/LF.
|
||||
TOKEN=$$(printf '%s' "$$WIKI_BOT_TOKEN" | tr -d '\r\n')
|
||||
git -c http.extraheader="AUTHORIZATION: token $$TOKEN" \
|
||||
clone --depth 50 --branch main https://gitea.renner.uno/k.renner/notes.git notes
|
||||
echo "raw backlog: $$(ls notes/raw/*.md | grep -v _index.md | wc -l) file(s)"
|
||||
|
||||
- name: distill
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
|
||||
pull: true
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
TYPESAFE_API_KEY:
|
||||
from_secret: typesafe_api_key
|
||||
WIKI_INDEX_BASIC_AUTH:
|
||||
from_secret: wiki_index_basic_auth
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
ln -sfn "$$PWD/notes" "$$HOME/Notes"
|
||||
mkdir -p "$$HOME/.config/wiki-index"
|
||||
printf '%s' "$$WIKI_INDEX_BASIC_AUTH" | tr -d '\r\n' > "$$HOME/.config/wiki-index/basic-auth"
|
||||
chmod 600 "$$HOME/.config/wiki-index/basic-auth"
|
||||
"$$HOME/Notes/bin/wiki_rank.py" --probe-index
|
||||
- |
|
||||
set -eu
|
||||
MODEL=$$(sh scripts/ci/pick-model.sh "distill raw source notes into atomic wiki notes: read sources, check the wiki for existing coverage, write or merge notes; careful writing, moderate reasoning")
|
||||
if [ -n "$$MODEL" ]; then echo "distill: selected model $$MODEL"; else echo "distill: using the CLI default model"; fi
|
||||
cd notes
|
||||
cat ../ci/agents/wiki-maintenance/distill.md | claude $${MODEL:+--model $$MODEL} \
|
||||
--agent wiki-distill \
|
||||
--permission-mode acceptEdits \
|
||||
--allowedTools "Read,Edit,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*),Bash(mv /root/Notes/raw/:*),Bash(mkdir -p /root/Notes/:*)" \
|
||||
--settings ../ci/agents/wiki-maintenance/settings.json \
|
||||
-p
|
||||
|
||||
- name: audit
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
|
||||
pull: true
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
TYPESAFE_API_KEY:
|
||||
from_secret: typesafe_api_key
|
||||
WIKI_INDEX_BASIC_AUTH:
|
||||
from_secret: wiki_index_basic_auth
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
ln -sfn "$$PWD/notes" "$$HOME/Notes"
|
||||
mkdir -p "$$HOME/.config/wiki-index"
|
||||
printf '%s' "$$WIKI_INDEX_BASIC_AUTH" | tr -d '\r\n' > "$$HOME/.config/wiki-index/basic-auth"
|
||||
chmod 600 "$$HOME/.config/wiki-index/basic-auth"
|
||||
- |
|
||||
set -eu
|
||||
MODEL=$$(sh scripts/ci/pick-model.sh "structural audit of ~1700 markdown notes: frontmatter completeness, broken links, orphans; mechanical checks, little reasoning")
|
||||
if [ -n "$$MODEL" ]; then echo "audit: selected model $$MODEL"; else echo "audit: using the CLI default model"; fi
|
||||
cd notes
|
||||
cat ../ci/agents/wiki-maintenance/audit.md | claude $${MODEL:+--model $$MODEL} \
|
||||
--agent wiki-audit \
|
||||
--permission-mode acceptEdits \
|
||||
--allowedTools "Read,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*)" \
|
||||
--settings ../ci/agents/wiki-maintenance/settings.json \
|
||||
-p
|
||||
|
||||
- name: push-notes
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.278
|
||||
pull: true
|
||||
environment:
|
||||
WIKI_BOT_TOKEN:
|
||||
from_secret: wiki_bot_token
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
cd notes
|
||||
git config user.email "claude-bot@renner.uno"
|
||||
git config user.name "Claude Bot"
|
||||
if git diff --quiet && git diff --cached --quiet && [ -z "$$(git ls-files --others --exclude-standard)" ]; then
|
||||
echo "wiki-maintenance: no changes"
|
||||
exit 0
|
||||
fi
|
||||
# The workstation pushes main on its own (notes-push.path), so main moves
|
||||
# while the agents run (run 10241: non-fast-forward). The run's work goes
|
||||
# to a branch and a PR that this step merges through Gitea; a conflict
|
||||
# leaves the PR open for a human and fails the run.
|
||||
BRANCH="wiki-maintenance/$$(date -u +%F)-$$CI_PIPELINE_NUMBER"
|
||||
git checkout -q -b "$$BRANCH"
|
||||
git add -A
|
||||
git commit -q -m "wiki maintenance $$(date -u +%F): distill + audit (CI run $$CI_PIPELINE_NUMBER)"
|
||||
TOKEN=$$(printf '%s' "$$WIKI_BOT_TOKEN" | tr -d '\r\n')
|
||||
git -c http.extraheader="AUTHORIZATION: token $$TOKEN" push origin "HEAD:$$BRANCH"
|
||||
API="https://gitea.renner.uno/api/v1/repos/k.renner/notes"
|
||||
jq -n --arg title "wiki maintenance $$(date -u +%F) (CI run $$CI_PIPELINE_NUMBER)" \
|
||||
--arg body "Nightly distill + audit. Merged by the pipeline; open only if the merge failed." \
|
||||
--arg head "$$BRANCH" '{title: $$title, body: $$body, head: $$head, base: "main"}' > /tmp/pr.json
|
||||
code=$$(curl -s -o /tmp/pr-resp.json -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token $$TOKEN" -H "Content-Type: application/json" --data @/tmp/pr.json "$$API/pulls")
|
||||
echo "POST pull -> HTTP $$code"
|
||||
[ "$$code" -ge 200 ] && [ "$$code" -lt 300 ] || { cat /tmp/pr-resp.json; exit 1; }
|
||||
NUM=$$(jq -r '.number' /tmp/pr-resp.json)
|
||||
code=$$(curl -s -o /tmp/merge-resp.json -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token $$TOKEN" -H "Content-Type: application/json" \
|
||||
--data '{"Do":"merge","delete_branch_after_merge":true}' "$$API/pulls/$$NUM/merge")
|
||||
echo "POST merge #$$NUM -> HTTP $$code"
|
||||
[ "$$code" -ge 200 ] && [ "$$code" -lt 300 ] || { cat /tmp/merge-resp.json; echo "PR #$$NUM left open"; exit 1; }
|
||||
@@ -1,23 +0,0 @@
|
||||
---
|
||||
# Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
|
||||
#
|
||||
# A single encrypted VALUE rather than a whole-file vault like its siblings in
|
||||
# group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
|
||||
# an existing whole-file vault means decrypting that file first, which exposes
|
||||
# every other secret in it for the sake of appending one line. encrypt_string
|
||||
# keeps the blast radius at this one value.
|
||||
#
|
||||
# Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
|
||||
# the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
|
||||
# LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
|
||||
# one is NOT in the quarterly rotator, because nothing on these hosts re-reads
|
||||
# OpenBao on a schedule. Rotating it means updating OpenBao and re-running
|
||||
# plays/install-netbird-proxy.yml.
|
||||
vault_crowdsec_edge_bouncer_key: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
32653764333536316163303838333839616562663862666637626165303837633866643033393239
|
||||
3363376538656236643161386432326437633232643564630a633635306365323537346139626364
|
||||
36613033343938363366306239363638383466396337623333623136313234656366333366623433
|
||||
6437656632373364340a346566343866366433306533353431383064356362346536383136323663
|
||||
39333864643534326433653233303137373162663066633132623931316135323733333863306330
|
||||
6366623463666435636362376238313639666263323163303733
|
||||
@@ -26,7 +26,3 @@
|
||||
- docker-install
|
||||
- netbird-proxy
|
||||
- netbird-proxy-watchdog
|
||||
# Enforces the cluster's CrowdSec decisions here, where public traffic
|
||||
# enters. Deployed in dry-run: it logs what it would block and creates no
|
||||
# nftables rules until crowdsec_bouncer_mode is flipped per host.
|
||||
- crowdsec-edge-bouncer
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
---
|
||||
# CrowdSec firewall bouncer on the NetBird proxy hosts.
|
||||
#
|
||||
# Enforces the decisions the in-cluster CrowdSec already detects, at the point
|
||||
# where public traffic actually enters: these hosts terminate TLS for every
|
||||
# *.renner.uno name. A drop here never reaches the mesh, the relay or the
|
||||
# cluster.
|
||||
#
|
||||
# Detection stays in the cluster. The reverse-proxy produces no parseable
|
||||
# access log and there is no CrowdSec collection for it, while Traefik has a
|
||||
# working one — and every scenario in use needs HTTP detail (paths, status
|
||||
# codes, user agents) that does not exist at this layer.
|
||||
|
||||
# Pinned. A release tarball URL is immutable, and this checksum is of the
|
||||
# amd64 asset for exactly this tag.
|
||||
crowdsec_bouncer_version: v0.0.36
|
||||
crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
|
||||
|
||||
# The LAPI over the public path these hosts already serve, see
|
||||
# kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
|
||||
# reach it over the mesh: the reverse-proxy container runs non-root, so its
|
||||
# embedded NetBird client terminates the tunnel in userspace and the tunnel
|
||||
# belongs to that process rather than to the host netns.
|
||||
#
|
||||
# No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
|
||||
# and the bouncer adds ca_cert_path to the system pool rather than replacing
|
||||
# it — the system pool already trusts LE.
|
||||
crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
|
||||
crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
|
||||
|
||||
# Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
|
||||
# crowdsec6) holding the decisions it pulled, touches no existing chain and
|
||||
# sets no chain policy — SSH and :443 stay open unless the source is banned.
|
||||
#
|
||||
# `dry-run` is the other value: it pulls and logs what it would block without
|
||||
# creating a single rule. That is what a new host should be given before it
|
||||
# enforces, and what an existing one can be dropped back to, per host through
|
||||
# inventory/host_vars, without disarming the rest.
|
||||
#
|
||||
# Worth knowing before arming anything new: the first bouncer on a path
|
||||
# enforces the whole CAPI set at once — around 15k addresses today — so the
|
||||
# false-positive window belongs before the switch, not after it. And because
|
||||
# this chain sits on the INPUT hook while these hosts run no other firewall,
|
||||
# there is no conntrack rule accepting established flows ahead of it: a banned
|
||||
# address is dropped even when it is answering our own outbound connection.
|
||||
# That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
|
||||
crowdsec_bouncer_mode: nftables
|
||||
|
||||
crowdsec_bouncer_update_frequency: 10s
|
||||
crowdsec_bouncer_log_level: info
|
||||
|
||||
# Bound to loopback deliberately. These hosts are public and have no metrics
|
||||
# collection path at all today — opening :60601 to the internet to gain a
|
||||
# scrape target is the wrong trade. Revisit when host metrics exist.
|
||||
crowdsec_bouncer_metrics_listen: 127.0.0.1
|
||||
crowdsec_bouncer_metrics_port: 60601
|
||||
@@ -1,10 +0,0 @@
|
||||
---
|
||||
# A restart drops both nftables tables and rebuilds them from the next
|
||||
# successful LAPI pull, so there is a short window with no blocklist. That is
|
||||
# the documented behaviour of a clean shutdown, not a defect — and it is why
|
||||
# retry_initial_connect matters.
|
||||
- name: Restart crowdsec firewall bouncer
|
||||
ansible.builtin.systemd:
|
||||
name: crowdsec-firewall-bouncer.service
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
@@ -1,111 +0,0 @@
|
||||
---
|
||||
# See defaults/main.yml for why this exists and why it starts inert.
|
||||
#
|
||||
# The static release tarball, not a distribution package: the AUR package
|
||||
# `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
|
||||
# whole Security Engine onto hosts that only need the bouncer, the group is
|
||||
# mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
|
||||
# repo does not otherwise have. The tarball is the same artefact the AUR
|
||||
# package downloads.
|
||||
|
||||
- name: Ensure nftables is present (the enforcing backend)
|
||||
ansible.builtin.package:
|
||||
name: nftables
|
||||
state: present
|
||||
|
||||
- name: Ensure the bouncer config directory exists
|
||||
ansible.builtin.file:
|
||||
path: /etc/crowdsec/bouncers
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
# Fetched on the CONTROLLER and pushed, not pulled by the host.
|
||||
#
|
||||
# proxy-1 is IPv6-only, and neither github.com nor objects.githubusercontent.com
|
||||
# publishes an AAAA record — a get_url on the host fails with "Network is
|
||||
# unreachable" and no amount of retrying fixes it. Mirroring into
|
||||
# gitea.renner.uno is no help either: its public records are the internal LB
|
||||
# VIP (10.2.99.86 / fd99::86), so it is unreachable from outside the LAN. And
|
||||
# upstream ships no container image for this bouncer, which is expected —
|
||||
# it manipulates host netfilter.
|
||||
#
|
||||
# The CI runner has v4, so it does the fetching. The checksum is verified
|
||||
# there, before anything is copied to a host.
|
||||
- name: Fetch the pinned bouncer release on the controller
|
||||
ansible.builtin.get_url:
|
||||
url: >-
|
||||
https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
|
||||
crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
|
||||
dest: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
|
||||
checksum: sha256:{{ crowdsec_bouncer_sha256 }}
|
||||
mode: "0644"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
|
||||
# Only the download had to move to the controller — the extraction stays on
|
||||
# the host. The ci-ansible image ships a tar that ansible rejects ("detected as
|
||||
# tar type None. GNU tar required"), while both host distributions have GNU
|
||||
# tar. Shipping the archive rather than the unpacked binary also keeps the
|
||||
# artefact that the checksum was verified against intact all the way to the
|
||||
# host.
|
||||
- name: Copy the bouncer archive to the host
|
||||
ansible.builtin.copy:
|
||||
src: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
|
||||
dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
|
||||
mode: "0644"
|
||||
|
||||
# Only the binary. The tarball also carries install.sh / upgrade.sh /
|
||||
# uninstall.sh, which manage systemd and config themselves — that is this
|
||||
# role's job, and letting them run would leave two owners for the same files.
|
||||
- name: Unpack the bouncer binary
|
||||
ansible.builtin.unarchive:
|
||||
src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
|
||||
dest: /opt
|
||||
remote_src: true
|
||||
include:
|
||||
- crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
|
||||
creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
|
||||
|
||||
- name: Install the bouncer binary
|
||||
ansible.builtin.copy:
|
||||
src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
|
||||
dest: /usr/local/bin/crowdsec-firewall-bouncer
|
||||
remote_src: true
|
||||
mode: "0755"
|
||||
notify: Restart crowdsec firewall bouncer
|
||||
|
||||
# 0600: this file carries the LAPI key.
|
||||
- name: Deploy the bouncer configuration
|
||||
ansible.builtin.template:
|
||||
src: crowdsec-firewall-bouncer.yaml.j2
|
||||
dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
|
||||
mode: "0600"
|
||||
notify: Restart crowdsec firewall bouncer
|
||||
|
||||
- name: Deploy the bouncer systemd unit
|
||||
ansible.builtin.template:
|
||||
src: crowdsec-firewall-bouncer.service.j2
|
||||
dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
|
||||
mode: "0644"
|
||||
notify: Restart crowdsec firewall bouncer
|
||||
|
||||
- name: Enable and start the bouncer
|
||||
ansible.builtin.systemd:
|
||||
name: crowdsec-firewall-bouncer.service
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
|
||||
- name: Report how to read it
|
||||
ansible.builtin.debug:
|
||||
msg: |
|
||||
crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
|
||||
|
||||
What it would block: journalctl -u crowdsec-firewall-bouncer -f
|
||||
What it does block: nft list table ip crowdsec ; nft list table ip6 crowdsec6
|
||||
Metrics: curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
|
||||
|
||||
Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
|
||||
both nftables tables by itself. Only after a kill -9 do they need
|
||||
removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6
|
||||
@@ -1,24 +0,0 @@
|
||||
# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
|
||||
#
|
||||
# Upstream's shipped unit orders itself After=crowdsec.service. There is no
|
||||
# local Security Engine on these hosts — the LAPI is in the cluster — so that
|
||||
# dependency is dropped; nss-lookup instead, because api_url is a public name
|
||||
# that has to resolve before the first connect.
|
||||
[Unit]
|
||||
Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
|
||||
After=network-online.target nss-lookup.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
# -t validates the config and exits non-zero on a bad one, so a broken config
|
||||
# fails the start instead of taking the running rules down with it.
|
||||
ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
|
||||
ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
LimitNOFILE=65536
|
||||
KillMode=mixed
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,55 +0,0 @@
|
||||
# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
|
||||
mode: {{ crowdsec_bouncer_mode }}
|
||||
update_frequency: {{ crowdsec_bouncer_update_frequency }}
|
||||
|
||||
# stdout, so journald owns the log and there is no second rotation policy to
|
||||
# keep in sync. The shipped example writes to /var/log; that only matters for
|
||||
# hosts without journald.
|
||||
log_mode: stdout
|
||||
log_level: {{ crowdsec_bouncer_log_level }}
|
||||
|
||||
api_url: {{ crowdsec_bouncer_api_url }}
|
||||
api_key: {{ crowdsec_bouncer_api_key }}
|
||||
|
||||
# Without this the process EXITS when the LAPI is unreachable at startup —
|
||||
# there is no retry on the first connect. And because a clean shutdown deletes
|
||||
# the nftables tables, the systemd restart loop would leave the host with no
|
||||
# blocklist at all until the first successful connect. Present in the code,
|
||||
# absent from the shipped example config.
|
||||
retry_initial_connect: true
|
||||
|
||||
disable_ipv4: false
|
||||
disable_ipv6: false
|
||||
deny_action: DROP
|
||||
|
||||
supported_decisions_types:
|
||||
- ban
|
||||
|
||||
# Separate tables per family; the bouncer never touches an existing table or
|
||||
# chain, and never sets a chain policy. Priority -10 puts it ahead of anything
|
||||
# at the default filter priority.
|
||||
nftables:
|
||||
ipv4:
|
||||
enabled: true
|
||||
set-only: false
|
||||
table: crowdsec
|
||||
chain: crowdsec-chain
|
||||
priority: -10
|
||||
ipv6:
|
||||
enabled: true
|
||||
set-only: false
|
||||
table: crowdsec6
|
||||
chain: crowdsec6-chain
|
||||
priority: -10
|
||||
|
||||
# input is the traffic this exists for: the reverse-proxy is host-networked and
|
||||
# owns :443, so its packets traverse INPUT. forward covers the bridge-networked
|
||||
# http-redirect sidecar, whose packets never do.
|
||||
nftables_hooks:
|
||||
- input
|
||||
- forward
|
||||
|
||||
prometheus:
|
||||
enabled: true
|
||||
listen_addr: {{ crowdsec_bouncer_metrics_listen }}
|
||||
listen_port: {{ crowdsec_bouncer_metrics_port }}
|
||||
@@ -26,16 +26,8 @@ edge_open3e_poll_seconds: 30
|
||||
# entries when added), so the multi-frame read is cheap at a 30 s poll.
|
||||
# Records faults/info events only (NormalOperation, LagDeviceDtcReported,
|
||||
# LegionellaProtectionTemperatureReached) — NOT setpoint or program changes.
|
||||
# onebase.*: the 29 paired ZigBee slots on the One Base gateway (ECU 0x6c3) --
|
||||
# 8 radiator valves, 9 climate sensors, 9 underfloor actuators, 3 floor
|
||||
# thermostats. Per-device room temperature, active setpoint, humidity, valve
|
||||
# opening/demand, battery and RSSI; the local replacement for the ViCare cloud
|
||||
# room sensors. Needs the onebase overlay (files/open3e-onebase.py) -- the stock
|
||||
# codec declares 57 bytes and a paired slot answers with 68.
|
||||
# Only CurrentValues are polled. The room and device registries (1884.., 2084..)
|
||||
# are static configuration and are read on demand, not every 30 s.
|
||||
edge_open3e_datapoints: >-
|
||||
vitocal.259,vitocal.260,vitocal.268,vitocal.269,vitocal.271,vitocal.274,vitocal.286,vitocal.288,vitocal.318,vitocal.321,vitocal.324,vitocal.335,vitocal.336,vitocal.396,vitocal.402,vitocal.403,vitocal.476,vitocal.477,vitocal.543,vitocal.548,vitocal.565,vitocal.1007,vitocal.1043,vitocal.1190,vitocal.1211,vitocal.1391,vitocal.1416,vitocal.1417,vitocal.2320,vitocal.2346,vitocal.2350,vitocal.2351,vitocal.2352,vitocal.2369,vitocal.2370,vitocal.2486,vitocal.2487,vitocal.2488,vitocal.2543,vitocal.2560,vitocal.2622,vitocal.2624,vitocal.2625,vitocal.2735,vitocal.3016,vx3.266,vx3.378,vx3.379,vx3.380,vx3.535,vx3.1603,vx3.1664,vx3.1690,vx3.1801,vx3.1802,vx3.1810,vx3.1828,vx3.1830,vx3.1577,vx3.1578,vx3.1579,vx3.1580,vx3.1581,vx3.1582,vx3.1834,vx3.1836,vx3.2214,vx3.2240,vx3.3085,vx3.3086,vx3.3087,vx3.3088,vx3.3089,vx3.3090,vcmu.320,vcmu.322,vcmu.325,vcmu.355,vcmu.389,vcmu.391,vcmu.1772,vcmu.1773,vcmu.1775,vcmu.1776,vcmu.2256,vcmu.2259,vcmu.2333,vcmu.2334,vcmu.2569,vcmu.2571,vitocal.2791,onebase.2086,onebase.2089,onebase.2092,onebase.2095,onebase.2098,onebase.2101,onebase.2104,onebase.2107,onebase.2110,onebase.2113,onebase.2116,onebase.2119,onebase.2122,onebase.2125,onebase.2128,onebase.2131,onebase.2134,onebase.2137,onebase.2140,onebase.2143,onebase.2262,onebase.2265,onebase.2268,onebase.2271,onebase.2274,onebase.2277,onebase.2280,onebase.2283,onebase.2286
|
||||
vitocal.259,vitocal.260,vitocal.268,vitocal.269,vitocal.271,vitocal.274,vitocal.286,vitocal.288,vitocal.318,vitocal.321,vitocal.324,vitocal.335,vitocal.336,vitocal.396,vitocal.402,vitocal.403,vitocal.476,vitocal.477,vitocal.543,vitocal.548,vitocal.565,vitocal.1007,vitocal.1043,vitocal.1190,vitocal.1211,vitocal.1391,vitocal.1416,vitocal.1417,vitocal.2320,vitocal.2346,vitocal.2350,vitocal.2351,vitocal.2352,vitocal.2369,vitocal.2370,vitocal.2486,vitocal.2487,vitocal.2488,vitocal.2543,vitocal.2560,vitocal.2622,vitocal.2624,vitocal.2625,vitocal.2735,vitocal.3016,vx3.266,vx3.378,vx3.379,vx3.380,vx3.535,vx3.1603,vx3.1664,vx3.1690,vx3.1801,vx3.1802,vx3.1810,vx3.1828,vx3.1830,vx3.1577,vx3.1578,vx3.1579,vx3.1580,vx3.1581,vx3.1582,vx3.1834,vx3.1836,vx3.2214,vx3.2240,vx3.3085,vx3.3086,vx3.3087,vx3.3088,vx3.3089,vx3.3090,vcmu.320,vcmu.322,vcmu.325,vcmu.355,vcmu.389,vcmu.391,vcmu.1772,vcmu.1773,vcmu.1775,vcmu.1776,vcmu.2256,vcmu.2259,vcmu.2333,vcmu.2334,vcmu.2569,vcmu.2571,vitocal.2791
|
||||
# ECU addresses; -dev selects codec set, tx selects the UDS address.
|
||||
# vcmu (refrigerant-circuit controller) has no dedicated codec set —
|
||||
# omitting dpList makes open3e fall back to the generic datapoint list,
|
||||
@@ -46,9 +38,6 @@ edge_open3e_devices:
|
||||
# files/open3e-vx3plus.py) — die Stock-vx3-Liste filtert sie sonst weg.
|
||||
vx3: { tx: "0x6a1", dpList: "/config/vx3plus.py" }
|
||||
vcmu: { tx: "0x68c" }
|
||||
# One Base ZigBee gateway. Overlay re-declares the paired ZigBee slots at
|
||||
# their real 68-byte length (see files/open3e-onebase.py).
|
||||
onebase: { tx: "0x6c3", dpList: "/config/onebase.py" }
|
||||
|
||||
edge_e3oncan_gap_seconds: 10
|
||||
edge_e3oncan_mqtt_format: "{device}_{didNumber:04d}_{didName}"
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
# Managed by ansible (roles/heating-can). Datapoint overlay for the One Base
|
||||
# ZigBee gateway on ECU 0x6c3 (BACKENDGATEWAY).
|
||||
#
|
||||
# Upstream declares ZigBee<N>DeviceCurrentValues as 57 bytes. A PAIRED device
|
||||
# slot on this gateway answers with 68: the 57 known bytes plus the parent
|
||||
# device serial and three trailing bytes. udsoncan reads only the declared
|
||||
# length and then treats the next two bytes -- the first half of that serial --
|
||||
# as a foreign data identifier, so every paired slot fails to decode with
|
||||
# "Server returned values for data identifier 0x.... that was not requested".
|
||||
# The bogus identifier differs per device because the serial does.
|
||||
#
|
||||
# This overlay re-declares the 29 paired slots at 68 bytes, reusing upstream's
|
||||
# sub-type list so the per-device-type O3ESwitch (climate sensor / TRV / floor
|
||||
# thermostat / actuator, incl. DeviceTemperatureSetpoint) is inherited rather
|
||||
# than copied.
|
||||
#
|
||||
# EMPTY slots answer with 57 and decode against the stock codec, so they are
|
||||
# left alone and are not polled. A slot that is later unpaired therefore starts
|
||||
# reporting a decode error on its topic instead of a value.
|
||||
#
|
||||
# Verified on ECU 0x6c3, 2026-09-17: 29 paired slots, all 68 bytes.
|
||||
from open3e.Open3Ecodecs import O3EComplexType, RawCodec
|
||||
from open3e.Open3Edatapoints import dataIdentifiers as _master
|
||||
|
||||
# ZigBee<N>DeviceCurrentValues of the paired slots: devices 1-20 run 2086..2143
|
||||
# in steps of 3, devices 21-29 continue at 2262..2286.
|
||||
_PAIRED_CURRENT_VALUES = [2086 + 3 * n for n in range(20)] + [
|
||||
2262 + 3 * n for n in range(9)
|
||||
]
|
||||
|
||||
|
||||
def _with_parent_serial(did):
|
||||
stock = _master["dids"][did]
|
||||
return O3EComplexType(
|
||||
68,
|
||||
stock.id,
|
||||
list(stock.subTypes)
|
||||
+ [
|
||||
RawCodec(8, "ParentDeviceSerialnumber"),
|
||||
RawCodec(2, "Unknown_65_66"),
|
||||
RawCodec(1, "Unknown_67"),
|
||||
],
|
||||
desc=stock.desc,
|
||||
info=stock.info,
|
||||
acc=stock.acc,
|
||||
)
|
||||
|
||||
|
||||
dataIdentifiers = {
|
||||
"dids": {
|
||||
**_master["dids"],
|
||||
**{did: _with_parent_serial(did) for did in _PAIRED_CURRENT_VALUES},
|
||||
}
|
||||
}
|
||||
@@ -129,13 +129,6 @@
|
||||
mode: "0644"
|
||||
notify: Restart heating-can
|
||||
|
||||
- name: Deploy open3e One Base datapoint overlay (68-byte ZigBee slots)
|
||||
ansible.builtin.copy:
|
||||
src: open3e-onebase.py
|
||||
dest: "{{ heating_can_base_dir }}/onebase.py"
|
||||
mode: "0644"
|
||||
notify: Restart heating-can
|
||||
|
||||
# --- Compose project ---------------------------------------------------------
|
||||
- name: Deploy the broker credential env file
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -17,7 +17,6 @@ services:
|
||||
volumes:
|
||||
- ./devices.json:/config/devices.json:ro
|
||||
- ./vx3plus.py:/config/vx3plus.py:ro
|
||||
- ./onebase.py:/config/onebase.py:ro
|
||||
env_file:
|
||||
# Broker credential, 0600 -- see heating-can.env.j2.
|
||||
- heating-can.env
|
||||
|
||||
@@ -32,13 +32,8 @@ netbird_watchdog_fail_threshold: 3
|
||||
|
||||
# Minimum seconds between watchdog-issued restarts, so a genuinely broken
|
||||
# backend can't put the proxy into a restart loop. Shared by both the URL-probe
|
||||
# watchdog and the flood watchdog below. One minute: the route-manager wedge
|
||||
# recurs on every routing-peer loss, and rolling node reboots lose one peer
|
||||
# every few minutes; a suppressed restart there leaves the wedge standing
|
||||
# until the URL probe's three rounds fire (3.5 min of 000 on every domain,
|
||||
# measured 2026-09-07). A minute still caps a loop at one restart per
|
||||
# probe interval.
|
||||
netbird_watchdog_cooldown_seconds: 60
|
||||
# watchdog and the flood watchdog below.
|
||||
netbird_watchdog_cooldown_seconds: 600
|
||||
|
||||
# Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain
|
||||
# substring in the proxy's docker logs by the flood watchdog. On this build the
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
You are running as the scheduled wiki-maintenance job in CI. The Notes vault is checked out at ~/Notes (the repository k.renner/notes).
|
||||
|
||||
Run the standard audit pass over ~/Notes/wiki/ and write the dated report to ~/Notes/output/. Do not modify any wiki note. Do not commit or push; the pipeline does that.
|
||||
|
||||
End with two lines: notes scanned, and issues found by category.
|
||||
@@ -1,7 +0,0 @@
|
||||
You are running as the scheduled wiki-maintenance job in CI. The Notes vault is checked out at ~/Notes (the repository k.renner/notes). Do exactly this, nothing else:
|
||||
|
||||
Glob ~/Notes/raw/*.md (top level only, never ~/Notes/raw/_processed/). Distill every file you find into ~/Notes/wiki/ following your process, including the ranker check for every concept. Move each fully processed source to ~/Notes/raw/_processed/. If raw/ has no top-level files besides _index.md, say so and stop.
|
||||
|
||||
Do not commit or push; the pipeline does that after you finish. Do not touch anything outside ~/Notes/wiki/ and ~/Notes/raw/.
|
||||
|
||||
End with one line per file: which wiki notes it produced or merged into, or "nothing distillable".
|
||||
@@ -1,37 +0,0 @@
|
||||
{
|
||||
"//": "Permission scope for the wiki-maintenance agents (wiki-distill, wiki-audit) in CI.",
|
||||
"//2": "The step's environment holds a Gitea push token, a TypeSafe key and the wiki-index",
|
||||
"//3": "client credential, so the Read tool is kept away from /proc (env dumps), /root outside",
|
||||
"//4": "the vault symlink target, /etc and the checkout's .git/. The vault itself is the work.",
|
||||
"//5": "Mirrors ci/agents/issue-fix/settings.json; absolute globs because the vault is reached",
|
||||
"//6": "through the ~/Notes symlink the pipeline creates.",
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Read(//root/Notes/**)",
|
||||
"Edit(//root/Notes/wiki/**)",
|
||||
"Edit(//root/Notes/output/**)",
|
||||
"Edit(//root/Notes/raw/**)",
|
||||
"Bash(/root/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(/root/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(mv /root/Notes/raw/:*)",
|
||||
"Bash(mkdir -p /root/Notes/:*)",
|
||||
"Bash(ls:*)",
|
||||
"Bash(date:*)",
|
||||
"Bash(wc:*)",
|
||||
"Bash(head:*)",
|
||||
"Bash(tail:*)"
|
||||
],
|
||||
"deny": [
|
||||
"Read(//proc/**)",
|
||||
"Read(//etc/**)",
|
||||
"Read(//root/.config/**)",
|
||||
"Read(//root/Notes/.git/**)",
|
||||
"Read(//root/Notes/_archive/**)",
|
||||
"Edit(//root/Notes/_archive/**)",
|
||||
"Bash(rm:*)",
|
||||
"Bash(git:*)",
|
||||
"Bash(curl:*)",
|
||||
"Bash(wget:*)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
|
||||
|
||||
# bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
|
||||
# step — the image is the binary plus its runtime.
|
||||
#
|
||||
# 3.22 rather than the 3.21 the other images here pin: bird2 first appears in
|
||||
# 3.22/community.
|
||||
#
|
||||
# Config is read from /etc/bird/bird.conf, not the package default
|
||||
# /etc/bird.conf, so the mount cannot shadow what the package ships.
|
||||
#
|
||||
# kubectl renders the init container's per-node config: go-template-file for
|
||||
# the templating, a field selector for excluding the node itself, in-cluster
|
||||
# config for the auth. From the distro, so there is no version literal to drift
|
||||
# and no download to trust. It is only ever used to read core/v1 Nodes.
|
||||
RUN apk add --no-cache bird2 kubectl
|
||||
|
||||
# Fails the build rather than the first pod if a binary or its path is wrong.
|
||||
RUN /usr/sbin/bird --version \
|
||||
&& kubectl version --client > /dev/null
|
||||
|
||||
ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]
|
||||
@@ -13,9 +13,7 @@ RUN apk add --no-cache \
|
||||
git \
|
||||
jq \
|
||||
openssh-client \
|
||||
openssl \
|
||||
python3 \
|
||||
py3-yaml
|
||||
openssl
|
||||
|
||||
# Install Claude Code, then make the resulting layer byte-stable so that a
|
||||
# rebuild of the SAME version DEDUPS in the Gitea registry instead of orphaning
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
|
||||
|
||||
ARG TALOS_VERSION=v1.13.9
|
||||
ARG TALOS_VERSION=v1.12.6
|
||||
ARG VELERO_VERSION=v1.16.0
|
||||
ARG KUBECTL_VERSION=v1.36.3
|
||||
ARG FLUX_VERSION=2.9.5
|
||||
ARG KUBECTL_VERSION=v1.36.2
|
||||
ARG FLUX_VERSION=2.9.2
|
||||
|
||||
RUN apk add --no-cache \
|
||||
aws-cli \
|
||||
@@ -18,7 +18,7 @@ RUN apk add --no-cache \
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/talosctl \
|
||||
"https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
|
||||
&& echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& echo "4c41a3b10b075292e64b182283d558e9d8e935f43239de1ed4e4b14359593efb /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/talosctl
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
@@ -31,7 +31,7 @@ RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
|
||||
RUN curl -sfL -o /tmp/flux.tar.gz \
|
||||
"https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \
|
||||
&& echo "b853df82adfd7736f580692f9f734473d571606307139f8fd20c2a80dd1ff473 /tmp/flux.tar.gz" | sha256sum -c - \
|
||||
&& echo "4092b367fd060097976fb7261601f79420ecf7266328417dfd8dfee27de0e6a3 /tmp/flux.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/flux.tar.gz -C /usr/local/bin flux \
|
||||
&& chmod +x /usr/local/bin/flux \
|
||||
&& rm -f /tmp/flux.tar.gz
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
# netbird-proxy-services: reconciles NetBird reverse-proxy services (public
|
||||
# domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
|
||||
# public label. Standard library only, so the image is the interpreter plus one
|
||||
# file; runs unprivileged as its own user.
|
||||
FROM python:3.13-slim
|
||||
|
||||
RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
|
||||
COPY controller.py /opt/controller.py
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
USER controller
|
||||
ENTRYPOINT ["python3", "/opt/controller.py"]
|
||||
@@ -1,275 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""netbird-proxy-services: reconcile NetBird reverse-proxy services from Ingresses.
|
||||
|
||||
Source of truth: every Ingress carrying the public label (dns.renner.uno/public).
|
||||
For each host on such an Ingress the NetBird management gets one reverse-proxy
|
||||
service in TLS-passthrough mode on :443, targeting Traefik's ClusterIP through a
|
||||
subnet resource that covers the cluster's service CIDR. SNI and Host reach
|
||||
Traefik untouched, the proxy terminates nothing, and no domain resource is
|
||||
involved -- the routing peers stay ordinary IP routers. The target sends a
|
||||
PROXY protocol v2 header so Traefik (and CrowdSec behind its access log) sees
|
||||
the WAN client address instead of the routing peer's pod IP.
|
||||
|
||||
Ownership is derived from NetBird itself, not from local state: a service is
|
||||
managed iff its mode is "tls" and its target is this controller's subnet
|
||||
resource. Services that do not match (the hand-made HTTP/domain ones) are never
|
||||
touched; a desired host held by such a service is reported and skipped until
|
||||
that service is removed by hand.
|
||||
|
||||
Credentials: the NetBird PAT is read from OpenBao on every pass through the
|
||||
Kubernetes auth role, exactly like the setup-key rotator; it never lands in a
|
||||
Kubernetes Secret or in a log line.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
LOG = logging.getLogger("netbird-proxy-services")
|
||||
|
||||
NB = os.environ.get("NB_MANAGEMENT_URL", "https://netbird.renner.uno").rstrip("/")
|
||||
BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
|
||||
BAO_ROLE = os.environ.get("BAO_ROLE", "netbird-proxy-services")
|
||||
BAO_PAT_PATH = os.environ.get("BAO_PAT_PATH", "homelab/data/infrastructure/secret-rotators-bootstrap/netbird")
|
||||
BAO_PAT_KEY = os.environ.get("BAO_PAT_KEY", "pat")
|
||||
SERVICE_CIDR = os.environ.get("SERVICE_CIDR", "10.96.0.0/12")
|
||||
NETWORK_NAME = os.environ.get("NETWORK_NAME", "k8s public ingress")
|
||||
ROUTER_GROUP = os.environ.get("ROUTER_GROUP_NAME", "k8s-ingress-router")
|
||||
SOURCE_GROUP = os.environ.get("SOURCE_GROUP_NAME", "All")
|
||||
TRAEFIK_NS, TRAEFIK_SVC = os.environ.get("TRAEFIK_SERVICE", "ingress/traefik").split("/", 1)
|
||||
PUBLIC_LABEL = os.environ.get("PUBLIC_LABEL", "dns.renner.uno/public")
|
||||
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "443"))
|
||||
INTERVAL = int(os.environ.get("INTERVAL_SECONDS", "60"))
|
||||
HTTP_TIMEOUT = 20
|
||||
|
||||
K8S_API = "https://kubernetes.default.svc"
|
||||
K8S_TOKEN_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
K8S_CA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def http(method, url, body=None, headers=None, cafile=None):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
req = urllib.request.Request(url, data=data, method=method)
|
||||
req.add_header("Accept", "application/json")
|
||||
if data is not None:
|
||||
req.add_header("Content-Type", "application/json")
|
||||
for k, v in (headers or {}).items():
|
||||
req.add_header(k, v)
|
||||
ctx = None
|
||||
if cafile:
|
||||
import ssl
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT, context=ctx) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = e.read().decode(errors="replace")[:300]
|
||||
raise ApiError(f"{method} {url} -> {e.code}: {detail}") from None
|
||||
|
||||
|
||||
# --- Kubernetes -------------------------------------------------------------
|
||||
|
||||
def k8s(path):
|
||||
with open(K8S_TOKEN_FILE) as f:
|
||||
token = f.read().strip()
|
||||
return http("GET", K8S_API + path, headers={"Authorization": f"Bearer {token}"}, cafile=K8S_CA_FILE)
|
||||
|
||||
|
||||
def desired_hosts():
|
||||
sel = urllib.parse.quote(f"{PUBLIC_LABEL}=true")
|
||||
ingresses = k8s(f"/apis/networking.k8s.io/v1/ingresses?labelSelector={sel}")
|
||||
hosts = {}
|
||||
for ing in ingresses.get("items", []):
|
||||
ref = f'{ing["metadata"]["namespace"]}/{ing["metadata"]["name"]}'
|
||||
for rule in ing.get("spec", {}).get("rules", []):
|
||||
host = rule.get("host")
|
||||
if not host or host.startswith("*"):
|
||||
continue
|
||||
hosts.setdefault(host, ref)
|
||||
return hosts
|
||||
|
||||
|
||||
def traefik_cluster_ip():
|
||||
svc = k8s(f"/api/v1/namespaces/{TRAEFIK_NS}/services/{TRAEFIK_SVC}")
|
||||
for ip in svc["spec"].get("clusterIPs", []):
|
||||
if ":" not in ip:
|
||||
return ip
|
||||
raise ApiError(f"{TRAEFIK_NS}/{TRAEFIK_SVC} has no IPv4 ClusterIP")
|
||||
|
||||
|
||||
# --- OpenBao ----------------------------------------------------------------
|
||||
|
||||
def netbird_pat():
|
||||
with open(K8S_TOKEN_FILE) as f:
|
||||
jwt = f.read().strip()
|
||||
login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
|
||||
bao_token = login["auth"]["client_token"]
|
||||
kv = http("GET", f"{BAO_ADDR}/v1/{BAO_PAT_PATH}", headers={"X-Vault-Token": bao_token})
|
||||
pat = kv["data"]["data"].get(BAO_PAT_KEY)
|
||||
if not pat:
|
||||
raise ApiError(f"OpenBao {BAO_PAT_PATH} has no key {BAO_PAT_KEY}")
|
||||
return pat
|
||||
|
||||
|
||||
# --- NetBird management -----------------------------------------------------
|
||||
|
||||
class NetBird:
|
||||
def __init__(self, pat):
|
||||
self.h = {"Authorization": f"Token {pat}"}
|
||||
|
||||
def get(self, path):
|
||||
return http("GET", f"{NB}/api/{path}", headers=self.h)
|
||||
|
||||
def list(self, path):
|
||||
# An empty collection comes back as an empty body / null, not [].
|
||||
return self.get(path) or []
|
||||
|
||||
def post(self, path, body):
|
||||
return http("POST", f"{NB}/api/{path}", body, headers=self.h)
|
||||
|
||||
def put(self, path, body):
|
||||
return http("PUT", f"{NB}/api/{path}", body, headers=self.h)
|
||||
|
||||
def delete(self, path):
|
||||
return http("DELETE", f"{NB}/api/{path}", headers=self.h)
|
||||
|
||||
|
||||
def group_id(nb, name):
|
||||
for g in nb.list("groups"):
|
||||
if g["name"] == name:
|
||||
return g["id"]
|
||||
raise ApiError(f"NetBird group {name!r} not found")
|
||||
|
||||
|
||||
def ensure_network(nb):
|
||||
for n in nb.list("networks"):
|
||||
if n["name"] == NETWORK_NAME:
|
||||
return n["id"]
|
||||
n = nb.post("networks", {"name": NETWORK_NAME,
|
||||
"description": "managed by netbird-proxy-services; public ingress via TLS passthrough"})
|
||||
LOG.info("created network %s (%s)", NETWORK_NAME, n["id"])
|
||||
return n["id"]
|
||||
|
||||
|
||||
def ensure_resource(nb, net):
|
||||
for r in nb.list(f"networks/{net}/resources"):
|
||||
if r.get("address") == SERVICE_CIDR:
|
||||
return r["id"]
|
||||
r = nb.post(f"networks/{net}/resources", {"name": "cluster service CIDR", "address": SERVICE_CIDR,
|
||||
"description": "managed by netbird-proxy-services",
|
||||
"enabled": True, "groups": []})
|
||||
LOG.info("created subnet resource %s (%s)", SERVICE_CIDR, r["id"])
|
||||
return r["id"]
|
||||
|
||||
|
||||
def ensure_router(nb, net, router_group):
|
||||
for r in nb.list(f"networks/{net}/routers"):
|
||||
if router_group in (r.get("peer_groups") or []):
|
||||
return r["id"]
|
||||
r = nb.post(f"networks/{net}/routers", {"peer_groups": [router_group], "metric": 9999,
|
||||
"masquerade": True, "enabled": True})
|
||||
LOG.info("created router on group %s (%s)", ROUTER_GROUP, r["id"])
|
||||
return r["id"]
|
||||
|
||||
|
||||
def ensure_policy(nb, resource, source_group):
|
||||
name = f"{NETWORK_NAME} access"
|
||||
for p in nb.list("policies"):
|
||||
if p["name"] == name:
|
||||
return p["id"]
|
||||
p = nb.post("policies", {"name": name, "enabled": True, "rules": [{
|
||||
"name": name, "enabled": True, "action": "accept", "protocol": "tcp",
|
||||
"ports": [str(LISTEN_PORT)], "bidirectional": False, "sources": [source_group],
|
||||
"destinationResource": {"id": resource, "type": "subnet"}}]})
|
||||
LOG.info("created policy %s (%s)", name, p["id"])
|
||||
return p["id"]
|
||||
|
||||
|
||||
def service_body(host, resource, target_ip):
|
||||
# proxy_protocol: PROXY v2 header carries the WAN client address to
|
||||
# Traefik (proxyProtocol.trustedIPs on websecure); TCP/TLS modes only.
|
||||
return {"name": host, "domain": host, "mode": "tls", "listen_port": LISTEN_PORT, "enabled": True,
|
||||
"targets": [{"target_type": "subnet", "target_id": resource, "protocol": "tcp",
|
||||
"host": target_ip, "port": 443, "enabled": True,
|
||||
"options": {"proxy_protocol": True}}]}
|
||||
|
||||
|
||||
def is_managed(svc, resource):
|
||||
return svc.get("mode") == "tls" and any(t.get("target_id") == resource for t in svc.get("targets") or [])
|
||||
|
||||
|
||||
def target_matches(svc, target_ip):
|
||||
if svc.get("listen_port") != LISTEN_PORT or not svc.get("enabled"):
|
||||
return False
|
||||
ts = svc.get("targets") or []
|
||||
return len(ts) == 1 and ts[0].get("host") == target_ip and ts[0].get("port") == 443 \
|
||||
and ts[0].get("protocol") == "tcp" and ts[0].get("enabled") \
|
||||
and (ts[0].get("options") or {}).get("proxy_protocol") is True
|
||||
|
||||
|
||||
def reconcile():
|
||||
hosts = desired_hosts()
|
||||
target_ip = traefik_cluster_ip()
|
||||
nb = NetBird(netbird_pat())
|
||||
|
||||
router_group = group_id(nb, ROUTER_GROUP)
|
||||
source_group = group_id(nb, SOURCE_GROUP)
|
||||
net = ensure_network(nb)
|
||||
resource = ensure_resource(nb, net)
|
||||
ensure_router(nb, net, router_group)
|
||||
ensure_policy(nb, resource, source_group)
|
||||
|
||||
by_domain = {s["domain"]: s for s in nb.list("reverse-proxies/services")}
|
||||
created = updated = deleted = held = 0
|
||||
|
||||
for host, ref in sorted(hosts.items()):
|
||||
svc = by_domain.get(host)
|
||||
if svc is None:
|
||||
nb.post("reverse-proxies/services", service_body(host, resource, target_ip))
|
||||
LOG.info("created %s (from %s) -> %s:443", host, ref, target_ip)
|
||||
created += 1
|
||||
elif not is_managed(svc, resource):
|
||||
LOG.warning("%s is held by unmanaged service %s (mode %s); remove it to migrate",
|
||||
host, svc["id"], svc.get("mode"))
|
||||
held += 1
|
||||
elif not target_matches(svc, target_ip):
|
||||
nb.put(f"reverse-proxies/services/{svc['id']}", service_body(host, resource, target_ip))
|
||||
LOG.info("updated %s -> %s:443", host, target_ip)
|
||||
updated += 1
|
||||
|
||||
for domain, svc in by_domain.items():
|
||||
if is_managed(svc, resource) and domain not in hosts:
|
||||
nb.delete(f"reverse-proxies/services/{svc['id']}")
|
||||
LOG.info("deleted %s (no public Ingress carries it any more)", domain)
|
||||
deleted += 1
|
||||
|
||||
LOG.info("reconciled: %d hosts desired, %d created, %d updated, %d deleted, %d held by unmanaged services",
|
||||
len(hosts), created, updated, deleted, held)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), stream=sys.stdout,
|
||||
format="%(asctime)s %(levelname)s %(message)s")
|
||||
if os.environ.get("ONCE") == "1":
|
||||
reconcile()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
reconcile()
|
||||
except Exception as e: # keep running; the next pass retries
|
||||
LOG.error("reconcile failed: %s", e)
|
||||
time.sleep(INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,84 +0,0 @@
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
|
||||
|
||||
# Everything `bootstrap.sh vmtest` needs, so the rebuild test can run as a Job in
|
||||
# the cluster instead of on a workstation. The workstation has 31G and the test
|
||||
# wants ~14.5G for four VMs — a single run left under 1G headroom, and two
|
||||
# overlapping runs OOM-killed the session. The nodes have ~94G with 70G free.
|
||||
#
|
||||
# Separate from ci-k8s rather than built on it: that image pins talosctl v1.12.6,
|
||||
# and this test creates a cluster at the version talos/patches/cluster.yaml
|
||||
# installs. A talosctl older than the Talos it provisions is exactly the kind of
|
||||
# skew the test exists to catch, so it must not be baked in here.
|
||||
#
|
||||
# KEEP TALOS_VERSION IN STEP with the installer image tag in
|
||||
# kubernetes/talos/patches/cluster.yaml. The phase reads the version from that
|
||||
# file at run time and passes it to the provisioner; this pin only decides which
|
||||
# client talks to it.
|
||||
ARG TALOS_VERSION=v1.13.9
|
||||
ARG KUBECTL_VERSION=v1.36.2
|
||||
ARG FLUX_VERSION=2.9.2
|
||||
ARG KOPIA_VERSION=0.23.1
|
||||
|
||||
# qemu-system-x86_64 and qemu-img are the provisioner's actual backend, and ovmf
|
||||
# supplies the UEFI firmware those VMs boot from — without it the provisioner
|
||||
# stops with "the required flash image was not found in any of the expected
|
||||
# paths", having already downloaded the ISO and generated the PKI. Note the
|
||||
# firmware comes from ovmf specifically: qemu-system-x86_64 ships an edk2 CODE
|
||||
# image but no VARS, which is the half the provisioner needs.
|
||||
#
|
||||
# iptables is shelled out to by the CNI plugins that build the VM network; the
|
||||
# provisioner fails with "error accessing iptables" without it. The rest is
|
||||
# what the phase shells out to: helm for the Cilium install, yq to read the
|
||||
# HelmRelease and the installer image, jq for OpenBao's seal status.
|
||||
#
|
||||
# Versions come from the digest-pinned base rather than being pinned one by one,
|
||||
# the same way ci-ansible does it.
|
||||
RUN apk add --no-cache \
|
||||
bash \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
helm \
|
||||
iptables \
|
||||
jq \
|
||||
openssl \
|
||||
ovmf \
|
||||
qemu-img \
|
||||
qemu-system-x86_64 \
|
||||
yq
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/talosctl \
|
||||
"https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
|
||||
&& echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/talosctl
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
"https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
|
||||
&& curl -sfL -o /tmp/kubectl.sha256 \
|
||||
"https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" \
|
||||
&& echo "$(cat /tmp/kubectl.sha256) /usr/local/bin/kubectl" | sha256sum -c - \
|
||||
&& rm -f /tmp/kubectl.sha256 \
|
||||
&& chmod +x /usr/local/bin/kubectl
|
||||
|
||||
RUN curl -sfL -o /tmp/flux.tar.gz \
|
||||
"https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \
|
||||
&& echo "4092b367fd060097976fb7261601f79420ecf7266328417dfd8dfee27de0e6a3 /tmp/flux.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/flux.tar.gz -C /usr/local/bin flux \
|
||||
&& chmod +x /usr/local/bin/flux \
|
||||
&& rm -f /tmp/flux.tar.gz
|
||||
|
||||
# kopia reads the Velero backup repository directly, which is what the OpenBao
|
||||
# restore does when there is no Velero yet to do it: the credentials Velero
|
||||
# needs live in OpenBao, so OpenBao cannot be restored by Velero. Its absence
|
||||
# was found by a rebuild rehearsal reaching the restore for the first time and
|
||||
# stopping at "kopia not found".
|
||||
#
|
||||
# Same checksum-verified download as the tools above; the tarball unpacks into
|
||||
# a versioned directory, so the binary is pulled out by name.
|
||||
RUN curl -sfL -o /tmp/kopia.tar.gz \
|
||||
"https://github.com/kopia/kopia/releases/download/v${KOPIA_VERSION}/kopia-${KOPIA_VERSION}-linux-x64.tar.gz" \
|
||||
&& echo "416d0f84a3dbb321a8b2d8f0997b1a0a6e915babe79ee76fa6e4d2bd1e1c5178 /tmp/kopia.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/kopia.tar.gz -C /tmp \
|
||||
&& mv "/tmp/kopia-${KOPIA_VERSION}-linux-x64/kopia" /usr/local/bin/kopia \
|
||||
&& chmod +x /usr/local/bin/kopia \
|
||||
&& rm -rf /tmp/kopia.tar.gz "/tmp/kopia-${KOPIA_VERSION}-linux-x64"
|
||||
@@ -1,10 +0,0 @@
|
||||
# wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea,
|
||||
# queried by ~/Notes/bin/wiki_rank.py. Standard library only; runs unprivileged.
|
||||
FROM python:3.13-slim
|
||||
|
||||
RUN useradd --system --no-create-home --shell /usr/sbin/nologin wikiindex
|
||||
COPY wiki_index.py /opt/wiki_index.py
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
USER wikiindex
|
||||
ENTRYPOINT ["python3", "/opt/wiki_index.py"]
|
||||
@@ -1,178 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unit tests for wiki_index. Run: python3 -m unittest test_wiki_index"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import wiki_index as wi
|
||||
|
||||
NOTE = """---
|
||||
title: 'UDM intercepts DNS'
|
||||
type: wiki
|
||||
tags: [dns, unifi]
|
||||
aliases: [udm dns interception, dig 1.1.1.1 wrong]
|
||||
---
|
||||
|
||||
# UDM intercepts DNS
|
||||
|
||||
The UDM answers every UDP/53 query itself.
|
||||
|
||||
## Details
|
||||
|
||||
More.
|
||||
"""
|
||||
|
||||
|
||||
class ParseTests(unittest.TestCase):
|
||||
def test_note_text(self):
|
||||
title, text = wi.note_text("udm-dns", NOTE)
|
||||
self.assertEqual(title, "UDM intercepts DNS")
|
||||
self.assertIn("udm dns UDM intercepts DNS udm dns interception", text)
|
||||
self.assertIn("dns unifi", text)
|
||||
self.assertTrue(text.endswith("The UDM answers every UDP/53 query itself."))
|
||||
self.assertLessEqual(len(text), wi.EMBED_CHARS)
|
||||
|
||||
def test_credentials_are_stripped(self):
|
||||
original = wi.http
|
||||
wi.http = lambda method, url, body=None, headers=None, timeout=60.0: (
|
||||
{"auth": {"client_token": "t"}} if url.endswith("/login")
|
||||
else {"data": {"data": {"gitea_token": "abc\n", "webhook_secret": " s3cret\n"}}})
|
||||
original_sa = wi.SA_TOKEN
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
wi.SA_TOKEN = Path(d) / "token"
|
||||
wi.SA_TOKEN.write_text("jwt\n")
|
||||
self.assertEqual(wi.Credentials().fetch(), {"gitea_token": "abc", "webhook_secret": "s3cret"})
|
||||
finally:
|
||||
wi.http, wi.SA_TOKEN = original, original_sa
|
||||
|
||||
def test_tree_paging_stops_on_short_last_page(self):
|
||||
# Gitea: truncated=true on every page including the last, "tree": null past it.
|
||||
pages = {
|
||||
1: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000)], "truncated": True, "total_count": 1192},
|
||||
2: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000, 1192)] + [{"path": "raw/x.md", "type": "blob", "sha": "r"}, {"path": "wiki/sub/y.md", "type": "blob", "sha": "y"}], "truncated": True, "total_count": 1192},
|
||||
3: {"tree": None, "truncated": False, "total_count": 1192},
|
||||
}
|
||||
calls = []
|
||||
original = wi.http
|
||||
|
||||
def fake_http(method, url, body=None, headers=None, timeout=60.0):
|
||||
page = int(url.rsplit("page=", 1)[1])
|
||||
calls.append(page)
|
||||
return pages[page]
|
||||
|
||||
wi.http = fake_http
|
||||
try:
|
||||
tree = wi.Gitea("t").tree("abc")
|
||||
finally:
|
||||
wi.http = original
|
||||
self.assertEqual(calls, [1, 2])
|
||||
self.assertEqual(len(tree), 1192)
|
||||
self.assertNotIn("raw/x.md", tree)
|
||||
self.assertNotIn("wiki/sub/y.md", tree)
|
||||
self.assertEqual(tree["wiki/n1191.md"], "s1191")
|
||||
|
||||
def test_signature(self):
|
||||
body = b'{"ref":"refs/heads/master"}'
|
||||
sig = hmac.new(b"s3cret", body, hashlib.sha256).hexdigest()
|
||||
self.assertTrue(wi.verify_signature("s3cret", body, sig))
|
||||
self.assertFalse(wi.verify_signature("other", body, sig))
|
||||
self.assertFalse(wi.verify_signature("s3cret", body, ""))
|
||||
|
||||
|
||||
class FakeGitea:
|
||||
def __init__(self, head, tree, files):
|
||||
self._head, self._tree, self.files = head, tree, files
|
||||
self.raw_calls = []
|
||||
|
||||
def head(self):
|
||||
return self._head
|
||||
|
||||
def tree(self, commit):
|
||||
return self._tree
|
||||
|
||||
def raw(self, path, ref):
|
||||
self.raw_calls.append(path)
|
||||
return self.files[path]
|
||||
|
||||
|
||||
class IndexTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.path = Path(self.tmp.name) / "index.json"
|
||||
self.original_embed = wi.embed
|
||||
self.embed_calls = []
|
||||
|
||||
def fake_embed(texts, batch=2):
|
||||
self.embed_calls.append(list(texts))
|
||||
return [wi.unit([1.0, float(len(t) % 7)]) for t in texts]
|
||||
|
||||
wi.embed = fake_embed
|
||||
|
||||
def tearDown(self):
|
||||
wi.embed = self.original_embed
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_sync_is_incremental_and_persists(self):
|
||||
files = {"wiki/udm-dns.md": NOTE, "wiki/ceph.md": "---\ntitle: Ceph\n---\n# Ceph\n\nOSD.\n", "wiki/_index.md": "---\ntitle: Wiki\n---\n"}
|
||||
tree = {"wiki/udm-dns.md": "aaa", "wiki/ceph.md": "bbb", "wiki/_index.md": "ccc"}
|
||||
g = FakeGitea("c1", tree, files)
|
||||
idx = wi.Index(self.path)
|
||||
s = idx.sync(g)
|
||||
self.assertEqual((s["changed"], s["removed"], s["notes"]), (2, 0, 2))
|
||||
self.assertEqual(set(idx.notes), {"udm-dns", "ceph"})
|
||||
self.assertEqual(idx.notes["udm-dns"]["title"], "UDM intercepts DNS")
|
||||
self.assertTrue(self.path.exists())
|
||||
|
||||
# head unchanged: nothing fetched
|
||||
s = idx.sync(g)
|
||||
self.assertEqual(s.get("skipped"), "head unchanged")
|
||||
self.assertEqual(len(g.raw_calls), 2)
|
||||
|
||||
# one note changed, one removed, new head
|
||||
g2 = FakeGitea("c2", {"wiki/udm-dns.md": "aaa2"}, {"wiki/udm-dns.md": NOTE.replace("itself", "itself!")})
|
||||
s = idx.sync(g2)
|
||||
self.assertEqual((s["changed"], s["removed"], s["notes"]), (1, 1, 1))
|
||||
self.assertEqual(g2.raw_calls, ["wiki/udm-dns.md"])
|
||||
|
||||
# restart loads the persisted index
|
||||
idx2 = wi.Index(self.path)
|
||||
self.assertEqual(idx2.commit, "c2")
|
||||
self.assertEqual(set(idx2.notes), {"udm-dns"})
|
||||
|
||||
def test_rank_orders_by_cosine(self):
|
||||
idx = wi.Index(self.path)
|
||||
idx.notes = {"a": {"sha": "1", "title": "A", "vector": wi.unit([1.0, 0.0])},
|
||||
"b": {"sha": "2", "title": "B", "vector": wi.unit([0.0, 1.0])}}
|
||||
wi.embed = lambda texts, batch=2: [wi.unit([0.2, 1.0])]
|
||||
rows = idx.rank("q", k=5)
|
||||
self.assertEqual([r["slug"] for r in rows], ["b", "a"])
|
||||
self.assertGreater(rows[0]["score"], rows[1]["score"])
|
||||
|
||||
def test_status_reports_age_and_errors(self):
|
||||
idx = wi.Index(self.path)
|
||||
self.assertFalse(idx.status()["ready"])
|
||||
g = FakeGitea("c1", {"wiki/a.md": "x"}, {"wiki/a.md": "---\ntitle: A\n---\n# A\n\nt\n"})
|
||||
idx.sync(g)
|
||||
st = idx.status()
|
||||
self.assertTrue(st["ready"])
|
||||
self.assertEqual(st["notes"], 1)
|
||||
self.assertEqual(st["commit"], "c1")
|
||||
|
||||
class Broken(FakeGitea):
|
||||
def head(self):
|
||||
raise wi.ApiError("gitea down")
|
||||
|
||||
with self.assertRaises(wi.ApiError):
|
||||
idx.sync(Broken("c1", {}, {}))
|
||||
self.assertIn("gitea down", idx.status()["last_error"])
|
||||
self.assertEqual(idx.status()["notes"], 1) # last good index kept
|
||||
self.assertTrue(idx.status()["ready"]) # and still served
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,408 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea.
|
||||
|
||||
Source of truth is the notes repository on Gitea. The index holds, per wiki
|
||||
note, the git blob sha, title and a bge-m3 vector of slug/title/aliases/tags/
|
||||
first paragraph. A sync fetches the repo tree, embeds every note whose blob sha
|
||||
changed (via wiki-embed, in-cluster) and drops notes that left the tree; it
|
||||
runs at start, on every push webhook from Gitea, and every RESYNC_SECONDS.
|
||||
|
||||
HTTP:
|
||||
POST /rank {"query": str, "k": int} -> {"rows": [{"slug","title","score"}], "index": {...}}
|
||||
GET /health -> 200 always (liveness) with {"ready", "notes", "commit", "synced_at", "age_s", "syncing", "last_error"}
|
||||
GET /ready -> 200 when an index exists, else 503 (readiness)
|
||||
POST /webhook Gitea push event, HMAC-SHA256 in X-Gitea-Signature -> {"queued": true}
|
||||
|
||||
Credentials (Gitea token, webhook secret) are read from OpenBao through the
|
||||
Kubernetes auth role on every use; they never land in a Secret or a log line.
|
||||
Standard library only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
LOG = logging.getLogger("wiki-index")
|
||||
|
||||
GITEA_URL = os.environ.get("GITEA_URL", "https://gitea.renner.uno").rstrip("/")
|
||||
NOTES_REPO = os.environ.get("NOTES_REPO", "k.renner/notes")
|
||||
NOTES_BRANCH = os.environ.get("NOTES_BRANCH", "master")
|
||||
WIKI_DIR = os.environ.get("WIKI_DIR", "wiki").strip("/")
|
||||
EMBED_URL = os.environ.get("EMBED_URL", "http://wiki-embed.ai.svc.cluster.local:8080").rstrip("/")
|
||||
EMBED_MODEL = os.environ.get("EMBED_MODEL", "bge-m3")
|
||||
EMBED_CHARS = int(os.environ.get("EMBED_CHARS", "1500"))
|
||||
INDEX_PATH = Path(os.environ.get("INDEX_PATH", "/data/index.json"))
|
||||
RESYNC_SECONDS = int(os.environ.get("RESYNC_SECONDS", "1800"))
|
||||
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
|
||||
BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
|
||||
BAO_ROLE = os.environ.get("BAO_ROLE", "wiki-index")
|
||||
BAO_SECRET_PATH = os.environ.get("BAO_SECRET_PATH", "homelab/data/apps/ai/wiki-index")
|
||||
SA_TOKEN = Path(os.environ.get("SA_TOKEN", "/var/run/secrets/kubernetes.io/serviceaccount/token"))
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- http helpers
|
||||
|
||||
|
||||
def http(method: str, url: str, body=None, headers=None, timeout: float = 60.0):
|
||||
data = None
|
||||
hdrs = dict(headers or {})
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs.setdefault("Content-Type", "application/json")
|
||||
req = urllib.request.Request(url, data=data, method=method, headers=hdrs)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
ctype = resp.headers.get("Content-Type", "")
|
||||
except urllib.error.HTTPError as e:
|
||||
raise ApiError(f"{method} {url}: HTTP {e.code}: {e.read().decode('utf-8', 'replace')[:300]}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise ApiError(f"{method} {url}: {e.reason}") from e
|
||||
if "json" in ctype:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
return raw
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- credentials
|
||||
|
||||
|
||||
class Credentials:
|
||||
"""Gitea token and webhook secret from OpenBao, fetched per use."""
|
||||
|
||||
def __init__(self, static: dict | None = None):
|
||||
self.static = static # tests / local runs
|
||||
|
||||
def fetch(self) -> dict:
|
||||
if self.static is not None:
|
||||
return self.static
|
||||
jwt = SA_TOKEN.read_text(encoding="utf-8").strip()
|
||||
login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
|
||||
token = login["auth"]["client_token"]
|
||||
kv = http("GET", f"{BAO_ADDR}/v1/{BAO_SECRET_PATH}", headers={"X-Vault-Token": token})
|
||||
data = {k: str(v).strip() for k, v in kv["data"]["data"].items()} # `bao kv put key=@file` keeps the file's newline
|
||||
for key in ("gitea_token", "webhook_secret"):
|
||||
if not data.get(key):
|
||||
raise ApiError(f"OpenBao {BAO_SECRET_PATH} has no key {key}")
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- gitea
|
||||
|
||||
|
||||
class Gitea:
|
||||
def __init__(self, token: str, base: str = GITEA_URL, repo: str = NOTES_REPO, branch: str = NOTES_BRANCH):
|
||||
self.base, self.repo, self.branch = base, repo, branch
|
||||
self.headers = {"Authorization": f"token {token}"}
|
||||
|
||||
def api(self, path: str, **params):
|
||||
q = ("?" + urllib.parse.urlencode(params)) if params else ""
|
||||
return http("GET", f"{self.base}/api/v1/repos/{self.repo}{path}{q}", headers=self.headers)
|
||||
|
||||
def head(self) -> str:
|
||||
return self.api(f"/branches/{urllib.parse.quote(self.branch, safe='')}")["commit"]["id"]
|
||||
|
||||
def tree(self, commit: str) -> dict[str, str]:
|
||||
"""path -> blob sha for every wiki note at that commit."""
|
||||
out: dict[str, str] = {}
|
||||
page, per_page, seen = 1, 1000, 0
|
||||
while True:
|
||||
t = self.api(f"/git/trees/{commit}", recursive="true", per_page=per_page, page=page)
|
||||
entries = t.get("tree") or [] # Gitea answers "tree": null past the last page
|
||||
for e in entries:
|
||||
p = e.get("path", "")
|
||||
if e.get("type") == "blob" and p.startswith(WIKI_DIR + "/") and p.endswith(".md") and "/" not in p[len(WIKI_DIR) + 1:]:
|
||||
out[p] = e["sha"]
|
||||
seen += len(entries)
|
||||
total = int(t.get("total_count") or 0)
|
||||
if not entries or len(entries) < per_page or (total and seen >= total):
|
||||
break
|
||||
page += 1
|
||||
if page > 50:
|
||||
raise ApiError("tree paging runaway")
|
||||
return out
|
||||
|
||||
def raw(self, path: str, ref: str) -> str:
|
||||
data = http("GET", f"{self.base}/api/v1/repos/{self.repo}/raw/{urllib.parse.quote(path)}?ref={urllib.parse.quote(ref, safe='')}",
|
||||
headers=self.headers)
|
||||
return data.decode("utf-8") if isinstance(data, bytes) else json.dumps(data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- notes
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict, str]:
|
||||
m = re.match(r"^---[ \t]*\n(.*?)\n---[ \t]*(?:\n|$)", text, re.DOTALL)
|
||||
if not m:
|
||||
return {}, text
|
||||
fm: dict = {}
|
||||
for line in m.group(1).splitlines():
|
||||
mm = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
||||
if mm:
|
||||
fm[mm.group(1)] = mm.group(2).strip()
|
||||
return fm, text[m.end():]
|
||||
|
||||
|
||||
def as_list(value: str) -> list[str]:
|
||||
s = (value or "").strip()
|
||||
if s.startswith("[") and s.endswith("]"):
|
||||
s = s[1:-1]
|
||||
return [p.strip().strip("'\"") for p in s.split(",") if p.strip()]
|
||||
|
||||
|
||||
def first_paragraph(body: str) -> str:
|
||||
para: list[str] = []
|
||||
seen_h1 = False
|
||||
for line in body.splitlines():
|
||||
if line.startswith("# "):
|
||||
seen_h1 = True
|
||||
continue
|
||||
if not seen_h1:
|
||||
continue
|
||||
if line.strip() == "" or line.startswith("#"):
|
||||
if para:
|
||||
break
|
||||
continue
|
||||
para.append(line.strip())
|
||||
return " ".join(para)
|
||||
|
||||
|
||||
def note_text(slug: str, text: str) -> tuple[str, str]:
|
||||
"""(title, embedding text) for one note file."""
|
||||
fm, body = parse_frontmatter(text)
|
||||
title = (fm.get("title") or slug).strip().strip("'\"")
|
||||
parts = [slug.replace("-", " "), title, " ".join(as_list(fm.get("aliases", ""))),
|
||||
" ".join(as_list(fm.get("tags", ""))), first_paragraph(body)]
|
||||
return title, " ".join(parts)[:EMBED_CHARS]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- embeddings
|
||||
|
||||
|
||||
def embed(texts: list[str], batch: int = 2) -> list[list[float]]:
|
||||
out: list[list[float]] = []
|
||||
for i in range(0, len(texts), batch):
|
||||
resp = http("POST", f"{EMBED_URL}/v1/embeddings", {"model": EMBED_MODEL, "input": texts[i:i + batch]}, timeout=120)
|
||||
rows = sorted(resp["data"], key=lambda d: d["index"])
|
||||
out.extend(unit(d["embedding"]) for d in rows)
|
||||
return out
|
||||
|
||||
|
||||
def unit(v: list[float]) -> list[float]:
|
||||
n = sum(x * x for x in v) ** 0.5 or 1.0
|
||||
return [round(x / n, 5) for x in v]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- index
|
||||
|
||||
|
||||
class Index:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.lock = threading.Lock()
|
||||
self.notes: dict[str, dict] = {} # slug -> {sha, title, vector}
|
||||
self.commit = ""
|
||||
self.synced_at = 0.0
|
||||
self.syncing = False
|
||||
self.last_error = ""
|
||||
if path.exists():
|
||||
try:
|
||||
d = json.loads(path.read_text(encoding="utf-8"))
|
||||
self.notes, self.commit, self.synced_at = d.get("notes", {}), d.get("commit", ""), d.get("synced_at", 0.0)
|
||||
LOG.info("loaded index: %d notes at %s", len(self.notes), self.commit[:8])
|
||||
except (OSError, ValueError) as e:
|
||||
LOG.warning("index unreadable (%s); starting empty", e)
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = self.path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps({"notes": self.notes, "commit": self.commit, "synced_at": self.synced_at}), encoding="utf-8")
|
||||
tmp.replace(self.path)
|
||||
|
||||
def sync(self, gitea: Gitea) -> dict:
|
||||
"""Bring the index to the branch head; returns a summary."""
|
||||
with self.lock:
|
||||
self.syncing = True
|
||||
try:
|
||||
head = gitea.head()
|
||||
if head == self.commit and self.notes:
|
||||
summary = {"commit": head, "changed": 0, "removed": 0, "notes": len(self.notes), "skipped": "head unchanged"}
|
||||
self.synced_at = time.time()
|
||||
self.save()
|
||||
return summary
|
||||
tree = gitea.tree(head)
|
||||
slugs = {p[len(WIKI_DIR) + 1:-3]: (p, sha) for p, sha in tree.items() if not p.endswith("/_index.md")}
|
||||
todo = [s for s, (p, sha) in slugs.items() if self.notes.get(s, {}).get("sha") != sha]
|
||||
removed = [s for s in list(self.notes) if s not in slugs]
|
||||
LOG.info("sync %s: %d notes, %d to embed, %d removed", head[:8], len(slugs), len(todo), len(removed))
|
||||
fresh: dict[str, dict] = {}
|
||||
for i in range(0, len(todo), 20):
|
||||
chunk = todo[i:i + 20]
|
||||
texts, titles = [], {}
|
||||
for s in chunk:
|
||||
title, text = note_text(s, gitea.raw(slugs[s][0], head))
|
||||
titles[s] = title
|
||||
texts.append(text)
|
||||
for s, vec in zip(chunk, embed(texts)):
|
||||
fresh[s] = {"sha": slugs[s][1], "title": titles[s], "vector": vec}
|
||||
with self.lock:
|
||||
for s in removed:
|
||||
self.notes.pop(s, None)
|
||||
self.notes.update(fresh)
|
||||
self.commit = head
|
||||
self.synced_at = time.time()
|
||||
self.last_error = ""
|
||||
self.save()
|
||||
return {"commit": head, "changed": len(todo), "removed": len(removed), "notes": len(self.notes)}
|
||||
except Exception as e:
|
||||
with self.lock:
|
||||
self.last_error = str(e)[:300]
|
||||
raise
|
||||
finally:
|
||||
with self.lock:
|
||||
self.syncing = False
|
||||
|
||||
def rank(self, query: str, k: int = 60) -> list[dict]:
|
||||
q = embed([query])[0]
|
||||
with self.lock:
|
||||
rows = [(s, n["title"], sum(a * b for a, b in zip(q, n["vector"]))) for s, n in self.notes.items()]
|
||||
rows.sort(key=lambda r: -r[2])
|
||||
return [{"slug": s, "title": t, "score": round(sc, 4)} for s, t, sc in rows[:k]]
|
||||
|
||||
def status(self) -> dict:
|
||||
"""`ready` = an index exists and can answer /rank; a failed sync keeps the last good index."""
|
||||
with self.lock:
|
||||
return {"ready": bool(self.notes), "notes": len(self.notes), "commit": self.commit,
|
||||
"synced_at": self.synced_at, "age_s": round(time.time() - self.synced_at) if self.synced_at else None,
|
||||
"syncing": self.syncing, "last_error": self.last_error or None}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- syncer
|
||||
|
||||
|
||||
class Syncer(threading.Thread):
|
||||
"""One worker: syncs on demand (webhook) and every RESYNC_SECONDS."""
|
||||
|
||||
def __init__(self, index: Index, creds: Credentials, interval: int):
|
||||
super().__init__(daemon=True, name="syncer")
|
||||
self.index, self.creds, self.interval = index, creds, interval
|
||||
self.wake = threading.Event()
|
||||
|
||||
def trigger(self) -> None:
|
||||
self.wake.set()
|
||||
|
||||
RETRY_SECONDS = 60
|
||||
|
||||
def run(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
data = self.creds.fetch()
|
||||
summary = self.index.sync(Gitea(data["gitea_token"]))
|
||||
LOG.info("sync done: %s", summary)
|
||||
wait = self.interval
|
||||
except Exception as e: # keep serving the last good index; retry soon, not next interval
|
||||
LOG.error("sync failed: %s", e)
|
||||
wait = self.RETRY_SECONDS
|
||||
self.wake.wait(wait)
|
||||
self.wake.clear()
|
||||
|
||||
|
||||
def verify_signature(secret: str, body: bytes, signature: str) -> bool:
|
||||
digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(digest, (signature or "").strip())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- server
|
||||
|
||||
|
||||
def make_handler(index: Index, syncer: Syncer, creds: Credentials):
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
LOG.info("%s %s", self.address_string(), fmt % args)
|
||||
|
||||
def send_json(self, code: int, obj) -> None:
|
||||
data = json.dumps(obj).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def body(self) -> bytes:
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
return self.rfile.read(n) if n else b""
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health": # liveness: the process serves
|
||||
return self.send_json(200, index.status())
|
||||
if self.path == "/ready": # readiness: an index exists
|
||||
st = index.status()
|
||||
return self.send_json(200 if st["ready"] else 503, st)
|
||||
self.send_json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self):
|
||||
if self.path == "/rank":
|
||||
try:
|
||||
req = json.loads(self.body() or b"{}")
|
||||
query = str(req.get("query", "")).strip()
|
||||
k = max(1, min(int(req.get("k", 60)), 500))
|
||||
if not query:
|
||||
return self.send_json(400, {"error": "query required"})
|
||||
rows = index.rank(query, k)
|
||||
return self.send_json(200, {"rows": rows, "index": index.status()})
|
||||
except ApiError as e:
|
||||
return self.send_json(502, {"error": str(e)})
|
||||
except (ValueError, TypeError) as e:
|
||||
return self.send_json(400, {"error": str(e)})
|
||||
if self.path == "/webhook":
|
||||
raw = self.body()
|
||||
try:
|
||||
secret = creds.fetch()["webhook_secret"]
|
||||
except ApiError as e:
|
||||
return self.send_json(503, {"error": str(e)})
|
||||
if not verify_signature(secret, raw, self.headers.get("X-Gitea-Signature", "")):
|
||||
return self.send_json(403, {"error": "bad signature"})
|
||||
try:
|
||||
event = json.loads(raw or b"{}")
|
||||
except ValueError:
|
||||
return self.send_json(400, {"error": "bad json"})
|
||||
ref = event.get("ref", "")
|
||||
if ref and ref != f"refs/heads/{NOTES_BRANCH}":
|
||||
return self.send_json(200, {"queued": False, "ref": ref})
|
||||
syncer.trigger()
|
||||
return self.send_json(200, {"queued": True})
|
||||
self.send_json(404, {"error": "not found"})
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout)
|
||||
index = Index(INDEX_PATH)
|
||||
creds = Credentials()
|
||||
syncer = Syncer(index, creds, RESYNC_SECONDS)
|
||||
syncer.start()
|
||||
server = ThreadingHTTPServer(("", LISTEN_PORT), make_handler(index, syncer, creds))
|
||||
LOG.info("listening on :%d, repo %s@%s, embed %s", LISTEN_PORT, NOTES_REPO, NOTES_BRANCH, EMBED_URL)
|
||||
server.serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -50,18 +50,15 @@ pass insert homelab/openbao/root-token
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator unseal # paste key
|
||||
```
|
||||
|
||||
**Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
|
||||
the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
|
||||
from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
|
||||
itself. The shamir key above became the **recovery key** in the migration; it
|
||||
is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
|
||||
runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
|
||||
one is neither a voter nor a standby, so a replica that stays sealed after a
|
||||
restart — wrong or missing seal-key Secret — costs redundancy, two cost
|
||||
quorum. The trade-off taken with the static seal: the key that unlocks the
|
||||
store sits in the same etcd as the Secrets ESO already syncs there in
|
||||
plaintext. Symptom of sealed: ESO stores go NotReady, synced Secrets keep
|
||||
their last value (they are real k8s Secrets — pods keep running).
|
||||
**Manual unseal is required after every pod restart** (node drain, upgrade,
|
||||
crash), **per replica** — the server runs three raft peers (see
|
||||
`infrastructure/openbao/helmrelease.yaml`), and a sealed one is neither a
|
||||
voter nor a standby, so it stays out of the cluster until unsealed:
|
||||
`kubectl -n openbao exec -it openbao-<N> -- bao operator unseal`, same key for
|
||||
all three. Three peers mean one sealed replica is survivable; two are not.
|
||||
Accepted for now; auto-unseal needs a second trust root (transit/KMS) and is
|
||||
explicitly out of scope. Symptom of sealed: ESO stores go NotReady, synced
|
||||
Secrets keep their last value (they are real k8s Secrets — pods keep running).
|
||||
|
||||
## Phase 2 — configure + ClusterSecretStore (follow-up PR)
|
||||
|
||||
@@ -237,19 +234,16 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
|
||||
- **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
|
||||
dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
|
||||
`openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
|
||||
A raft snapshot restore needs the matching seal key — which lives in
|
||||
A raft snapshot restore needs the matching unseal key — which lives in
|
||||
`pass`, outside the cluster (GPG key backed up externally 2026-07-14).
|
||||
A failed snapshot Job is the sealed-after-restart alarm. Still owed:
|
||||
one tested restore before phase 4.
|
||||
- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
|
||||
openbao (empty) → init → restore raft snapshot → replicas unseal themselves
|
||||
→ ClusterSecretStore Ready → ESO materializes Secrets → apps start
|
||||
(`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
|
||||
hand). Until then, secret-consuming apps simply stay Pending on missing
|
||||
Secrets — same as any missing dependency.
|
||||
- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
|
||||
offline-capable). With those, everything is recoverable from a raft
|
||||
snapshot alone.
|
||||
- **Cluster rebuild order:** Flux brings up openbao (sealed, empty) → restore
|
||||
raft snapshot → unseal (key from `pass`) → ClusterSecretStore Ready → ESO
|
||||
materializes Secrets → apps start. Until unseal, secret-consuming apps
|
||||
simply stay Pending on missing Secrets — same as any missing dependency.
|
||||
- **Break-glass:** `pass` holds unseal key + root token (GPG, offline-capable).
|
||||
With those two, everything is recoverable from a raft snapshot alone.
|
||||
- **Ansible Vault password:** `ansible/vault-password.sh` reads
|
||||
`ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
|
||||
2026-07-14. It gates decryption of several Ansible-Vault files (PBS
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
# Traffic classes on the k8s nodes
|
||||
|
||||
Issue #576. Three of the four classes are separated; management is not.
|
||||
|
||||
## Where each class runs
|
||||
|
||||
| class | link | mechanism |
|
||||
|---|---|---|
|
||||
| Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) |
|
||||
| world→pod | `ingress0`, VLAN 47 | BGP session sourced there, so the advertised next hop is that address |
|
||||
| pod→world | `ingress0`, VLAN 47 | routing rule on the pod source prefix → table 47 |
|
||||
| pod↔pod | `pod0`, VLAN 48 | routing rule on pod source **and** destination → table 48 |
|
||||
| management | `mgmt0`, VLAN 40 | not started |
|
||||
|
||||
Hard constraint, set by the owner: inter-node and pod traffic never traverses
|
||||
the UDM. Everything above stays on the 10G switch.
|
||||
|
||||
## Why policy routing rather than moving the node IP
|
||||
|
||||
Cilium reaches a peer's pods at that peer's Kubernetes InternalIP, in native
|
||||
and tunnel mode alike. `--ipv4-node` is overwritten from the Node object, and
|
||||
`nodePort.directRoutingDevice` selects the BPF NodePort device, not a next hop.
|
||||
Both were established by test (#837, #838), not by reading.
|
||||
|
||||
So the InternalIP decides where pod traffic goes — unless the lookup happens in
|
||||
a different table. The rules do that, and nothing about the node's identity
|
||||
moves. The routing decision runs before Cilium's masquerade hook, so a rule
|
||||
matching the pod source sees the pod address, and the masquerade then rewrites
|
||||
it to the egress link's.
|
||||
|
||||
## Rule order
|
||||
|
||||
Lower priority runs first.
|
||||
|
||||
| prio | match | table |
|
||||
|---|---|---|
|
||||
| 980 | src pod, dst pod | 48 |
|
||||
| 981 | src pod, dst pod (v4) | 48 |
|
||||
| 990 | dst `fd00::/8` | main |
|
||||
| 991 | dst `10.0.0.0/8` | main |
|
||||
| 1000 | src pod | 47 |
|
||||
| 1001 | src pod (v4) | 47 |
|
||||
|
||||
980/981 match source **and** destination. Source alone would also catch the
|
||||
node's own traffic to a pod, which would then leave with a `pod0` source — an
|
||||
address `CiliumNode.spec.addresses` does not list, so Cilium classifies it as
|
||||
`world` rather than `host`.
|
||||
|
||||
990/991 are Cilium's `ipv4/ipv6NativeRoutingCIDR`, and they are what keeps
|
||||
cluster-internal traffic out of table 47. Without them a pod's packet to a
|
||||
service matches the source rule, finds only a default route, and leaves for the
|
||||
UDM — measured, in-cluster DNS stopped resolving.
|
||||
|
||||
Traffic to a pod on the same node falls through on its own: table 48 holds only
|
||||
the peers' CIDRs, so the lookup misses and 990/991 send it to main.
|
||||
|
||||
## Where each table comes from
|
||||
|
||||
- **47** — two default routes declared per node in `talos/patches/k8s-*.yaml`.
|
||||
The IPv6 gateway is the UDM's configured address on the segment, not the
|
||||
RA-learned link-local; `accept_ra_defrtr` stays `0` on `ingress0` so the RA
|
||||
default does not land in main.
|
||||
- **48** — filled by the `bird` DaemonSet: an iBGP mesh over `pod0`, one route
|
||||
per peer pod CIDR with the peer's `pod0` address as next hop.
|
||||
|
||||
## The bird mesh
|
||||
|
||||
`kubernetes/infrastructure/bird/`. Each node advertises only its own pod CIDR
|
||||
and installs what it learns into table 48. Cilium's routes in main are
|
||||
untouched, which is why the two coexist.
|
||||
|
||||
No address is written into this repo twice. Each node publishes its `pod0`
|
||||
addresses as `nodeAnnotations`, beside the `VLANConfig` that assigns them; the
|
||||
pod CIDRs come from the Node objects. An init container renders both from the
|
||||
API with `kubectl -o go-template-file`, excludes the node itself with a field
|
||||
selector, and ends with `bird -p` so BIRD's own parser gates startup.
|
||||
|
||||
The node's own CIDR is **originated**, not read back from the kernel. Reading
|
||||
it back made BIRD depend on the shape Cilium installs, and the IPv4 route is
|
||||
rejected outright: its next hop sits inside the prefix it describes, which KRT
|
||||
refuses with `strange next-hop`. The IPv6 route is a device route and passed,
|
||||
so the failure was silent on one family only.
|
||||
|
||||
## Failure behaviour
|
||||
|
||||
Benign at every layer. An empty or absent table 48 makes 980/981 fall through
|
||||
to main, and pod traffic returns to VLAN 40 — unseparated, not broken. The same
|
||||
holds if the DaemonSet cannot start, which is the case during a cold rebuild
|
||||
until Gitea serves its image.
|
||||
|
||||
## Management, the remaining class
|
||||
|
||||
Not started. The levers differ per service and none of them is the node IP:
|
||||
|
||||
| service | can leave the node IP? | lever |
|
||||
|---|---|---|
|
||||
| etcd | yes | `cluster.etcd.advertisedSubnets`; needs a reboot per node |
|
||||
| kube-apiserver endpoint | yes | it is a VIP — `Layer2VIPConfig.link` |
|
||||
| Talos API | n/a | `apid` answers on every node address; a firewall question |
|
||||
| kubelet | no | it *is* the node IP |
|
||||
|
||||
Two approaches were rejected by the owner and are not to be proposed again:
|
||||
moving the node IP and etcd to `pod0` (#838), and relabelling VLAN 40 as the
|
||||
cluster VLAN while moving etcd and the API VIP to a new management VLAN.
|
||||
Replacing the CNI (#839, Antrea) was shelved.
|
||||
|
||||
Note that the reasoning which produced that list also concluded pod↔pod was
|
||||
blocked, which policy routing disproved. Treat it as a record of decisions, not
|
||||
as an exhaustive analysis.
|
||||
|
||||
## Sources
|
||||
|
||||
Cilium v1.20.1 `pkg/datapath/linux/node.go`, `pkg/node/types/node.go`,
|
||||
`pkg/node/sync/local_node_sync.go`; BIRD 2.17 user manual (kernel protocol,
|
||||
`learn all`, `strict bind`, `direct`); Talos v1.13 `RoutingRuleConfig` and
|
||||
`machine.nodeAnnotations`. PRs #831, #833, #835, #836, #837, #842, #852, #854,
|
||||
#855; closed unmerged: #830, #838, #839.
|
||||
@@ -0,0 +1,194 @@
|
||||
# Proposal — separate the four traffic classes onto their own VLANs
|
||||
|
||||
Status: draft. The question this started with is **answered** — from the Cilium
|
||||
source and flag reference, not from a test — and the answer changes the shape
|
||||
of the proposal rather than confirming it.
|
||||
|
||||
Supersedes the framing in issue #576, which described this as "give node
|
||||
management its own NIC, and Ceph public its own VLAN". That was two of four
|
||||
classes and one of them turned out already solved.
|
||||
|
||||
## The four classes
|
||||
|
||||
| class | what it is | where it goes today |
|
||||
|---|---|---|
|
||||
| **Ceph replication** | OSD-to-OSD heartbeat, replication, recovery | VLAN 49, its own X710 port — **separated** |
|
||||
| **cluster-internal** | pod-to-pod east-west, including Ceph client I/O | pod network, VXLAN over `mgmt0` |
|
||||
| **cluster-external** | ingress to LoadBalancer VIPs, egress to LAN and internet | `mgmt0` |
|
||||
| **management** | Talos API, etcd, kube-apiserver, kubelet | `mgmt0` |
|
||||
|
||||
Three of four share one interface. That is not a design decision — until the
|
||||
bare-metal conversion there was one NIC to work with.
|
||||
|
||||
Ceph client I/O living on the pod network is correct rather than a leftover:
|
||||
that traffic *is* cluster-internal. Ceph is done.
|
||||
|
||||
## Principle
|
||||
|
||||
**Each class gets its own VLAN. Which physical interface a VLAN lands on is a
|
||||
separate, changeable decision.**
|
||||
|
||||
Defining classes by VLAN means the separation survives recabling, port failures
|
||||
and capacity changes, and it does not require one port per class. Defining them
|
||||
by interface would freeze a capacity decision into the architecture.
|
||||
|
||||
## The finding that shapes everything else
|
||||
|
||||
The original open question was whether `nodePort.directRoutingDevice` moves the
|
||||
next-hop of Cilium's direct routes, or only selects an interface. It is
|
||||
answerable from the source, and the answer is: **only the interface.**
|
||||
|
||||
`installDirectRoute` takes the next-hop as a parameter:
|
||||
|
||||
```go
|
||||
func installDirectRoute(log *slog.Logger, prefix netip.Prefix, nodeIP net.IP,
|
||||
skipUnreachable bool) (routeSpec *netlink.Route, err error)
|
||||
```
|
||||
|
||||
and `nodeUpdate` supplies it as `newNode.GetNodeIP(false)`. `GetNodeIP` returns,
|
||||
in priority order, `NodeInternalIP` → `NodeExternalIP` → anything else. The
|
||||
same `newIP4` is what tunnel mode uses as the VXLAN endpoint.
|
||||
|
||||
`NodeInternalIP` comes from the Kubernetes Node object. `--ipv4-node` looks like
|
||||
an override but is not one: `initFromConfig()` calls `SetNodeInternalIP` and
|
||||
`initFromK8s()` runs **after** it and overwrites the value. In a Kubernetes
|
||||
cluster there is no path by which Cilium's node IP differs from the kubelet's
|
||||
`InternalIP`.
|
||||
|
||||
`--direct-routing-device` is documented as "Device name used to connect nodes in
|
||||
direct routing mode" — a device, not an address.
|
||||
|
||||
**So cluster-internal traffic is welded to the node IP, and the node IP is the
|
||||
kubelet InternalIP.** Two consequences:
|
||||
|
||||
- The class cannot be moved off whatever VLAN the node IP lives on.
|
||||
- **Routing mode is irrelevant to this problem.** Tunnel mode sources the VXLAN
|
||||
from the node IP; native mode uses it as the direct-route next-hop. Native
|
||||
routing buys nothing here, so the disruptive, non-rolling step this proposal
|
||||
previously contained is simply gone.
|
||||
|
||||
## Consequence: draw the line the other way
|
||||
|
||||
If cluster-internal cannot leave the node IP, then separation has to come from
|
||||
moving *management services* off it — and each one has to be checked
|
||||
individually, because they do not share a mechanism.
|
||||
|
||||
| management service | can it leave the node IP? | lever |
|
||||
|---|---|---|
|
||||
| etcd | **yes** | `cluster.etcd.advertisedSubnets` (+ `listenSubnets`, which defaults to it) |
|
||||
| kube-apiserver endpoint | **yes** | already a VIP — `Layer2VIPConfig` with a `link:` field |
|
||||
| Talos API | **n/a** | `apid` answers on every node address (measured); a client and firewall choice, not a bind setting |
|
||||
| kubelet | **no** | it *is* the node IP, by definition |
|
||||
|
||||
So four VLANs are reachable, but the boundary does not fall where the class
|
||||
names suggest: **kubelet stays with cluster-internal.** That is the honest
|
||||
result, and it is worth stating plainly rather than filing kubelet under
|
||||
management and hoping.
|
||||
|
||||
The remaining classes keep the levers established earlier:
|
||||
|
||||
- **cluster-external ingress** — `CiliumBGPNodeConfigOverride.spec.bgpInstances[].peers[].localAddress`.
|
||||
For eBGP the session source is the next-hop, so pinning the session pins
|
||||
where return traffic arrives. The session to the UDM has been running for
|
||||
months.
|
||||
- **cluster-external egress** — eBPF masquerading follows the output device, so
|
||||
this is a default-route decision plus `enable-masquerade-to-route-source`
|
||||
(today `false`).
|
||||
- **Ceph replication** — Multus macvlan on VLAN 49, done.
|
||||
|
||||
## The general escape hatch
|
||||
|
||||
Ceph is separated because its OSDs got a second interface and stopped using the
|
||||
pod network for replication. That is the *only* mechanism that fully detaches a
|
||||
workload from the node IP, and it generalises: anything that must not ride the
|
||||
node IP gets its own `NetworkAttachmentDefinition`.
|
||||
|
||||
It is also expensive per workload — an attachment, an IPAM pool, a pod spec
|
||||
change. Worth it for Ceph replication. Not worth it for east-west traffic in
|
||||
general.
|
||||
|
||||
## Recommended mapping
|
||||
|
||||
Two ways to arrange this, and they differ only in which VLAN gets relabelled:
|
||||
|
||||
**A — move the node IP to a new cluster VLAN.** VLAN 40 keeps etcd, the API VIP
|
||||
and Talos API. Cleanest match to the class names, but it changes every node's
|
||||
identity, which is the one address the whole cluster is administered through.
|
||||
|
||||
**B (recommended) — leave the node IP on VLAN 40 and move management off it.**
|
||||
VLAN 40 becomes the cluster VLAN; a new VLAN carries etcd and the API VIP.
|
||||
Identical end state, and no node ever changes its InternalIP.
|
||||
|
||||
Under B:
|
||||
|
||||
| VLAN | carries |
|
||||
|---|---|
|
||||
| 49 | Ceph replication |
|
||||
| 40 (relabelled *cluster*) | pod east-west, kubelet |
|
||||
| new *management* | etcd, kube-apiserver VIP, Talos API by convention |
|
||||
| new *external* | LoadBalancer ingress, egress default route |
|
||||
|
||||
The property the user asked for — management carries only management traffic —
|
||||
holds under B. The label moves; the property is what matters.
|
||||
|
||||
## Migration shape
|
||||
|
||||
Every step is rolling and reversible. The routing-mode change is gone.
|
||||
|
||||
1. **New VLANs** on the switch, trunked to the X710. Nothing uses them.
|
||||
2. **Addresses** on the new VLANs via Talos, one node at a time, `--mode=try`.
|
||||
Still unused; this only proves the L2 works.
|
||||
3. **etcd** to the management VLAN via `advertisedSubnets`. One node at a time,
|
||||
waiting for a healthy quorum between each.
|
||||
4. **API VIP** to the management VLAN via `Layer2VIPConfig.link`. DNS for
|
||||
`k8s.renner.uno` follows.
|
||||
5. **cluster-external**: BGP `localAddress`, default route, masquerade flag.
|
||||
Reversible by reverting the override; failure is visible immediately in
|
||||
`cilium-dbg shell -- bgp/peers`.
|
||||
6. Verify VLAN 40 by what is *absent* from it, not by what still works.
|
||||
|
||||
Step 3 is the one that deserves care — etcd is the only step where a mistake
|
||||
costs quorum rather than connectivity.
|
||||
|
||||
## What this does not buy
|
||||
|
||||
Management currently shares a 10 Gb link and is not contended. Nothing observed
|
||||
today is slow because of this. The case for the split is that traffic classes
|
||||
with different failure modes and security postures should not be
|
||||
indistinguishable — not that something is broken.
|
||||
|
||||
## Sources and measurements
|
||||
|
||||
Source, Cilium v1.20.1 (the running version):
|
||||
|
||||
- `pkg/datapath/linux/node.go` — `installDirectRoute(…, nodeIP net.IP, …)`,
|
||||
fed from `newIP4 = newNode.GetNodeIP(false)`; `createDirectRouteSpec` rejects
|
||||
a next-hop that is not directly reachable.
|
||||
- `pkg/node/types/node.go` — `GetNodeIP` prefers `NodeInternalIP`.
|
||||
- `pkg/node/sync/local_node_sync.go` — `initFromConfig()` applies
|
||||
`--ipv4-node`, `initFromK8s()` runs afterwards and overwrites it from the
|
||||
Kubernetes Node.
|
||||
- `cilium-agent` flag reference — `--direct-routing-device`: "Device name used
|
||||
to connect nodes in direct routing mode".
|
||||
|
||||
Talos:
|
||||
|
||||
- Multihoming guide — `cluster.etcd.advertisedSubnets`; `listenSubnets`
|
||||
defaults to it. kube-apiserver and Talos API listen addresses are not
|
||||
covered there.
|
||||
- `kubernetes/talos/patches/k8s-{1,2,3}.yaml` — `Layer2VIPConfig` for
|
||||
`10.2.40.60` with `link: mgmt0`.
|
||||
|
||||
Measured on the cluster:
|
||||
|
||||
- `apid` answered on a node address that is not the InternalIP, so the Talos
|
||||
API is not bound to one address.
|
||||
- k8s-1 currently holds one routable v4 address plus the VIP on `mgmt0`; its
|
||||
InternalIP had changed since the last session, which is why nothing here
|
||||
names an address as fixed.
|
||||
- Cilium runs `routing-mode: tunnel` with `enable-masquerade-to-route-source`
|
||||
unset and no direct-routing device configured.
|
||||
|
||||
Not verified: whether the kube-apiserver's `--advertise-address` follows the
|
||||
node IP independently of the VIP. It does not affect the recommendation, since
|
||||
clients reach the API through the VIP either way.
|
||||
@@ -2,12 +2,7 @@
|
||||
|
||||
Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
|
||||
metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
|
||||
after doing jupiter on 2026-08-25.
|
||||
|
||||
All three nodes are now on bare metal — `talosctl get systeminformation`
|
||||
reports Minisforum MS-01 hardware with distinct chassis serials for k8s-1,
|
||||
k8s-2 and k8s-3, with no hypervisor in between. Kept as the procedure for a
|
||||
rebuild, not as outstanding work.
|
||||
after doing jupiter on 2026-08-25; saturn and neptune remain.
|
||||
|
||||
Everything here that reads like an over-specific warning is something that
|
||||
actually happened. The conversion itself is straightforward — the failures were
|
||||
@@ -432,8 +427,8 @@ is why it belongs here.
|
||||
among its devices but not `ens19`, because `ens19` is deliberately
|
||||
address-less. Cilium therefore owns `mgmt0` and sees VLAN 40 frames already
|
||||
untagged by the 8021q layer, while VLAN 30 frames only ever meet the filter
|
||||
on the shared parent `uplink`, still tagged. Any future address-less VLAN
|
||||
carrying pod traffic will need the same treatment.
|
||||
on the shared parent `enp2s0f0np0`, still tagged. Any future address-less
|
||||
VLAN carrying pod traffic will need the same treatment.
|
||||
|
||||
The symptom does not look like a VLAN problem. A tc-layer drop reaches an
|
||||
AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports `no buffer space
|
||||
|
||||
@@ -67,12 +67,10 @@ These must exist outside the cluster or the rebuild cannot complete:
|
||||
`flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
|
||||
files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
|
||||
`MIGRATION-sops-openbao.md`.)
|
||||
- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
|
||||
store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
|
||||
former shamir key, now the recovery key — and `homelab/openbao/root-token`),
|
||||
and the GPG private key backed up off-machine. The seal key is the one
|
||||
the snapshots are encrypted under: without it, no OpenBao snapshot can be
|
||||
restored and no secret-consuming app can start.
|
||||
- [ ] **OpenBao unseal key + root token** — in the GPG `pass` store
|
||||
(`homelab/openbao/unseal-key`, `homelab/openbao/root-token`), and the GPG
|
||||
private key backed up off-machine. Without the unseal key, no OpenBao
|
||||
snapshot can be restored and no secret-consuming app can start.
|
||||
- [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
|
||||
(`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
|
||||
`openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
|
||||
@@ -296,65 +294,42 @@ output** — a green step over a still-forming Ceph is a known trap.
|
||||
> Steady-state (non-rebuild) bootstrap is just `kubernetes/bootstrap.sh flux`
|
||||
> with `GITEA_TOKEN` set; `FLUX_SOURCE` defaults to `gitea`.
|
||||
|
||||
## Step 4 — openbao: restore the snapshot
|
||||
## Step 4 — openbao: restore the snapshot + unseal
|
||||
|
||||
`bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
|
||||
`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
|
||||
the empty server, restores the newest raft snapshot straight out of the Velero
|
||||
kopia repository, and waits until every replica has unsealed itself from the
|
||||
static seal. What follows is the same procedure by hand, for when the script
|
||||
is not an option.
|
||||
|
||||
OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
|
||||
`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
|
||||
snapshots are encrypted under that key, so it must be the same key the backed
|
||||
up cluster used — a different one restores nothing. Until the snapshot is
|
||||
restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
|
||||
Secrets, and every secret-consuming app stays Pending.
|
||||
After Step 3, OpenBao comes up **sealed and empty**. Until it is restored and
|
||||
unsealed, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
|
||||
Secrets, and every secret-consuming app stays Pending. This step is **manual**
|
||||
(the unseal key lives in `pass`, off-cluster by design).
|
||||
|
||||
```sh
|
||||
# 1. The seal key, before the pods start (a pod without it never starts).
|
||||
kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl -n openbao create secret generic openbao-seal-key \
|
||||
--from-literal=key="$(pass show homelab/openbao/seal-key)"
|
||||
# 1. Wait for the OpenBao pods to exist (all three sealed, 0/1 Ready is expected).
|
||||
kubectl -n openbao get pods
|
||||
|
||||
# 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
|
||||
# initialise it. With an auto-seal, init hands out recovery keys and the
|
||||
# server unseals itself. These keys are throwaway: the restore replaces them.
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator init \
|
||||
-recovery-shares=1 -recovery-threshold=1
|
||||
|
||||
# 3. Restore the most recent raft snapshot. Source it from the snapshot PVC
|
||||
# 2. Restore the most recent raft snapshot. Source it from the snapshot PVC
|
||||
# (infrastructure/openbao/snapshot-cronjob.yaml) or, in a total-loss rebuild,
|
||||
# from the Velero `openbao-daily` offsite copy. Root token from the init
|
||||
# output. No -force: OpenBao checks the snapshot was written under this
|
||||
# server's seal key and refuses otherwise ("different autoseal key") — that
|
||||
# refusal is the signal the Secret is wrong. -force would skip the check and
|
||||
# leave a server that can never unseal.
|
||||
# from the Velero `openbao-daily` offsite copy. Copy it into the pod, then:
|
||||
kubectl -n openbao cp <snapshot>.snap openbao-0:/tmp/restore.snap
|
||||
kubectl -n openbao exec -it openbao-0 -- \
|
||||
env BAO_TOKEN=<init root token> bao operator raft snapshot restore /tmp/restore.snap
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator raft snapshot restore /tmp/restore.snap
|
||||
|
||||
# 4. The server seals against the snapshot's keyring and unseals itself from
|
||||
# the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
|
||||
# to type. Afterwards the tokens and recovery key from pass apply again.
|
||||
kubectl -n openbao get pods -w
|
||||
# 3. Unseal with the key from pass (threshold 1) — EVERY replica, same key.
|
||||
# A sealed server is neither a raft voter nor a standby, so unsealing only
|
||||
# openbao-0 leaves the three-peer cluster one node short of quorum.
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator unseal # paste homelab/openbao/unseal-key
|
||||
kubectl -n openbao exec -it openbao-1 -- bao operator unseal # same key
|
||||
kubectl -n openbao exec -it openbao-2 -- bao operator unseal # same key
|
||||
```
|
||||
|
||||
Verify the store recovers before moving on:
|
||||
|
||||
```sh
|
||||
kubectl -n openbao exec openbao-0 -- bao status # want: Seal Type static, Sealed false
|
||||
kubectl get clustersecretstore openbao # want: Valid / Ready
|
||||
kubectl -n <ns> get externalsecret # want: SecretSynced
|
||||
kubectl get clustersecretstore openbao # want: Valid / Ready
|
||||
kubectl -n <ns> get externalsecret # want: SecretSynced
|
||||
```
|
||||
|
||||
> A replica that stays **sealed** after a restart could not use the static
|
||||
> seal — the Secret is missing or does not match the key the data was
|
||||
> written under. Fix the Secret and restart the pod; `bao operator unseal`
|
||||
> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
|
||||
> unseal an auto-sealed server and is only for recovery-mode operations.
|
||||
> Details + rationale: `MIGRATION-sops-openbao.md`.
|
||||
> **Manual unseal is required after every OpenBao pod restart**, not just
|
||||
> during rebuilds, and for *each* replica that restarts — a rolling node
|
||||
> drain seals all three in turn. Details + rationale:
|
||||
> `MIGRATION-sops-openbao.md`.
|
||||
|
||||
## Step 5 — apps: let the rest reconcile
|
||||
|
||||
@@ -418,15 +393,14 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
|
||||
- **`bootstrap.sh talos` is dead** — it cannot generate a config (#581), and
|
||||
Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
|
||||
Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
|
||||
Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
|
||||
other secrets are read from `pass` or streamed in at run time and never live
|
||||
in CI.
|
||||
- **OpenBao unseals itself, but only under the seal key the snapshot was
|
||||
written with.** Three raft replicas mean the service survives losing one; a
|
||||
sealed server is neither a voter nor a standby. A pod that stays sealed after
|
||||
a restart has the wrong `openbao-seal-key` Secret — Step 4 covers this; it is
|
||||
repeated here because it is the step most likely to be half-done under
|
||||
pressure.
|
||||
It has also never had an `openbao` phase — Step 4 is manual because the
|
||||
unseal key must not live in CI; see issue #187 on whether to wire a
|
||||
semi-automated OpenBao restore.
|
||||
- **OpenBao seals on every restart and each replica must be unsealed
|
||||
individually.** Three raft replicas mean the service survives losing one, but
|
||||
a sealed server is neither a voter nor a standby — unsealing only one leaves
|
||||
the cluster short of quorum. Step 4 covers this; it is repeated here because
|
||||
it is the step most likely to be half-done under pressure.
|
||||
- Ceph's cluster network is a Multus attachment on VLAN 49 with whereabouts
|
||||
IPAM, not host networking. Nothing in the rebuild needs special handling for
|
||||
it — the nodes hold no address on that segment — but a rebuilt cluster will
|
||||
|
||||
@@ -21,9 +21,6 @@ metadata:
|
||||
namespace: audiobookshelf
|
||||
labels:
|
||||
app.kubernetes.io/name: audiobookshelf
|
||||
annotations:
|
||||
# envFrom does not restart the pod when the ConfigMap changes; Reloader does.
|
||||
configmap.reloader.stakater.com/reload: audiobookshelf-config
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
@@ -70,7 +67,7 @@ spec:
|
||||
mountPath: /ca
|
||||
containers:
|
||||
- name: audiobookshelf
|
||||
image: ghcr.io/advplyr/audiobookshelf:2.36.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
|
||||
image: ghcr.io/advplyr/audiobookshelf:2.36.0 # {"$imagepolicy": "flux-system:audiobookshelf"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 80
|
||||
|
||||
@@ -120,14 +120,7 @@ spec:
|
||||
server:
|
||||
resources:
|
||||
requests:
|
||||
# 500m, not 200m: the HPA below targets 70% of *this* request, so at
|
||||
# 200m the scale-up threshold sat at 140m — below the pod's own
|
||||
# ordinary peak. Measured over 7d, a warm server pod p99s at ~108m
|
||||
# and peaks at ~190m, and a freshly-started one burns ~300m booting
|
||||
# Django. That made the HPA oscillate 2<->3<->4 all day and pin at
|
||||
# maxReplicas (KubeHpaMaxedOut, issue #761). At 500m the threshold
|
||||
# is 350m/pod, above the boot burst, so only real load scales out.
|
||||
cpu: 500m
|
||||
cpu: 200m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
@@ -161,11 +154,7 @@ spec:
|
||||
worker:
|
||||
resources:
|
||||
requests:
|
||||
# Same reasoning as server above, and the worker is burstier still:
|
||||
# over 7d its per-pod p99 reaches ~336m and peaks hit ~630m while
|
||||
# running blueprint applies and scheduled tasks. 200m understated
|
||||
# that badly enough that routine task batches read as saturation.
|
||||
cpu: 500m
|
||||
cpu: 200m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
|
||||
@@ -16,6 +16,7 @@ resources:
|
||||
- prometheus-gate-egress.yaml
|
||||
- wordpress-gate-egress.yaml
|
||||
- llama-gate-egress.yaml
|
||||
- outpost-servicemonitors.yaml
|
||||
- ldap-outpost-cert.yaml
|
||||
- ldap-outpost-lb.yaml
|
||||
- ldap-outpost-cnp.yaml
|
||||
|
||||
@@ -40,7 +40,7 @@ metadata:
|
||||
# so this Service produces no public record — LDAP stays
|
||||
# LAN-internal, which matches the threat model of the cleartext
|
||||
# bind on 389.
|
||||
external-dns.kubernetes.io/hostname: ldap.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: ldap.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -3,4 +3,3 @@ kind: Kustomization
|
||||
resources:
|
||||
- externalsecret.yaml
|
||||
- helmrelease.yaml
|
||||
- networkpolicy.yaml
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
# The apps namespace is default-deny-ingress, so the ServiceMonitor added in
|
||||
# monitoring-rules/external-dns-servicemonitor.yaml scraped into a timeout.
|
||||
# The selector matches both releases — the Cloudflare one deployed here and
|
||||
# external-dns-unifi from infrastructure/ — since they share the chart's
|
||||
# app.kubernetes.io/name and land in the same namespace.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: external-dns-allow-metrics-scrape
|
||||
namespace: apps
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: monitoring
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "7979"
|
||||
protocol: TCP
|
||||
@@ -111,23 +111,9 @@ statistics max_id=13323904 max_start_ts=1787162400.0
|
||||
statistics_short_term max_id=159205762 max_start_ts=1787166000.0
|
||||
```
|
||||
|
||||
## Phase 2c — cutover: DONE
|
||||
## Phase 2c — cutover (REMAINING)
|
||||
|
||||
Verified 2026-09-02 against the live database:
|
||||
|
||||
| table | rows | span |
|
||||
|---|---|---|
|
||||
| `statistics` | 11,092,586 | 2024-03-13 20:00 → current |
|
||||
| `statistics_short_term` | 1,883,709 | rolling ~7 days |
|
||||
| `statistics_meta` | 1,480 | |
|
||||
|
||||
The 2024-03-13 origin is the proof that matters. It is unchanged from the
|
||||
pre-migration count in "What is at risk" above, so the history was copied
|
||||
across engines rather than started from zero — which is the failure this
|
||||
document exists to prevent. `states` carries current rows, so `db_url` points
|
||||
at Postgres.
|
||||
|
||||
The procedure below is kept as the record of what was done.
|
||||
Do not start until phase 1 is verified.
|
||||
|
||||
**Migrate statistics only.** Copying all 18M `states` rows is slow, needs a long
|
||||
HA outage, and buys a table that is discarded within 7 days anyway.
|
||||
|
||||
@@ -18,7 +18,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
lbipam.cilium.io/ips: "10.2.99.87,fd99::87"
|
||||
external-dns.kubernetes.io/hostname: ha.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: ha.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -56,9 +56,10 @@ spec:
|
||||
# Re-rendered on every pod start, so a CNPG password rotation takes
|
||||
# effect rather than silently leaving a stale URI behind.
|
||||
#
|
||||
# Load-bearing: `db_url: !secret recorder_db_url` in the homeassistant
|
||||
# repo's configuration.yaml reads this key, so HA does not start
|
||||
# without it. Removing this init container takes the recorder down.
|
||||
# INERT until the recorder is repointed: nothing reads this key until
|
||||
# `db_url: !secret recorder_db_url` lands in configuration.yaml in the
|
||||
# homeassistant repo. That ordering is required — HA refuses to start
|
||||
# when a referenced secret is missing.
|
||||
- name: render-db-url
|
||||
image: busybox:1.36
|
||||
command:
|
||||
@@ -95,7 +96,7 @@ spec:
|
||||
memory: 32Mi
|
||||
containers:
|
||||
- name: home-assistant
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.9.3 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
env:
|
||||
- name: TZ
|
||||
value: Europe/Berlin
|
||||
|
||||
@@ -18,12 +18,12 @@
|
||||
# That is the point of the move — it attacks the multiplier rather than trimming
|
||||
# what gets recorded, which would only hide the problem.
|
||||
#
|
||||
# This is the live recorder database. HA writes here; the SQLite file is no
|
||||
# longer the system of record. The 2.4 years of long-term statistics were
|
||||
# copied across engines rather than regenerated — HA does not migrate data
|
||||
# between database engines, so an empty Postgres would have silently started
|
||||
# from zero and taken the energy dashboard's history with it.
|
||||
# MIGRATION-recorder-postgres.md holds the sequence that was followed.
|
||||
# STAGING
|
||||
# Phase 1 (this manifest): the cluster comes up EMPTY and HA keeps using SQLite.
|
||||
# Nothing depends on it, so a failure here is inert. Verify Ready, WAL archiving
|
||||
# to Hetzner, and a successful base backup before going further.
|
||||
# Phase 2: migrate the statistics tables and repoint recorder.db_url.
|
||||
# See MIGRATION-recorder-postgres.md for the sequence and the rollback.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
|
||||
@@ -60,8 +60,9 @@ resources:
|
||||
- zigbee2mqtt-gate-service.yaml
|
||||
- zigbee2mqtt-ingress.yaml
|
||||
# Postgres backend for the recorder (MIGRATION-recorder-postgres.md).
|
||||
# Live: recorder.db_url in the homeassistant repo points here, and the
|
||||
# long-term statistics carry their full history from 2024.
|
||||
# PHASE 1: this cluster comes up EMPTY and nothing consumes it — HA is still
|
||||
# on SQLite. Deploying it is therefore inert; phase 2 migrates the statistics
|
||||
# tables and repoints recorder.db_url in the homeassistant repo.
|
||||
- ha-pg-hetzner-creds-externalsecret.yaml
|
||||
- ha-pg-objectstore.yaml
|
||||
- ha-pg-cluster.yaml
|
||||
|
||||
@@ -60,7 +60,7 @@ spec:
|
||||
# Two processes rather than one: socat carries a single address pair per
|
||||
# invocation, and the families need different receive and send addresses.
|
||||
# They are independent — one crashing does not blind the other — and each
|
||||
# carries its own supervisor loop.
|
||||
# is restarted by the kubelet on its own.
|
||||
containers:
|
||||
# -u is load-bearing on both: unidirectional, left to right. Without it
|
||||
# socat would also carry HA's replies back out, and this must stay a
|
||||
@@ -77,29 +77,17 @@ spec:
|
||||
# UDP4-SENDTO takes its A record and UDP6-SENDTO its AAAA. See
|
||||
# mdns-relay-service.yaml for why that indirection is required and why
|
||||
# the Service is dual-stack.
|
||||
#
|
||||
# Supervisor loop: socat exits 1 on the first datagram it cannot
|
||||
# deliver, and the ClusterIP has no backend while HA restarts. The
|
||||
# trap forwards SIGTERM so the pod still terminates promptly.
|
||||
- name: socat-ipv4
|
||||
image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
trap 'kill "$pid"; exit 0' TERM INT
|
||||
while :; do
|
||||
socat -u \
|
||||
"UDP4-RECV:5353,ip-add-membership=224.0.0.251:net1,so-bindtodevice=net1,reuseaddr" \
|
||||
"UDP4-SENDTO:home-assistant-mdns.home-assistant.svc.cluster.local:5353" &
|
||||
pid=$!
|
||||
wait "$pid"
|
||||
echo "mdns-relay: socat-ipv4 exited ($?); restarting in 2s" >&2
|
||||
sleep 2
|
||||
done
|
||||
# No probes. The supervisor loop is the liveness story; a probe here
|
||||
# could only re-assert that PID 1 is running.
|
||||
args:
|
||||
- -u
|
||||
- "UDP4-RECV:5353,ip-add-membership=224.0.0.251:net1,so-bindtodevice=net1,reuseaddr"
|
||||
- "UDP4-SENDTO:home-assistant-mdns.home-assistant.svc.cluster.local:5353"
|
||||
# No probes. socat is a single process with no health surface: if it
|
||||
# exits, the container exits and the kubelet restarts it, which is the
|
||||
# entire liveness story. A probe here could only re-assert that PID 1
|
||||
# is running.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
@@ -126,20 +114,10 @@ spec:
|
||||
- name: socat-ipv6
|
||||
image: alpine/socat:1.8.0.3@sha256:beb4a68d9e4fe6b0f21ea774a0fde6c31f580dde6368939ed70100c5385b015e
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
trap 'kill "$pid"; exit 0' TERM INT
|
||||
while :; do
|
||||
socat -u \
|
||||
"UDP6-RECV:5353,ipv6-join-group=[ff02::fb]:net1,so-bindtodevice=net1,ipv6-v6only=1,reuseaddr" \
|
||||
"UDP6-SENDTO:home-assistant-mdns.home-assistant.svc.cluster.local:5353" &
|
||||
pid=$!
|
||||
wait "$pid"
|
||||
echo "mdns-relay: socat-ipv6 exited ($?); restarting in 2s" >&2
|
||||
sleep 2
|
||||
done
|
||||
args:
|
||||
- -u
|
||||
- "UDP6-RECV:5353,ipv6-join-group=[ff02::fb]:net1,so-bindtodevice=net1,ipv6-v6only=1,reuseaddr"
|
||||
- "UDP6-SENDTO:home-assistant-mdns.home-assistant.svc.cluster.local:5353"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
|
||||
@@ -19,7 +19,7 @@ metadata:
|
||||
# LAN can connect with `mqtt.renner.uno` (matches the broker cert
|
||||
# SAN) instead of the bare IP. No Cloudflare label — kept off the
|
||||
# public DNS; UniFi serves a split-DNS answer for LAN clients.
|
||||
external-dns.kubernetes.io/hostname: mqtt.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: mqtt.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -57,14 +57,7 @@ spec:
|
||||
# and leaves this one untouched (tags are not deleted), so this keeps
|
||||
# resolving — but the image then came from the branch build rather than
|
||||
# master. Re-point it at a master-built tag when convenient.
|
||||
#
|
||||
# konairius/, not gitea_admin/: this init container runs during a cold
|
||||
# rebuild, when Gitea does not exist because it runs inside the
|
||||
# cluster being rebuilt. The node then falls back to a ghcr.io copy
|
||||
# (machine.registries.mirrors, applied by bootstrap.sh), and containerd
|
||||
# keeps the repository path when it falls through — so the path has to
|
||||
# exist on both registries. Steady state still pulls from Gitea.
|
||||
image: gitea.renner.uno/konairius/alpine-jq:2026-09-04
|
||||
image: gitea.renner.uno/gitea_admin/alpine-jq:2026-07-30
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
|
||||
@@ -10,15 +10,15 @@ metadata:
|
||||
# Kept permanently as a standing guard.
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
# A replica is viable because nothing here writes pg_vectors any more. Both
|
||||
# vector indexes run on vchordrq and the vectors extension is gone, so the
|
||||
# long pg_vectors/indexes/<uuid> paths are absent — pg_basebackup's tar
|
||||
# format cannot represent them, and that is what blocks a replica join.
|
||||
instances: 2
|
||||
# See ak-pg cluster.yaml for rationale. The barman-plugin shutdown wedge
|
||||
# (cloudnative-pg#9770) holds an instance up until SIGKILL, so bounding
|
||||
# terminationGracePeriod (stopDelay) is what keeps a failover from waiting
|
||||
# on it. smartShutdownTimeout keeps pod restarts quick.
|
||||
# Single instance during pgvecto.rs → VectorChord migration window;
|
||||
# pg_basebackup tar format chokes on the long pg_vectors/indexes/<uuid>
|
||||
# paths, blocking replica join. Bump back to 2 once Immich's in-app
|
||||
# vchord migration drops the vectors extension.
|
||||
instances: 1
|
||||
# See ak-pg cluster.yaml for rationale. Single-instance, so no failover, but
|
||||
# the barman-plugin shutdown wedge (cloudnative-pg#9770) takes the whole DB
|
||||
# down until SIGKILL — bounding terminationGracePeriod (stopDelay) matters
|
||||
# even without a standby. smartShutdownTimeout keeps pod restarts quick.
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration
|
||||
@@ -36,10 +36,8 @@ spec:
|
||||
database: immich
|
||||
owner: immich
|
||||
postgresql:
|
||||
# vchord only. The image also ships pgvecto.rs (vectors.so), but its
|
||||
# extension is not installed and preloading it would recreate pg_vectors
|
||||
# in PGDATA for nothing. pgvector needs no preloading.
|
||||
shared_preload_libraries:
|
||||
- vectors.so
|
||||
- vchord.so
|
||||
parameters:
|
||||
shared_buffers: "512MB"
|
||||
|
||||
@@ -32,7 +32,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: immich-machine-learning
|
||||
image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
|
||||
image: ghcr.io/immich-app/immich-machine-learning:v3.1.0 # {"$imagepolicy": "flux-system:immich-machine-learning"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3003
|
||||
|
||||
@@ -35,9 +35,6 @@ metadata:
|
||||
namespace: immich
|
||||
labels:
|
||||
app.kubernetes.io/name: immich-server
|
||||
annotations:
|
||||
# envFrom does not restart the pod when the ConfigMap changes; Reloader does.
|
||||
configmap.reloader.stakater.com/reload: immich-config
|
||||
spec:
|
||||
# Scaled to 0 alongside immich-machine-learning during single-node
|
||||
# maintenance; see the note there. The CNPG Postgres cluster stays running,
|
||||
@@ -87,7 +84,7 @@ spec:
|
||||
mountPath: /ca
|
||||
containers:
|
||||
- name: immich-server
|
||||
image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
|
||||
image: ghcr.io/immich-app/immich-server:v3.1.0 # {"$imagepolicy": "flux-system:immich-server"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3001
|
||||
|
||||
@@ -42,11 +42,9 @@ spec:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
# Seed the libation-config PVC from the bootstrap secret on
|
||||
# first run only — sentinel file gates re-runs so DB updates
|
||||
# from subsequent liberate passes are preserved. Refreshed
|
||||
# Audible tokens are preserved by the container args, not by
|
||||
# this sentinel: the entrypoint copies AccountsSettings.json
|
||||
# into ephemeral /config-internal rather than symlinking it.
|
||||
# first run only — sentinel file gates re-runs so token
|
||||
# refreshes in AccountsSettings.json and DB updates from
|
||||
# subsequent liberate passes are preserved.
|
||||
- name: seed-config
|
||||
image: busybox:1.37.0
|
||||
command: ["/bin/sh", "-eu", "-c"]
|
||||
@@ -68,11 +66,6 @@ spec:
|
||||
install -m 660 /bootstrap/AccountsSettings.json /libation-state/AccountsSettings.json
|
||||
install -m 660 /bootstrap/Settings.json /libation-state/Settings.json
|
||||
install -m 660 /bootstrap/LibationContext.db /libation-state/LibationContext.db
|
||||
# Without this key the seeded tokens cannot be decrypted:
|
||||
# TokenStorageMethod is Encrypted and the container has no
|
||||
# OS secret store, so a missing key file makes the
|
||||
# entrypoint mint a throwaway one on every run.
|
||||
install -m 660 /bootstrap/libation-master.key /libation-state/libation-master.key
|
||||
touch /libation-state/.bootstrapped
|
||||
ls -la /libation-state
|
||||
volumeMounts:
|
||||
@@ -92,23 +85,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: libation
|
||||
# Pinned by digest: Spegel caches mutable tag resolutions, so
|
||||
# :latest stayed on 13.7.11 despite imagePullPolicy Always.
|
||||
image: docker.io/rmcrackan/libation:14.2.1@sha256:6f9d8ba19dc3803dd6ba295ab5a7abfb6ff3bc06286eebf4ecd8ccbe43245888
|
||||
# The entrypoint stages config into ephemeral /config-internal
|
||||
# by copy (only the DB is symlinked to the PVC), so Audible
|
||||
# tokens refreshed during a run are lost at container exit.
|
||||
# Passing args makes the entrypoint set up the environment and
|
||||
# then exec this instead of its own loop; the trap copies the
|
||||
# refreshed file back to the PVC on every exit path.
|
||||
args:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
trap 'cp /config-internal/AccountsSettings.json /libation-state/AccountsSettings.json' EXIT
|
||||
set -e
|
||||
/libation/LibationCli scan
|
||||
/libation/LibationCli liberate
|
||||
image: docker.io/rmcrackan/libation:latest
|
||||
env:
|
||||
- name: LIBATION_CONFIG_DIR
|
||||
value: /libation-state
|
||||
|
||||
@@ -25,13 +25,6 @@ metadata:
|
||||
app.kubernetes.io/name: llama-server
|
||||
spec:
|
||||
replicas: 1
|
||||
# The startupProbe below sanctions a 40 min cold download, but the default
|
||||
# progressDeadlineSeconds is 600. Left implicit the Deployment declares
|
||||
# ProgressDeadlineExceeded at 10 minutes while the probe is still legitimately
|
||||
# waiting, and KubeDeploymentRolloutStuck fires 15 min after that on a rollout
|
||||
# that is fine. Kept above the startup budget, with room for the image pull
|
||||
# and scheduling that happen before the first probe.
|
||||
progressDeadlineSeconds: 3000
|
||||
# A rolling update would load a second copy of the weights onto the same
|
||||
# iGPU and into the same node's RAM before releasing the first.
|
||||
strategy:
|
||||
@@ -82,62 +75,11 @@ spec:
|
||||
# many small per-expert matmuls that sets the pace, not bandwidth.
|
||||
- --threads
|
||||
- "4"
|
||||
# 16k is room to spare, not a safety limit. The limit that matters
|
||||
# is not set here at all.
|
||||
#
|
||||
# Isolated by bisection on a scratch pod: this stack breaks on
|
||||
# PROMPT length, and the value of --ctx-size makes no difference to
|
||||
# it. Measured at --ctx-size 65536 with a single slot, so with four
|
||||
# times the headroom:
|
||||
#
|
||||
# 13588 prompt tokens coherent answer
|
||||
# 15086 one token, immediate EOS, empty content
|
||||
# 16088 CJK characters and digit runs
|
||||
#
|
||||
# A single request past that boundary then poisons the slot: the
|
||||
# same 13588-token prompt that answered cleanly a minute earlier
|
||||
# returns one empty token afterwards, and stays broken until the
|
||||
# pod restarts. That is why it looked intermittent, and why a
|
||||
# 650-token classification once came back as 1025 tokens of "!" —
|
||||
# it followed Home Assistant's ~19k prompts through the same slot.
|
||||
#
|
||||
# Nothing reports any of it: finish_reason "stop", a well-formed
|
||||
# OpenAI response, HTTP 200, /health green.
|
||||
#
|
||||
# So the operating rule is about callers, not this flag. Paperless
|
||||
# sits near 5k and is safe; Home Assistant's Assist prompt measured
|
||||
# 18910 and never had a chance. An earlier version of this comment
|
||||
# blamed --ctx-size 32768, which the bisection disproves — 32768
|
||||
# and 65536 both answer correctly as long as the prompt stays under
|
||||
# the boundary.
|
||||
#
|
||||
# The same stack also fails hard at --ubatch-size 2048, which is
|
||||
# the loud version of a limit this one keeps quiet about.
|
||||
#
|
||||
# 16k rather than 8k because the measured prefill falloff is mild
|
||||
# and this model's KV cache is unusually cheap — only 10 of its 40
|
||||
# layers use full attention.
|
||||
- --ctx-size
|
||||
- "16384"
|
||||
# One slot, so the 16384 belongs to one request at a time.
|
||||
#
|
||||
# Left on auto, llama.cpp opens four slots and — because auto is
|
||||
# also what enables --kv-unified — has them share a single buffer
|
||||
# of this size. Four Paperless requests at once wanted 6209, 6472,
|
||||
# 4303 and 1271 tokens against those 16384, and all four died with
|
||||
# "Context size has been exceeded".
|
||||
#
|
||||
# They were concurrent because a client timeout does not stop the
|
||||
# generation behind it: each abandoned request keeps consuming the
|
||||
# shared budget the next one needs, so one slow document is enough
|
||||
# to start the pile-up. Queueing is the behaviour worth having
|
||||
# here — nothing calls this server in parallel on purpose, and
|
||||
# Paperless's celery worker runs at concurrency 1.
|
||||
#
|
||||
# No liveness probe change goes with this. The server recovered on
|
||||
# its own once the batch drained, so a probe that restarted the pod
|
||||
# would have made the incident worse; with a single slot, one that
|
||||
# asked for a real completion would also queue behind live work and
|
||||
# kill healthy pods during a long document.
|
||||
- --parallel
|
||||
- "1"
|
||||
# A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
|
||||
# with ErrorDeviceLost.
|
||||
- --ubatch-size
|
||||
|
||||
@@ -19,11 +19,6 @@ metadata:
|
||||
app.kubernetes.io/name: llama-embed
|
||||
spec:
|
||||
replicas: 1
|
||||
# Same reasoning as llama-server: the startupProbe budget (600s) must stay
|
||||
# below progressDeadlineSeconds, whose default is also 600 — exactly equal, so
|
||||
# any cold download that uses most of its budget trips
|
||||
# KubeDeploymentRolloutStuck on a rollout that is still progressing normally.
|
||||
progressDeadlineSeconds: 900
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
@@ -46,7 +41,7 @@ spec:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
||||
args:
|
||||
- --hf-repo
|
||||
- gpustack/bge-m3-GGUF:Q8_0
|
||||
- Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0
|
||||
# Serves POST /v1/embeddings.
|
||||
- --embeddings
|
||||
# A short, stable id for consumers. Without it the model is
|
||||
@@ -55,41 +50,28 @@ spec:
|
||||
# One name only: llama-server keeps aliases in a sorted set and
|
||||
# reports the first as the id, so a second name can steal it.
|
||||
- --alias
|
||||
- bge-m3
|
||||
# bge-m3 is trained for CLS pooling — a different type from the
|
||||
# Qwen family this replaces, which uses last-token. Left unspecified
|
||||
# llama-server takes the GGUF default, which is not visible from
|
||||
# outside and would silently produce vectors the model was not
|
||||
# trained to emit. Read from the model's own 1_Pooling/config.json
|
||||
# rather than assumed; of the candidates measured, three different
|
||||
# pooling types were needed.
|
||||
- qwen3-embedding-0.6b
|
||||
# Qwen3-Embedding is trained for last-token pooling. Left
|
||||
# unspecified llama-server takes the GGUF's default, which is not
|
||||
# visible from outside and would silently produce vectors the model
|
||||
# was not trained to emit. Set before anything is indexed: changing
|
||||
# pooling later invalidates every stored embedding.
|
||||
- --pooling
|
||||
- cls
|
||||
- last
|
||||
- --n-gpu-layers
|
||||
- "99"
|
||||
- --threads
|
||||
- "2"
|
||||
# Paperless chunks at 1024 tokens, but counts them with its own
|
||||
# tokenizer while this server measures the batch in bge-m3's.
|
||||
# Those disagree: XLM-RoBERTa splits German into more pieces than
|
||||
# llama_index's counter does, so a chunk Paperless considers full
|
||||
# at 1024 arrived here as 1069 and the request failed outright —
|
||||
# "input (1069 tokens) is too large to process".
|
||||
#
|
||||
# The headroom therefore has to absorb a tokenizer ratio, not just
|
||||
# a rounding error, and 1.5x covers far more than the 1.044 that
|
||||
# broke it. The defaults are batch 2048 / ubatch 512, and with
|
||||
# --embeddings llama-server clamps both to the smaller rather than
|
||||
# run them mismatched, so both are set.
|
||||
#
|
||||
# 2048 stays avoided: that is where the chat model met
|
||||
# ErrorDeviceLost on this Vulkan stack. That failure came under a
|
||||
# 20 GB model's memory pressure and may not bind a 600 MB one, but
|
||||
# this does not need to find out.
|
||||
# Paperless chunks documents at 1024 tokens. The defaults are
|
||||
# batch 2048 / ubatch 512, and with --embeddings llama-server
|
||||
# clamps both down to 512 rather than run them mismatched — below
|
||||
# the chunk size. Matching both to 1024 fits a whole chunk in one
|
||||
# pass; 2048 is deliberately avoided, since that is where this
|
||||
# Vulkan stack fails with ErrorDeviceLost.
|
||||
- --batch-size
|
||||
- "1536"
|
||||
- "1024"
|
||||
- --ubatch-size
|
||||
- "1536"
|
||||
- "1024"
|
||||
# Room for a full chunk plus overhead rather than exactly one.
|
||||
- --ctx-size
|
||||
- "4096"
|
||||
@@ -109,22 +91,24 @@ spec:
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
# Measured, not estimated: embedding 400 real documents peaked at
|
||||
# 1.6 GiB (Prometheus max_over_time on the working set). The limit is
|
||||
# well above that because the footprint is dominated by the four slots
|
||||
# at 4096 context and the Vulkan working set rather than by the
|
||||
# weights — the Qwen model this replaces peaked at 10.6 GiB on the
|
||||
# same settings for the same 600 MiB of weights, and several
|
||||
# candidates were OOMKilled at limits that looked generous against
|
||||
# their model size.
|
||||
# The weights are only ~600 MiB, but that is not what sets the
|
||||
# footprint: four slots at 4096 context with a 1024-token micro-batch,
|
||||
# and Vulkan allocating the iGPU's working set from system RAM where
|
||||
# it counts against this cgroup.
|
||||
#
|
||||
# 4Gi was sized for the earlier 2048 context and 512 micro-batch. Both
|
||||
# were raised to match Paperless's chunking without raising this, and
|
||||
# the pod then survived interactive use and was OOMKilled (137) partway
|
||||
# through the first sustained run — roughly 300 documents in. Indexing
|
||||
# the archive would have hit it on day one.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 2Gi
|
||||
memory: 4Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
memory: 10Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
# ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
|
||||
# model needs, but still sized for a cold download.
|
||||
|
||||
@@ -6,5 +6,3 @@ resources:
|
||||
- embed.yaml
|
||||
- gate-service.yaml
|
||||
- ingress.yaml
|
||||
- wiki-embed.yaml
|
||||
- wiki-embed-ingress.yaml
|
||||
|
||||
@@ -1,20 +1,10 @@
|
||||
---
|
||||
# Holds the GGUF weights, downloaded once by llama-server itself via --hf-repo.
|
||||
# Prune-disabled so retiring this Kustomization does not throw away tens of GiB
|
||||
# of model files that would have to be pulled from Hugging Face again.
|
||||
# Holds the GGUF weights, downloaded once by the Deployment's initContainer.
|
||||
# Prune-disabled so retiring this Kustomization does not throw away several
|
||||
# GiB of model files that would have to be pulled from Hugging Face again.
|
||||
#
|
||||
# ReadWriteMany so a second replica, a one-off benchmark Job, or a candidate
|
||||
# model being evaluated can mount the same weights; the models are read-only at
|
||||
# inference time.
|
||||
#
|
||||
# Sized well beyond the two deployed models. The cache is shared by every
|
||||
# candidate under evaluation (scripts/embed-eval-real.sh), and on a slow
|
||||
# connection the download is the long pole — a shared cache lets the large
|
||||
# candidates pull in the background while the small ones are already being
|
||||
# measured, and means a model is never fetched twice across runs.
|
||||
#
|
||||
# CephFS supports online expansion (allowVolumeExpansion on the storage class),
|
||||
# so growing this does not disturb the running pods.
|
||||
# ReadWriteMany so a second replica or a one-off benchmark Job can mount the
|
||||
# same weights; the models are read-only at inference time.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
@@ -28,4 +18,4 @@ spec:
|
||||
storageClassName: ceph-filesystem
|
||||
resources:
|
||||
requests:
|
||||
storage: 120Gi
|
||||
storage: 40Gi
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
---
|
||||
# wiki-embed.renner.uno — LAN-only (no dns.renner.uno/public label), consumed by
|
||||
# bin/wiki_rank.py on the workstation and the nightly wiki jobs. Machine client,
|
||||
# so Traefik BasicAuth instead of the browser-only Authentik gate; same pattern
|
||||
# as kubernetes/apps/monitoring/loki-ingress.yaml.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: wiki-embed-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
basicAuth:
|
||||
secret: wiki-embed-basic-auth
|
||||
realm: wiki-embed
|
||||
---
|
||||
# OpenBao key apps/ai/wiki-embed-basic-auth, field `users` = htpasswd line.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wiki-embed-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: wiki-embed-basic-auth
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/ai/wiki-embed-basic-auth
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
traefik.ingress.kubernetes.io/router.middlewares: ai-wiki-embed-basic-auth@kubernetescrd
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- wiki-embed.renner.uno
|
||||
secretName: wiki-embed-tls
|
||||
rules:
|
||||
- host: wiki-embed.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: wiki-embed
|
||||
port:
|
||||
number: 8080
|
||||
@@ -1,117 +0,0 @@
|
||||
---
|
||||
# Embedding endpoint for the ~/Notes wiki ranker (bin/wiki_rank.py). Own
|
||||
# Deployment so wiki traffic never queues behind Paperless on llama-embed.
|
||||
# Same model and PVC as llama-embed, so no second download.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
replicas: 1
|
||||
# Above the 600s startupProbe budget; see embed.yaml.
|
||||
progressDeadlineSeconds: 900
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: wiki-embed
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
||||
args:
|
||||
- --hf-repo
|
||||
- gpustack/bge-m3-GGUF:Q8_0
|
||||
- --embeddings
|
||||
- --alias
|
||||
- bge-m3
|
||||
# bge-m3 is trained for CLS pooling; the GGUF default is not.
|
||||
- --pooling
|
||||
- cls
|
||||
- --n-gpu-layers
|
||||
- "99"
|
||||
- --threads
|
||||
- "2"
|
||||
# wiki_rank.py caps each note at 1500 chars and sends 2 per request.
|
||||
- --batch-size
|
||||
- "1536"
|
||||
- --ubatch-size
|
||||
- "1536"
|
||||
- --ctx-size
|
||||
- "4096"
|
||||
- --no-webui
|
||||
env:
|
||||
- name: LLAMA_CACHE
|
||||
value: /models
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
# Same sizing as llama-embed: 1.6 GiB measured peak, headroom for slots.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 2Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llama-server-models
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -1,46 +0,0 @@
|
||||
# Cilium's agent ServiceMonitor, declared here rather than by the chart.
|
||||
#
|
||||
# Reproduces what cilium 1.20.1 renders with prometheus.serviceMonitor.enabled,
|
||||
# taken field for field off the live object so scraping is unchanged: same name,
|
||||
# same namespace, same selector, same 10s interval and node relabeling.
|
||||
#
|
||||
# It lives here because the chart's template is gated on monitoring.coreos.com/v1
|
||||
# existing at install time, and on an empty cluster it does not — those CRDs
|
||||
# arrive with kube-prometheus-stack, which reconciles after infrastructure.
|
||||
# Cilium is one of infrastructure's healthChecks, so the chart aborting on the
|
||||
# missing CRD held the whole cluster: CNI waiting on monitoring waiting on CNI.
|
||||
#
|
||||
# velero/helmrelease.yaml already does exactly this for its own SM/PM and says
|
||||
# why; this is the same move for the component where it mattered most.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: cilium-agent
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app.kubernetes.io/name: cilium-agent
|
||||
app.kubernetes.io/part-of: cilium
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: cilium-agent
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- kube-system
|
||||
# The chart's Service keeps rendering (prometheus.enabled), so the target and
|
||||
# its `metrics` port are there regardless of who declares the ServiceMonitor.
|
||||
targetLabels:
|
||||
- k8s-app
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
interval: 10s
|
||||
honorLabels: true
|
||||
relabelings:
|
||||
# Agent metrics are per-node; without this every series lands without a
|
||||
# node label and BPF map pressure cannot be attributed to a machine.
|
||||
- action: replace
|
||||
sourceLabels:
|
||||
- __meta_kubernetes_pod_node_name
|
||||
targetLabel: node
|
||||
replacement: ${1}
|
||||
@@ -1,36 +0,0 @@
|
||||
# Scrapes CrowdSec. Until this landed, nothing did: `up{namespace="crowdsec"}`
|
||||
# returned zero series and Prometheus held no cs_* metric at all, so the
|
||||
# detection side of the ingress path was invisible — including whether the
|
||||
# agents were reading logs and whether any bouncer was pulling decisions.
|
||||
#
|
||||
# One monitor for both services. They carry the same shape (a `metrics` port on
|
||||
# 6060) and differ only in `app`, so a matchExpressions selector keeps this to
|
||||
# a single object rather than two near-identical ones:
|
||||
#
|
||||
# crowdsec-service the LAPI — decisions, bouncer + machine requests
|
||||
# crowdsec-agent-service the agents — acquisition, parsers, scenarios
|
||||
#
|
||||
# The crowdsec-ns CiliumNetworkPolicy already admits the monitoring namespace
|
||||
# without a port restriction, so no policy change is needed.
|
||||
#
|
||||
# `release: kube-prometheus-stack` is what the chart's Prometheus selects on.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: crowdsec
|
||||
namespace: crowdsec
|
||||
labels:
|
||||
app.kubernetes.io/name: crowdsec
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- crowdsec-service
|
||||
- crowdsec-agent-service
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
@@ -1,25 +0,0 @@
|
||||
# Both external-dns releases (Cloudflare and the UniFi webhook) carry
|
||||
# app.kubernetes.io/name: external-dns and differ only by instance, so one
|
||||
# monitor covers both. jobLabel keeps them apart as separate job values.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: external-dns
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobLabel: app.kubernetes.io/instance
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- apps
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
endpoints:
|
||||
# The controller's own metrics. Port 8080 on the unifi release is the
|
||||
# webhook sidecar and exports no external_dns_* series.
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: 60s
|
||||
@@ -1,20 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
endpoints:
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
relabelings:
|
||||
# The series would otherwise carry only the exporter pod's IP.
|
||||
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
targetLabel: node
|
||||
@@ -21,31 +21,10 @@ resources:
|
||||
# alerted if that stopped — HetznerBackupEndpointDown only probes the S3
|
||||
# endpoint, which proves reachability, not that anything was written.
|
||||
- prometheusrule-cnpg-backup.yaml
|
||||
# DNS publication. Nothing scraped external-dns, so it deleted six LAN
|
||||
# records on 2026-09-12 and stayed silent for five days.
|
||||
- prometheusrule-external-dns.yaml
|
||||
- external-dns-servicemonitor.yaml
|
||||
# Outcome check for the same failure: the names themselves, asked of the
|
||||
# UDM. Catches a cause external-dns cannot see, e.g. records removed there.
|
||||
- prometheusrule-dns-resolution.yaml
|
||||
- probes-dns.yaml
|
||||
- alloy-servicemonitor.yaml
|
||||
- cilium-servicemonitor.yaml
|
||||
- cnpg-podmonitors.yaml
|
||||
- crowdsec-servicemonitor.yaml
|
||||
# Velero's own monitors: monitoring.coreos.com CRDs arrive with
|
||||
# kube-prometheus-stack, which reconciles after infrastructure, so these
|
||||
# cannot live beside the Velero HelmRelease.
|
||||
- velero-podmonitor.yaml
|
||||
- velero-servicemonitor.yaml
|
||||
- authentik-outpost-servicemonitors.yaml
|
||||
# Rook's exporter/mgr monitors: the operator aborts its node-daemon reconcile
|
||||
# when it cannot create these, which held the CephCluster in Progressing and
|
||||
# deadlocked the whole apps layer. See the file for the cycle.
|
||||
- rook-servicemonitors.yaml
|
||||
- home-assistant-servicemonitor.yaml
|
||||
- immich-servicemonitor.yaml
|
||||
- loki-servicemonitor.yaml
|
||||
- smartctl-exporter-servicemonitor.yaml
|
||||
- intel-gpu-exporter-servicemonitor.yaml
|
||||
- probes-blackbox.yaml
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
# DNS probes for the names external-dns publishes to the UDM from
|
||||
# LoadBalancer Services. One Probe per name because the dns prober's
|
||||
# query_name is module-scoped; the target is the resolver, queried on both
|
||||
# families so a single-stack blip does not read as a missing record.
|
||||
#
|
||||
# The UDM is queried directly rather than through the cluster resolver:
|
||||
# k8s-gateway caches these with `serve_stale 1h immediate`, which would keep
|
||||
# a deleted record green for an hour.
|
||||
#
|
||||
# `release` is load-bearing — probeSelector matches on it.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ha
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ha
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
# instance is the resolver and identical across these Probes, so the
|
||||
# queried name has to come from a label.
|
||||
labels:
|
||||
dns_name: ha.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-mqtt
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_mqtt
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: mqtt.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ldap
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ldap
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: ldap.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-syslog
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_syslog
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: syslog.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ftp-scanner
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ftp_scanner
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: ftp.paperless.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-syncthing
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_syncthing
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: syncthing.paperless.renner.uno
|
||||
@@ -34,9 +34,7 @@ spec:
|
||||
# query parameter, and an exact match would break silently if the
|
||||
# instance label ever encodes it differently.
|
||||
expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
|
||||
# Two consecutive failed probes: enough to skip a single lost probe
|
||||
# through Traefik, not enough to sit on a sealed store.
|
||||
for: 1m
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: dns-resolution
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# ruleSelector matches on this label; without it the rule is created but
|
||||
# never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: dns-resolution.rules
|
||||
rules:
|
||||
# min by dns_name: both resolver families are probed, and the record
|
||||
# is only gone when neither answers. A single-family failure is the
|
||||
# UDM's dnsmasq refusing connections in bursts (k8s-gateway carries
|
||||
# the same note), which is why the window is 30m rather than the 10m
|
||||
# used for HTTP probes.
|
||||
- alert: LanDnsRecordMissing
|
||||
expr: min by (dns_name) (probe_success{job="blackbox-dns"}) == 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.dns_name }} no longer resolves to its LAN address"
|
||||
description: >-
|
||||
The UDM stopped answering for a name external-dns publishes from
|
||||
a LoadBalancer Service, or answered with the public
|
||||
*.renner.uno wildcard instead of the record. Clients that use
|
||||
the name rather than the address lose the service while
|
||||
everything in the cluster stays healthy — nothing else in the
|
||||
stack reports this. Check the external-dns alerts first, then
|
||||
the record on the UDM.
|
||||
|
||||
# Dead-man's-switch, mirroring BlackboxProbesAbsent: without probe
|
||||
# results the rule above cannot fire.
|
||||
- alert: LanDnsProbesAbsent
|
||||
expr: absent(probe_success{job="blackbox-dns"})
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No LAN DNS probe results are reaching Prometheus"
|
||||
description: >-
|
||||
The blackbox-dns Probes stopped being discovered or the exporter
|
||||
rejects the modules. A 400 from /probe yields no probe_* series
|
||||
at all, which looks the same as no Probe — check that every
|
||||
dns_lan_* module the Probes reference exists in the
|
||||
blackbox-exporter config.
|
||||
@@ -1,87 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: external-dns
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# ruleSelector matches on this label; without it the rule is created but
|
||||
# never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: external-dns.rules
|
||||
rules:
|
||||
# The signal that was missing on 2026-09-12: the sources stopped
|
||||
# producing endpoints they had produced before, and policy=sync
|
||||
# deleted the records. 10% below the 24h peak clears a single
|
||||
# retired Ingress (2 of ~48) and catches a whole class of names.
|
||||
- alert: ExternalDNSSourceEndpointsDropped
|
||||
expr: >-
|
||||
external_dns_source_endpoints_total
|
||||
< 0.9 * max_over_time(external_dns_source_endpoints_total[24h])
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is publishing {{ $value }} endpoints, below its 24h peak"
|
||||
description: >-
|
||||
external-dns stopped seeing endpoints it had before. With
|
||||
policy=sync the matching DNS records are deleted, so names stop
|
||||
resolving without anything else failing. Compare
|
||||
external_dns_source_endpoints_total against the last 24h, then
|
||||
check whether the resources still carry a hostname annotation
|
||||
the running version understands — a changed annotation prefix
|
||||
default empties the source silently. Deliberate removals also
|
||||
fire this and resolve themselves 24h later.
|
||||
|
||||
# Source errors mean the Kubernetes side failed; registry errors mean
|
||||
# the provider did. Either way desired state is computed from partial
|
||||
# input, which policy=sync acts on.
|
||||
- alert: ExternalDNSSourceErrors
|
||||
expr: increase(external_dns_source_errors_total[15m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is failing to read its sources"
|
||||
description: >-
|
||||
external-dns could not list the Kubernetes resources it derives
|
||||
records from. Check the pod log and its RBAC.
|
||||
|
||||
- alert: ExternalDNSRegistryErrors
|
||||
expr: increase(external_dns_registry_errors_total[15m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is failing to reach its DNS provider"
|
||||
description: >-
|
||||
external-dns could not read or write records at the provider.
|
||||
For external-dns-unifi that is the UniFi webhook sidecar and the
|
||||
controller behind it; for external-dns it is the Cloudflare API.
|
||||
|
||||
- alert: ExternalDNSDown
|
||||
expr: up{job=~"external-dns.*"} == 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is not being scraped"
|
||||
description: >-
|
||||
The external-dns instance is down or unreachable. Records already
|
||||
at the provider keep resolving, but nothing reconciles them any
|
||||
more and every other alert in this group is blind.
|
||||
|
||||
# Dead-man's-switch for the group, mirroring BlackboxProbesAbsent: a
|
||||
# dropped ServiceMonitor makes every rule above evaluate to nothing.
|
||||
- alert: ExternalDNSMetricsMissing
|
||||
expr: absent(external_dns_source_endpoints_total)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No external-dns metrics are reaching Prometheus"
|
||||
description: >-
|
||||
Either both external-dns instances are gone or the ServiceMonitor
|
||||
stopped being discovered. Verify it still carries
|
||||
release=kube-prometheus-stack.
|
||||
@@ -28,28 +28,24 @@ spec:
|
||||
# scheduling would also fire this — but both want a human too, so the
|
||||
# imprecision costs nothing.
|
||||
- alert: OpenBaoReplicaNotReady
|
||||
# A replica unseals itself from the static seal, so a restart is
|
||||
# Ready again in well under a minute; the duration only ignores that
|
||||
# ordinary rescheduling. A replica still sealed after it never
|
||||
# recovers on its own: the seal failed (openbao-seal-key Secret
|
||||
# missing or wrong).
|
||||
# max by: kube-state-metrics stamps instance/pod on the series, so a
|
||||
# rescheduled exporter starts a new series and resets the timer.
|
||||
expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 3
|
||||
# A replica that is sealed never recovers on its own, so the
|
||||
# duration is only there to ignore ordinary rescheduling: a healthy
|
||||
# pod moving between nodes is Ready again in well under a minute.
|
||||
expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "OpenBao has {{ $value }} of 3 replicas ready"
|
||||
description: >-
|
||||
An OpenBao replica has been unavailable for 10 minutes. A restart
|
||||
unseals itself from the static seal, so this is not the ordinary
|
||||
drain case. The service is still up on the remaining replicas,
|
||||
so nothing is broken yet, but redundancy is reduced and a second
|
||||
failure would cost quorum. Check `kubectl -n openbao get pods`,
|
||||
the pod log and `bao status` in the not-ready pod. If it is
|
||||
sealed, the static seal failed: verify the openbao-seal-key
|
||||
Secret matches `pass show homelab/openbao/seal-key`.
|
||||
An OpenBao replica has been unavailable for 10 minutes. The usual
|
||||
cause is a sealed replica after a node drain — every restart
|
||||
seals it, and it stays out of the raft cluster until unsealed by
|
||||
hand. The service is still up on the remaining replicas, so
|
||||
nothing is broken yet, but redundancy is reduced and a second
|
||||
drain would cost quorum. Check `kubectl -n openbao get pods` and
|
||||
`bao status` in the not-ready pod; unseal with the key from
|
||||
`pass` (homelab/openbao/unseal-key).
|
||||
|
||||
- alert: OpenBaoQuorumLost
|
||||
# Raft quorum of a three-node cluster is two. At one ready replica
|
||||
@@ -57,10 +53,11 @@ spec:
|
||||
# the ClusterSecretStore goes InvalidProviderConfig and no
|
||||
# ExternalSecret can refresh.
|
||||
#
|
||||
# No window: below quorum the secret store is down and every
|
||||
# ExternalSecret stalls with it, so waiting only delays the page.
|
||||
expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
|
||||
for: 0m
|
||||
# Shorter window than the warning above because this is an outage,
|
||||
# not a degradation — but not zero, so a rolling restart that
|
||||
# briefly dips does not page.
|
||||
expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
@@ -70,11 +67,9 @@ spec:
|
||||
quorum and the secret store is unavailable. ExternalSecrets stop
|
||||
refreshing; existing Secrets keep their last values, so failures
|
||||
surface only when something rotates or a new secret is needed.
|
||||
Replicas unseal themselves from the static seal; one that stays
|
||||
sealed could not. Check its log and the openbao-seal-key Secret
|
||||
against `pass show homelab/openbao/seal-key`, fix the Secret and
|
||||
restart the pod. The recovery key cannot unseal an auto-sealed
|
||||
server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
|
||||
Unseal the sealed replicas (key in `pass`,
|
||||
homelab/openbao/unseal-key) — each one individually, since a
|
||||
sealed server cannot be a standby.
|
||||
|
||||
- alert: OpenBaoMetricsMissing
|
||||
# Guards the two alerts above. Both are threshold comparisons, so if
|
||||
|
||||
@@ -58,30 +58,10 @@ spec:
|
||||
# VeleroBackupFailed alert fires on the first failure regardless — so
|
||||
# nothing is hidden, only the escalation to critical is deferred
|
||||
# until a second cycle confirms it.
|
||||
#
|
||||
# The `and on(schedule) velero_backup_last_status` gate exists because
|
||||
# velero_backup_last_successful_timestamp is an in-process gauge that
|
||||
# velero does NOT retract when a Schedule is deleted: the series just
|
||||
# stops being updated, ages past 60h and pages critical forever, for a
|
||||
# schedule that no longer exists. That is what decommissioning
|
||||
# netbird-backup-daily did on 2026-09-01.
|
||||
#
|
||||
# velero_backup_last_status is the same shape but IS retracted on
|
||||
# Schedule deletion, so it works as a liveness set for "this schedule
|
||||
# still exists". It hides nothing: velero repopulates it for every
|
||||
# existing Schedule on its first scrape after a restart (verified
|
||||
# across the three pods of the 2026-08-27 rollout), and it is written
|
||||
# on failed runs too (0) as well as successful ones (1) — so a
|
||||
# schedule that exists and is failing, or that exists and has stopped
|
||||
# firing, still carries the series and still alerts.
|
||||
expr: |
|
||||
(
|
||||
time() - velero_backup_last_successful_timestamp{schedule!=""} > 60 * 3600
|
||||
and
|
||||
velero_backup_last_successful_timestamp{schedule!=""} > 0
|
||||
)
|
||||
and on(schedule)
|
||||
velero_backup_last_status
|
||||
time() - velero_backup_last_successful_timestamp{schedule!=""} > 60 * 3600
|
||||
and
|
||||
velero_backup_last_successful_timestamp{schedule!=""} > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
# Rook's two ServiceMonitors, declared here rather than by the operator.
|
||||
#
|
||||
# Reproduced field for field off the live objects in the production cluster, so
|
||||
# scraping is unchanged: same names, same namespace, same selectors, same 10s
|
||||
# interval and `cluster` relabeling. The only thing dropped is the CephCluster
|
||||
# ownerReference, which Rook set and Flux cannot.
|
||||
#
|
||||
# They live here because CephCluster.spec.monitoring.enabled is now false, and
|
||||
# that is not a monitoring decision — it breaks a deadlock. Rook's node-daemon
|
||||
# reconcile creates this ServiceMonitor and, unlike the exporter DaemonSet next
|
||||
# to it, treats a failure as fatal (pkg/operator/ceph/cluster/nodedaemon/
|
||||
# reconcile.go, v1.20.7):
|
||||
#
|
||||
# if !cephCluster.Spec.Monitoring.MetricsDisabled {
|
||||
# // exporter reconciled; its errors are logged and reconcile continues
|
||||
# if cephCluster.Spec.Monitoring.Enabled {
|
||||
# if err := EnableCephExporterServiceMonitor(...); err != nil {
|
||||
# return errors.Wrap(err, "failed to enable service monitor")
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# On an empty cluster monitoring.coreos.com/v1 does not exist yet, so that call
|
||||
# fails with "the server could not find the requested resource (post
|
||||
# servicemonitors.monitoring.coreos.com)", the whole node-daemon reconcile
|
||||
# aborts, and the CephCluster never leaves status.phase Progressing. That is
|
||||
# what infrastructure-ceph-cluster's healthCheckExpr waits on — and the chain
|
||||
# closes on itself:
|
||||
#
|
||||
# infrastructure-ceph-cluster → CephCluster phase Ready
|
||||
# → Rook reconcile settles
|
||||
# → monitoring.coreos.com CRDs exist
|
||||
# → kube-prometheus-stack (monitoring)
|
||||
# → grafana-db
|
||||
# → infrastructure-ceph-cluster
|
||||
#
|
||||
# Nothing outside that loop can break it, so a cold rebuild stopped here with
|
||||
# every one of the twenty app Kustomizations behind it.
|
||||
#
|
||||
# Same treatment velero/helmrelease.yaml and cilium/helmrelease.yaml already
|
||||
# get, and for the same reason: the metrics stay where they are, only the
|
||||
# scrape wiring moves to the layer where its CRDs exist.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: rook-ceph-exporter
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
team: rook
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: rook-ceph-exporter
|
||||
rook_cluster: rook-ceph
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- rook-ceph
|
||||
endpoints:
|
||||
- port: ceph-exporter-http-metrics
|
||||
path: /metrics
|
||||
interval: 10s
|
||||
honorLabels: true
|
||||
# Rook stamps the cluster name so series from several Ceph clusters stay
|
||||
# separable. One cluster here, but dropping it would rename every series.
|
||||
relabelings:
|
||||
- action: replace
|
||||
replacement: rook-ceph
|
||||
targetLabel: cluster
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: rook-ceph-mgr
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
team: rook
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: rook-ceph-mgr
|
||||
rook_cluster: rook-ceph
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- rook-ceph
|
||||
endpoints:
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 10s
|
||||
honorLabels: true
|
||||
relabelings:
|
||||
- action: replace
|
||||
replacement: rook-ceph
|
||||
targetLabel: cluster
|
||||
@@ -1,163 +0,0 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"description": "CrowdSec: Erkennung im Cluster, Durchsetzung am NetBird-Edge",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Aktive Ban-Entscheidungen in der LAPI, ganz überwiegend aus den CAPI-Community-Blocklisten. Das ist die Menge, die ein scharfgeschalteter Bouncer durchsetzt.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(cs_active_decisions)", "refId": "A"}],
|
||||
"title": "Aktive Decisions",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Bouncer, die in den letzten 10 Minuten Decisions abgeholt haben. Fällt das unter die erwartete Zahl, holt ein Bouncer nichts mehr — und setzt damit einen veralteten Stand durch, ohne dass irgendetwas ausfällt.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "short"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
|
||||
"id": 2,
|
||||
"options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[10m])) > 0)", "refId": "A"}],
|
||||
"title": "Bouncer, die abholen",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Vom Agent gelesene Logzeilen pro Sekunde. Steht das auf 0, detektiert CrowdSec nichts mehr — der häufigste stille Ausfall, weil alles andere weiterläuft.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 0.01}]}, "unit": "reqps"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
|
||||
"id": 3,
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_filesource_hits_total[5m]))", "refId": "A"}],
|
||||
"title": "Gelesene Logzeilen/s",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Szenario-Overflows der letzten 24 h — also lokal erkannte Angriffe, im Gegensatz zu den von CAPI übernommenen.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
|
||||
"id": 4,
|
||||
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(increase(cs_bucket_overflowed_total[24h]))", "refId": "A"}],
|
||||
"title": "Lokale Overflows (24 h)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Abholrate je Bouncer. Der Firewall-Bouncer auf den Proxies fragt alle 10 s. Die LAPI legt bei jeder neuen beobachteten Client-Adresse einen Eintrag `<name>@<ip>` an, und beobachtet wird die Pod-IP des netbird-ingress-Peers — deshalb erscheint ein logischer Bouncer hier mehrfach und nach Pod-Rollen unter neuen Namen.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
|
||||
"id": 5,
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[5m]))", "legendFormat": "{{bouncer}}", "refId": "A"}],
|
||||
"title": "Bouncer-Abholrate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Aktive Decisions nach Grund. `http:scan` dominiert und stammt aus den CAPI-Blocklisten.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
|
||||
"id": 6,
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (cs_active_decisions)", "legendFormat": "{{reason}}", "refId": "A"}],
|
||||
"title": "Aktive Decisions nach Grund",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Welche Szenarien lokal auslösen. Das ist das, was CrowdSec selbst aus den Traefik-Logs erkennt.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 70, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
|
||||
"id": 7,
|
||||
"options": {"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (name) (increase(cs_bucket_overflowed_total[1h]))", "legendFormat": "{{name}}", "refId": "A"}],
|
||||
"title": "Szenario-Overflows (je Stunde)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Geparste gegen nicht geparste Zeilen. Ein Anstieg bei `ko` heißt meist, dass sich ein Logformat geändert hat und der Parser nicht mehr greift — CrowdSec liest dann weiter, erkennt aber nichts mehr.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
|
||||
"id": 8,
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ok_total[5m]))", "legendFormat": "geparst", "refId": "A"},
|
||||
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ko_total[5m]))", "legendFormat": "nicht geparst", "refId": "B"}
|
||||
],
|
||||
"title": "Parser",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Whitelist-Treffer nach Grund. Nützlich beim Scharfschalten: was hier zählt, wäre sonst ein Kandidat für einen Ban gewesen.",
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
|
||||
"id": 9,
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (rate(cs_node_wl_hits_ok_total[5m]))", "legendFormat": "{{reason}}", "refId": "A"}],
|
||||
"title": "Whitelist-Treffer",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
|
||||
"content": "## Was dieses Dashboard NICHT zeigt\n\nAlle Panels stammen aus der LAPI und den Agents im Cluster. Die **Bouncer auf proxy-1 und proxy-2 exportieren eigene Metriken**, die hier fehlen:\n\n- `fw_bouncer_active_decisions` — was tatsächlich in den nftables-Sets steht\n- `fw_bouncer_dropped_packets` / `_bytes` — was wirklich verworfen wurde\n- `lapi_requests_total` / `_failures_total` — aus Sicht des Bouncers\n\nSie liegen auf `127.0.0.1:60601` der Proxy-Hosts. Die Hosts sind öffentlich und haben bisher keinerlei Metrik-Pfad; den Port ins Internet zu öffnen, nur um ein Scrape-Ziel zu bekommen, wäre der falsche Tausch.\n\n**Konsequenz:** „Bouncer-Abholrate\" oben zeigt, dass ein Bouncer *fragt* — nicht, dass er die Antwort auch angewandt hat. Solange der Modus `dry-run` ist, wendet er ohnehin nichts an.\n\nEin Scrape-Pfad für die Hosts ist eigene Arbeit und Voraussetzung dafür, dass „was wird wirklich geblockt\" beantwortbar wird."
|
||||
},
|
||||
"title": "Blinder Fleck",
|
||||
"type": "text"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["homelab", "crowdsec", "security"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-6h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "CrowdSec Overview",
|
||||
"uid": "crowdsec-overview",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -1,368 +0,0 @@
|
||||
{
|
||||
"title": "Intel iGPU",
|
||||
"uid": "intel-gpu",
|
||||
"tags": ["gpu", "intel", "i915"],
|
||||
"schemaVersion": 39,
|
||||
"editable": true,
|
||||
"timezone": "",
|
||||
"refresh": "1m",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "node",
|
||||
"label": "Node",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": {
|
||||
"query": "label_values(gpumon_power, node)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"definition": "label_values(gpumon_power, node)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"current": { "selected": true, "text": ["All"], "value": ["$__all"] }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Render/3D busy now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "GPU power now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"decimals": 1,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "GT frequency now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "suffix: MHz",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Render/3D busy",
|
||||
"description": "Compute and 3D. llama.cpp's Vulkan backend runs here; 0 % during an inference means the model is on the CPU.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Video busy",
|
||||
"description": "Fixed-function decode/encode (QuickSync).",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Video\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "VideoEnhance busy",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"VideoEnhance\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Blitter busy",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Blitter\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GT frequency (actual)",
|
||||
"description": "0 while the GT sits in RC6.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "suffix: MHz",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GPU power",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Package power",
|
||||
"description": "Whole SoC (CPU + GPU + uncore), from the same PMU.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"pkg\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory controller bandwidth",
|
||||
"description": "Reads solid, writes dashed. The iGPU shares DDR5 with the CPU, so this is the whole package's traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "MiBs",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/write/" }, "properties": [{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_imc_read{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} read",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "gpumon_imc_write{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} write",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -38,9 +38,6 @@ configMapGenerator:
|
||||
- name: dashboard-blackbox-overview
|
||||
files:
|
||||
- blackbox-overview.json
|
||||
- name: dashboard-crowdsec-overview
|
||||
files:
|
||||
- crowdsec-overview.json
|
||||
- name: dashboard-disk-health
|
||||
files:
|
||||
- disk-health.json
|
||||
@@ -77,9 +74,6 @@ configMapGenerator:
|
||||
- name: dashboard-infrastructure-overview
|
||||
files:
|
||||
- infrastructure-overview.json
|
||||
- name: dashboard-intel-gpu
|
||||
files:
|
||||
- intel-gpu.json
|
||||
- name: dashboard-netbird-overview
|
||||
files:
|
||||
- netbird-overview.json
|
||||
|
||||
@@ -232,7 +232,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
io.cilium/lb-ipam-ips: "10.2.99.84,fd99::84"
|
||||
external-dns.kubernetes.io/hostname: syslog.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: syslog.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -72,73 +72,3 @@ spec:
|
||||
tcp:
|
||||
preferred_ip_protocol: ip6
|
||||
ip_protocol_fallback: true
|
||||
# One module per name: the dns prober takes the RESOLVER as its
|
||||
# target and reads query_name from the module, so a name cannot be
|
||||
# passed per scrape. The Probes in monitoring-rules/probes-dns.yaml
|
||||
# point these at the UDM.
|
||||
#
|
||||
# Both answer checks are load-bearing. valid_rcodes alone passes a
|
||||
# deleted record two labels deep (NOERROR, no answer), and the public
|
||||
# *.renner.uno wildcard answers one label deep with a CNAME to
|
||||
# proxy.renner.uno — NOERROR with the WAN proxy behind it. Requiring
|
||||
# an A and rejecting a CNAME separates a real record from both.
|
||||
dns_lan_ha:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ha.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_mqtt:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: mqtt.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_ldap:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ldap.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_syslog:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: syslog.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_ftp_scanner:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ftp.paperless.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_syncthing:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: syncthing.paperless.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# Intel iGPU utilisation, frequency and power for every node (#752).
|
||||
#
|
||||
# Not privileged: /dev/dri comes from the device plugin's monitoring resource
|
||||
# (every GPU on the node, no hostPath), and the i915 PMU that intel_gpu_top
|
||||
# reads needs CAP_PERFMON — the one capability granted. No hostPID, no
|
||||
# hostNetwork.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
# Every node is a control plane; without this the exporter covers nothing.
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: intel-gpu-exporter
|
||||
image: ghcr.io/clambin/intel-gpu-exporter:0.7.4@sha256:d78761fef018834c759db154fe9dcf3dd9db5974703fa50ca21f6abc35221be1
|
||||
args:
|
||||
# No -device: intel_gpu_top picks the only GPU the plugin injects.
|
||||
- -interval=5s
|
||||
ports:
|
||||
- name: http-metrics
|
||||
containerPort: 9100
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
# Root so the PERFMON capability is effective without file
|
||||
# capabilities on the binary.
|
||||
runAsUser: 0
|
||||
runAsNonRoot: false
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["PERFMON"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
gpu.intel.com/monitoring: 1
|
||||
limits:
|
||||
memory: 128Mi
|
||||
gpu.intel.com/monitoring: 1
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: http-metrics
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
ports:
|
||||
- name: http-metrics
|
||||
port: 9100
|
||||
targetPort: http-metrics
|
||||
protocol: TCP
|
||||
@@ -16,7 +16,6 @@ resources:
|
||||
- prometheus-gate-ingress.yaml
|
||||
- ci-diagnostics-rbac.yaml
|
||||
- smartctl-exporter.yaml
|
||||
- intel-gpu-exporter.yaml
|
||||
- dashboards/
|
||||
# prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
|
||||
# separate Flux Kustomization that dependsOn `monitoring` — otherwise the
|
||||
|
||||
@@ -28,7 +28,7 @@ spec:
|
||||
# net1 instead. master: ens19 resolves on every k8s-{1,2,3}/edge
|
||||
# worker, though it is no longer the same kind of device everywhere:
|
||||
# on the bare-metal control-plane nodes it is a VLAN 30 sub-interface
|
||||
# of uplink rather than a NIC, which is why the datapath needs
|
||||
# of enp2s0f0np0 rather than a NIC, which is why the datapath needs
|
||||
# bpf.vlanBypass to carry it (issue #592).
|
||||
k8s.v1.cni.cncf.io/networks: music-assistant-iot
|
||||
spec:
|
||||
@@ -55,7 +55,7 @@ spec:
|
||||
# stdlib only; Flux keeps the tag in lockstep via the shared image
|
||||
# policy) — no extra dependency.
|
||||
- name: reconcile-publish-ip
|
||||
image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.10.1 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
@@ -65,7 +65,7 @@ spec:
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: music-assistant
|
||||
image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.10.1 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
env:
|
||||
- name: LOG_LEVEL
|
||||
value: info
|
||||
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
mountPath: /consume
|
||||
containers:
|
||||
- name: vsftpd
|
||||
image: gitea.renner.uno/k.renner/vsftpd:build.1789630783 # {"$imagepolicy": "flux-system:vsftpd"}
|
||||
image: gitea.renner.uno/k.renner/vsftpd:build.1784731469 # {"$imagepolicy": "flux-system:vsftpd"}
|
||||
ports:
|
||||
- name: ftp-ctrl
|
||||
containerPort: 21
|
||||
@@ -70,19 +70,24 @@ spec:
|
||||
- name: consume
|
||||
mountPath: /home/scanner
|
||||
subPath: scanner
|
||||
# Asserts the control socket is listening without opening an FTP
|
||||
# connection: vsftpd 3.0.5 segfaults its listener on a fraction of
|
||||
# the connections it accepts. The passive range is negotiated per
|
||||
# transfer and is not listening at rest, so it is not probed.
|
||||
# FTP has no health verb, so the control port accepting a connection
|
||||
# is what can be asserted. It is enough for the failure that matters:
|
||||
# if vsftpd dies the scanner cannot deposit anything and nothing else
|
||||
# in the chain notices, because paperless only ever sees an empty
|
||||
# consume directory — which is indistinguishable from nobody having
|
||||
# scanned.
|
||||
#
|
||||
# Only the control port is probed. The passive range is negotiated
|
||||
# per transfer and is not listening at rest.
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "netstat -lnt | grep -q ':21 '"]
|
||||
tcpSocket:
|
||||
port: ftp-ctrl
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "netstat -lnt | grep -q ':21 '"]
|
||||
tcpSocket:
|
||||
port: ftp-ctrl
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -119,7 +124,7 @@ metadata:
|
||||
# label — FTP must not be exposed via the cargo tunnel (clear-text
|
||||
# protocol, public exposure not appropriate). The brother scanner
|
||||
# uses this hostname instead of the LB IP.
|
||||
external-dns.kubernetes.io/hostname: ftp.paperless.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: ftp.paperless.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -22,7 +22,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: gotenberg
|
||||
image: gotenberg/gotenberg:8.37.0 # {"$imagepolicy": "flux-system:gotenberg"}
|
||||
image: gotenberg/gotenberg:8.36.0 # {"$imagepolicy": "flux-system:gotenberg"}
|
||||
args:
|
||||
- "gotenberg"
|
||||
- "--chromium-disable-javascript=true"
|
||||
|
||||
@@ -25,15 +25,18 @@ resources:
|
||||
configMapGenerator:
|
||||
# vsftpd config for the ftp-scanner deployment. The content-hash name
|
||||
# suffix is intentionally LEFT ENABLED (unlike home-assistant's nginx
|
||||
# config): vsftpd reads these into vsftpd.conf at container start, so a
|
||||
# changed value must roll the pod, which the renamed ConfigMap does.
|
||||
#
|
||||
# No PASV_ADDRESS: the LB address is declared once, on the Service
|
||||
# (io.cilium/lb-ipam-ips). The image default 0.0.0.0 tells PASV clients
|
||||
# to reuse the control-connection address.
|
||||
# config): vsftpd templates pasv_address from PASV_ADDRESS at container
|
||||
# start, so a changed value must roll the pod. With the hash suffix,
|
||||
# editing any value here renames the generated ConfigMap, which mutates
|
||||
# the Deployment's pod template and triggers an automatic rolling
|
||||
# restart. Without it, an IP change (e.g. an LB re-allocation) silently
|
||||
# leaves the running pod advertising the stale PASV address, which
|
||||
# breaks classic-PASV clients (older Brother scanners) while EPSV
|
||||
# clients keep working.
|
||||
- name: ftp-scanner-config
|
||||
namespace: paperless
|
||||
literals:
|
||||
- PASV_ADDRESS=10.2.99.82
|
||||
- PASV_MIN_PORT=21100
|
||||
- PASV_MAX_PORT=21104
|
||||
- FTP_USER=scanner
|
||||
|
||||
@@ -82,10 +82,7 @@ data:
|
||||
# primarily English archives — the better fit for a German corpus.
|
||||
PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
|
||||
PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
|
||||
# Must match --alias in kubernetes/apps/llama-server/embed.yaml. Changing the
|
||||
# embedding model invalidates every stored vector, so a swap means the LLM
|
||||
# index has to be rebuilt.
|
||||
PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
|
||||
PAPERLESS_AI_LLM_EMBEDDING_MODEL: "qwen3-embedding-0.6b"
|
||||
PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
@@ -143,20 +140,7 @@ spec:
|
||||
# on every paperless pod start: tools belong in the image, not
|
||||
# installed at runtime. Dated tag so a change is a reviewable diff
|
||||
# here, matching the ci-ansible / alpine-jq convention.
|
||||
#
|
||||
# konairius/, not gitea_admin/, and that is not cosmetic. This is an
|
||||
# INIT container, so paperless does not start without it — and during
|
||||
# a cold rebuild gitea.renner.uno does not exist, because Gitea runs
|
||||
# inside the cluster being rebuilt. A rehearsal left this pod in
|
||||
# Init:ErrImagePull on exactly this image.
|
||||
#
|
||||
# The node falls back to a ghcr.io copy in that window
|
||||
# (machine.registries.mirrors, applied by bootstrap.sh). Containerd
|
||||
# keeps the repository path when it falls through, so the path has to
|
||||
# exist on both registries — hence konairius/, which matches the ghcr
|
||||
# namespace. Steady state still pulls from Gitea and never leaves the
|
||||
# network.
|
||||
image: gitea.renner.uno/konairius/alpine-openssl:2026-09-04
|
||||
image: gitea.renner.uno/gitea_admin/alpine-openssl:2026-07-30
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -183,7 +167,7 @@ spec:
|
||||
drop: ["ALL"]
|
||||
containers:
|
||||
- name: paperless
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:3.2.1 # {"$imagepolicy": "flux-system:paperless-ngx"}
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2 # {"$imagepolicy": "flux-system:paperless-ngx"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
@@ -338,6 +322,7 @@ spec:
|
||||
- port: 80
|
||||
targetPort: http
|
||||
---
|
||||
---
|
||||
# ExternalName mirror so the paperless-namespace Ingress can backend to
|
||||
# the gate outpost in the authentik namespace (cross-namespace Service
|
||||
# refs aren't supported in standard Ingress).
|
||||
|
||||
@@ -145,7 +145,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
io.cilium/lb-ipam-ips: "10.2.99.85,fd99::85"
|
||||
external-dns.kubernetes.io/hostname: syncthing.paperless.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: syncthing.paperless.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
# trmnl
|
||||
|
||||
Terminus (self-hosted TRMNL BYOS server) plus the headless renderer that
|
||||
screenshots the Home Assistant entrance dashboard and pushes it in. One
|
||||
device: a TRMNL X at the entrance, 1872x1404, 16 greys.
|
||||
|
||||
## Device firmware
|
||||
|
||||
Terminus mirrors one model-agnostic binary and it is the **ESP32-C3** (OG)
|
||||
build. This panel is a TRMNL X, i.e. ESP32-S3, so that file is not the
|
||||
artefact for a manual flash here. Use
|
||||
`https://trmnl-fw.s3.us-east-2.amazonaws.com/trmnl_x/flash/FW<version>.bin`
|
||||
— a merged image at offset `0x0` — and run `esptool image-info` on
|
||||
anything before writing it.
|
||||
|
||||
The panel runs 1.8.16, flashed over USB on 2026-09-05. It shipped on
|
||||
1.8.6, where turning on `firmware_update` froze the screen for as long as
|
||||
an update was offered (fixed upstream in 1.8.7, commit `9b7a0f7`).
|
||||
|
||||
## Image automation
|
||||
|
||||
Both images are enrolled (`infrastructure/image-automation/terminus.yaml`,
|
||||
`trmnl-renderer.yaml`) and track the latest `0.x`. A bad bump on either
|
||||
surfaces as a **blank or corrupted panel**, not as a failed rollout — no
|
||||
readiness probe can tell that a rendered screen became unreadable. Look at
|
||||
the panel after a bump lands; revert the tag if it is wrong.
|
||||
|
||||
## Device endpoint
|
||||
|
||||
`trmnl-device.renner.uno` is the device-facing name, TLS via Traefik
|
||||
(`device-ingress.yaml`). `API_URI` must carry the same scheme the device
|
||||
polls with, because Terminus bakes it into the absolute `image_url` it
|
||||
returns. The LoadBalancer in `device-service.yaml` keeps `10.2.99.89`
|
||||
without a hostname as a cleartext fallback that depends on neither
|
||||
Traefik nor certificate renewal — the captive portal accepts a bare
|
||||
address.
|
||||
@@ -1,52 +0,0 @@
|
||||
# Device-facing endpoint, over TLS.
|
||||
#
|
||||
# This replaces the reason the cleartext LoadBalancer in device-service.yaml
|
||||
# was given a hostname at all. That file claimed the firmware "would reject
|
||||
# the internal CA" — but nothing here is signed by an internal CA:
|
||||
# letsencrypt-cluster solves DNS-01 against the renner.uno Cloudflare zone,
|
||||
# so trmnl.renner.uno already serves a public Let's Encrypt certificate
|
||||
# (issuer CN=YR2) that any client with a normal CA bundle trusts. A
|
||||
# LAN-only name gets the same treatment — DNS-01 never needs the host to be
|
||||
# publicly reachable — so the device can speak HTTPS like everything else.
|
||||
#
|
||||
# The hostname moves here from the Service's external-dns annotation rather
|
||||
# than being duplicated: two sources claiming trmnl-device.renner.uno would
|
||||
# have external-dns flapping the record between the Traefik VIP and the
|
||||
# device LB. The LoadBalancer keeps its address and stays reachable at
|
||||
# http://10.2.99.89 as a nameless fallback if TLS ever turns out to be the
|
||||
# thing that breaks.
|
||||
#
|
||||
# CONSEQUENCE, and the reason API_URI in terminus.yaml changes with this:
|
||||
# Traefik's `web` entrypoint carries a global http -> websecure redirect
|
||||
# (helmrelease.yaml: `to: websecure, scheme: https, permanent: true`), so
|
||||
# once this name points at Traefik, http://trmnl-device.renner.uno answers
|
||||
# 308 rather than serving. Whether the firmware follows that is not
|
||||
# documented, so the device is provisioned with the https URL and Terminus
|
||||
# hands back https image_urls to match. No mixed-scheme fetch.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: terminus-device
|
||||
namespace: trmnl
|
||||
# No external-dns hostname annotation: external-dns-unifi runs with
|
||||
# --source=ingress, so it registers the hosts in spec.rules on its own.
|
||||
# Same as the terminus Ingress next door.
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- trmnl-device.renner.uno
|
||||
secretName: trmnl-device-tls
|
||||
rules:
|
||||
- host: trmnl-device.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: terminus
|
||||
port:
|
||||
number: 80
|
||||
@@ -1,24 +1,15 @@
|
||||
# Cleartext fallback for the device, reachable by address only.
|
||||
# Device-facing endpoint. Cleartext HTTP, deliberately.
|
||||
#
|
||||
# This used to be THE device endpoint, on the premise that the firmware
|
||||
# "cannot be handed the internal CA". The premise was wrong: nothing here
|
||||
# hands out an internal CA for these names — letsencrypt-cluster solves
|
||||
# DNS-01 against the renner.uno Cloudflare zone and issues publicly-trusted
|
||||
# certificates for LAN-only hosts, which is why trmnl.renner.uno has always
|
||||
# served an issuer CN=YR2 certificate. The device speaks HTTPS via
|
||||
# device-ingress.yaml, which now owns the trmnl-device.renner.uno name.
|
||||
#
|
||||
# Kept, without a hostname, because it is the one path to Terminus that
|
||||
# depends on neither Traefik nor certificate renewal: http://10.2.99.89
|
||||
# still serves /api/setup and /api/display. If TLS is ever what breaks the
|
||||
# panel, the recovery is to type that address into the captive portal.
|
||||
#
|
||||
# Scope of the exposure is unchanged: LAN-only (no public DNS label, no
|
||||
# Cloudflare record), no admin session — the device authenticates with its
|
||||
# MAC in the `ID` header plus a per-device api_key.
|
||||
# The TRMNL firmware initiates every request and cannot be handed the
|
||||
# internal CA, so terminating TLS here would simply break polling. Scope of
|
||||
# the exposure: this Service is LAN-only (no public DNS label, no Cloudflare
|
||||
# record), it carries no admin session — the device authenticates with its
|
||||
# MAC in the `ID` header plus a per-device api_key — and the Terminus admin
|
||||
# UI stays on the TLS route at trmnl.renner.uno.
|
||||
#
|
||||
# Same shape as home-assistant-bgp: a dedicated address on the Cilium BGP
|
||||
# pool.
|
||||
# pool rather than a TLS-less Ingress, which would be a new pattern for this
|
||||
# repo. The device is provisioned with the HOSTNAME, not the address.
|
||||
#
|
||||
# The address is requested explicitly, matching every other LB service here.
|
||||
# Left to allocate freely, Cilium IPAM handed out the first free address in
|
||||
@@ -39,10 +30,8 @@ metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: terminus
|
||||
lb-pool: bgp
|
||||
# No external-dns hostname: device-ingress.yaml claims that name. Two
|
||||
# sources claiming trmnl-device.renner.uno would flap the record between
|
||||
# the Traefik VIP and this address (external-dns runs --policy=sync).
|
||||
annotations:
|
||||
external-dns.alpha.kubernetes.io/hostname: trmnl-device.renner.uno
|
||||
lbipam.cilium.io/ips: "10.2.99.89,fd99::89"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
|
||||
@@ -6,9 +6,6 @@ resources:
|
||||
- valkey.yaml
|
||||
- terminus.yaml
|
||||
- ingress.yaml
|
||||
# TLS device endpoint (owns trmnl-device.renner.uno) plus the nameless
|
||||
# cleartext LoadBalancer it falls back to.
|
||||
- device-ingress.yaml
|
||||
- device-service.yaml
|
||||
# Headless HA screenshotter. Lives in this namespace because Terminus
|
||||
# FETCHES rendered screens back from it (uri delivery mode), so the
|
||||
|
||||
@@ -177,14 +177,7 @@ spec:
|
||||
# webhook start failing with "expired JWT access token" about
|
||||
# thirty minutes after seeding.
|
||||
- name: render-schedule
|
||||
# konairius/, not gitea_admin/: this container starts during a cold
|
||||
# rebuild, when Gitea does not exist because it runs inside the
|
||||
# cluster being rebuilt — a rehearsal left it in Init:ImagePullBackOff.
|
||||
# The node falls back to a ghcr.io copy then
|
||||
# (machine.registries.mirrors, applied by bootstrap.sh), and containerd
|
||||
# keeps the repository path, so it has to exist on both registries.
|
||||
# Steady state still pulls from Gitea.
|
||||
image: gitea.renner.uno/konairius/alpine-jq:2026-09-04
|
||||
image: gitea.renner.uno/gitea_admin/alpine-jq:2026-07-30
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -234,12 +227,16 @@ spec:
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
containers:
|
||||
# Enrolled in image automation (flux-system:trmnl-renderer). A bad
|
||||
# bump shows up as a corrupted screen, not a failed rollout — check
|
||||
# the panel after one lands. ARCH-SUFFIXED image: upstream publishes
|
||||
# no multi-arch manifest.
|
||||
# Pinned, not enrolled in image automation — same reasoning as
|
||||
# Terminus itself. This drives a wall-mounted panel and upstream
|
||||
# is pre-1.0; an unattended bump that changes the dithering
|
||||
# pipeline or the webhook payload shows up as a corrupted
|
||||
# screen, not as a failed rollout.
|
||||
#
|
||||
# Note the image is ARCH-SUFFIXED: upstream publishes no
|
||||
# multi-arch manifest, only trmnl-ha-amd64 and trmnl-ha-aarch64.
|
||||
- name: renderer
|
||||
image: ghcr.io/usetrmnl/trmnl-ha-amd64:0.10.3 # {"$imagepolicy": "flux-system:trmnl-renderer"}
|
||||
image: ghcr.io/usetrmnl/trmnl-ha-amd64:0.9.5
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 10000
|
||||
|
||||
@@ -7,17 +7,11 @@ data:
|
||||
TZ: "Europe/Berlin"
|
||||
HANAMI_PORT: "2345"
|
||||
# What the device is told to fetch from. Terminus bakes this into the
|
||||
# absolute `image_url` it hands back on /api/display, so it has to be a
|
||||
# name the DEVICE can resolve and fetch — and it has to carry the same
|
||||
# scheme the device polls with, or every screen fetch is a second,
|
||||
# cleartext connection. No trailing slash: the same string is typed into
|
||||
# the captive portal at provisioning time.
|
||||
#
|
||||
# https since device-ingress.yaml took over trmnl-device.renner.uno. The
|
||||
# earlier value was http on the theory that the firmware would reject an
|
||||
# internal CA; there is no internal CA here — letsencrypt-cluster issues
|
||||
# publicly-trusted certs for LAN-only names via DNS-01.
|
||||
API_URI: "https://trmnl-device.renner.uno"
|
||||
# absolute `image_url` it hands back on /api/display, so it MUST be the
|
||||
# cleartext LAN name, not trmnl.renner.uno — the firmware speaks plain
|
||||
# HTTP and would reject the internal CA. No trailing slash: the same
|
||||
# string is typed into the captive portal at provisioning time.
|
||||
API_URI: "http://trmnl-device.renner.uno"
|
||||
KEYVALUE_URL: "redis://terminus-valkey:6379"
|
||||
# Rack::Attack rate-limit bypass. Beyond upstream's RFC1918 defaults this
|
||||
# adds fc00::/7, which covers the pod CIDR (fd10:244::/48), the Cilium LB
|
||||
@@ -134,7 +128,7 @@ spec:
|
||||
# is a device-bricking class of risk. Bumps are a reviewed, manual
|
||||
# act — see kubernetes/infrastructure/image-automation/README.md.
|
||||
- name: terminus
|
||||
image: ghcr.io/usetrmnl/terminus:0.74.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
image: ghcr.io/usetrmnl/terminus:0.68.0
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 2345
|
||||
@@ -197,7 +191,7 @@ spec:
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
- name: sidekiq
|
||||
image: ghcr.io/usetrmnl/terminus:0.74.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
image: ghcr.io/usetrmnl/terminus:0.68.0
|
||||
command: ["bundle", "exec", "sidekiq", "-r", "./config/sidekiq.rb"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
@@ -18,7 +18,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: whisper
|
||||
image: rhasspy/wyoming-whisper:3.8.1 # {"$imagepolicy": "flux-system:wyoming-whisper"}
|
||||
image: rhasspy/wyoming-whisper:3.7.0 # {"$imagepolicy": "flux-system:wyoming-whisper"}
|
||||
args:
|
||||
- --model=base-int8
|
||||
- --language=de
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
---
|
||||
# ServiceAccount is the OpenBao identity (auth/kubernetes/role/wiki-index); no
|
||||
# Kubernetes API access needed.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
---
|
||||
# Index (~17 MB JSON for 1700 notes). Survives restarts so a pod start does not
|
||||
# re-embed the wiki; a lost volume only costs one full sync (~5 min).
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: wiki-index-data
|
||||
namespace: ai
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
# Embedding index over the Notes repo's wiki/ (ci/images/wiki-index). Syncs from
|
||||
# Gitea on push webhook and every RESYNC_SECONDS, embeds via wiki-embed, serves
|
||||
# POST /rank to ~/Notes/bin/wiki_rank.py through wiki-index.renner.uno.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
serviceAccountName: wiki-index
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: wiki-index
|
||||
# Per-commit tag (build-ci-images.yml): immutable, no same-day overwrite.
|
||||
image: gitea.renner.uno/gitea_admin/wiki-index:2026-09-20-4b529cd1
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: GITEA_URL
|
||||
value: https://gitea.renner.uno
|
||||
- name: NOTES_REPO
|
||||
value: k.renner/notes
|
||||
- name: NOTES_BRANCH
|
||||
value: main
|
||||
- name: EMBED_URL
|
||||
value: http://wiki-embed.ai.svc.cluster.local:8080
|
||||
- name: BAO_ADDR
|
||||
value: http://openbao.openbao.svc.cluster.local:8200
|
||||
- name: BAO_ROLE
|
||||
value: wiki-index
|
||||
- name: BAO_SECRET_PATH
|
||||
value: homelab/data/apps/ai/wiki-index
|
||||
- name: INDEX_PATH
|
||||
value: /data/index.json
|
||||
- name: RESYNC_SECONDS
|
||||
value: "1800"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
# Pure-Python cosine over 1700 x 1024 floats per query; the index itself is ~60 MB in memory.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
# /health: the process serves (liveness). /ready: an index exists; a cold
|
||||
# start on an empty PVC embeds every note (~5 min) before traffic arrives.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
failureThreshold: 12
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 60
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 3
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: wiki-index-data
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -1,56 +0,0 @@
|
||||
---
|
||||
# wiki-index.renner.uno — LAN-only (no dns.renner.uno/public label), machine
|
||||
# client (bin/wiki_rank.py), so Traefik BasicAuth like wiki-embed.renner.uno.
|
||||
# Gitea's webhook does not pass here; it targets the ClusterIP Service directly.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: wiki-index-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
basicAuth:
|
||||
secret: wiki-index-basic-auth
|
||||
realm: wiki-index
|
||||
---
|
||||
# OpenBao key apps/ai/wiki-index-basic-auth, field `users` = htpasswd line.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wiki-index-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: wiki-index-basic-auth
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/ai/wiki-index-basic-auth
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
traefik.ingress.kubernetes.io/router.middlewares: ai-wiki-index-basic-auth@kubernetescrd
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- wiki-index.renner.uno
|
||||
secretName: wiki-index-tls
|
||||
rules:
|
||||
- host: wiki-index.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: wiki-index
|
||||
port:
|
||||
number: 8080
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
@@ -34,7 +34,7 @@ spec:
|
||||
# Image automation, range 3.x (see image-automation/woodpecker.yaml).
|
||||
# MUST stay version-matched with woodpecker-server (gRPC parity) — both
|
||||
# track the same range/filter so they bump together on each release.
|
||||
image: woodpeckerci/woodpecker-agent:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
env:
|
||||
- name: WOODPECKER_SERVER
|
||||
value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
|
||||
|
||||
@@ -1,3 +1,18 @@
|
||||
# Dedicated namespace where the kubernetes backend spawns pipeline step pods,
|
||||
# isolated from the apps namespace.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: woodpecker-ci
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
# Image-build steps run a privileged docker/buildx plugin pod (see the
|
||||
# server's WOODPECKER_PLUGINS_PRIVILEGED). Talos enforces Pod Security;
|
||||
# without this the privileged step pods are rejected at admission. The
|
||||
# namespace is dedicated to ephemeral CI step pods, so scoping the
|
||||
# exemption here keeps it off apps/.
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
# minor + patch auto-update within 3.x; only the 4.0 major is held for
|
||||
# a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
|
||||
# -bump hazard is gone). Stays in lockstep with the agent.
|
||||
image: woodpeckerci/woodpecker-server:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
env:
|
||||
- name: WOODPECKER_HOST
|
||||
value: https://ci.renner.uno
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user