Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  claude-botandClaude Fable 5.1
|
fb27f3909a | ||
|
claude-botandClaude Fable 5.1
|
6d4c4e3a99 | ||
|
claude-botandClaude Fable 5.1
|
af7004e85c |
@@ -42,7 +42,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: fetch-alerts
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
@@ -101,7 +101,7 @@ steps:
|
||||
bash scripts/ci/make-diag-kubeconfig.sh
|
||||
|
||||
- name: alert-triage-agent
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
|
||||
@@ -89,7 +89,7 @@ steps:
|
||||
--destination gitea.renner.uno/gitea_admin/ci-k8s:latest
|
||||
|
||||
- name: build-vmtest
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
directory: ci/images/vmtest
|
||||
depends_on: []
|
||||
when:
|
||||
@@ -349,87 +349,3 @@ steps:
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/heating-can:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/heating-can:latest
|
||||
|
||||
# netbird-service-sync: the controller that reconciles NetBird reverse-proxy
|
||||
# services (public Ingress hosts -> TLS passthrough -> Traefik ClusterIP).
|
||||
# Runtime artifact like heating-can; same registry so the audit tracks it.
|
||||
- name: build-netbird-service-sync
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/netbird-service-sync
|
||||
depends_on: []
|
||||
when:
|
||||
- event: deployment
|
||||
- event: push
|
||||
path: ["ci/images/netbird-service-sync/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/netbird-service-sync:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/netbird-service-sync:latest
|
||||
|
||||
# wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea and
|
||||
# queried by ~/Notes/bin/wiki_rank.py. Runtime artifact like netbird-service-sync.
|
||||
- name: build-wiki-index
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/wiki-index
|
||||
depends_on: []
|
||||
when:
|
||||
- event: deployment
|
||||
- event: push
|
||||
path: ["ci/images/wiki-index/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
# Also a per-commit tag: several builds a day (branch pushes and the
|
||||
# merge) all rewrite the dated tag, and the deployment pin raced them
|
||||
# on 2026-09-20. The commit tag is immutable and needs no digest lookup.
|
||||
- SHORT=$(printf '%s' "$CI_COMMIT_SHA" | cut -c1-8)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/wiki-index:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/wiki-index:$TAG-$SHORT \
|
||||
--destination gitea.renner.uno/gitea_admin/wiki-index:latest
|
||||
|
||||
# bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime
|
||||
# artifact like heating-can; same registry so the audit tracks it.
|
||||
- name: build-bird
|
||||
image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
|
||||
directory: ci/images/bird
|
||||
depends_on: []
|
||||
when:
|
||||
- event: deployment
|
||||
- event: push
|
||||
path: ["ci/images/bird/**", ".woodpecker/build-ci-images.yml"]
|
||||
environment:
|
||||
GITEA_USER: gitea_admin
|
||||
GITEA_TOKEN:
|
||||
from_secret: ci_images_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
|
||||
- printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
|
||||
- TAG=$(date +%Y-%m-%d)
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--dockerfile Containerfile \
|
||||
--destination gitea.renner.uno/gitea_admin/bird:$TAG \
|
||||
--destination gitea.renner.uno/gitea_admin/bird:latest
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
# and ci-claude:latest. The Containerfile makes this layer
|
||||
# byte-reproducible, so rebuilding an unchanged version dedups
|
||||
# in the registry instead of orphaning a fresh ~260 MB copy.
|
||||
# 3. bump-pr — (only when any pin is off the latest version) rewrite every
|
||||
# consumer pin to ci-claude:<version> and open a PR. A human
|
||||
# 3. bump-pr — (only when the version actually changed) rewrite the 5
|
||||
# consumer pins to ci-claude:<version> and open a PR. A human
|
||||
# reviews and merges it; nothing auto-merges.
|
||||
#
|
||||
# TAG SCHEME — deliberately version tags (ci-claude:<claude-version>), NOT the dated
|
||||
@@ -107,22 +107,31 @@ steps:
|
||||
[0-9]*.[0-9]*.[0-9]*) : ;;
|
||||
*) echo "npm returned an implausible version '$$NEW' — aborting."; exit 1 ;;
|
||||
esac
|
||||
# Distinct tags the consumers currently reference. Divergent pins are
|
||||
# normal (a pipeline bumped on its own); the bump rewrites all of them.
|
||||
CUR=$$(grep -hoE 'ci-claude:[0-9][0-9A-Za-z._-]*' $$CONSUMERS | sed 's/^ci-claude://' | sort -u | tr '\n' ' ' | sed 's/ $$//')
|
||||
[ -n "$$CUR" ] || { echo "No ci-claude pin found in the consumers — aborting."; exit 1; }
|
||||
echo "latest=$$NEW current=[$$CUR] event=$$CI_PIPELINE_EVENT"
|
||||
# The tag every consumer currently references. They are kept identical,
|
||||
# so this must resolve to EXACTLY ONE non-empty tag; anything else
|
||||
# (no match, or a partial/disagreeing set) aborts rather than risk a
|
||||
# corrupting sed later. Counting with `grep -c .` (non-empty lines)
|
||||
# instead of `wc -l` is deliberate: an empty result yields 0 here but
|
||||
# 1 under wc -l, which would slip past the guard.
|
||||
CUR=$$(grep -hoE 'ci-claude:[0-9A-Za-z._-]+' $$CONSUMERS | sed 's/^ci-claude://' | sort -u)
|
||||
NTAGS=$$(printf '%s' "$$CUR" | grep -c . || true)
|
||||
if [ -z "$$CUR" ] || [ "$$NTAGS" -ne 1 ]; then
|
||||
echo "Expected exactly one ci-claude tag across the consumers, got [$$CUR]."
|
||||
echo "Refusing to guess — fix the pins by hand."; exit 1
|
||||
fi
|
||||
echo "latest=$$NEW current=$$CUR event=$$CI_PIPELINE_EVENT"
|
||||
printf '%s' "$$NEW" > "$$CI_WORKSPACE/.ci-refresh/version"
|
||||
printf '%s' "$$CUR" > "$$CI_WORKSPACE/.ci-refresh/current"
|
||||
# Build when any pin is off latest, or on a push (Containerfile edit must
|
||||
# ship even if the version is unchanged). Bump when any pin is off latest.
|
||||
# Build when the version changed, or on a push (Containerfile edit must
|
||||
# ship even if the version is unchanged). Bump the pins only when the
|
||||
# version the consumers reference is not the latest.
|
||||
if [ "$$CUR" != "$$NEW" ] || [ "$$CI_PIPELINE_EVENT" = "push" ]; then
|
||||
touch "$$CI_WORKSPACE/.ci-refresh/do-build"; echo "will build $$NEW"
|
||||
else
|
||||
echo "already on $$NEW — nothing to build"
|
||||
fi
|
||||
if [ "$$CUR" != "$$NEW" ]; then
|
||||
touch "$$CI_WORKSPACE/.ci-refresh/do-bump"; echo "will open bump PR [$$CUR] -> $$NEW"
|
||||
touch "$$CI_WORKSPACE/.ci-refresh/do-bump"; echo "will open bump PR $$CUR -> $$NEW"
|
||||
else
|
||||
echo "consumers already pin $$NEW — no PR"
|
||||
fi
|
||||
@@ -166,9 +175,6 @@ steps:
|
||||
- |
|
||||
[ -f "$$CI_WORKSPACE/.ci-refresh/do-bump" ] || { echo "no version change — no PR"; exit 0; }
|
||||
set -eu
|
||||
# Secret values can carry a trailing newline; it breaks the push URL and the auth header.
|
||||
PUSH_TOKEN=$$(printf '%s' "$$PUSH_TOKEN" | tr -d '\r\n')
|
||||
[ -n "$$PUSH_TOKEN" ] || { echo "PUSH_TOKEN is empty — aborting"; exit 1; }
|
||||
NEW=$$(cat "$$CI_WORKSPACE/.ci-refresh/version")
|
||||
CUR=$$(cat "$$CI_WORKSPACE/.ci-refresh/current")
|
||||
# Both were validated non-empty by the decide step; re-assert before a
|
||||
@@ -184,11 +190,9 @@ steps:
|
||||
# is not a maintained list.
|
||||
CONSUMERS=$$(grep -rlE 'ci-claude:[0-9]' .woodpecker)
|
||||
[ -n "$$CONSUMERS" ] || { echo "No ci-claude consumers found — refusing to continue."; exit 1; }
|
||||
# NEW goes into a sed replacement: allow only a plain x.y.z version.
|
||||
printf '%s' "$$NEW" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$$' || { echo "refusing non-x.y.z version '$$NEW'"; exit 1; }
|
||||
# Every version-pinned tag becomes NEW, whatever it was; consumers only.
|
||||
# Exact-string swap of the current tag for the new one, consumers only.
|
||||
for f in $$CONSUMERS; do
|
||||
sed -i "s#ci-claude:[0-9][0-9A-Za-z._-]*#ci-claude:$$NEW#g" "$$f"
|
||||
sed -i "s#ci-claude:$$CUR#ci-claude:$$NEW#g" "$$f"
|
||||
done
|
||||
# Author is claude-bot (the PAT's account); the commit subject makes
|
||||
# the automated-bump nature clear.
|
||||
|
||||
@@ -60,7 +60,7 @@ steps:
|
||||
# run that never touched them. Tracked in #461.
|
||||
- name: kustomize-build
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
@@ -158,7 +158,7 @@ steps:
|
||||
# whenever it trails the deployed server. If you bump the server, bump here.
|
||||
- name: woodpecker-lint
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: woodpeckerci/woodpecker-cli:v3.18.0-alpine
|
||||
image: woodpeckerci/woodpecker-cli:v3.17.0-alpine
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
@@ -246,7 +246,7 @@ steps:
|
||||
# this step fails permanently on its own documentation.
|
||||
- name: no-digest-pins
|
||||
depends_on: [] # see note above `steps:`
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
@@ -17,6 +17,6 @@ when:
|
||||
|
||||
steps:
|
||||
- name: model-selection-tests
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
|
||||
commands:
|
||||
- sh scripts/ci/test-model-selection.sh
|
||||
|
||||
@@ -4,7 +4,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: e2e
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
+2
-2
@@ -26,12 +26,12 @@ steps:
|
||||
# fetch-models.sh response mapping. No cluster, no secrets, no network — the
|
||||
# ci-claude image just provides sh + jq. Keeps the flag-injection regression
|
||||
# test from rotting (it exercises the exact injection shapes).
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
|
||||
commands:
|
||||
- sh scripts/ci/test-model-selection.sh
|
||||
|
||||
- name: e2e-test
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
|
||||
@@ -49,7 +49,7 @@ steps:
|
||||
# (the issue-pick logic needs jq, which only the ci-claude image has);
|
||||
# minting a 1h read-only token per run is cheap. Degrades to
|
||||
# snapshot-only if the ci-diagnostics ServiceAccount isn't reconciled.
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-08-10
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
@@ -62,7 +62,7 @@ steps:
|
||||
- bash scripts/ci/make-diag-kubeconfig.sh
|
||||
|
||||
- name: issue-fix-agent
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
|
||||
@@ -65,7 +65,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: pr-review-agent
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
|
||||
@@ -70,7 +70,7 @@ steps:
|
||||
# this step is skipped — otherwise we'd fork-bomb PRs.
|
||||
when:
|
||||
- branch: master
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
|
||||
@@ -30,7 +30,7 @@ when:
|
||||
|
||||
steps:
|
||||
- name: update-audit-agent
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
# Nightly wiki maintenance for the Notes vault (k.renner/notes), replacing the
|
||||
# workstation's wiki-maintenance and wiki-consolidate timers: distill the raw/
|
||||
# backlog into wiki/ (wiki-distill agent), audit wiki/ with the vault's own
|
||||
# script (structural checks, no model), then consolidate (wiki-consolidate
|
||||
# agent: verifies a cohort of notes against sources, repos and the live
|
||||
# cluster and edits them). The pipeline commits and pushes as Claude Bot; the
|
||||
# push webhook then refreshes the wiki-index service.
|
||||
#
|
||||
# The agents address the vault as ~/Notes and the repos as ~/source/<repo>, so
|
||||
# each agent step links those into the workspace instead of rewriting them.
|
||||
when:
|
||||
- event: cron
|
||||
cron: wiki-maintenance
|
||||
# Hand-started runs: `woodpecker pipeline create k.renner/homelab --branch master`.
|
||||
# The typesafe_api_key secret is scoped to cron and manual, not deployment.
|
||||
- event: manual
|
||||
|
||||
steps:
|
||||
- name: checkout-notes
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
pull: true
|
||||
environment:
|
||||
WIKI_BOT_TOKEN:
|
||||
from_secret: wiki_bot_token
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
# Token only on the wire, never in .git/config. Secrets set from files carry
|
||||
# a trailing newline (run 10220: "could not read Username"); strip CR/LF.
|
||||
TOKEN=$$(printf '%s' "$$WIKI_BOT_TOKEN" | tr -d '\r\n')
|
||||
git -c http.extraheader="AUTHORIZATION: token $$TOKEN" \
|
||||
clone --depth 50 --branch main https://gitea.renner.uno/k.renner/notes.git notes
|
||||
echo "raw backlog: $$(ls notes/raw/*.md | grep -v _index.md | wc -l) file(s)"
|
||||
# Full history for the consolidate agent's `git log` checks; the
|
||||
# workspace clone is shallow and tree-filtered.
|
||||
mkdir -p sources
|
||||
git clone --quiet https://gitea.renner.uno/k.renner/homelab.git sources/homelab
|
||||
git -c http.extraheader="AUTHORIZATION: token $$TOKEN" \
|
||||
clone --quiet https://gitea.renner.uno/k.renner/homeassistant.git sources/homeassistant
|
||||
|
||||
- name: fetch-diag
|
||||
# Read-only live-cluster access for the consolidate agent, same mechanism
|
||||
# as issue-fix.yml: ./.ci-diag/kubectl plus a 1h ci-diagnostics token.
|
||||
image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
|
||||
environment:
|
||||
SOPS_AGE_KEY:
|
||||
from_secret: sops_age_key
|
||||
S3_ACCESS_KEY_ID:
|
||||
from_secret: hetzner_s3_access_key
|
||||
S3_SECRET_ACCESS_KEY:
|
||||
from_secret: hetzner_s3_secret_key
|
||||
commands:
|
||||
- bash scripts/ci/fetch-kubeconfig.sh
|
||||
- bash scripts/ci/make-diag-kubeconfig.sh
|
||||
|
||||
- name: distill
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
pull: true
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
TYPESAFE_API_KEY:
|
||||
from_secret: typesafe_api_key
|
||||
WIKI_INDEX_BASIC_AUTH:
|
||||
from_secret: wiki_index_basic_auth
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
# An empty queue must not start the agent: its startup context alone
|
||||
# costs more than the whole audit.
|
||||
BACKLOG=$$(ls notes/raw/*.md | grep -v _index.md | wc -l)
|
||||
if [ "$$BACKLOG" -eq 0 ]; then
|
||||
echo "distill: raw/ is empty, skipping the agent"
|
||||
exit 0
|
||||
fi
|
||||
ln -sfn "$$PWD/notes" "$$HOME/Notes"
|
||||
mkdir -p "$$HOME/.config/wiki-index"
|
||||
printf '%s' "$$WIKI_INDEX_BASIC_AUTH" | tr -d '\r\n' > "$$HOME/.config/wiki-index/basic-auth"
|
||||
chmod 600 "$$HOME/.config/wiki-index/basic-auth"
|
||||
"$$HOME/Notes/bin/wiki_rank.py" --probe-index
|
||||
MODEL=$$(sh scripts/ci/pick-model.sh "distill raw source notes into atomic wiki notes: read sources, check the wiki for existing coverage, write or merge notes; careful writing, moderate reasoning")
|
||||
if [ -n "$$MODEL" ]; then echo "distill: selected model $$MODEL"; else echo "distill: using the CLI default model"; fi
|
||||
cd notes
|
||||
cat ../ci/agents/wiki-maintenance/distill.md | claude $${MODEL:+--model $$MODEL} \
|
||||
--agent wiki-distill \
|
||||
--permission-mode acceptEdits \
|
||||
--allowedTools "Read,Edit,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*),Bash(mv /root/Notes/raw/:*),Bash(mkdir -p /root/Notes/:*),Bash(~/Notes/bin/wiki_rank.py:*),Bash(~/Notes/bin/wiki_ask.py:*),Bash(mv ~/Notes/raw/:*),Bash(mkdir -p ~/Notes/:*)" \
|
||||
--settings ../ci/agents/wiki-maintenance/settings.json \
|
||||
-p
|
||||
|
||||
- name: audit
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
pull: true
|
||||
commands:
|
||||
# Findings are report lines in output/, not a pipeline failure.
|
||||
- python3 -B notes/bin/wiki_audit.py --vault "$$PWD/notes"
|
||||
|
||||
- name: consolidate
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
pull: true
|
||||
environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN:
|
||||
from_secret: claude_code_oauth_token
|
||||
TYPESAFE_API_KEY:
|
||||
from_secret: typesafe_api_key
|
||||
WIKI_INDEX_BASIC_AUTH:
|
||||
from_secret: wiki_index_basic_auth
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
ln -sfn "$$PWD/notes" "$$HOME/Notes"
|
||||
ln -sfn "$$PWD/sources" "$$HOME/source"
|
||||
mkdir -p "$$HOME/.config/wiki-index"
|
||||
printf '%s' "$$WIKI_INDEX_BASIC_AUTH" | tr -d '\r\n' > "$$HOME/.config/wiki-index/basic-auth"
|
||||
chmod 600 "$$HOME/.config/wiki-index/basic-auth"
|
||||
if [ -f .ci-diag/kubeconfig ]; then
|
||||
ln -sf "$$PWD/.ci-diag/kubectl" /usr/local/bin/kubectl
|
||||
export KUBECONFIG="$$PWD/.ci-diag/kubeconfig"
|
||||
else
|
||||
echo "consolidate: no ci-diag kubeconfig, live cluster checks unavailable"
|
||||
fi
|
||||
cd notes
|
||||
# JSON output: the agent's report and the run's cost come out of one
|
||||
# record at the end instead of a 45-minute silent log.
|
||||
# One source for the Bash allowlist: the settings file (run 10608: the
|
||||
# agent writes ~/Notes, the rules carried only /root/Notes).
|
||||
BASH_RULES=$$(jq -r '[.permissions.allow[] | select(startswith("Bash("))] | join(",")' ../ci/agents/wiki-maintenance/consolidate-settings.json)
|
||||
rc=0
|
||||
cat ../ci/agents/wiki-maintenance/consolidate.md | claude \
|
||||
--agent wiki-consolidate \
|
||||
--permission-mode acceptEdits \
|
||||
--allowedTools "Read,Edit,Write,Glob,Grep,Agent,$$BASH_RULES" \
|
||||
--settings ../ci/agents/wiki-maintenance/consolidate-settings.json \
|
||||
--output-format json \
|
||||
-p > /tmp/consolidate.json || rc=$$?
|
||||
# A killed or crashed claude leaves no valid JSON; report rc, not a jq error.
|
||||
if [ -s /tmp/consolidate.json ] && jq -e . /tmp/consolidate.json > /dev/null; then
|
||||
jq -r '.result // empty' /tmp/consolidate.json
|
||||
jq -c '{duration_ms, num_turns, total_cost_usd, is_error}' /tmp/consolidate.json
|
||||
else
|
||||
echo "consolidate: no result record (exit $$rc)"; head -c 2000 /tmp/consolidate.json || true
|
||||
fi
|
||||
echo "consolidate: git log --oneline of the run:"
|
||||
git log --oneline origin/main..HEAD
|
||||
exit $$rc
|
||||
|
||||
- name: push-notes
|
||||
image: gitea.renner.uno/gitea_admin/ci-claude:2.1.280
|
||||
pull: true
|
||||
environment:
|
||||
WIKI_BOT_TOKEN:
|
||||
from_secret: wiki_bot_token
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
cd notes
|
||||
git config user.email "claude-bot@renner.uno"
|
||||
git config user.name "Claude Bot"
|
||||
# Only the vault content is committed; anything an agent leaves elsewhere
|
||||
# in the checkout (run 10362: scratch scripts in the root) stays behind.
|
||||
# The consolidate agent commits its own snapshots, so HEAD may already be
|
||||
# ahead of origin/main with a clean tree.
|
||||
git add -A -- wiki raw output .wiki-consolidation-state.json
|
||||
git diff --cached --quiet || git commit -q -m "wiki maintenance $$(date -u +%F): distill + audit + consolidate (CI run $$CI_PIPELINE_NUMBER)"
|
||||
if [ "$$(git rev-list --count origin/main..HEAD)" -eq 0 ]; then
|
||||
echo "wiki-maintenance: no changes"
|
||||
exit 0
|
||||
fi
|
||||
# The workstation pushes main on its own (notes-push.path), so main moves
|
||||
# while the agents run (run 10241: non-fast-forward). The run's work goes
|
||||
# to a branch and a PR that this step merges through Gitea; a conflict
|
||||
# leaves the PR open for a human and fails the run.
|
||||
BRANCH="wiki-maintenance/$$(date -u +%F)-$$CI_PIPELINE_NUMBER"
|
||||
git checkout -q -b "$$BRANCH"
|
||||
TOKEN=$$(printf '%s' "$$WIKI_BOT_TOKEN" | tr -d '\r\n')
|
||||
git -c http.extraheader="AUTHORIZATION: token $$TOKEN" push origin "HEAD:$$BRANCH"
|
||||
API="https://gitea.renner.uno/api/v1/repos/k.renner/notes"
|
||||
jq -n --arg title "wiki maintenance $$(date -u +%F) (CI run $$CI_PIPELINE_NUMBER)" \
|
||||
--arg body "Nightly distill + audit + consolidate. Merged by the pipeline; open only if the merge failed." \
|
||||
--arg head "$$BRANCH" '{title: $$title, body: $$body, head: $$head, base: "main"}' > /tmp/pr.json
|
||||
code=$$(curl -s -o /tmp/pr-resp.json -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token $$TOKEN" -H "Content-Type: application/json" --data @/tmp/pr.json "$$API/pulls")
|
||||
echo "POST pull -> HTTP $$code"
|
||||
[ "$$code" -ge 200 ] && [ "$$code" -lt 300 ] || { cat /tmp/pr-resp.json; exit 1; }
|
||||
NUM=$$(jq -r '.number' /tmp/pr-resp.json)
|
||||
# Merge without delete_branch_after_merge: run 10254 got HTTP 500 from that
|
||||
# call although the merge itself had landed. Verify the PR state instead of
|
||||
# trusting the status code, then drop the branch as a separate, best-effort step.
|
||||
code=$$(curl -s -o /tmp/merge-resp.json -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token $$TOKEN" -H "Content-Type: application/json" \
|
||||
--data '{"Do":"merge"}' "$$API/pulls/$$NUM/merge")
|
||||
echo "POST merge #$$NUM -> HTTP $$code"
|
||||
MERGED=$$(curl -s -H "Authorization: token $$TOKEN" "$$API/pulls/$$NUM" | jq -r '.merged')
|
||||
echo "PR #$$NUM merged: $$MERGED"
|
||||
[ "$$MERGED" = "true" ] || { cat /tmp/merge-resp.json; echo "PR #$$NUM left open"; exit 1; }
|
||||
code=$$(curl -s -o /tmp/del-resp.json -w '%{http_code}' -X DELETE -H "Authorization: token $$TOKEN" \
|
||||
"$$API/branches/$$(printf '%s' "$$BRANCH" | sed 's|/|%2F|g')")
|
||||
echo "DELETE branch $$BRANCH -> HTTP $$code (best effort)"
|
||||
@@ -26,16 +26,8 @@ edge_open3e_poll_seconds: 30
|
||||
# entries when added), so the multi-frame read is cheap at a 30 s poll.
|
||||
# Records faults/info events only (NormalOperation, LagDeviceDtcReported,
|
||||
# LegionellaProtectionTemperatureReached) — NOT setpoint or program changes.
|
||||
# onebase.*: the 29 paired ZigBee slots on the One Base gateway (ECU 0x6c3) --
|
||||
# 8 radiator valves, 9 climate sensors, 9 underfloor actuators, 3 floor
|
||||
# thermostats. Per-device room temperature, active setpoint, humidity, valve
|
||||
# opening/demand, battery and RSSI; the local replacement for the ViCare cloud
|
||||
# room sensors. Needs the onebase overlay (files/open3e-onebase.py) -- the stock
|
||||
# codec declares 57 bytes and a paired slot answers with 68.
|
||||
# Only CurrentValues are polled. The room and device registries (1884.., 2084..)
|
||||
# are static configuration and are read on demand, not every 30 s.
|
||||
edge_open3e_datapoints: >-
|
||||
vitocal.259,vitocal.260,vitocal.268,vitocal.269,vitocal.271,vitocal.274,vitocal.286,vitocal.288,vitocal.318,vitocal.321,vitocal.324,vitocal.335,vitocal.336,vitocal.396,vitocal.402,vitocal.403,vitocal.476,vitocal.477,vitocal.543,vitocal.548,vitocal.565,vitocal.1007,vitocal.1043,vitocal.1190,vitocal.1211,vitocal.1391,vitocal.1416,vitocal.1417,vitocal.2320,vitocal.2346,vitocal.2350,vitocal.2351,vitocal.2352,vitocal.2369,vitocal.2370,vitocal.2486,vitocal.2487,vitocal.2488,vitocal.2543,vitocal.2560,vitocal.2622,vitocal.2624,vitocal.2625,vitocal.2735,vitocal.3016,vx3.266,vx3.378,vx3.379,vx3.380,vx3.535,vx3.1603,vx3.1664,vx3.1690,vx3.1801,vx3.1802,vx3.1810,vx3.1828,vx3.1830,vx3.1577,vx3.1578,vx3.1579,vx3.1580,vx3.1581,vx3.1582,vx3.1834,vx3.1836,vx3.2214,vx3.2240,vx3.3085,vx3.3086,vx3.3087,vx3.3088,vx3.3089,vx3.3090,vcmu.320,vcmu.322,vcmu.325,vcmu.355,vcmu.389,vcmu.391,vcmu.1772,vcmu.1773,vcmu.1775,vcmu.1776,vcmu.2256,vcmu.2259,vcmu.2333,vcmu.2334,vcmu.2569,vcmu.2571,vitocal.2791,onebase.2086,onebase.2089,onebase.2092,onebase.2095,onebase.2098,onebase.2101,onebase.2104,onebase.2107,onebase.2110,onebase.2113,onebase.2116,onebase.2119,onebase.2122,onebase.2125,onebase.2128,onebase.2131,onebase.2134,onebase.2137,onebase.2140,onebase.2143,onebase.2262,onebase.2265,onebase.2268,onebase.2271,onebase.2274,onebase.2277,onebase.2280,onebase.2283,onebase.2286
|
||||
vitocal.259,vitocal.260,vitocal.268,vitocal.269,vitocal.271,vitocal.274,vitocal.286,vitocal.288,vitocal.318,vitocal.321,vitocal.324,vitocal.335,vitocal.336,vitocal.396,vitocal.402,vitocal.403,vitocal.476,vitocal.477,vitocal.543,vitocal.548,vitocal.565,vitocal.1007,vitocal.1043,vitocal.1190,vitocal.1211,vitocal.1391,vitocal.1416,vitocal.1417,vitocal.2320,vitocal.2346,vitocal.2350,vitocal.2351,vitocal.2352,vitocal.2369,vitocal.2370,vitocal.2486,vitocal.2487,vitocal.2488,vitocal.2543,vitocal.2560,vitocal.2622,vitocal.2624,vitocal.2625,vitocal.2735,vitocal.3016,vx3.266,vx3.378,vx3.379,vx3.380,vx3.535,vx3.1603,vx3.1664,vx3.1690,vx3.1801,vx3.1802,vx3.1810,vx3.1828,vx3.1830,vx3.1577,vx3.1578,vx3.1579,vx3.1580,vx3.1581,vx3.1582,vx3.1834,vx3.1836,vx3.2214,vx3.2240,vx3.3085,vx3.3086,vx3.3087,vx3.3088,vx3.3089,vx3.3090,vcmu.320,vcmu.322,vcmu.325,vcmu.355,vcmu.389,vcmu.391,vcmu.1772,vcmu.1773,vcmu.1775,vcmu.1776,vcmu.2256,vcmu.2259,vcmu.2333,vcmu.2334,vcmu.2569,vcmu.2571,vitocal.2791
|
||||
# ECU addresses; -dev selects codec set, tx selects the UDS address.
|
||||
# vcmu (refrigerant-circuit controller) has no dedicated codec set —
|
||||
# omitting dpList makes open3e fall back to the generic datapoint list,
|
||||
@@ -46,9 +38,6 @@ edge_open3e_devices:
|
||||
# files/open3e-vx3plus.py) — die Stock-vx3-Liste filtert sie sonst weg.
|
||||
vx3: { tx: "0x6a1", dpList: "/config/vx3plus.py" }
|
||||
vcmu: { tx: "0x68c" }
|
||||
# One Base ZigBee gateway. Overlay re-declares the paired ZigBee slots at
|
||||
# their real 68-byte length (see files/open3e-onebase.py).
|
||||
onebase: { tx: "0x6c3", dpList: "/config/onebase.py" }
|
||||
|
||||
edge_e3oncan_gap_seconds: 10
|
||||
edge_e3oncan_mqtt_format: "{device}_{didNumber:04d}_{didName}"
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
# Managed by ansible (roles/heating-can). Datapoint overlay for the One Base
|
||||
# ZigBee gateway on ECU 0x6c3 (BACKENDGATEWAY).
|
||||
#
|
||||
# Upstream declares ZigBee<N>DeviceCurrentValues as 57 bytes. A PAIRED device
|
||||
# slot on this gateway answers with 68: the 57 known bytes plus the parent
|
||||
# device serial and three trailing bytes. udsoncan reads only the declared
|
||||
# length and then treats the next two bytes -- the first half of that serial --
|
||||
# as a foreign data identifier, so every paired slot fails to decode with
|
||||
# "Server returned values for data identifier 0x.... that was not requested".
|
||||
# The bogus identifier differs per device because the serial does.
|
||||
#
|
||||
# This overlay re-declares the 29 paired slots at 68 bytes, reusing upstream's
|
||||
# sub-type list so the per-device-type O3ESwitch (climate sensor / TRV / floor
|
||||
# thermostat / actuator, incl. DeviceTemperatureSetpoint) is inherited rather
|
||||
# than copied.
|
||||
#
|
||||
# EMPTY slots answer with 57 and decode against the stock codec, so they are
|
||||
# left alone and are not polled. A slot that is later unpaired therefore starts
|
||||
# reporting a decode error on its topic instead of a value.
|
||||
#
|
||||
# Verified on ECU 0x6c3, 2026-09-17: 29 paired slots, all 68 bytes.
|
||||
from open3e.Open3Ecodecs import O3EComplexType, RawCodec
|
||||
from open3e.Open3Edatapoints import dataIdentifiers as _master
|
||||
|
||||
# ZigBee<N>DeviceCurrentValues of the paired slots: devices 1-20 run 2086..2143
|
||||
# in steps of 3, devices 21-29 continue at 2262..2286.
|
||||
_PAIRED_CURRENT_VALUES = [2086 + 3 * n for n in range(20)] + [
|
||||
2262 + 3 * n for n in range(9)
|
||||
]
|
||||
|
||||
|
||||
def _with_parent_serial(did):
|
||||
stock = _master["dids"][did]
|
||||
return O3EComplexType(
|
||||
68,
|
||||
stock.id,
|
||||
list(stock.subTypes)
|
||||
+ [
|
||||
RawCodec(8, "ParentDeviceSerialnumber"),
|
||||
RawCodec(2, "Unknown_65_66"),
|
||||
RawCodec(1, "Unknown_67"),
|
||||
],
|
||||
desc=stock.desc,
|
||||
info=stock.info,
|
||||
acc=stock.acc,
|
||||
)
|
||||
|
||||
|
||||
dataIdentifiers = {
|
||||
"dids": {
|
||||
**_master["dids"],
|
||||
**{did: _with_parent_serial(did) for did in _PAIRED_CURRENT_VALUES},
|
||||
}
|
||||
}
|
||||
@@ -129,13 +129,6 @@
|
||||
mode: "0644"
|
||||
notify: Restart heating-can
|
||||
|
||||
- name: Deploy open3e One Base datapoint overlay (68-byte ZigBee slots)
|
||||
ansible.builtin.copy:
|
||||
src: open3e-onebase.py
|
||||
dest: "{{ heating_can_base_dir }}/onebase.py"
|
||||
mode: "0644"
|
||||
notify: Restart heating-can
|
||||
|
||||
# --- Compose project ---------------------------------------------------------
|
||||
- name: Deploy the broker credential env file
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -17,7 +17,6 @@ services:
|
||||
volumes:
|
||||
- ./devices.json:/config/devices.json:ro
|
||||
- ./vx3plus.py:/config/vx3plus.py:ro
|
||||
- ./onebase.py:/config/onebase.py:ro
|
||||
env_file:
|
||||
# Broker credential, 0600 -- see heating-can.env.j2.
|
||||
- heating-can.env
|
||||
|
||||
@@ -32,13 +32,8 @@ netbird_watchdog_fail_threshold: 3
|
||||
|
||||
# Minimum seconds between watchdog-issued restarts, so a genuinely broken
|
||||
# backend can't put the proxy into a restart loop. Shared by both the URL-probe
|
||||
# watchdog and the flood watchdog below. One minute: the route-manager wedge
|
||||
# recurs on every routing-peer loss, and rolling node reboots lose one peer
|
||||
# every few minutes; a suppressed restart there leaves the wedge standing
|
||||
# until the URL probe's three rounds fire (3.5 min of 000 on every domain,
|
||||
# measured 2026-09-07). A minute still caps a loop at one restart per
|
||||
# probe interval.
|
||||
netbird_watchdog_cooldown_seconds: 60
|
||||
# watchdog and the flood watchdog below.
|
||||
netbird_watchdog_cooldown_seconds: 600
|
||||
|
||||
# Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain
|
||||
# substring in the proxy's docker logs by the flood watchdog. On this build the
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
{
|
||||
"//": "Permission scope for the wiki-consolidate agent in CI: read the vault and the",
|
||||
"//2": "source repos cloned under /root/source, read the live cluster through the",
|
||||
"//3": "ci-diagnostics kubeconfig, edit wiki/, output/ and the rotation state only.",
|
||||
"//4": "Deny rules win; git and kubectl are read-only subcommands, no network tools.",
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Read(//root/Notes/**)",
|
||||
"Read(//root/source/**)",
|
||||
"Edit(//root/Notes/wiki/**)",
|
||||
"Edit(//root/Notes/output/**)",
|
||||
"Edit(//root/Notes/.wiki-consolidation-state.json)",
|
||||
"Bash(/root/Notes/bin/wiki-snapshot.sh:*)",
|
||||
"Bash(/root/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(/root/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(~/Notes/bin/wiki-snapshot.sh:*)",
|
||||
"Bash(~/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(~/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(~/Notes/bin/wiki-delete.sh:*)",
|
||||
"Bash(git -C /root/Notes log:*)",
|
||||
"Bash(git -C /root/Notes show:*)",
|
||||
"Bash(git -C /root/Notes diff:*)",
|
||||
"Bash(git -C /root/Notes status:*)",
|
||||
"Bash(git -C /root/Notes rev-parse:*)",
|
||||
"Bash(git -C /root/Notes ls-files:*)",
|
||||
"Bash(git -C ~/Notes log:*)",
|
||||
"Bash(git -C ~/Notes show:*)",
|
||||
"Bash(git -C ~/Notes diff:*)",
|
||||
"Bash(git -C ~/Notes status:*)",
|
||||
"Bash(git -C ~/Notes rev-parse:*)",
|
||||
"Bash(git -C ~/Notes ls-files:*)",
|
||||
"Bash(kubectl get:*)",
|
||||
"Bash(kubectl describe:*)",
|
||||
"Bash(kubectl version:*)",
|
||||
"Bash(kubectl api-resources:*)",
|
||||
"Bash(kubectl explain:*)",
|
||||
"Bash(git log:*)",
|
||||
"Bash(git show:*)",
|
||||
"Bash(git status:*)",
|
||||
"Bash(git diff:*)",
|
||||
"Bash(git rev-parse:*)",
|
||||
"Bash(git ls-files:*)",
|
||||
"Bash(git -C /root/source/homelab log:*)",
|
||||
"Bash(git -C /root/source/homelab show:*)",
|
||||
"Bash(git -C /root/source/homelab diff:*)",
|
||||
"Bash(git -C /root/source/homelab status:*)",
|
||||
"Bash(git -C /root/source/homelab rev-parse:*)",
|
||||
"Bash(git -C /root/source/homelab ls-files:*)",
|
||||
"Bash(git -C /root/source/homeassistant log:*)",
|
||||
"Bash(git -C /root/source/homeassistant show:*)",
|
||||
"Bash(git -C /root/source/homeassistant diff:*)",
|
||||
"Bash(git -C /root/source/homeassistant status:*)",
|
||||
"Bash(git -C /root/source/homeassistant rev-parse:*)",
|
||||
"Bash(git -C /root/source/homeassistant ls-files:*)",
|
||||
"Bash(git -C ~/source/homelab log:*)",
|
||||
"Bash(git -C ~/source/homelab show:*)",
|
||||
"Bash(git -C ~/source/homelab diff:*)",
|
||||
"Bash(git -C ~/source/homelab status:*)",
|
||||
"Bash(git -C ~/source/homelab rev-parse:*)",
|
||||
"Bash(git -C ~/source/homelab ls-files:*)",
|
||||
"Bash(git -C ~/source/homeassistant log:*)",
|
||||
"Bash(git -C ~/source/homeassistant show:*)",
|
||||
"Bash(git -C ~/source/homeassistant diff:*)",
|
||||
"Bash(git -C ~/source/homeassistant status:*)",
|
||||
"Bash(git -C ~/source/homeassistant rev-parse:*)",
|
||||
"Bash(git -C ~/source/homeassistant ls-files:*)",
|
||||
"Bash(/root/Notes/bin/wiki-delete.sh:*)",
|
||||
"Bash(ls:*)",
|
||||
"Bash(date:*)",
|
||||
"Bash(wc:*)",
|
||||
"Bash(stat:*)",
|
||||
"Bash(head:*)",
|
||||
"Bash(tail:*)"
|
||||
],
|
||||
"deny": [
|
||||
"Read(//proc/**)",
|
||||
"Read(//etc/**)",
|
||||
"Read(//root/.config/**)",
|
||||
"Read(//root/.kube/**)",
|
||||
"Read(//woodpecker/**/.ci-diag/**)",
|
||||
"Read(//root/Notes/.git/**)",
|
||||
"Read(//root/source/*/.git/**)",
|
||||
"Read(//root/Notes/_archive/**)",
|
||||
"Edit(//root/Notes/raw/**)",
|
||||
"Edit(//root/Notes/_archive/**)",
|
||||
"Edit(//root/source/**)",
|
||||
"Bash(kubectl apply:*)",
|
||||
"Bash(kubectl delete:*)",
|
||||
"Bash(kubectl patch:*)",
|
||||
"Bash(kubectl edit:*)",
|
||||
"Bash(kubectl scale:*)",
|
||||
"Bash(kubectl rollout:*)",
|
||||
"Bash(kubectl exec:*)",
|
||||
"Bash(kubectl cp:*)",
|
||||
"Bash(kubectl drain:*)",
|
||||
"Bash(kubectl cordon:*)",
|
||||
"Bash(kubectl label:*)",
|
||||
"Bash(kubectl annotate:*)",
|
||||
"Bash(git push:*)",
|
||||
"Bash(git reset:*)",
|
||||
"Bash(git checkout:*)",
|
||||
"Bash(git clean:*)",
|
||||
"Bash(git rebase:*)",
|
||||
"Bash(git config:*)",
|
||||
"Bash(git -C /root/Notes config:*)",
|
||||
"Bash(git -C ~/Notes config:*)",
|
||||
"Bash(git -C /root/source/homelab config:*)",
|
||||
"Bash(git -C /root/source/homeassistant config:*)",
|
||||
"Bash(git -C ~/source/homelab config:*)",
|
||||
"Bash(git -C ~/source/homeassistant config:*)",
|
||||
"Bash(rm:*)",
|
||||
"Bash(rm -rf:*)",
|
||||
"Bash(curl:*)",
|
||||
"Bash(wget:*)",
|
||||
"Bash(ssh:*)",
|
||||
"Bash(tea:*)",
|
||||
"Bash(woodpecker:*)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
You are running as the scheduled wiki-maintenance job in CI. The Notes vault is checked out at ~/Notes (the repository k.renner/notes); the homelab and homeassistant repositories are checked out under ~/source/. `kubectl` reads the live cluster with a read-only identity.
|
||||
|
||||
Run the wiki-consolidate agent now for today's scheduled consolidation pass. Follow its instructions exactly: snapshot the vault to git first, abort if that fails, select today's rotating cohort of 20 from the state file, verify claims against both their sources and the live systems, act on the verdicts, write the dated report to ~/Notes/output/, update the state file, and snapshot again at the end. Do not push; the pipeline pushes the run's commits.
|
||||
|
||||
Report the counts and the coverage progress when you are done.
|
||||
@@ -1,7 +0,0 @@
|
||||
You are running as the scheduled wiki-maintenance job in CI. The Notes vault is checked out at ~/Notes (the repository k.renner/notes). Do exactly this, nothing else:
|
||||
|
||||
Glob ~/Notes/raw/*.md (top level only, never ~/Notes/raw/_processed/). Distill every file you find into ~/Notes/wiki/ following your process, including the ranker check for every concept. Move each fully processed source to ~/Notes/raw/_processed/. If raw/ has no top-level files besides _index.md, say so and stop.
|
||||
|
||||
Do not commit or push; the pipeline does that after you finish. Do not touch anything outside ~/Notes/wiki/ and ~/Notes/raw/.
|
||||
|
||||
End with one line per file: which wiki notes it produced or merged into, or "nothing distillable".
|
||||
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"//": "Permission scope for the wiki-maintenance agent (wiki-distill) in CI.",
|
||||
"//2": "The step's environment holds a Gitea push token, a TypeSafe key and the wiki-index",
|
||||
"//3": "client credential, so the Read tool is kept away from /proc (env dumps), /root outside",
|
||||
"//4": "the vault symlink target, /etc and the checkout's .git/. The vault itself is the work.",
|
||||
"//5": "Mirrors ci/agents/issue-fix/settings.json; absolute globs because the vault is reached",
|
||||
"//6": "through the ~/Notes symlink the pipeline creates.",
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Read(//root/Notes/**)",
|
||||
"Edit(//root/Notes/wiki/**)",
|
||||
"Edit(//root/Notes/output/**)",
|
||||
"Edit(//root/Notes/raw/**)",
|
||||
"Bash(/root/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(/root/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(mv /root/Notes/raw/:*)",
|
||||
"Bash(mkdir -p /root/Notes/:*)",
|
||||
"Bash(~/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(~/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(mv ~/Notes/raw/:*)",
|
||||
"Bash(mkdir -p ~/Notes/:*)",
|
||||
"Bash(ls:*)",
|
||||
"Bash(date:*)",
|
||||
"Bash(wc:*)",
|
||||
"Bash(head:*)",
|
||||
"Bash(tail:*)"
|
||||
],
|
||||
"deny": [
|
||||
"Read(//proc/**)",
|
||||
"Read(//etc/**)",
|
||||
"Read(//root/.config/**)",
|
||||
"Read(//root/Notes/.git/**)",
|
||||
"Read(//root/Notes/_archive/**)",
|
||||
"Edit(//root/Notes/_archive/**)",
|
||||
"Bash(rm:*)",
|
||||
"Bash(git:*)",
|
||||
"Bash(curl:*)",
|
||||
"Bash(wget:*)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
|
||||
|
||||
# bird: BGP daemon for the pod-network mesh over pod0. Distro package, no build
|
||||
# step — the image is the binary plus its runtime.
|
||||
#
|
||||
# 3.22 rather than the 3.21 the other images here pin: bird2 first appears in
|
||||
# 3.22/community.
|
||||
#
|
||||
# Config is read from /etc/bird/bird.conf, not the package default
|
||||
# /etc/bird.conf, so the mount cannot shadow what the package ships.
|
||||
#
|
||||
# kubectl renders the init container's per-node config: go-template-file for
|
||||
# the templating, a field selector for excluding the node itself, in-cluster
|
||||
# config for the auth. From the distro, so there is no version literal to drift
|
||||
# and no download to trust. It is only ever used to read core/v1 Nodes.
|
||||
RUN apk add --no-cache bird2 kubectl
|
||||
|
||||
# Fails the build rather than the first pod if a binary or its path is wrong.
|
||||
RUN /usr/sbin/bird --version \
|
||||
&& kubectl version --client > /dev/null
|
||||
|
||||
ENTRYPOINT ["/usr/sbin/bird", "-f", "-c", "/etc/bird/bird.conf"]
|
||||
@@ -13,9 +13,7 @@ RUN apk add --no-cache \
|
||||
git \
|
||||
jq \
|
||||
openssh-client \
|
||||
openssl \
|
||||
python3 \
|
||||
py3-yaml
|
||||
openssl
|
||||
|
||||
# Install Claude Code, then make the resulting layer byte-stable so that a
|
||||
# rebuild of the SAME version DEDUPS in the Gitea registry instead of orphaning
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
|
||||
|
||||
ARG TALOS_VERSION=v1.13.10
|
||||
ARG TALOS_VERSION=v1.12.6
|
||||
ARG VELERO_VERSION=v1.16.0
|
||||
ARG KUBECTL_VERSION=v1.36.3
|
||||
ARG FLUX_VERSION=2.9.5
|
||||
ARG KUBECTL_VERSION=v1.36.2
|
||||
ARG FLUX_VERSION=2.9.2
|
||||
|
||||
RUN apk add --no-cache \
|
||||
aws-cli \
|
||||
@@ -18,7 +18,7 @@ RUN apk add --no-cache \
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/talosctl \
|
||||
"https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
|
||||
&& echo "28e9ccd9aebd690fa03d8cf2582221d87ea38b2a1704f411d75a03589c58f2aa /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& echo "4c41a3b10b075292e64b182283d558e9d8e935f43239de1ed4e4b14359593efb /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/talosctl
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
@@ -31,7 +31,7 @@ RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
|
||||
RUN curl -sfL -o /tmp/flux.tar.gz \
|
||||
"https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \
|
||||
&& echo "b853df82adfd7736f580692f9f734473d571606307139f8fd20c2a80dd1ff473 /tmp/flux.tar.gz" | sha256sum -c - \
|
||||
&& echo "4092b367fd060097976fb7261601f79420ecf7266328417dfd8dfee27de0e6a3 /tmp/flux.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/flux.tar.gz -C /usr/local/bin flux \
|
||||
&& chmod +x /usr/local/bin/flux \
|
||||
&& rm -f /tmp/flux.tar.gz
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
# netbird-service-sync: reconciles NetBird reverse-proxy services (public
|
||||
# domains -> TLS passthrough -> Traefik ClusterIP) from Ingresses carrying the
|
||||
# public label. Standard library only, so the image is the interpreter plus one
|
||||
# file; runs unprivileged as its own user.
|
||||
FROM python:3.13-slim
|
||||
|
||||
RUN useradd --system --no-create-home --shell /usr/sbin/nologin controller
|
||||
COPY controller.py /opt/controller.py
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
USER controller
|
||||
ENTRYPOINT ["python3", "/opt/controller.py"]
|
||||
@@ -1,275 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""netbird-service-sync: reconcile NetBird reverse-proxy services from Ingresses.
|
||||
|
||||
Source of truth: every Ingress carrying the public label (dns.renner.uno/public).
|
||||
For each host on such an Ingress the NetBird management gets one reverse-proxy
|
||||
service in TLS-passthrough mode on :443, targeting Traefik's ClusterIP through a
|
||||
subnet resource that covers the cluster's service CIDR. SNI and Host reach
|
||||
Traefik untouched, the proxy terminates nothing, and no domain resource is
|
||||
involved -- the routing peers stay ordinary IP routers. The target sends a
|
||||
PROXY protocol v2 header so Traefik (and CrowdSec behind its access log) sees
|
||||
the WAN client address instead of the routing peer's pod IP.
|
||||
|
||||
Ownership is derived from NetBird itself, not from local state: a service is
|
||||
managed iff its mode is "tls" and its target is this controller's subnet
|
||||
resource. Services that do not match (the hand-made HTTP/domain ones) are never
|
||||
touched; a desired host held by such a service is reported and skipped until
|
||||
that service is removed by hand.
|
||||
|
||||
Credentials: the NetBird PAT is read from OpenBao on every pass through the
|
||||
Kubernetes auth role, exactly like the setup-key rotator; it never lands in a
|
||||
Kubernetes Secret or in a log line.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
LOG = logging.getLogger("netbird-service-sync")
|
||||
|
||||
NB = os.environ.get("NB_MANAGEMENT_URL", "https://netbird.renner.uno").rstrip("/")
|
||||
BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
|
||||
BAO_ROLE = os.environ.get("BAO_ROLE", "netbird-service-sync")
|
||||
BAO_PAT_PATH = os.environ.get("BAO_PAT_PATH", "homelab/data/infrastructure/secret-rotators-bootstrap/netbird")
|
||||
BAO_PAT_KEY = os.environ.get("BAO_PAT_KEY", "pat")
|
||||
SERVICE_CIDR = os.environ.get("SERVICE_CIDR", "10.96.0.0/12")
|
||||
NETWORK_NAME = os.environ.get("NETWORK_NAME", "k8s public ingress")
|
||||
ROUTER_GROUP = os.environ.get("ROUTER_GROUP_NAME", "k8s-ingress-router")
|
||||
SOURCE_GROUP = os.environ.get("SOURCE_GROUP_NAME", "All")
|
||||
TRAEFIK_NS, TRAEFIK_SVC = os.environ.get("TRAEFIK_SERVICE", "ingress/traefik").split("/", 1)
|
||||
PUBLIC_LABEL = os.environ.get("PUBLIC_LABEL", "dns.renner.uno/public")
|
||||
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "443"))
|
||||
INTERVAL = int(os.environ.get("INTERVAL_SECONDS", "60"))
|
||||
HTTP_TIMEOUT = 20
|
||||
|
||||
K8S_API = "https://kubernetes.default.svc"
|
||||
K8S_TOKEN_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
K8S_CA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def http(method, url, body=None, headers=None, cafile=None):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
req = urllib.request.Request(url, data=data, method=method)
|
||||
req.add_header("Accept", "application/json")
|
||||
if data is not None:
|
||||
req.add_header("Content-Type", "application/json")
|
||||
for k, v in (headers or {}).items():
|
||||
req.add_header(k, v)
|
||||
ctx = None
|
||||
if cafile:
|
||||
import ssl
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT, context=ctx) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = e.read().decode(errors="replace")[:300]
|
||||
raise ApiError(f"{method} {url} -> {e.code}: {detail}") from None
|
||||
|
||||
|
||||
# --- Kubernetes -------------------------------------------------------------
|
||||
|
||||
def k8s(path):
|
||||
with open(K8S_TOKEN_FILE) as f:
|
||||
token = f.read().strip()
|
||||
return http("GET", K8S_API + path, headers={"Authorization": f"Bearer {token}"}, cafile=K8S_CA_FILE)
|
||||
|
||||
|
||||
def desired_hosts():
|
||||
sel = urllib.parse.quote(f"{PUBLIC_LABEL}=true")
|
||||
ingresses = k8s(f"/apis/networking.k8s.io/v1/ingresses?labelSelector={sel}")
|
||||
hosts = {}
|
||||
for ing in ingresses.get("items", []):
|
||||
ref = f'{ing["metadata"]["namespace"]}/{ing["metadata"]["name"]}'
|
||||
for rule in ing.get("spec", {}).get("rules", []):
|
||||
host = rule.get("host")
|
||||
if not host or host.startswith("*"):
|
||||
continue
|
||||
hosts.setdefault(host, ref)
|
||||
return hosts
|
||||
|
||||
|
||||
def traefik_cluster_ip():
|
||||
svc = k8s(f"/api/v1/namespaces/{TRAEFIK_NS}/services/{TRAEFIK_SVC}")
|
||||
for ip in svc["spec"].get("clusterIPs", []):
|
||||
if ":" not in ip:
|
||||
return ip
|
||||
raise ApiError(f"{TRAEFIK_NS}/{TRAEFIK_SVC} has no IPv4 ClusterIP")
|
||||
|
||||
|
||||
# --- OpenBao ----------------------------------------------------------------
|
||||
|
||||
def netbird_pat():
|
||||
with open(K8S_TOKEN_FILE) as f:
|
||||
jwt = f.read().strip()
|
||||
login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
|
||||
bao_token = login["auth"]["client_token"]
|
||||
kv = http("GET", f"{BAO_ADDR}/v1/{BAO_PAT_PATH}", headers={"X-Vault-Token": bao_token})
|
||||
pat = kv["data"]["data"].get(BAO_PAT_KEY)
|
||||
if not pat:
|
||||
raise ApiError(f"OpenBao {BAO_PAT_PATH} has no key {BAO_PAT_KEY}")
|
||||
return pat
|
||||
|
||||
|
||||
# --- NetBird management -----------------------------------------------------
|
||||
|
||||
class NetBird:
|
||||
def __init__(self, pat):
|
||||
self.h = {"Authorization": f"Token {pat}"}
|
||||
|
||||
def get(self, path):
|
||||
return http("GET", f"{NB}/api/{path}", headers=self.h)
|
||||
|
||||
def list(self, path):
|
||||
# An empty collection comes back as an empty body / null, not [].
|
||||
return self.get(path) or []
|
||||
|
||||
def post(self, path, body):
|
||||
return http("POST", f"{NB}/api/{path}", body, headers=self.h)
|
||||
|
||||
def put(self, path, body):
|
||||
return http("PUT", f"{NB}/api/{path}", body, headers=self.h)
|
||||
|
||||
def delete(self, path):
|
||||
return http("DELETE", f"{NB}/api/{path}", headers=self.h)
|
||||
|
||||
|
||||
def group_id(nb, name):
|
||||
for g in nb.list("groups"):
|
||||
if g["name"] == name:
|
||||
return g["id"]
|
||||
raise ApiError(f"NetBird group {name!r} not found")
|
||||
|
||||
|
||||
def ensure_network(nb):
|
||||
for n in nb.list("networks"):
|
||||
if n["name"] == NETWORK_NAME:
|
||||
return n["id"]
|
||||
n = nb.post("networks", {"name": NETWORK_NAME,
|
||||
"description": "managed by netbird-service-sync; public ingress via TLS passthrough"})
|
||||
LOG.info("created network %s (%s)", NETWORK_NAME, n["id"])
|
||||
return n["id"]
|
||||
|
||||
|
||||
def ensure_resource(nb, net):
|
||||
for r in nb.list(f"networks/{net}/resources"):
|
||||
if r.get("address") == SERVICE_CIDR:
|
||||
return r["id"]
|
||||
r = nb.post(f"networks/{net}/resources", {"name": "cluster service CIDR", "address": SERVICE_CIDR,
|
||||
"description": "managed by netbird-service-sync",
|
||||
"enabled": True, "groups": []})
|
||||
LOG.info("created subnet resource %s (%s)", SERVICE_CIDR, r["id"])
|
||||
return r["id"]
|
||||
|
||||
|
||||
def ensure_router(nb, net, router_group):
|
||||
for r in nb.list(f"networks/{net}/routers"):
|
||||
if router_group in (r.get("peer_groups") or []):
|
||||
return r["id"]
|
||||
r = nb.post(f"networks/{net}/routers", {"peer_groups": [router_group], "metric": 9999,
|
||||
"masquerade": True, "enabled": True})
|
||||
LOG.info("created router on group %s (%s)", ROUTER_GROUP, r["id"])
|
||||
return r["id"]
|
||||
|
||||
|
||||
def ensure_policy(nb, resource, source_group):
|
||||
name = f"{NETWORK_NAME} access"
|
||||
for p in nb.list("policies"):
|
||||
if p["name"] == name:
|
||||
return p["id"]
|
||||
p = nb.post("policies", {"name": name, "enabled": True, "rules": [{
|
||||
"name": name, "enabled": True, "action": "accept", "protocol": "tcp",
|
||||
"ports": [str(LISTEN_PORT)], "bidirectional": False, "sources": [source_group],
|
||||
"destinationResource": {"id": resource, "type": "subnet"}}]})
|
||||
LOG.info("created policy %s (%s)", name, p["id"])
|
||||
return p["id"]
|
||||
|
||||
|
||||
def service_body(host, resource, target_ip):
|
||||
# proxy_protocol: PROXY v2 header carries the WAN client address to
|
||||
# Traefik (proxyProtocol.trustedIPs on websecure); TCP/TLS modes only.
|
||||
return {"name": host, "domain": host, "mode": "tls", "listen_port": LISTEN_PORT, "enabled": True,
|
||||
"targets": [{"target_type": "subnet", "target_id": resource, "protocol": "tcp",
|
||||
"host": target_ip, "port": 443, "enabled": True,
|
||||
"options": {"proxy_protocol": True}}]}
|
||||
|
||||
|
||||
def is_managed(svc, resource):
|
||||
return svc.get("mode") == "tls" and any(t.get("target_id") == resource for t in svc.get("targets") or [])
|
||||
|
||||
|
||||
def target_matches(svc, target_ip):
|
||||
if svc.get("listen_port") != LISTEN_PORT or not svc.get("enabled"):
|
||||
return False
|
||||
ts = svc.get("targets") or []
|
||||
return len(ts) == 1 and ts[0].get("host") == target_ip and ts[0].get("port") == 443 \
|
||||
and ts[0].get("protocol") == "tcp" and ts[0].get("enabled") \
|
||||
and (ts[0].get("options") or {}).get("proxy_protocol") is True
|
||||
|
||||
|
||||
def reconcile():
|
||||
hosts = desired_hosts()
|
||||
target_ip = traefik_cluster_ip()
|
||||
nb = NetBird(netbird_pat())
|
||||
|
||||
router_group = group_id(nb, ROUTER_GROUP)
|
||||
source_group = group_id(nb, SOURCE_GROUP)
|
||||
net = ensure_network(nb)
|
||||
resource = ensure_resource(nb, net)
|
||||
ensure_router(nb, net, router_group)
|
||||
ensure_policy(nb, resource, source_group)
|
||||
|
||||
by_domain = {s["domain"]: s for s in nb.list("reverse-proxies/services")}
|
||||
created = updated = deleted = held = 0
|
||||
|
||||
for host, ref in sorted(hosts.items()):
|
||||
svc = by_domain.get(host)
|
||||
if svc is None:
|
||||
nb.post("reverse-proxies/services", service_body(host, resource, target_ip))
|
||||
LOG.info("created %s (from %s) -> %s:443", host, ref, target_ip)
|
||||
created += 1
|
||||
elif not is_managed(svc, resource):
|
||||
LOG.warning("%s is held by unmanaged service %s (mode %s); remove it to migrate",
|
||||
host, svc["id"], svc.get("mode"))
|
||||
held += 1
|
||||
elif not target_matches(svc, target_ip):
|
||||
nb.put(f"reverse-proxies/services/{svc['id']}", service_body(host, resource, target_ip))
|
||||
LOG.info("updated %s -> %s:443", host, target_ip)
|
||||
updated += 1
|
||||
|
||||
for domain, svc in by_domain.items():
|
||||
if is_managed(svc, resource) and domain not in hosts:
|
||||
nb.delete(f"reverse-proxies/services/{svc['id']}")
|
||||
LOG.info("deleted %s (no public Ingress carries it any more)", domain)
|
||||
deleted += 1
|
||||
|
||||
LOG.info("reconciled: %d hosts desired, %d created, %d updated, %d deleted, %d held by unmanaged services",
|
||||
len(hosts), created, updated, deleted, held)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), stream=sys.stdout,
|
||||
format="%(asctime)s %(levelname)s %(message)s")
|
||||
if os.environ.get("ONCE") == "1":
|
||||
reconcile()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
reconcile()
|
||||
except Exception as e: # keep running; the next pass retries
|
||||
LOG.error("reconcile failed: %s", e)
|
||||
time.sleep(INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,7 +14,7 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
|
||||
# kubernetes/talos/patches/cluster.yaml. The phase reads the version from that
|
||||
# file at run time and passes it to the provisioner; this pin only decides which
|
||||
# client talks to it.
|
||||
ARG TALOS_VERSION=v1.13.10
|
||||
ARG TALOS_VERSION=v1.13.9
|
||||
ARG KUBECTL_VERSION=v1.36.2
|
||||
ARG FLUX_VERSION=2.9.2
|
||||
ARG KOPIA_VERSION=0.23.1
|
||||
@@ -49,7 +49,7 @@ RUN apk add --no-cache \
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/talosctl \
|
||||
"https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
|
||||
&& echo "28e9ccd9aebd690fa03d8cf2582221d87ea38b2a1704f411d75a03589c58f2aa /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/talosctl
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
# wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea,
|
||||
# queried by ~/Notes/bin/wiki_rank.py. Standard library only; runs unprivileged.
|
||||
FROM python:3.13-slim
|
||||
|
||||
RUN useradd --system --no-create-home --shell /usr/sbin/nologin wikiindex
|
||||
COPY wiki_index.py /opt/wiki_index.py
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
USER wikiindex
|
||||
ENTRYPOINT ["python3", "/opt/wiki_index.py"]
|
||||
@@ -1,178 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unit tests for wiki_index. Run: python3 -m unittest test_wiki_index"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import wiki_index as wi
|
||||
|
||||
NOTE = """---
|
||||
title: 'UDM intercepts DNS'
|
||||
type: wiki
|
||||
tags: [dns, unifi]
|
||||
aliases: [udm dns interception, dig 1.1.1.1 wrong]
|
||||
---
|
||||
|
||||
# UDM intercepts DNS
|
||||
|
||||
The UDM answers every UDP/53 query itself.
|
||||
|
||||
## Details
|
||||
|
||||
More.
|
||||
"""
|
||||
|
||||
|
||||
class ParseTests(unittest.TestCase):
|
||||
def test_note_text(self):
|
||||
title, text = wi.note_text("udm-dns", NOTE)
|
||||
self.assertEqual(title, "UDM intercepts DNS")
|
||||
self.assertIn("udm dns UDM intercepts DNS udm dns interception", text)
|
||||
self.assertIn("dns unifi", text)
|
||||
self.assertTrue(text.endswith("The UDM answers every UDP/53 query itself."))
|
||||
self.assertLessEqual(len(text), wi.EMBED_CHARS)
|
||||
|
||||
def test_credentials_are_stripped(self):
|
||||
original = wi.http
|
||||
wi.http = lambda method, url, body=None, headers=None, timeout=60.0: (
|
||||
{"auth": {"client_token": "t"}} if url.endswith("/login")
|
||||
else {"data": {"data": {"gitea_token": "abc\n", "webhook_secret": " s3cret\n"}}})
|
||||
original_sa = wi.SA_TOKEN
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
wi.SA_TOKEN = Path(d) / "token"
|
||||
wi.SA_TOKEN.write_text("jwt\n")
|
||||
self.assertEqual(wi.Credentials().fetch(), {"gitea_token": "abc", "webhook_secret": "s3cret"})
|
||||
finally:
|
||||
wi.http, wi.SA_TOKEN = original, original_sa
|
||||
|
||||
def test_tree_paging_stops_on_short_last_page(self):
|
||||
# Gitea: truncated=true on every page including the last, "tree": null past it.
|
||||
pages = {
|
||||
1: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000)], "truncated": True, "total_count": 1192},
|
||||
2: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000, 1192)] + [{"path": "raw/x.md", "type": "blob", "sha": "r"}, {"path": "wiki/sub/y.md", "type": "blob", "sha": "y"}], "truncated": True, "total_count": 1192},
|
||||
3: {"tree": None, "truncated": False, "total_count": 1192},
|
||||
}
|
||||
calls = []
|
||||
original = wi.http
|
||||
|
||||
def fake_http(method, url, body=None, headers=None, timeout=60.0):
|
||||
page = int(url.rsplit("page=", 1)[1])
|
||||
calls.append(page)
|
||||
return pages[page]
|
||||
|
||||
wi.http = fake_http
|
||||
try:
|
||||
tree = wi.Gitea("t").tree("abc")
|
||||
finally:
|
||||
wi.http = original
|
||||
self.assertEqual(calls, [1, 2])
|
||||
self.assertEqual(len(tree), 1192)
|
||||
self.assertNotIn("raw/x.md", tree)
|
||||
self.assertNotIn("wiki/sub/y.md", tree)
|
||||
self.assertEqual(tree["wiki/n1191.md"], "s1191")
|
||||
|
||||
def test_signature(self):
|
||||
body = b'{"ref":"refs/heads/master"}'
|
||||
sig = hmac.new(b"s3cret", body, hashlib.sha256).hexdigest()
|
||||
self.assertTrue(wi.verify_signature("s3cret", body, sig))
|
||||
self.assertFalse(wi.verify_signature("other", body, sig))
|
||||
self.assertFalse(wi.verify_signature("s3cret", body, ""))
|
||||
|
||||
|
||||
class FakeGitea:
|
||||
def __init__(self, head, tree, files):
|
||||
self._head, self._tree, self.files = head, tree, files
|
||||
self.raw_calls = []
|
||||
|
||||
def head(self):
|
||||
return self._head
|
||||
|
||||
def tree(self, commit):
|
||||
return self._tree
|
||||
|
||||
def raw(self, path, ref):
|
||||
self.raw_calls.append(path)
|
||||
return self.files[path]
|
||||
|
||||
|
||||
class IndexTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.path = Path(self.tmp.name) / "index.json"
|
||||
self.original_embed = wi.embed
|
||||
self.embed_calls = []
|
||||
|
||||
def fake_embed(texts, batch=2):
|
||||
self.embed_calls.append(list(texts))
|
||||
return [wi.unit([1.0, float(len(t) % 7)]) for t in texts]
|
||||
|
||||
wi.embed = fake_embed
|
||||
|
||||
def tearDown(self):
|
||||
wi.embed = self.original_embed
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_sync_is_incremental_and_persists(self):
|
||||
files = {"wiki/udm-dns.md": NOTE, "wiki/ceph.md": "---\ntitle: Ceph\n---\n# Ceph\n\nOSD.\n", "wiki/_index.md": "---\ntitle: Wiki\n---\n"}
|
||||
tree = {"wiki/udm-dns.md": "aaa", "wiki/ceph.md": "bbb", "wiki/_index.md": "ccc"}
|
||||
g = FakeGitea("c1", tree, files)
|
||||
idx = wi.Index(self.path)
|
||||
s = idx.sync(g)
|
||||
self.assertEqual((s["changed"], s["removed"], s["notes"]), (2, 0, 2))
|
||||
self.assertEqual(set(idx.notes), {"udm-dns", "ceph"})
|
||||
self.assertEqual(idx.notes["udm-dns"]["title"], "UDM intercepts DNS")
|
||||
self.assertTrue(self.path.exists())
|
||||
|
||||
# head unchanged: nothing fetched
|
||||
s = idx.sync(g)
|
||||
self.assertEqual(s.get("skipped"), "head unchanged")
|
||||
self.assertEqual(len(g.raw_calls), 2)
|
||||
|
||||
# one note changed, one removed, new head
|
||||
g2 = FakeGitea("c2", {"wiki/udm-dns.md": "aaa2"}, {"wiki/udm-dns.md": NOTE.replace("itself", "itself!")})
|
||||
s = idx.sync(g2)
|
||||
self.assertEqual((s["changed"], s["removed"], s["notes"]), (1, 1, 1))
|
||||
self.assertEqual(g2.raw_calls, ["wiki/udm-dns.md"])
|
||||
|
||||
# restart loads the persisted index
|
||||
idx2 = wi.Index(self.path)
|
||||
self.assertEqual(idx2.commit, "c2")
|
||||
self.assertEqual(set(idx2.notes), {"udm-dns"})
|
||||
|
||||
def test_rank_orders_by_cosine(self):
|
||||
idx = wi.Index(self.path)
|
||||
idx.notes = {"a": {"sha": "1", "title": "A", "vector": wi.unit([1.0, 0.0])},
|
||||
"b": {"sha": "2", "title": "B", "vector": wi.unit([0.0, 1.0])}}
|
||||
wi.embed = lambda texts, batch=2: [wi.unit([0.2, 1.0])]
|
||||
rows = idx.rank("q", k=5)
|
||||
self.assertEqual([r["slug"] for r in rows], ["b", "a"])
|
||||
self.assertGreater(rows[0]["score"], rows[1]["score"])
|
||||
|
||||
def test_status_reports_age_and_errors(self):
|
||||
idx = wi.Index(self.path)
|
||||
self.assertFalse(idx.status()["ready"])
|
||||
g = FakeGitea("c1", {"wiki/a.md": "x"}, {"wiki/a.md": "---\ntitle: A\n---\n# A\n\nt\n"})
|
||||
idx.sync(g)
|
||||
st = idx.status()
|
||||
self.assertTrue(st["ready"])
|
||||
self.assertEqual(st["notes"], 1)
|
||||
self.assertEqual(st["commit"], "c1")
|
||||
|
||||
class Broken(FakeGitea):
|
||||
def head(self):
|
||||
raise wi.ApiError("gitea down")
|
||||
|
||||
with self.assertRaises(wi.ApiError):
|
||||
idx.sync(Broken("c1", {}, {}))
|
||||
self.assertIn("gitea down", idx.status()["last_error"])
|
||||
self.assertEqual(idx.status()["notes"], 1) # last good index kept
|
||||
self.assertTrue(idx.status()["ready"]) # and still served
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,408 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea.
|
||||
|
||||
Source of truth is the notes repository on Gitea. The index holds, per wiki
|
||||
note, the git blob sha, title and a bge-m3 vector of slug/title/aliases/tags/
|
||||
first paragraph. A sync fetches the repo tree, embeds every note whose blob sha
|
||||
changed (via wiki-embed, in-cluster) and drops notes that left the tree; it
|
||||
runs at start, on every push webhook from Gitea, and every RESYNC_SECONDS.
|
||||
|
||||
HTTP:
|
||||
POST /rank {"query": str, "k": int} -> {"rows": [{"slug","title","score"}], "index": {...}}
|
||||
GET /health -> 200 always (liveness) with {"ready", "notes", "commit", "synced_at", "age_s", "syncing", "last_error"}
|
||||
GET /ready -> 200 when an index exists, else 503 (readiness)
|
||||
POST /webhook Gitea push event, HMAC-SHA256 in X-Gitea-Signature -> {"queued": true}
|
||||
|
||||
Credentials (Gitea token, webhook secret) are read from OpenBao through the
|
||||
Kubernetes auth role on every use; they never land in a Secret or a log line.
|
||||
Standard library only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
LOG = logging.getLogger("wiki-index")
|
||||
|
||||
GITEA_URL = os.environ.get("GITEA_URL", "https://gitea.renner.uno").rstrip("/")
|
||||
NOTES_REPO = os.environ.get("NOTES_REPO", "k.renner/notes")
|
||||
NOTES_BRANCH = os.environ.get("NOTES_BRANCH", "master")
|
||||
WIKI_DIR = os.environ.get("WIKI_DIR", "wiki").strip("/")
|
||||
EMBED_URL = os.environ.get("EMBED_URL", "http://wiki-embed.ai.svc.cluster.local:8080").rstrip("/")
|
||||
EMBED_MODEL = os.environ.get("EMBED_MODEL", "bge-m3")
|
||||
EMBED_CHARS = int(os.environ.get("EMBED_CHARS", "1500"))
|
||||
INDEX_PATH = Path(os.environ.get("INDEX_PATH", "/data/index.json"))
|
||||
RESYNC_SECONDS = int(os.environ.get("RESYNC_SECONDS", "1800"))
|
||||
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
|
||||
BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/")
|
||||
BAO_ROLE = os.environ.get("BAO_ROLE", "wiki-index")
|
||||
BAO_SECRET_PATH = os.environ.get("BAO_SECRET_PATH", "homelab/data/apps/ai/wiki-index")
|
||||
SA_TOKEN = Path(os.environ.get("SA_TOKEN", "/var/run/secrets/kubernetes.io/serviceaccount/token"))
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- http helpers
|
||||
|
||||
|
||||
def http(method: str, url: str, body=None, headers=None, timeout: float = 60.0):
|
||||
data = None
|
||||
hdrs = dict(headers or {})
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs.setdefault("Content-Type", "application/json")
|
||||
req = urllib.request.Request(url, data=data, method=method, headers=hdrs)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
ctype = resp.headers.get("Content-Type", "")
|
||||
except urllib.error.HTTPError as e:
|
||||
raise ApiError(f"{method} {url}: HTTP {e.code}: {e.read().decode('utf-8', 'replace')[:300]}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise ApiError(f"{method} {url}: {e.reason}") from e
|
||||
if "json" in ctype:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
return raw
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- credentials
|
||||
|
||||
|
||||
class Credentials:
|
||||
"""Gitea token and webhook secret from OpenBao, fetched per use."""
|
||||
|
||||
def __init__(self, static: dict | None = None):
|
||||
self.static = static # tests / local runs
|
||||
|
||||
def fetch(self) -> dict:
|
||||
if self.static is not None:
|
||||
return self.static
|
||||
jwt = SA_TOKEN.read_text(encoding="utf-8").strip()
|
||||
login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt})
|
||||
token = login["auth"]["client_token"]
|
||||
kv = http("GET", f"{BAO_ADDR}/v1/{BAO_SECRET_PATH}", headers={"X-Vault-Token": token})
|
||||
data = {k: str(v).strip() for k, v in kv["data"]["data"].items()} # `bao kv put key=@file` keeps the file's newline
|
||||
for key in ("gitea_token", "webhook_secret"):
|
||||
if not data.get(key):
|
||||
raise ApiError(f"OpenBao {BAO_SECRET_PATH} has no key {key}")
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- gitea
|
||||
|
||||
|
||||
class Gitea:
|
||||
def __init__(self, token: str, base: str = GITEA_URL, repo: str = NOTES_REPO, branch: str = NOTES_BRANCH):
|
||||
self.base, self.repo, self.branch = base, repo, branch
|
||||
self.headers = {"Authorization": f"token {token}"}
|
||||
|
||||
def api(self, path: str, **params):
|
||||
q = ("?" + urllib.parse.urlencode(params)) if params else ""
|
||||
return http("GET", f"{self.base}/api/v1/repos/{self.repo}{path}{q}", headers=self.headers)
|
||||
|
||||
def head(self) -> str:
|
||||
return self.api(f"/branches/{urllib.parse.quote(self.branch, safe='')}")["commit"]["id"]
|
||||
|
||||
def tree(self, commit: str) -> dict[str, str]:
|
||||
"""path -> blob sha for every wiki note at that commit."""
|
||||
out: dict[str, str] = {}
|
||||
page, per_page, seen = 1, 1000, 0
|
||||
while True:
|
||||
t = self.api(f"/git/trees/{commit}", recursive="true", per_page=per_page, page=page)
|
||||
entries = t.get("tree") or [] # Gitea answers "tree": null past the last page
|
||||
for e in entries:
|
||||
p = e.get("path", "")
|
||||
if e.get("type") == "blob" and p.startswith(WIKI_DIR + "/") and p.endswith(".md") and "/" not in p[len(WIKI_DIR) + 1:]:
|
||||
out[p] = e["sha"]
|
||||
seen += len(entries)
|
||||
total = int(t.get("total_count") or 0)
|
||||
if not entries or len(entries) < per_page or (total and seen >= total):
|
||||
break
|
||||
page += 1
|
||||
if page > 50:
|
||||
raise ApiError("tree paging runaway")
|
||||
return out
|
||||
|
||||
def raw(self, path: str, ref: str) -> str:
|
||||
data = http("GET", f"{self.base}/api/v1/repos/{self.repo}/raw/{urllib.parse.quote(path)}?ref={urllib.parse.quote(ref, safe='')}",
|
||||
headers=self.headers)
|
||||
return data.decode("utf-8") if isinstance(data, bytes) else json.dumps(data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- notes
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict, str]:
|
||||
m = re.match(r"^---[ \t]*\n(.*?)\n---[ \t]*(?:\n|$)", text, re.DOTALL)
|
||||
if not m:
|
||||
return {}, text
|
||||
fm: dict = {}
|
||||
for line in m.group(1).splitlines():
|
||||
mm = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
||||
if mm:
|
||||
fm[mm.group(1)] = mm.group(2).strip()
|
||||
return fm, text[m.end():]
|
||||
|
||||
|
||||
def as_list(value: str) -> list[str]:
|
||||
s = (value or "").strip()
|
||||
if s.startswith("[") and s.endswith("]"):
|
||||
s = s[1:-1]
|
||||
return [p.strip().strip("'\"") for p in s.split(",") if p.strip()]
|
||||
|
||||
|
||||
def first_paragraph(body: str) -> str:
|
||||
para: list[str] = []
|
||||
seen_h1 = False
|
||||
for line in body.splitlines():
|
||||
if line.startswith("# "):
|
||||
seen_h1 = True
|
||||
continue
|
||||
if not seen_h1:
|
||||
continue
|
||||
if line.strip() == "" or line.startswith("#"):
|
||||
if para:
|
||||
break
|
||||
continue
|
||||
para.append(line.strip())
|
||||
return " ".join(para)
|
||||
|
||||
|
||||
def note_text(slug: str, text: str) -> tuple[str, str]:
|
||||
"""(title, embedding text) for one note file."""
|
||||
fm, body = parse_frontmatter(text)
|
||||
title = (fm.get("title") or slug).strip().strip("'\"")
|
||||
parts = [slug.replace("-", " "), title, " ".join(as_list(fm.get("aliases", ""))),
|
||||
" ".join(as_list(fm.get("tags", ""))), first_paragraph(body)]
|
||||
return title, " ".join(parts)[:EMBED_CHARS]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- embeddings
|
||||
|
||||
|
||||
def embed(texts: list[str], batch: int = 2) -> list[list[float]]:
|
||||
out: list[list[float]] = []
|
||||
for i in range(0, len(texts), batch):
|
||||
resp = http("POST", f"{EMBED_URL}/v1/embeddings", {"model": EMBED_MODEL, "input": texts[i:i + batch]}, timeout=120)
|
||||
rows = sorted(resp["data"], key=lambda d: d["index"])
|
||||
out.extend(unit(d["embedding"]) for d in rows)
|
||||
return out
|
||||
|
||||
|
||||
def unit(v: list[float]) -> list[float]:
|
||||
n = sum(x * x for x in v) ** 0.5 or 1.0
|
||||
return [round(x / n, 5) for x in v]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- index
|
||||
|
||||
|
||||
class Index:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.lock = threading.Lock()
|
||||
self.notes: dict[str, dict] = {} # slug -> {sha, title, vector}
|
||||
self.commit = ""
|
||||
self.synced_at = 0.0
|
||||
self.syncing = False
|
||||
self.last_error = ""
|
||||
if path.exists():
|
||||
try:
|
||||
d = json.loads(path.read_text(encoding="utf-8"))
|
||||
self.notes, self.commit, self.synced_at = d.get("notes", {}), d.get("commit", ""), d.get("synced_at", 0.0)
|
||||
LOG.info("loaded index: %d notes at %s", len(self.notes), self.commit[:8])
|
||||
except (OSError, ValueError) as e:
|
||||
LOG.warning("index unreadable (%s); starting empty", e)
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = self.path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps({"notes": self.notes, "commit": self.commit, "synced_at": self.synced_at}), encoding="utf-8")
|
||||
tmp.replace(self.path)
|
||||
|
||||
def sync(self, gitea: Gitea) -> dict:
|
||||
"""Bring the index to the branch head; returns a summary."""
|
||||
with self.lock:
|
||||
self.syncing = True
|
||||
try:
|
||||
head = gitea.head()
|
||||
if head == self.commit and self.notes:
|
||||
summary = {"commit": head, "changed": 0, "removed": 0, "notes": len(self.notes), "skipped": "head unchanged"}
|
||||
self.synced_at = time.time()
|
||||
self.save()
|
||||
return summary
|
||||
tree = gitea.tree(head)
|
||||
slugs = {p[len(WIKI_DIR) + 1:-3]: (p, sha) for p, sha in tree.items() if not p.endswith("/_index.md")}
|
||||
todo = [s for s, (p, sha) in slugs.items() if self.notes.get(s, {}).get("sha") != sha]
|
||||
removed = [s for s in list(self.notes) if s not in slugs]
|
||||
LOG.info("sync %s: %d notes, %d to embed, %d removed", head[:8], len(slugs), len(todo), len(removed))
|
||||
fresh: dict[str, dict] = {}
|
||||
for i in range(0, len(todo), 20):
|
||||
chunk = todo[i:i + 20]
|
||||
texts, titles = [], {}
|
||||
for s in chunk:
|
||||
title, text = note_text(s, gitea.raw(slugs[s][0], head))
|
||||
titles[s] = title
|
||||
texts.append(text)
|
||||
for s, vec in zip(chunk, embed(texts)):
|
||||
fresh[s] = {"sha": slugs[s][1], "title": titles[s], "vector": vec}
|
||||
with self.lock:
|
||||
for s in removed:
|
||||
self.notes.pop(s, None)
|
||||
self.notes.update(fresh)
|
||||
self.commit = head
|
||||
self.synced_at = time.time()
|
||||
self.last_error = ""
|
||||
self.save()
|
||||
return {"commit": head, "changed": len(todo), "removed": len(removed), "notes": len(self.notes)}
|
||||
except Exception as e:
|
||||
with self.lock:
|
||||
self.last_error = str(e)[:300]
|
||||
raise
|
||||
finally:
|
||||
with self.lock:
|
||||
self.syncing = False
|
||||
|
||||
def rank(self, query: str, k: int = 60) -> list[dict]:
|
||||
q = embed([query])[0]
|
||||
with self.lock:
|
||||
rows = [(s, n["title"], sum(a * b for a, b in zip(q, n["vector"]))) for s, n in self.notes.items()]
|
||||
rows.sort(key=lambda r: -r[2])
|
||||
return [{"slug": s, "title": t, "score": round(sc, 4)} for s, t, sc in rows[:k]]
|
||||
|
||||
def status(self) -> dict:
|
||||
"""`ready` = an index exists and can answer /rank; a failed sync keeps the last good index."""
|
||||
with self.lock:
|
||||
return {"ready": bool(self.notes), "notes": len(self.notes), "commit": self.commit,
|
||||
"synced_at": self.synced_at, "age_s": round(time.time() - self.synced_at) if self.synced_at else None,
|
||||
"syncing": self.syncing, "last_error": self.last_error or None}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- syncer
|
||||
|
||||
|
||||
class Syncer(threading.Thread):
|
||||
"""One worker: syncs on demand (webhook) and every RESYNC_SECONDS."""
|
||||
|
||||
def __init__(self, index: Index, creds: Credentials, interval: int):
|
||||
super().__init__(daemon=True, name="syncer")
|
||||
self.index, self.creds, self.interval = index, creds, interval
|
||||
self.wake = threading.Event()
|
||||
|
||||
def trigger(self) -> None:
|
||||
self.wake.set()
|
||||
|
||||
RETRY_SECONDS = 60
|
||||
|
||||
def run(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
data = self.creds.fetch()
|
||||
summary = self.index.sync(Gitea(data["gitea_token"]))
|
||||
LOG.info("sync done: %s", summary)
|
||||
wait = self.interval
|
||||
except Exception as e: # keep serving the last good index; retry soon, not next interval
|
||||
LOG.error("sync failed: %s", e)
|
||||
wait = self.RETRY_SECONDS
|
||||
self.wake.wait(wait)
|
||||
self.wake.clear()
|
||||
|
||||
|
||||
def verify_signature(secret: str, body: bytes, signature: str) -> bool:
|
||||
digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(digest, (signature or "").strip())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- server
|
||||
|
||||
|
||||
def make_handler(index: Index, syncer: Syncer, creds: Credentials):
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
LOG.info("%s %s", self.address_string(), fmt % args)
|
||||
|
||||
def send_json(self, code: int, obj) -> None:
|
||||
data = json.dumps(obj).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def body(self) -> bytes:
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
return self.rfile.read(n) if n else b""
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health": # liveness: the process serves
|
||||
return self.send_json(200, index.status())
|
||||
if self.path == "/ready": # readiness: an index exists
|
||||
st = index.status()
|
||||
return self.send_json(200 if st["ready"] else 503, st)
|
||||
self.send_json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self):
|
||||
if self.path == "/rank":
|
||||
try:
|
||||
req = json.loads(self.body() or b"{}")
|
||||
query = str(req.get("query", "")).strip()
|
||||
k = max(1, min(int(req.get("k", 60)), 500))
|
||||
if not query:
|
||||
return self.send_json(400, {"error": "query required"})
|
||||
rows = index.rank(query, k)
|
||||
return self.send_json(200, {"rows": rows, "index": index.status()})
|
||||
except ApiError as e:
|
||||
return self.send_json(502, {"error": str(e)})
|
||||
except (ValueError, TypeError) as e:
|
||||
return self.send_json(400, {"error": str(e)})
|
||||
if self.path == "/webhook":
|
||||
raw = self.body()
|
||||
try:
|
||||
secret = creds.fetch()["webhook_secret"]
|
||||
except ApiError as e:
|
||||
return self.send_json(503, {"error": str(e)})
|
||||
if not verify_signature(secret, raw, self.headers.get("X-Gitea-Signature", "")):
|
||||
return self.send_json(403, {"error": "bad signature"})
|
||||
try:
|
||||
event = json.loads(raw or b"{}")
|
||||
except ValueError:
|
||||
return self.send_json(400, {"error": "bad json"})
|
||||
ref = event.get("ref", "")
|
||||
if ref and ref != f"refs/heads/{NOTES_BRANCH}":
|
||||
return self.send_json(200, {"queued": False, "ref": ref})
|
||||
syncer.trigger()
|
||||
return self.send_json(200, {"queued": True})
|
||||
self.send_json(404, {"error": "not found"})
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout)
|
||||
index = Index(INDEX_PATH)
|
||||
creds = Credentials()
|
||||
syncer = Syncer(index, creds, RESYNC_SECONDS)
|
||||
syncer.start()
|
||||
server = ThreadingHTTPServer(("", LISTEN_PORT), make_handler(index, syncer, creds))
|
||||
LOG.info("listening on :%d, repo %s@%s, embed %s", LISTEN_PORT, NOTES_REPO, NOTES_BRANCH, EMBED_URL)
|
||||
server.serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -50,18 +50,15 @@ pass insert homelab/openbao/root-token
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator unseal # paste key
|
||||
```
|
||||
|
||||
**Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
|
||||
the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
|
||||
from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
|
||||
itself. The shamir key above became the **recovery key** in the migration; it
|
||||
is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
|
||||
runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
|
||||
one is neither a voter nor a standby, so a replica that stays sealed after a
|
||||
restart — wrong or missing seal-key Secret — costs redundancy, two cost
|
||||
quorum. The trade-off taken with the static seal: the key that unlocks the
|
||||
store sits in the same etcd as the Secrets ESO already syncs there in
|
||||
plaintext. Symptom of sealed: ESO stores go NotReady, synced Secrets keep
|
||||
their last value (they are real k8s Secrets — pods keep running).
|
||||
**Manual unseal is required after every pod restart** (node drain, upgrade,
|
||||
crash), **per replica** — the server runs three raft peers (see
|
||||
`infrastructure/openbao/helmrelease.yaml`), and a sealed one is neither a
|
||||
voter nor a standby, so it stays out of the cluster until unsealed:
|
||||
`kubectl -n openbao exec -it openbao-<N> -- bao operator unseal`, same key for
|
||||
all three. Three peers mean one sealed replica is survivable; two are not.
|
||||
Accepted for now; auto-unseal needs a second trust root (transit/KMS) and is
|
||||
explicitly out of scope. Symptom of sealed: ESO stores go NotReady, synced
|
||||
Secrets keep their last value (they are real k8s Secrets — pods keep running).
|
||||
|
||||
## Phase 2 — configure + ClusterSecretStore (follow-up PR)
|
||||
|
||||
@@ -237,19 +234,16 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
|
||||
- **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
|
||||
dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
|
||||
`openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
|
||||
A raft snapshot restore needs the matching seal key — which lives in
|
||||
A raft snapshot restore needs the matching unseal key — which lives in
|
||||
`pass`, outside the cluster (GPG key backed up externally 2026-07-14).
|
||||
A failed snapshot Job is the sealed-after-restart alarm. Still owed:
|
||||
one tested restore before phase 4.
|
||||
- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
|
||||
openbao (empty) → init → restore raft snapshot → replicas unseal themselves
|
||||
→ ClusterSecretStore Ready → ESO materializes Secrets → apps start
|
||||
(`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
|
||||
hand). Until then, secret-consuming apps simply stay Pending on missing
|
||||
Secrets — same as any missing dependency.
|
||||
- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
|
||||
offline-capable). With those, everything is recoverable from a raft
|
||||
snapshot alone.
|
||||
- **Cluster rebuild order:** Flux brings up openbao (sealed, empty) → restore
|
||||
raft snapshot → unseal (key from `pass`) → ClusterSecretStore Ready → ESO
|
||||
materializes Secrets → apps start. Until unseal, secret-consuming apps
|
||||
simply stay Pending on missing Secrets — same as any missing dependency.
|
||||
- **Break-glass:** `pass` holds unseal key + root token (GPG, offline-capable).
|
||||
With those two, everything is recoverable from a raft snapshot alone.
|
||||
- **Ansible Vault password:** `ansible/vault-password.sh` reads
|
||||
`ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
|
||||
2026-07-14. It gates decryption of several Ansible-Vault files (PBS
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
# Traffic classes on the k8s nodes
|
||||
|
||||
Issue #576. Five classes, each on its own VLAN.
|
||||
|
||||
## Where each class runs
|
||||
|
||||
| class | link | mechanism |
|
||||
|---|---|---|
|
||||
| Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) |
|
||||
| pod↔pod | `pod0`, VLAN 48 | pod-sourced traffic is routed from table 47; bird puts the peers' pod CIDRs there via `pod0` |
|
||||
| world↔pod | `ingress0`, VLAN 47 | BGP session sourced there (next hop of every LoadBalancer VIP); pod egress takes table 47's default |
|
||||
| node↔node | `node0`, VLAN 46 | the node InternalIP and etcd live there; so do apiserver↔kubelet, cilium-health and host→remote-pod traffic |
|
||||
| world↔node | `mgmt0`, VLAN 40 | API VIP for kubectl, talosctl from outside, node egress (DNS, NTP, image pulls) |
|
||||
|
||||
Hard constraint, set by the owner: inter-node and pod traffic never traverses
|
||||
the UDM. Everything but world↔node stays on the 10G switch. VLAN 46 and 48 are
|
||||
statically addressed, without a gateway; VLAN 46 is isolated in UniFi.
|
||||
|
||||
## Pod traffic: one table
|
||||
|
||||
Two rules, one per family, send every packet with a pod source to table 47
|
||||
(`talos/patches/site-homelab.yaml`, priorities 1000/1001, below main). The
|
||||
table decides by longest prefix; no rule carves out exceptions.
|
||||
|
||||
bird (`kubernetes/infrastructure/bird/`) owns table 47 and fills it with:
|
||||
|
||||
- the node's attached networks — `mgmt0`, `ingress0`, `pod0`, `node0` —
|
||||
taken from the interfaces;
|
||||
- the peers' pod CIDRs, learned over an iBGP mesh on `pod0`, next hop the
|
||||
peer's `pod0` address;
|
||||
- the node's own pod CIDR via `cilium_host`, originated rather than read back
|
||||
from the kernel (reading it back made BIRD depend on the shape of Cilium's
|
||||
route, and KRT rejects the IPv4 one);
|
||||
- the egress default via the UDM on `ingress0`, from the node annotations
|
||||
`net.renner.uno/egress-gw-v4/v6`.
|
||||
|
||||
`node0` has to be in the table: a pod reaching a node address — the
|
||||
`kubernetes` ClusterIP, which socket LB rewrites to the apiserver's advertise
|
||||
address, or a hostNetwork pod — would otherwise take the default to the UDM.
|
||||
|
||||
The routing decision runs before Cilium's masquerade hook, so the rule sees the
|
||||
pod address and the masquerade then rewrites it to the egress link's.
|
||||
|
||||
Node-originated traffic does not match the rules and uses main, where Cilium
|
||||
routes each peer's pod CIDR via the peer's InternalIP — on `node0`.
|
||||
|
||||
No address is written into this repo twice. Each node publishes its `pod0`
|
||||
addresses as `nodeAnnotations` beside the `VLANConfig` that assigns them; the
|
||||
pod CIDRs come from the Node objects. bird's init container renders both from
|
||||
the API and ends with `bird -p`, so BIRD's own parser gates startup.
|
||||
|
||||
## Node↔node
|
||||
|
||||
`kubelet.nodeIP.validSubnets` and `cluster.etcd.advertisedSubnets` point at
|
||||
VLAN 46. Talos picks one InternalIP per family, IPv4 first — the service CIDR
|
||||
is IPv4, and the apiserver's advertise address must match it.
|
||||
|
||||
The API VIP stays on `mgmt0` for kubectl. No in-cluster client uses it:
|
||||
kubelet, scheduler, controller-manager and Cilium go through KubePrism
|
||||
(`localhost:7445`), which balances over every address of every control-plane
|
||||
member; pods use the ClusterIP.
|
||||
|
||||
Talos never updates an existing etcd member's peer URLs
|
||||
(siderolabs/talos#12720). Changing `advertisedSubnets` means, per member:
|
||||
stage the config, `talosctl etcd forfeit-leadership`, `talosctl etcd leave`,
|
||||
reboot. `leave` removes only `/var/lib/etcd`; the node rejoins as a learner
|
||||
and is promoted. EPHEMERAL does not need a wipe.
|
||||
|
||||
## Moving node IPs: the mixed state
|
||||
|
||||
While some nodes have their InternalIP on the new network and some on the old,
|
||||
host→remote-pod traffic leaves with a node address that is not an InternalIP:
|
||||
the kernel picks the source from the route to the peer's InternalIP. Cilium
|
||||
classifies that as `world`, and every ingress policy that admits only
|
||||
`host`/`remote-node` (rook-ceph, monitoring, …) drops it. Kernel Ceph clients
|
||||
lose the OSDs on the other side; RBD mounts hang.
|
||||
|
||||
Bridge it for the duration with a temporary `CiliumClusterwideNetworkPolicy`
|
||||
admitting the node subnets of both networks, both families, with
|
||||
`enableDefaultDeny: false` so it restricts nothing else, and delete it once
|
||||
every node has moved. Keep all OSD nodes schedulable while one is drained:
|
||||
draining two takes Ceph below its data availability.
|
||||
|
||||
## Failure behaviour
|
||||
|
||||
If bird stops, table 47 empties, the rules find nothing and fall through to
|
||||
main: pod traffic returns to the node networks, unseparated rather than
|
||||
misrouted. The same holds during a cold rebuild until Gitea serves bird's
|
||||
image. The default route lives in table 47, not in the Talos config, so a
|
||||
default cannot outlive the specific routes and send pod traffic to the UDM.
|
||||
|
||||
## Sources
|
||||
|
||||
Cilium v1.20 `pkg/datapath/linux/node.go`; BIRD 2.17 user manual (kernel
|
||||
protocol, `strict bind`, `direct`); Talos v1.13 `RoutingRuleConfig`,
|
||||
`machine.nodeAnnotations`, configuration patching (lists are appended,
|
||||
`$patch: delete`), etcd `LeaveCluster`; KubePrism docs;
|
||||
siderolabs/talos#12720. PRs #831, #833, #835, #836, #837, #842, #852, #854,
|
||||
#855, #857, #858, #918, #919, #922; closed unmerged: #830, #838, #839.
|
||||
@@ -0,0 +1,304 @@
|
||||
# Proposal — a node roll without the ingress hiccup
|
||||
|
||||
Status: draft, grounded in one measured roll. On 2026-09-07 all three MS-01
|
||||
were drained, patched and rebooted back to back (06:52–07:08) for the SAN
|
||||
rollout. Everything below is read off that window: Traefik access logs and
|
||||
Cilium BGP logs in Loki, the drain logs, and the live cluster afterwards.
|
||||
|
||||
The short version: there are two ingresses and they failed differently.
|
||||
**The LAN ingress (Traefik behind the BGP VIP) did not hiccup** — it
|
||||
answered every request; the 5xx it returned were for backends mid-move.
|
||||
**The public edge (proxies → NetBird mesh → Traefik) went dark for 40–80 s
|
||||
on two of the three rolls, and would have for 3½ minutes on the third had
|
||||
its routes not recovered by luck** — a known client bug the repo already
|
||||
documents and works around with a watchdog, whose 600 s restart cooldown is
|
||||
what turns a 3-second blip into minutes when rolls follow each other. So
|
||||
this is two projects: make evictions cheap, and make a peer loss on the
|
||||
mesh cost seconds, deterministically.
|
||||
|
||||
## What was measured
|
||||
|
||||
Traefik, 06:48–07:12, 24 minutes, 3 391 requests, of which 5xx:
|
||||
|
||||
| router | status | count |
|
||||
|---|---|---|
|
||||
| gitea | 503 | 56 |
|
||||
| home-assistant | 503 | 36 |
|
||||
| shelly-manager | 502 | 25 |
|
||||
| immich | 503 | 19 |
|
||||
| openbao | 503 | 13 |
|
||||
| grafana | 503 | 10 |
|
||||
| music-assistant | 503 | 9 |
|
||||
| crowdsec-lapi | 503 | 7 |
|
||||
| wordpress | 503 | 6 |
|
||||
| woodpecker | 503 | 4 |
|
||||
|
||||
The quiet 24 minutes before the roll had zero 5xx. Every entry above is a
|
||||
single-replica workload (or its single database) that was evicted and had to
|
||||
come up on another node — pod reschedule, RBD volume detach and re-attach,
|
||||
process start. That is 30–90 s of 503 per app per drain.
|
||||
|
||||
Two things made it worse than one move per app:
|
||||
|
||||
- **Double moves.** The drain of k8s-1 pushed pods onto k8s-2, which was
|
||||
drained next: `ha-pg-1`, `ak-pg-1`, `im-pg-1`, `pprl-pg-2`, `tr-pg-1`,
|
||||
`wp-pg-1`, `influxdb`, `protonmail-bridge`, `prometheus`, `wordpress-mariadb`
|
||||
each appear in two drain logs. Half the downtime was paid twice.
|
||||
- **`--disable-eviction`.** Ten PodDisruptionBudgets allow zero disruptions
|
||||
(every CNPG primary, gitea, influxdb, syncthing, protonmail-bridge,
|
||||
woodpecker-server). A plain `kubectl drain` hangs on them, so the roll used
|
||||
`--disable-eviction`, which deletes pods outright. For the two-instance CNPG
|
||||
clusters (authentik, immich, paperless) that skipped the switchover CNPG
|
||||
would have done: the operator's documented behaviour is "if a node is to be
|
||||
drained and contains a cluster's primary instance, a switchover happens
|
||||
ahead of the drain" — but only when the drain goes through the eviction API.
|
||||
We turned a 2-second switchover into a 30–60 s primary restart.
|
||||
|
||||
Beside the HTTP path, one reboot side effect took a dependency down for
|
||||
half an hour: openbao-0 came back **sealed**, and because the StatefulSet is
|
||||
`OrderedReady`, openbao-1/2 were never recreated after their eviction until
|
||||
openbao-0 was unsealed by hand. ESO could not refresh a secret for 35 minutes.
|
||||
|
||||
### The public edge
|
||||
|
||||
Public traffic reaches Traefik with client address `127.0.0.1` (the rootless
|
||||
netbird-ingress peer forwards from inside its pod through socket-LB), LAN
|
||||
traffic with `10.2.x`. That splits the access log by path. At that hour the
|
||||
only public traffic was the proxies' own watchdog — two hosts probed once a
|
||||
minute from each of two proxies, a flat 4 requests/min — which makes it a
|
||||
clean synthetic probe of the whole external chain:
|
||||
|
||||
| minute (UTC) | LAN req | public req | what was happening |
|
||||
|---|---|---|---|
|
||||
| 04:55 | 128 | 4 | k8s-1 rebooting (04:55:10) |
|
||||
| 04:56 | 109 | 2 | k8s-1 back, its peer re-enrolled 04:56:24 |
|
||||
| 04:57 | 127 | **0** | — |
|
||||
| 04:58 | 126 | 4 | proxies restarted by their watchdog |
|
||||
| 05:00–05:03 | 186–233 | 4 | k8s-2 roll: **no dip** |
|
||||
| 05:06–05:07 | 97–135 | 2 | k8s-3 roll: half |
|
||||
|
||||
The external chain was down for roughly two minutes at the k8s-1 roll, one
|
||||
at k8s-3, and not at all at k8s-2. The mechanism is in
|
||||
`ansible/roles/netbird-proxy-watchdog/defaults/main.yml`: the proxy's
|
||||
embedded NetBird client pins each domain route to one routing-peer
|
||||
*identity*; the netbird-ingress DaemonSet enrols with a setup key, so every
|
||||
pod start is a **new identity** (the k8s-1 pod registered fresh at
|
||||
04:56:24); and upstream's dns-interceptor never rebinds a route whose
|
||||
owning peer is gone ("already routed by peer … HA routing disabled"). The
|
||||
proxy then serves 502 for every domain until its container restarts —
|
||||
which the watchdog does after three failed one-minute probes, or at once
|
||||
when the flood watchdog sees the wedge in the proxy log. Which roll hurts
|
||||
depends on which of the three peers the proxies happened to pin. k8s-2's
|
||||
peer was pinned by neither.
|
||||
|
||||
Note what did *not* help: the DaemonSet's preStop (`netbird down`) took the
|
||||
peer out of the mesh cleanly at 04:55:11, and the peer was back 73 s later.
|
||||
Neither matters to a client that will not rebind and will not recognise
|
||||
the returning pod as the same peer.
|
||||
|
||||
## What was not the cause
|
||||
|
||||
**BGP.** Per node the UDM sessions were down for 47–51 s (Peer Down at the
|
||||
Cilium shutdown, Peer Up ~50 s later). That gap did not carry traffic: every
|
||||
LoadBalancer here is `externalTrafficPolicy: Local`, so a VIP is only
|
||||
advertised from nodes with a local endpoint, and the drain had already
|
||||
evicted those endpoints — Cilium withdrew the routes while the session was
|
||||
still healthy (the "soft reset out" bursts during each drain). The 90 s hold
|
||||
time and 120 s graceful-restart window never came into play. Leave them.
|
||||
|
||||
**Traefik.** Two replicas on k8s-1/k8s-2, one evicted per roll, the other
|
||||
carried the VIP alone. No PDB, no spread constraint, no preStop — which held
|
||||
by luck (the two pods happened to sit on different nodes), and is fixed below
|
||||
because luck is not a design.
|
||||
|
||||
## Target state, in order of measured impact
|
||||
|
||||
### 1. Drain in a way that lets operators do their job (procedure only)
|
||||
|
||||
- **Cordon the next node before draining the current one.** Evicted pods
|
||||
then land on the node that stays up, and nothing moves twice. Order:
|
||||
cordon B → drain A → reboot A → uncordon A → cordon C → drain B → …
|
||||
- **Never `--disable-eviction`.** It exists in the roll only because ten
|
||||
PDBs say "never". Stage 2 makes those PDBs honest, after which a plain
|
||||
`kubectl drain --ignore-daemonsets --delete-emptydir-data` completes and
|
||||
CNPG switches primaries over before its replicas are evicted.
|
||||
- **After every reboot, three gates before uncordon:** `00-multus.conf` on
|
||||
the host and Multus + whereabouts Running (k8s-1's OSD came up
|
||||
Cilium-only today because it beat the Multus DaemonSet by 30 s); OpenBao
|
||||
unsealed (`bao status` on every pod that exists, then the ones the
|
||||
StatefulSet creates after it); Ceph back at baseline.
|
||||
- This is `rollout-node.sh` from today's window, minus `--disable-eviction`,
|
||||
plus the OpenBao gate, checked into `scripts/`. A reboot runbook that lives
|
||||
in a scratchpad is a runbook nobody else can run.
|
||||
|
||||
### 2. PodDisruptionBudgets that mean what they say
|
||||
|
||||
A `minAvailable: 1` PDB on a single-replica workload does not protect
|
||||
anything; it only forces the operator to bypass all PDBs. Two changes:
|
||||
|
||||
- **CNPG single-instance clusters** (grafana-db, home-assistant, trmnl,
|
||||
woodpecker): `spec.enablePDB: false`. CNPG documents exactly this — "for
|
||||
single-instance clusters the operator will prevent draining the node" and
|
||||
recommends moving to direct control of disruption budgets. The instance
|
||||
restarts on drain (it does today too, just through the back door), and the
|
||||
multi-instance clusters keep their switchover-before-eviction behaviour.
|
||||
- **gitea, influxdb, syncthing, protonmail-bridge, woodpecker-server**: drop
|
||||
the `minAvailable: 1` PDBs. They are single replicas with RWO volumes; the
|
||||
PDB cannot buy availability, only stalls.
|
||||
|
||||
### 3. The ingress itself: three replicas, one per node, drained gracefully
|
||||
|
||||
Traefik goes to `replicas: 3` with a required `podAntiAffinity` on
|
||||
hostname, `PodDisruptionBudget maxUnavailable: 1`, and a preStop that lets
|
||||
in-flight requests finish before the endpoint is withdrawn:
|
||||
|
||||
```yaml
|
||||
# chart keys as in traefik/values.yaml
|
||||
deployment:
|
||||
replicas: 3
|
||||
lifecycle:
|
||||
preStop:
|
||||
sleep:
|
||||
seconds: 5
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- topologyKey: kubernetes.io/hostname
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: traefik
|
||||
podDisruptionBudget:
|
||||
enabled: true
|
||||
maxUnavailable: 1
|
||||
ports:
|
||||
websecure:
|
||||
transport:
|
||||
lifeCycle:
|
||||
graceTimeOut: 15s
|
||||
```
|
||||
|
||||
With `Local` traffic policy every node then advertises the VIP; a roll takes
|
||||
one of three ECMP paths away instead of one of two, and the UDM sees the
|
||||
withdrawal before the pod stops accepting. Also gives the UDM a route from
|
||||
k8s-3, which today advertises nothing for the ingress.
|
||||
|
||||
### 4. Databases the apps actually wait on
|
||||
|
||||
The 503 table is a priority list. Each of these is a single Postgres today:
|
||||
|
||||
| app | 5xx | cluster | today |
|
||||
|---|---|---|---|
|
||||
| home-assistant | 36 | ha-pg | `instances: 1` |
|
||||
| grafana | 10 | gf-pg | `instances: 1` |
|
||||
| woodpecker | 4 | wp-pg | `instances: 1` |
|
||||
| trmnl | — | tr-pg | `instances: 1` |
|
||||
|
||||
`instances: 2` for ha-pg and gf-pg turns a drain into a CNPG switchover
|
||||
(seconds). The cost is one more Postgres pod and a second RBD volume each.
|
||||
trmnl and woodpecker can stay at one: they were barely visible.
|
||||
|
||||
gitea (56) is the largest number and has no cheap answer: one StatefulSet,
|
||||
one RWO volume, an application that does not run two instances against the
|
||||
same data directory without shared storage. It moves once per roll (with
|
||||
stage 1) and that is its floor unless its repositories move to CephFS.
|
||||
|
||||
### 5. The public edge: what the proxies' own journals say
|
||||
|
||||
This is the only *complete* outage in the window, so despite the numbering
|
||||
it ships right after stage 1. It was measured twice: from the proxies'
|
||||
watchdog journals for the morning roll, and in a controlled afternoon
|
||||
experiment (three routing-peer kills, probed from proxy-2 itself at 2 s —
|
||||
the LAN cannot probe this path, the UDM hairpin-DNATs the proxy addresses
|
||||
straight to the VIP). Proxy clocks run ~3 min behind the cluster.
|
||||
|
||||
**What a peer loss does.** When the routing peer the proxy has pinned
|
||||
disappears, the proxy's route manager wedges (`couldn't fetch peer state`
|
||||
burst) and the *whole* process stops answering — every domain, and in the
|
||||
experiment also services that never used a domain route. The flood
|
||||
watchdog restarts the container within 2 s of the burst; the probe saw
|
||||
one failed round (≈3 s) at the first two kills. The third kill fell inside
|
||||
the flood watchdog's **600 s restart cooldown**: restart suppressed, wedge
|
||||
left standing, and the URL watchdog needed its three one-minute rounds —
|
||||
**3½ minutes of 000 on every public domain**. The morning roll had the
|
||||
same shape: k8s-1 (both proxies restarted, 40–80 s), k8s-2 (wedge at
|
||||
06:58, restart suppressed on both proxies, 278 s after the previous one —
|
||||
the routes happened to recover on their own), k8s-3 (restart, ~10 s).
|
||||
Three rolls in sixteen minutes guarantee the suppressed case.
|
||||
|
||||
- **5a. Cooldown 600 s → 60 s** in
|
||||
`ansible/roles/netbird-proxy-watchdog/defaults/main.yml`. The cooldown
|
||||
exists so a genuinely broken backend cannot restart-loop the proxy; a
|
||||
minute still does that (the URL watchdog needs three minutes to fire
|
||||
anyway). With it, every peer loss — rolls back to back included — is a
|
||||
~3 s blip instead of a lottery between 3 s and 3½ min. Cheapest change
|
||||
in this document, ships first.
|
||||
- **5b. Persist the peer identity.** Replace the DaemonSet with a
|
||||
StatefulSet of three, required anti-affinity on hostname,
|
||||
`PodDisruptionBudget maxUnavailable: 1`, and a small RBD volume per
|
||||
replica holding NetBird's state directory. A drain then evicts the pod
|
||||
through the PDB (the DaemonSet pod is not evicted at all today, it dies
|
||||
with the reboot) and it returns within ~30 s **as the same peer**. Whether
|
||||
the route manager then un-wedges without a restart is the hypothesis to
|
||||
measure; the setup-key constraint (ephemeral peers reaped after >10 min
|
||||
offline, NeedsLogin liveness restart) holds unchanged.
|
||||
- **5c. The structural exit: no domain routes at all.** The subnet-target
|
||||
experiment settled what works: an HTTP-mode service with an IP target
|
||||
forwards `Host: <ip>` and Traefik answers 404 (NetBird rejects a Host
|
||||
override); an HTTP-mode service with a domain target needs mesh DNS on
|
||||
the proxy, which answers SERVFAIL there; a **TLS-passthrough service
|
||||
(`mode: tls`, `listen_port: 443`) with a subnet target works** — SNI and
|
||||
Host reach Traefik untouched, it coexists with the HTTP listener on 443,
|
||||
no `skip_tls_verify`. If every public service became such a passthrough
|
||||
over a subnet resource covering the Cilium LB pool (`10.2.99.0/24`,
|
||||
`fd99::/64`), there would be no domain resource left to wedge on, and
|
||||
peer loss would be NetBird's documented HA failover (seconds). Costs: the
|
||||
target's `host` is the VIP, so a reconciler (the external-dns-unifi
|
||||
pattern, `PUT /api/reverse-proxies/services/{id}` from the Traefik
|
||||
Service's `status.loadBalancer.ingress`) has to own it; the proxy's L7
|
||||
features (auth, path rules) are lost — none are in use; and the pure
|
||||
failover time is unmeasured, because today every kill wedged the domain
|
||||
routes first. Decide after 5a and 5b have been measured on a real roll.
|
||||
- **Track the upstream bug.** The repo's note dates the dns-interceptor
|
||||
behaviour to 2026-07; a search of NetBird's releases and issues turned up
|
||||
no fix for the "already routed by peer / HA routing disabled" path.
|
||||
Re-check on each NetBird bump.
|
||||
- **Side findings, each its own small change:** both proxy clocks are
|
||||
~3 min slow (the proxy logs SCT timestamps "in the future"; TLS
|
||||
validity windows care) — NTP on proxy-1/proxy-2; a new service's
|
||||
certificate reaches the IPv6-only member only after ~20 min of ACME
|
||||
retries (Let's Encrypt validates over IPv4 and lands on the other member,
|
||||
which has no token for it); proxy-1 and proxy-2 do not ship syslog to
|
||||
Loki (edge-1 does), so every public-edge timeline is reconstructed over
|
||||
SSH — point them at the alloy-syslog VIP.
|
||||
|
||||
## Expected result
|
||||
|
||||
| stage | removes |
|
||||
|---|---|
|
||||
| 1 | every double move (half of today's 5xx), the primary restarts, the 35-minute OpenBao/ESO outage, the Multus race |
|
||||
| 2 | the need for `--disable-eviction` at all |
|
||||
| 3 | the single-node ingress exposure; ECMP loses 1/3 instead of 1/2 |
|
||||
| 4 | home-assistant and grafana 503s entirely |
|
||||
| 5a | the 3½-minute public-edge blackout of a second peer loss within 10 min; every peer loss becomes a ~3 s restart |
|
||||
| 5b/5c | the restart itself, if the returning identity (5b) or the absence of domain routes (5c) proves to un-wedge — to be measured |
|
||||
|
||||
What remains after all of it: one move per single-replica app per roll
|
||||
(gitea, immich, shelly-manager, music-assistant, wordpress), 30–60 s each,
|
||||
once, and a few seconds of public edge per roll. That is the honest floor
|
||||
for single-replica services on RWO storage and a mesh client that pins.
|
||||
|
||||
## Transition
|
||||
|
||||
1. Stage 5a is one Ansible default and a proxy deploy — ships first.
|
||||
2. Stage 1 is a script and a runbook change — no cluster change.
|
||||
3. Stage 5b's StatefulSet changes how the public edge enrols, so it gets
|
||||
its own roll to measure, probed from a proxy.
|
||||
4. Stage 2 is one field on four CNPG clusters and five deleted PDBs; each
|
||||
Kustomization applies without disruption.
|
||||
5. Stage 3 is a Traefik HelmRelease change; the rollout itself is a rolling
|
||||
update under the new PDB.
|
||||
6. Stage 4 is `instances: 2` on two clusters; CNPG adds the replica online.
|
||||
7. Stage 5c only if 5a+5b leave a measurable public gap.
|
||||
|
||||
Measure the next roll the same way: 5xx by router, the proxies' watchdog
|
||||
journals (or their syslog in Loki), BGP Down→Up gaps, pods per drain log.
|
||||
@@ -0,0 +1,194 @@
|
||||
# Proposal — separate the four traffic classes onto their own VLANs
|
||||
|
||||
Status: draft. The question this started with is **answered** — from the Cilium
|
||||
source and flag reference, not from a test — and the answer changes the shape
|
||||
of the proposal rather than confirming it.
|
||||
|
||||
Supersedes the framing in issue #576, which described this as "give node
|
||||
management its own NIC, and Ceph public its own VLAN". That was two of four
|
||||
classes and one of them turned out already solved.
|
||||
|
||||
## The four classes
|
||||
|
||||
| class | what it is | where it goes today |
|
||||
|---|---|---|
|
||||
| **Ceph replication** | OSD-to-OSD heartbeat, replication, recovery | VLAN 49, its own X710 port — **separated** |
|
||||
| **cluster-internal** | pod-to-pod east-west, including Ceph client I/O | pod network, VXLAN over `mgmt0` |
|
||||
| **cluster-external** | ingress to LoadBalancer VIPs, egress to LAN and internet | `mgmt0` |
|
||||
| **management** | Talos API, etcd, kube-apiserver, kubelet | `mgmt0` |
|
||||
|
||||
Three of four share one interface. That is not a design decision — until the
|
||||
bare-metal conversion there was one NIC to work with.
|
||||
|
||||
Ceph client I/O living on the pod network is correct rather than a leftover:
|
||||
that traffic *is* cluster-internal. Ceph is done.
|
||||
|
||||
## Principle
|
||||
|
||||
**Each class gets its own VLAN. Which physical interface a VLAN lands on is a
|
||||
separate, changeable decision.**
|
||||
|
||||
Defining classes by VLAN means the separation survives recabling, port failures
|
||||
and capacity changes, and it does not require one port per class. Defining them
|
||||
by interface would freeze a capacity decision into the architecture.
|
||||
|
||||
## The finding that shapes everything else
|
||||
|
||||
The original open question was whether `nodePort.directRoutingDevice` moves the
|
||||
next-hop of Cilium's direct routes, or only selects an interface. It is
|
||||
answerable from the source, and the answer is: **only the interface.**
|
||||
|
||||
`installDirectRoute` takes the next-hop as a parameter:
|
||||
|
||||
```go
|
||||
func installDirectRoute(log *slog.Logger, prefix netip.Prefix, nodeIP net.IP,
|
||||
skipUnreachable bool) (routeSpec *netlink.Route, err error)
|
||||
```
|
||||
|
||||
and `nodeUpdate` supplies it as `newNode.GetNodeIP(false)`. `GetNodeIP` returns,
|
||||
in priority order, `NodeInternalIP` → `NodeExternalIP` → anything else. The
|
||||
same `newIP4` is what tunnel mode uses as the VXLAN endpoint.
|
||||
|
||||
`NodeInternalIP` comes from the Kubernetes Node object. `--ipv4-node` looks like
|
||||
an override but is not one: `initFromConfig()` calls `SetNodeInternalIP` and
|
||||
`initFromK8s()` runs **after** it and overwrites the value. In a Kubernetes
|
||||
cluster there is no path by which Cilium's node IP differs from the kubelet's
|
||||
`InternalIP`.
|
||||
|
||||
`--direct-routing-device` is documented as "Device name used to connect nodes in
|
||||
direct routing mode" — a device, not an address.
|
||||
|
||||
**So cluster-internal traffic is welded to the node IP, and the node IP is the
|
||||
kubelet InternalIP.** Two consequences:
|
||||
|
||||
- The class cannot be moved off whatever VLAN the node IP lives on.
|
||||
- **Routing mode is irrelevant to this problem.** Tunnel mode sources the VXLAN
|
||||
from the node IP; native mode uses it as the direct-route next-hop. Native
|
||||
routing buys nothing here, so the disruptive, non-rolling step this proposal
|
||||
previously contained is simply gone.
|
||||
|
||||
## Consequence: draw the line the other way
|
||||
|
||||
If cluster-internal cannot leave the node IP, then separation has to come from
|
||||
moving *management services* off it — and each one has to be checked
|
||||
individually, because they do not share a mechanism.
|
||||
|
||||
| management service | can it leave the node IP? | lever |
|
||||
|---|---|---|
|
||||
| etcd | **yes** | `cluster.etcd.advertisedSubnets` (+ `listenSubnets`, which defaults to it) |
|
||||
| kube-apiserver endpoint | **yes** | already a VIP — `Layer2VIPConfig` with a `link:` field |
|
||||
| Talos API | **n/a** | `apid` answers on every node address (measured); a client and firewall choice, not a bind setting |
|
||||
| kubelet | **no** | it *is* the node IP, by definition |
|
||||
|
||||
So four VLANs are reachable, but the boundary does not fall where the class
|
||||
names suggest: **kubelet stays with cluster-internal.** That is the honest
|
||||
result, and it is worth stating plainly rather than filing kubelet under
|
||||
management and hoping.
|
||||
|
||||
The remaining classes keep the levers established earlier:
|
||||
|
||||
- **cluster-external ingress** — `CiliumBGPNodeConfigOverride.spec.bgpInstances[].peers[].localAddress`.
|
||||
For eBGP the session source is the next-hop, so pinning the session pins
|
||||
where return traffic arrives. The session to the UDM has been running for
|
||||
months.
|
||||
- **cluster-external egress** — eBPF masquerading follows the output device, so
|
||||
this is a default-route decision plus `enable-masquerade-to-route-source`
|
||||
(today `false`).
|
||||
- **Ceph replication** — Multus macvlan on VLAN 49, done.
|
||||
|
||||
## The general escape hatch
|
||||
|
||||
Ceph is separated because its OSDs got a second interface and stopped using the
|
||||
pod network for replication. That is the *only* mechanism that fully detaches a
|
||||
workload from the node IP, and it generalises: anything that must not ride the
|
||||
node IP gets its own `NetworkAttachmentDefinition`.
|
||||
|
||||
It is also expensive per workload — an attachment, an IPAM pool, a pod spec
|
||||
change. Worth it for Ceph replication. Not worth it for east-west traffic in
|
||||
general.
|
||||
|
||||
## Recommended mapping
|
||||
|
||||
Two ways to arrange this, and they differ only in which VLAN gets relabelled:
|
||||
|
||||
**A — move the node IP to a new cluster VLAN.** VLAN 40 keeps etcd, the API VIP
|
||||
and Talos API. Cleanest match to the class names, but it changes every node's
|
||||
identity, which is the one address the whole cluster is administered through.
|
||||
|
||||
**B (recommended) — leave the node IP on VLAN 40 and move management off it.**
|
||||
VLAN 40 becomes the cluster VLAN; a new VLAN carries etcd and the API VIP.
|
||||
Identical end state, and no node ever changes its InternalIP.
|
||||
|
||||
Under B:
|
||||
|
||||
| VLAN | carries |
|
||||
|---|---|
|
||||
| 49 | Ceph replication |
|
||||
| 40 (relabelled *cluster*) | pod east-west, kubelet |
|
||||
| new *management* | etcd, kube-apiserver VIP, Talos API by convention |
|
||||
| new *external* | LoadBalancer ingress, egress default route |
|
||||
|
||||
The property the user asked for — management carries only management traffic —
|
||||
holds under B. The label moves; the property is what matters.
|
||||
|
||||
## Migration shape
|
||||
|
||||
Every step is rolling and reversible. The routing-mode change is gone.
|
||||
|
||||
1. **New VLANs** on the switch, trunked to the X710. Nothing uses them.
|
||||
2. **Addresses** on the new VLANs via Talos, one node at a time, `--mode=try`.
|
||||
Still unused; this only proves the L2 works.
|
||||
3. **etcd** to the management VLAN via `advertisedSubnets`. One node at a time,
|
||||
waiting for a healthy quorum between each.
|
||||
4. **API VIP** to the management VLAN via `Layer2VIPConfig.link`. DNS for
|
||||
`k8s.renner.uno` follows.
|
||||
5. **cluster-external**: BGP `localAddress`, default route, masquerade flag.
|
||||
Reversible by reverting the override; failure is visible immediately in
|
||||
`cilium-dbg shell -- bgp/peers`.
|
||||
6. Verify VLAN 40 by what is *absent* from it, not by what still works.
|
||||
|
||||
Step 3 is the one that deserves care — etcd is the only step where a mistake
|
||||
costs quorum rather than connectivity.
|
||||
|
||||
## What this does not buy
|
||||
|
||||
Management currently shares a 10 Gb link and is not contended. Nothing observed
|
||||
today is slow because of this. The case for the split is that traffic classes
|
||||
with different failure modes and security postures should not be
|
||||
indistinguishable — not that something is broken.
|
||||
|
||||
## Sources and measurements
|
||||
|
||||
Source, Cilium v1.20.1 (the running version):
|
||||
|
||||
- `pkg/datapath/linux/node.go` — `installDirectRoute(…, nodeIP net.IP, …)`,
|
||||
fed from `newIP4 = newNode.GetNodeIP(false)`; `createDirectRouteSpec` rejects
|
||||
a next-hop that is not directly reachable.
|
||||
- `pkg/node/types/node.go` — `GetNodeIP` prefers `NodeInternalIP`.
|
||||
- `pkg/node/sync/local_node_sync.go` — `initFromConfig()` applies
|
||||
`--ipv4-node`, `initFromK8s()` runs afterwards and overwrites it from the
|
||||
Kubernetes Node.
|
||||
- `cilium-agent` flag reference — `--direct-routing-device`: "Device name used
|
||||
to connect nodes in direct routing mode".
|
||||
|
||||
Talos:
|
||||
|
||||
- Multihoming guide — `cluster.etcd.advertisedSubnets`; `listenSubnets`
|
||||
defaults to it. kube-apiserver and Talos API listen addresses are not
|
||||
covered there.
|
||||
- `kubernetes/talos/patches/k8s-{1,2,3}.yaml` — `Layer2VIPConfig` for
|
||||
`10.2.40.60` with `link: mgmt0`.
|
||||
|
||||
Measured on the cluster:
|
||||
|
||||
- `apid` answered on a node address that is not the InternalIP, so the Talos
|
||||
API is not bound to one address.
|
||||
- k8s-1 currently holds one routable v4 address plus the VIP on `mgmt0`; its
|
||||
InternalIP had changed since the last session, which is why nothing here
|
||||
names an address as fixed.
|
||||
- Cilium runs `routing-mode: tunnel` with `enable-masquerade-to-route-source`
|
||||
unset and no direct-routing device configured.
|
||||
|
||||
Not verified: whether the kube-apiserver's `--advertise-address` follows the
|
||||
node IP independently of the VIP. It does not affect the recommendation, since
|
||||
clients reach the API through the VIP either way.
|
||||
@@ -1,316 +0,0 @@
|
||||
# Wiki: local web interface and agent API, as a modular .NET solution
|
||||
|
||||
## Goal
|
||||
|
||||
A web interface for the Notes wiki that runs entirely in the cluster, and an
|
||||
HTTP API through which agents ask the wiki typed questions and deposit facts.
|
||||
Built as one C# solution with a MudBlazor front end, and built so that every
|
||||
external dependency is replaceable: where the notes live, which model embeds
|
||||
them, what judges a typed question, what runs an agent, how the vector index
|
||||
is stored. The first adapters are what the homelab uses today. The point of
|
||||
the design is that the next one is a project and a config line, not a
|
||||
rewrite.
|
||||
|
||||
## Where we are
|
||||
|
||||
- `wiki-index` (Python, ns `ai`) syncs `wiki/` from `k.renner/notes` by blob
|
||||
SHA, embeds through `wiki-embed` (bge-m3 on llama-server), answers
|
||||
`POST /rank`. It keeps slug, title and vector, not the text. LAN-only behind
|
||||
Traefik BasicAuth; the Gitea webhook hits the ClusterIP.
|
||||
- `~/Notes/bin/wiki_rank.py` and `wiki_ask.py` rank through the index, build a
|
||||
document context from a local vault checkout, send it with typed questions
|
||||
to TypeSafe, gate on `covered`. The wiki-synth agent composes answers from
|
||||
those judgements.
|
||||
- New knowledge enters as a file in `raw/` (facts with provenance, never
|
||||
hypotheses), distilled nightly in CI.
|
||||
- The house C# project, shelly-manager, sets the conventions: net10.0, an
|
||||
`.slnx` with `src/` and `test/`, xUnit, MudBlazor 9, Blazor Web App with
|
||||
interactive server rendering, minimal APIs, an Authentik proxy-header
|
||||
authentication handler for humans and an API-key handler for machines, a
|
||||
multi-stage Dockerfile, `dotnet test` on every push and a kaniko release
|
||||
pipeline to the Gitea registry. Human-facing apps sit behind an Authentik
|
||||
outpost (`<app>-gate`) with a CNP that admits only that outpost.
|
||||
|
||||
## Architecture: ports in the core, adapters around it
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────┐
|
||||
│ Wiki.Host (ASP.NET Core + Blazor/Mud) │ composition root,
|
||||
│ /api/v1, /rank, /webhook, UI pages │ config, auth
|
||||
└───────────────┬──────────────────────────┘
|
||||
│ uses
|
||||
┌───────────────▼──────────────────────────┐
|
||||
│ Wiki.Application (use cases) │ Sync, Rank, Ask,
|
||||
│ │ DepositFact, Queue-
|
||||
└───────────────┬──────────────────────────┘ Question, Evaluate
|
||||
│ depends only on
|
||||
┌───────────────▼──────────────────────────┐
|
||||
│ Wiki.Core (domain + ports) │ Note, RawSource,
|
||||
│ no package references │ Report, LinkGraph,
|
||||
└──────────────────────────────────────────┘ the interfaces
|
||||
implemented by ▲ ▲ ▲ ▲ ▲
|
||||
Adapters.Gitea Adapters.LlamaCpp Adapters.TypeSafe Adapters.FileIndex Adapters.Anthropic
|
||||
(later: Database, Ollama/Onnx, local Jev, pgvector/Qdrant, OpenAI-compatible answer models)
|
||||
```
|
||||
|
||||
Rules that keep it modular, enforced by project references and a test:
|
||||
|
||||
- `Wiki.Core` references nothing. It holds the domain types and the port
|
||||
interfaces. Adapters reference Core only; they never reference each other
|
||||
or the host. `Wiki.Host` is the only project that knows which adapters
|
||||
exist.
|
||||
- Every port has a **contract test suite** in `test/Wiki.Contracts` that
|
||||
runs against every adapter of that port. A new adapter is done when the
|
||||
suite is green. That is what makes "swap the storage" a safe operation.
|
||||
- Adapters are selected by configuration and registered by name (keyed
|
||||
services), so several can be active at once. The UI's lab page and the
|
||||
evaluator use that to compare, the API uses the one marked active.
|
||||
- Nothing in Core or Application knows about HTTP, Gitea, TypeSafe, files or
|
||||
Kubernetes.
|
||||
|
||||
### Ports
|
||||
|
||||
| port | what it abstracts | first adapter | examples of the next ones |
|
||||
|---|---|---|---|
|
||||
| `INoteStore` | read notes, raw sources and reports; enumerate by kind; observe changes (content hash per document) | Gitea repository API (`k.renner/notes`, blob SHA) | git checkout on a volume; a database (CNPG Postgres) with the vault imported; an Obsidian sync folder |
|
||||
| `IFactSink` | append a new raw source with provenance | Gitea contents API, new file on `main` | database insert; git commit on a checkout |
|
||||
| `IEmbeddingModel` | text → vector, with a model id and dimension | llama-server `/v1/embeddings` (bge-m3 via wiki-embed, the model that won the German-document comparison on accuracy per cost) | the qwen3 embedding variants from that comparison; ONNX in-process; any OpenAI-compatible endpoint |
|
||||
| `IVectorIndex` | store vectors per (model id, document), query top-k | in-memory cosine with a file snapshot per model, as today | pgvector in the database; Qdrant |
|
||||
| `IChunker` | split a document into addressable units for typed questions | paragraph splitter (headings, fences), as in `wiki_ask.py` | sentence windows; semantic chunks |
|
||||
| `IJudge` | typed questions over a state: Choice, Noul, Score, with probabilities | TypeSafe System One (`jev-latest`, cloud; the wiki records no locally hosted Jev today) | a locally hosted Jev with the same request shape, should TypeSafe offer one; a constrained-output local LLM adapter, flagged as uncalibrated |
|
||||
| `IAnswerModel` | prose from a question plus evidence: the selected paragraphs with their notes and the judgements; must cite by slug; streaming | Anthropic Messages API (Claude, model from config, Sonnet by default) | any OpenAI-compatible endpoint, including the local llama-server; a different Claude model per question |
|
||||
| `IEvaluator` | measure a retrieval or judge configuration against the labelled set (questions and cited sources in `output/`) | recall@k, hit rate, MRR, as `wiki_rank_eval.py` | judge accuracy against hand-labelled paragraphs |
|
||||
| `IIdentity` | who is calling, for attribution of facts and saved answers | Authentik proxy headers (humans), API key (machines), as in shelly-manager | OIDC bearer tokens |
|
||||
|
||||
Two ports deserve a sentence of scope each. The judge returns judgements
|
||||
with probabilities, never prose, and the UI never presents its output as an
|
||||
answer, only as ranked, quoted evidence. The answer model turns that
|
||||
evidence into words and is allowed nothing else: it sees the selected
|
||||
paragraphs and the judgements, not the whole context, and its output is
|
||||
shown next to the evidence it was given. Both are single model calls inside
|
||||
the request. The service runs no agent: no tool loop, no vault edits. The
|
||||
wiki-synth agent in a Claude session remains the path for artifacts and for
|
||||
multi-question synthesis, and it can call `/questions` itself.
|
||||
|
||||
### Solution layout
|
||||
|
||||
```
|
||||
Wiki.slnx
|
||||
src/
|
||||
Wiki.Core/ domain types, ports, frontmatter and link parsing
|
||||
Wiki.Application/ use cases; depends on Core only
|
||||
Wiki.Adapters.Gitea/ INoteStore, IFactSink
|
||||
Wiki.Adapters.LlamaCpp/ IEmbeddingModel (OpenAI-compatible embeddings)
|
||||
Wiki.Adapters.TypeSafe/ IJudge (System One client)
|
||||
Wiki.Adapters.FileIndex/ IVectorIndex (memory + snapshot per model)
|
||||
Wiki.Adapters.Anthropic/ IAnswerModel (Messages API, streaming)
|
||||
Wiki.Host/ ASP.NET Core: API, webhook, MudBlazor UI, auth,
|
||||
composition root, appsettings
|
||||
test/
|
||||
Wiki.Core.Tests/ parsing, link graph, chunker
|
||||
Wiki.Contracts/ one suite per port, parameterised over adapters
|
||||
Wiki.Application.Tests/ use cases with fake ports
|
||||
Wiki.Host.Tests/ API surface, auth handlers
|
||||
```
|
||||
|
||||
Own repository `k.renner/wiki`, same shape as shelly-manager: `Dockerfile`,
|
||||
`.woodpecker/test.yml` on pull requests, `.woodpecker/release.yml` building
|
||||
the image with kaniko into the Gitea registry on push to `main`, tagged
|
||||
`build.<unix-ts>` plus the commit SHA so Flux image automation can follow
|
||||
it. The homelab repo holds only the manifests. `claude-bot` needs write
|
||||
access to the new repository.
|
||||
|
||||
Two things the shelly-manager history settles in advance: the TypeSafe
|
||||
client is written from the documented field contract (endpoint, bearer key,
|
||||
`jev-latest`, Noul/Choice/Score, 255-option cap, 32k-token state budget),
|
||||
there is no prior C# client to copy; and the UI needs no login page, because
|
||||
identity comes from the outpost's headers, which avoids the Blazor-circuit
|
||||
sign-in constraint shelly-manager ran into.
|
||||
|
||||
### Configuration
|
||||
|
||||
Adapters are declared in `appsettings` and selected by name; secrets come
|
||||
through environment variables, filled by ExternalSecret, never from the file.
|
||||
|
||||
```json
|
||||
"Wiki": {
|
||||
"NoteStore": { "Active": "gitea", "gitea": { "Repository": "k.renner/notes", "Branch": "main" } },
|
||||
"Embeddings": {
|
||||
"Active": "bge-m3",
|
||||
"Models": {
|
||||
"bge-m3": { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc:8080", "Model": "bge-m3" },
|
||||
"e5-large": { "Adapter": "Ollama", "Url": "http://ollama.ai.svc:11434", "Model": "multilingual-e5-large" }
|
||||
}
|
||||
},
|
||||
"Judges": {
|
||||
"Active": "typesafe",
|
||||
"Models": {
|
||||
"typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" },
|
||||
"jev-local": { "Adapter": "TypeSafe", "Url": "http://jev.ai.svc:8080/v1/systemone", "Model": "jev" } // illustration: same adapter, other endpoint
|
||||
}
|
||||
},
|
||||
"Index": { "Active": "file", "file": { "Path": "/data/index" } },
|
||||
"AnswerModels": {
|
||||
"Active": "claude-sonnet",
|
||||
"Models": {
|
||||
"claude-sonnet": { "Adapter": "Anthropic", "Model": "claude-sonnet-5" },
|
||||
"qwen-local": { "Adapter": "OpenAI", "Url": "http://llama-server.ai.svc:8080/v1", "Model": "qwen3.6-35b-a3b" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Every embedding model gets its own index namespace. Switching the active
|
||||
model is a config change after the new index has been built in the
|
||||
background; until then both exist and can be compared. Nothing is
|
||||
re-embedded in place.
|
||||
|
||||
## The web interface (MudBlazor)
|
||||
|
||||
Blazor Web App, interactive server rendering, hosted in `Wiki.Host`. Static
|
||||
assets vendored into the image, no CDN. Pages:
|
||||
|
||||
1. **Search**: a query box, results ranked by the active index with score and
|
||||
tags; for a phrased question, optionally the paragraph-level hits from the
|
||||
judge with their probabilities, shown as quoted evidence.
|
||||
2. **Note**: frontmatter, rendered body with `[[wikilinks]]` resolved,
|
||||
backlinks, sources linking to their raw file.
|
||||
3. **Browse**: by tag, by `updated`, recently changed, orphans.
|
||||
4. **Answers**: a question box. The answer streams in within seconds: rank,
|
||||
judge, compose, all in the request. Below it the evidence: the paragraphs
|
||||
the judge selected with their notes and probabilities, and the `covered`
|
||||
gate; a low gate is shown as "the wiki does not cover this" instead of an
|
||||
answer. A save button writes the answer to `output/` with the wiki-synth
|
||||
frontmatter (`question`, `sources`, `confidence`), so it joins the
|
||||
labelled set. Underneath, the saved answers, newest first, including the
|
||||
ones the wiki-synth agent wrote from a session.
|
||||
5. **Maintenance**: the raw queue with ages, the last consolidation and audit
|
||||
reports, index and sync status.
|
||||
6. **Lab**: pick any registered embedding model and judge, run the same query
|
||||
through them side by side, and run the evaluator on the labelled set to
|
||||
get recall@k per configuration. This is the page for "play around with
|
||||
different models".
|
||||
7. **Settings**: read-only view of the active adapters and their health.
|
||||
|
||||
No editing of notes in the browser. The vault stays a git repository edited
|
||||
by the consolidate agent and by hand.
|
||||
|
||||
## API v1
|
||||
|
||||
Reads are authenticated by the hostname's gate (humans) or API key
|
||||
(machines); writes always need an API key so a fact carries its author.
|
||||
|
||||
| method and path | purpose |
|
||||
|---|---|
|
||||
| `GET /api/v1/status` | index commit and age, counts, active adapters |
|
||||
| `GET /api/v1/notes?q=&tag=&limit=` | list; `q` ranks through the active index |
|
||||
| `GET /api/v1/notes/{slug}` | frontmatter, body, links, backlinks, sources |
|
||||
| `GET /api/v1/tags` | tags with counts |
|
||||
| `POST /api/v1/rank` | `{question, k, model?}` → ranked slugs; also served at `/rank` for the existing scripts |
|
||||
| `POST /api/v1/ask` | `{queries, questions, k, judge?, model?}` → context, typed judgements with the answering paragraphs verbatim, `covered`. For agents and code. |
|
||||
| `POST /api/v1/questions` | `{question, k?, judge?, model?, answerModel?, save?}` → the answer text with `[[slug]]` citations, the evidence (paragraphs, probabilities), `covered`, model and token usage; streamed as server-sent events when requested; with `save`, the `output/` path and commit |
|
||||
| `POST /api/v1/facts` | deposit a raw source: title, source, `kind: fact`, observed_at, content, tags → path and commit |
|
||||
| `GET /api/v1/raw` | the distill queue with ages |
|
||||
| `GET /api/v1/reports` | `output/` documents by type, newest first |
|
||||
| `POST /api/v1/lab/eval` | run the evaluator for a (model, judge) pair |
|
||||
| `POST /webhook` | Gitea push, HMAC as today |
|
||||
|
||||
`POST /api/v1/facts` is rejected without `source`, with a `kind` other than
|
||||
`fact`, or with content under a minimum length. The raw rule is enforced by
|
||||
shape: there is no field for a hypothesis. The file gets the vault's
|
||||
provenance frontmatter with `captured_by` set from the API key's identity.
|
||||
|
||||
`POST /api/v1/ask` is `wiki_ask.py` moved server-side, so the scripts become
|
||||
thin clients and agents no longer need a vault checkout. Its result is
|
||||
judgements and quotations, never prose.
|
||||
|
||||
`POST /api/v1/questions` is `/ask` plus one answer-model call, in process:
|
||||
rank the question, build the context, run the default typed questions
|
||||
(answered, where, covered), hand the selected paragraphs and judgements to
|
||||
the answer model, stream the prose back. Below the `covered` threshold it
|
||||
returns the gap instead of an answer. Time is the sum of the parts, about
|
||||
0.2 s rank, 1 to 4 s judge, a few seconds of generation. Cost is one model
|
||||
call over a few thousand tokens, a few cents on Sonnet, against a quarter
|
||||
dollar for a full agent run.
|
||||
|
||||
## Auth and network
|
||||
|
||||
- Humans: `wiki.renner.uno`, LAN-only (no `dns.renner.uno/public` label),
|
||||
Ingress backend `ak-outpost-wiki-gate:9000`, a `wiki-gate-egress` CNP in
|
||||
`authentik`, the proxy-header handler trusts `X-Authentik-*` only from that
|
||||
outpost.
|
||||
- Machines: `wiki-index.renner.uno` with the API-key handler; the workstation
|
||||
key stays where it is. CI reaches the ClusterIP.
|
||||
- Secrets (Gitea token, webhook secret, API keys, TypeSafe key, Anthropic
|
||||
key) reach the
|
||||
pod as environment variables from Kubernetes Secrets filled by
|
||||
ExternalSecret from the `openbao` ClusterSecretStore. The pod itself has
|
||||
no OpenBao access. The Python service fetched its secrets from OpenBao
|
||||
directly through its own Kubernetes-auth role; that role, its policy and
|
||||
the `ai` → `openbao` CNP entry are retired with it.
|
||||
- Egress from the pod: Gitea, wiki-embed, `api.typesafe.ai:443`,
|
||||
`api.anthropic.com:443`. Every
|
||||
further adapter that talks to a new endpoint gets its own CNP line, which
|
||||
is the one place adding an adapter touches the cluster.
|
||||
|
||||
## Health endpoints
|
||||
|
||||
Three endpoints with three different meanings, wired to the .NET health
|
||||
checks with tags so that the probes and the humans see the right thing:
|
||||
|
||||
| endpoint | meaning | probe |
|
||||
|---|---|---|
|
||||
| `GET /health` | the process serves HTTP. Always 200 once Kestrel listens; no dependency checks. | liveness |
|
||||
| `GET /ready` | the active note store has synced at least once and the active index is loaded; 503 while the first sync or index build runs. A failed later sync keeps the last good state and stays ready. | readiness, plus a startup probe with enough budget for the first sync (the Python service was killed mid-build when its `/health` returned 503 during the initial sync) |
|
||||
| `GET /api/v1/status` | the full picture for humans and the Settings page: index commit and age, counts, queue lengths, and one entry per registered adapter with its last check result and latency. | none |
|
||||
|
||||
Adapter checks (Gitea reachable, embeddings endpoint answering, judge
|
||||
endpoint answering, answer model answering) are reported in `/status` and
|
||||
on the Settings page and are never part of `/ready`: a judge outage must not
|
||||
take the reader down. Their failures show up as red entries, not as
|
||||
restarts.
|
||||
|
||||
## Phases
|
||||
|
||||
One PR each, verified before the next. The Python `wiki-index` keeps running
|
||||
until phase 3 proves parity.
|
||||
|
||||
0. **Skeleton.** Repository, solution, `Wiki.Core` with the ports and domain
|
||||
types, `Wiki.Contracts` with the suites, `Wiki.Host` serving `/health`,
|
||||
`/ready` and `/api/v1/status`. CI: test and release pipelines. Verify:
|
||||
image in the registry, pipeline green, the Deployment's three probes
|
||||
wired and the pod Ready.
|
||||
1. **Parity with the Python service.** Gitea, LlamaCpp and FileIndex
|
||||
adapters, the sync use case, `/rank` and `/webhook`. Verify: the 40-query
|
||||
recall eval gives the same numbers as the Python service on the same
|
||||
commit; `wiki_rank.py --probe-index` against the new service passes.
|
||||
2. **Read API and UI.** Notes, tags, raw, reports; Search, Note, Browse,
|
||||
Maintenance pages; `wiki-gate`. Verify: a note renders with resolved
|
||||
links; an unauthenticated request is redirected to Authentik.
|
||||
3. **Ask.** Chunker, TypeSafe adapter, `/api/v1/ask`, scripts as clients.
|
||||
Verify: `test_wiki_ask.py` passes against the API; a wiki-synth run with
|
||||
the definition unchanged. Then the Python service is retired and the
|
||||
`wiki-index` hostname points at the new one.
|
||||
4. **Facts.** API keys as an ExternalSecret, `IFactSink`, `/api/v1/facts`. Verify:
|
||||
a fact posted from the workstation shows as a `claude-bot` commit and in
|
||||
the next distill.
|
||||
5. **Questions.** Anthropic adapter, `/api/v1/questions` with streaming,
|
||||
the Answers page with save. Verify: a question answered in under ten
|
||||
seconds with citations that resolve to the shown evidence; a question the
|
||||
wiki does not cover returns the gap; a saved answer appears in `output/`
|
||||
with the wiki-synth frontmatter and is picked up by the evaluator.
|
||||
6. **Lab.** Evaluator, a second embedding model and a second answer model
|
||||
registered, the Lab page. Verify: two models compared on the labelled set
|
||||
from the UI.
|
||||
|
||||
## Not in scope
|
||||
|
||||
- Editing notes in the browser.
|
||||
- Replacing Obsidian; the vault stays a git repository and `notes-push.path`
|
||||
keeps pushing.
|
||||
- Public exposure; LAN and VPN only.
|
||||
- Running or triggering agents. The service makes single model calls inside
|
||||
a request; it never runs a tool loop or edits the vault. Distill,
|
||||
consolidate and multi-question synthesis stay Claude processes in CI or in
|
||||
a session; the service shows their inputs and outputs and serves their API
|
||||
calls.
|
||||
@@ -432,8 +432,8 @@ is why it belongs here.
|
||||
among its devices but not `ens19`, because `ens19` is deliberately
|
||||
address-less. Cilium therefore owns `mgmt0` and sees VLAN 40 frames already
|
||||
untagged by the 8021q layer, while VLAN 30 frames only ever meet the filter
|
||||
on the shared parent `uplink`, still tagged. Any future address-less VLAN
|
||||
carrying pod traffic will need the same treatment.
|
||||
on the shared parent `enp2s0f0np0`, still tagged. Any future address-less
|
||||
VLAN carrying pod traffic will need the same treatment.
|
||||
|
||||
The symptom does not look like a VLAN problem. A tc-layer drop reaches an
|
||||
AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports `no buffer space
|
||||
|
||||
@@ -67,12 +67,10 @@ These must exist outside the cluster or the rebuild cannot complete:
|
||||
`flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
|
||||
files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
|
||||
`MIGRATION-sops-openbao.md`.)
|
||||
- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
|
||||
store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
|
||||
former shamir key, now the recovery key — and `homelab/openbao/root-token`),
|
||||
and the GPG private key backed up off-machine. The seal key is the one
|
||||
the snapshots are encrypted under: without it, no OpenBao snapshot can be
|
||||
restored and no secret-consuming app can start.
|
||||
- [ ] **OpenBao unseal key + root token** — in the GPG `pass` store
|
||||
(`homelab/openbao/unseal-key`, `homelab/openbao/root-token`), and the GPG
|
||||
private key backed up off-machine. Without the unseal key, no OpenBao
|
||||
snapshot can be restored and no secret-consuming app can start.
|
||||
- [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
|
||||
(`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
|
||||
`openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
|
||||
@@ -107,7 +105,7 @@ config, and the nodes are addressed by name.
|
||||
## Step 1b — boot all three machines from install media
|
||||
|
||||
```
|
||||
https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.13.10/metal-amd64.iso
|
||||
https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.13.9/metal-amd64.iso
|
||||
```
|
||||
|
||||
Write it to a USB stick and boot each MS-01 from it. Each comes up in
|
||||
@@ -296,65 +294,42 @@ output** — a green step over a still-forming Ceph is a known trap.
|
||||
> Steady-state (non-rebuild) bootstrap is just `kubernetes/bootstrap.sh flux`
|
||||
> with `GITEA_TOKEN` set; `FLUX_SOURCE` defaults to `gitea`.
|
||||
|
||||
## Step 4 — openbao: restore the snapshot
|
||||
## Step 4 — openbao: restore the snapshot + unseal
|
||||
|
||||
`bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
|
||||
`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
|
||||
the empty server, restores the newest raft snapshot straight out of the Velero
|
||||
kopia repository, and waits until every replica has unsealed itself from the
|
||||
static seal. What follows is the same procedure by hand, for when the script
|
||||
is not an option.
|
||||
|
||||
OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
|
||||
`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
|
||||
snapshots are encrypted under that key, so it must be the same key the backed
|
||||
up cluster used — a different one restores nothing. Until the snapshot is
|
||||
restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
|
||||
Secrets, and every secret-consuming app stays Pending.
|
||||
After Step 3, OpenBao comes up **sealed and empty**. Until it is restored and
|
||||
unsealed, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
|
||||
Secrets, and every secret-consuming app stays Pending. This step is **manual**
|
||||
(the unseal key lives in `pass`, off-cluster by design).
|
||||
|
||||
```sh
|
||||
# 1. The seal key, before the pods start (a pod without it never starts).
|
||||
kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl -n openbao create secret generic openbao-seal-key \
|
||||
--from-literal=key="$(pass show homelab/openbao/seal-key)"
|
||||
# 1. Wait for the OpenBao pods to exist (all three sealed, 0/1 Ready is expected).
|
||||
kubectl -n openbao get pods
|
||||
|
||||
# 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
|
||||
# initialise it. With an auto-seal, init hands out recovery keys and the
|
||||
# server unseals itself. These keys are throwaway: the restore replaces them.
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator init \
|
||||
-recovery-shares=1 -recovery-threshold=1
|
||||
|
||||
# 3. Restore the most recent raft snapshot. Source it from the snapshot PVC
|
||||
# 2. Restore the most recent raft snapshot. Source it from the snapshot PVC
|
||||
# (infrastructure/openbao/snapshot-cronjob.yaml) or, in a total-loss rebuild,
|
||||
# from the Velero `openbao-daily` offsite copy. Root token from the init
|
||||
# output. No -force: OpenBao checks the snapshot was written under this
|
||||
# server's seal key and refuses otherwise ("different autoseal key") — that
|
||||
# refusal is the signal the Secret is wrong. -force would skip the check and
|
||||
# leave a server that can never unseal.
|
||||
# from the Velero `openbao-daily` offsite copy. Copy it into the pod, then:
|
||||
kubectl -n openbao cp <snapshot>.snap openbao-0:/tmp/restore.snap
|
||||
kubectl -n openbao exec -it openbao-0 -- \
|
||||
env BAO_TOKEN=<init root token> bao operator raft snapshot restore /tmp/restore.snap
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator raft snapshot restore /tmp/restore.snap
|
||||
|
||||
# 4. The server seals against the snapshot's keyring and unseals itself from
|
||||
# the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
|
||||
# to type. Afterwards the tokens and recovery key from pass apply again.
|
||||
kubectl -n openbao get pods -w
|
||||
# 3. Unseal with the key from pass (threshold 1) — EVERY replica, same key.
|
||||
# A sealed server is neither a raft voter nor a standby, so unsealing only
|
||||
# openbao-0 leaves the three-peer cluster one node short of quorum.
|
||||
kubectl -n openbao exec -it openbao-0 -- bao operator unseal # paste homelab/openbao/unseal-key
|
||||
kubectl -n openbao exec -it openbao-1 -- bao operator unseal # same key
|
||||
kubectl -n openbao exec -it openbao-2 -- bao operator unseal # same key
|
||||
```
|
||||
|
||||
Verify the store recovers before moving on:
|
||||
|
||||
```sh
|
||||
kubectl -n openbao exec openbao-0 -- bao status # want: Seal Type static, Sealed false
|
||||
kubectl get clustersecretstore openbao # want: Valid / Ready
|
||||
kubectl -n <ns> get externalsecret # want: SecretSynced
|
||||
kubectl get clustersecretstore openbao # want: Valid / Ready
|
||||
kubectl -n <ns> get externalsecret # want: SecretSynced
|
||||
```
|
||||
|
||||
> A replica that stays **sealed** after a restart could not use the static
|
||||
> seal — the Secret is missing or does not match the key the data was
|
||||
> written under. Fix the Secret and restart the pod; `bao operator unseal`
|
||||
> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
|
||||
> unseal an auto-sealed server and is only for recovery-mode operations.
|
||||
> Details + rationale: `MIGRATION-sops-openbao.md`.
|
||||
> **Manual unseal is required after every OpenBao pod restart**, not just
|
||||
> during rebuilds, and for *each* replica that restarts — a rolling node
|
||||
> drain seals all three in turn. Details + rationale:
|
||||
> `MIGRATION-sops-openbao.md`.
|
||||
|
||||
## Step 5 — apps: let the rest reconcile
|
||||
|
||||
@@ -418,15 +393,14 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
|
||||
- **`bootstrap.sh talos` is dead** — it cannot generate a config (#581), and
|
||||
Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
|
||||
Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
|
||||
Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
|
||||
other secrets are read from `pass` or streamed in at run time and never live
|
||||
in CI.
|
||||
- **OpenBao unseals itself, but only under the seal key the snapshot was
|
||||
written with.** Three raft replicas mean the service survives losing one; a
|
||||
sealed server is neither a voter nor a standby. A pod that stays sealed after
|
||||
a restart has the wrong `openbao-seal-key` Secret — Step 4 covers this; it is
|
||||
repeated here because it is the step most likely to be half-done under
|
||||
pressure.
|
||||
It has also never had an `openbao` phase — Step 4 is manual because the
|
||||
unseal key must not live in CI; see issue #187 on whether to wire a
|
||||
semi-automated OpenBao restore.
|
||||
- **OpenBao seals on every restart and each replica must be unsealed
|
||||
individually.** Three raft replicas mean the service survives losing one, but
|
||||
a sealed server is neither a voter nor a standby — unsealing only one leaves
|
||||
the cluster short of quorum. Step 4 covers this; it is repeated here because
|
||||
it is the step most likely to be half-done under pressure.
|
||||
- Ceph's cluster network is a Multus attachment on VLAN 49 with whereabouts
|
||||
IPAM, not host networking. Nothing in the rebuild needs special handling for
|
||||
it — the nodes hold no address on that segment — but a rebuilt cluster will
|
||||
|
||||
@@ -21,9 +21,6 @@ metadata:
|
||||
namespace: audiobookshelf
|
||||
labels:
|
||||
app.kubernetes.io/name: audiobookshelf
|
||||
annotations:
|
||||
# envFrom does not restart the pod when the ConfigMap changes; Reloader does.
|
||||
configmap.reloader.stakater.com/reload: audiobookshelf-config
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
@@ -70,7 +67,7 @@ spec:
|
||||
mountPath: /ca
|
||||
containers:
|
||||
- name: audiobookshelf
|
||||
image: ghcr.io/advplyr/audiobookshelf:2.36.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
|
||||
image: ghcr.io/advplyr/audiobookshelf:2.36.0 # {"$imagepolicy": "flux-system:audiobookshelf"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 80
|
||||
|
||||
@@ -40,7 +40,7 @@ metadata:
|
||||
# so this Service produces no public record — LDAP stays
|
||||
# LAN-internal, which matches the threat model of the cleartext
|
||||
# bind on 389.
|
||||
external-dns.kubernetes.io/hostname: ldap.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: ldap.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -3,4 +3,3 @@ kind: Kustomization
|
||||
resources:
|
||||
- externalsecret.yaml
|
||||
- helmrelease.yaml
|
||||
- networkpolicy.yaml
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
# The apps namespace is default-deny-ingress, so the ServiceMonitor added in
|
||||
# monitoring-rules/external-dns-servicemonitor.yaml scraped into a timeout.
|
||||
# The selector matches both releases — the Cloudflare one deployed here and
|
||||
# external-dns-unifi from infrastructure/ — since they share the chart's
|
||||
# app.kubernetes.io/name and land in the same namespace.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: external-dns-allow-metrics-scrape
|
||||
namespace: apps
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: monitoring
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "7979"
|
||||
protocol: TCP
|
||||
@@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: gitea
|
||||
namespace: gitea
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
annotations:
|
||||
|
||||
@@ -2,7 +2,7 @@ apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRouteTCP
|
||||
metadata:
|
||||
name: gitea-ssh
|
||||
namespace: gitea
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
spec:
|
||||
|
||||
@@ -6,6 +6,7 @@ kind: Kustomization
|
||||
# and verified. See MIGRATION-gitea-woodpecker.md.
|
||||
resources:
|
||||
- statefulset.yaml
|
||||
- pdb.yaml
|
||||
- service.yaml
|
||||
- ingressroute-ssh.yaml
|
||||
- ingress.yaml
|
||||
|
||||
@@ -1,8 +1,28 @@
|
||||
# Prometheus scrape and Traefik are admitted by the namespace baseline in
|
||||
# infrastructure/network-policies/gitea-ns.yaml.
|
||||
#
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: gitea-allow-metrics-scrape
|
||||
namespace: apps
|
||||
spec:
|
||||
# Prometheus (monitoring ns) scrapes gitea's /metrics on :3000. The apps
|
||||
# default-deny-ingress only allows same-ns / home-assistant / host, so the
|
||||
# scrape is dropped (context deadline exceeded). This became necessary only
|
||||
# after the LXC->in-cluster cutover — the old static target was external, so
|
||||
# no in-cluster policy applied. Scoped to the gitea pod + :3000.
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gitea
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": monitoring
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "3000"
|
||||
protocol: TCP
|
||||
---
|
||||
# Secret rotation: the flux-git rotator (openbao ns) reaches gitea to mint a new
|
||||
# flux PAT. It uses the in-cluster Service (gitea.gitea.svc:3000) — the public
|
||||
# flux PAT. It uses the in-cluster Service (gitea.apps.svc:3000) — the public
|
||||
# gitea.renner.uno name hairpins off the ingress LB from a pod (same failure as
|
||||
# pve.renner.uno). Scoped to the gitea pod :3000 and the rotator label; the
|
||||
# egress half is rotator-gitea-flux-egress in network-policies/openbao-ns.yaml.
|
||||
@@ -10,7 +30,7 @@ apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: gitea-allow-flux-rotator
|
||||
namespace: gitea
|
||||
namespace: apps
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
@@ -34,7 +54,7 @@ apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: gitea-allow-review-bot-rotator
|
||||
namespace: gitea
|
||||
namespace: apps
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
---
|
||||
# Eviction protection for the single-replica Gitea StatefulSet.
|
||||
#
|
||||
# Gitea runs on SQLite (single writer) and its repos + DB live on a single
|
||||
# ReadWriteOnce PVC, so it is deliberately replicas:1 and cannot fail over.
|
||||
# With no PDB, a node drain / maintenance evicts the pod abruptly, mid-write,
|
||||
# with zero coordination. minAvailable:1 on a 1-replica workload makes the
|
||||
# eviction API refuse *voluntary* evictions: `kubectl drain` blocks and
|
||||
# surfaces this pod, forcing a conscious decision (drain with
|
||||
# --disable-eviction, or scale down first) instead of yanking the live DB.
|
||||
# It does NOT block involuntary disruptions (node crash, OOM).
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: gitea
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gitea
|
||||
@@ -2,7 +2,7 @@ apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: gitea
|
||||
namespace: gitea
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
spec:
|
||||
|
||||
@@ -2,7 +2,7 @@ apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: gitea
|
||||
namespace: gitea
|
||||
namespace: apps
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
spec:
|
||||
@@ -152,7 +152,7 @@ spec:
|
||||
# NOTE: volumeClaimTemplates is immutable on an existing StatefulSet,
|
||||
# so bumping this value cannot be `kubectl apply`-ed in place — Flux
|
||||
# will reject the whole object. Landing this size change requires a
|
||||
# one-time `kubectl delete statefulset gitea -n gitea --cascade=orphan`
|
||||
# one-time `kubectl delete statefulset gitea -n apps --cascade=orphan`
|
||||
# (pod + PVC keep running) so Flux re-creates the STS from this
|
||||
# template and re-adopts them. After the cleanup_packages GC reclaims
|
||||
# the ~10G of orphaned blobs, real usage is ~9G, so 40Gi is headroom.
|
||||
|
||||
@@ -4,9 +4,7 @@ metadata:
|
||||
name: gf-pg
|
||||
namespace: monitoring
|
||||
spec:
|
||||
instances: 2
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
|
||||
@@ -18,7 +18,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
lbipam.cilium.io/ips: "10.2.99.87,fd99::87"
|
||||
external-dns.kubernetes.io/hostname: ha.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: ha.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -95,7 +95,7 @@ spec:
|
||||
memory: 32Mi
|
||||
containers:
|
||||
- name: home-assistant
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.9.3 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.9.1 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
env:
|
||||
- name: TZ
|
||||
value: Europe/Berlin
|
||||
|
||||
@@ -30,9 +30,7 @@ metadata:
|
||||
name: ha-pg
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
instances: 2
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
@@ -126,7 +124,8 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volumes are lost outright.
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
# Fewer checkpoints => fewer full-page writes, since a page is only
|
||||
# fully-logged on its first modification after a checkpoint.
|
||||
|
||||
@@ -19,7 +19,7 @@ metadata:
|
||||
# LAN can connect with `mqtt.renner.uno` (matches the broker cert
|
||||
# SAN) instead of the bare IP. No Cloudflare label — kept off the
|
||||
# public DNS; UniFi serves a split-DNS answer for LAN clients.
|
||||
external-dns.kubernetes.io/hostname: mqtt.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: mqtt.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -59,7 +59,8 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volumes are lost outright.
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
|
||||
@@ -32,7 +32,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: immich-machine-learning
|
||||
image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
|
||||
image: ghcr.io/immich-app/immich-machine-learning:v3.1.0 # {"$imagepolicy": "flux-system:immich-machine-learning"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3003
|
||||
|
||||
@@ -35,9 +35,6 @@ metadata:
|
||||
namespace: immich
|
||||
labels:
|
||||
app.kubernetes.io/name: immich-server
|
||||
annotations:
|
||||
# envFrom does not restart the pod when the ConfigMap changes; Reloader does.
|
||||
configmap.reloader.stakater.com/reload: immich-config
|
||||
spec:
|
||||
# Scaled to 0 alongside immich-machine-learning during single-node
|
||||
# maintenance; see the note there. The CNPG Postgres cluster stays running,
|
||||
@@ -87,7 +84,7 @@ spec:
|
||||
mountPath: /ca
|
||||
containers:
|
||||
- name: immich-server
|
||||
image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
|
||||
image: ghcr.io/immich-app/immich-server:v3.1.0 # {"$imagepolicy": "flux-system:immich-server"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3001
|
||||
|
||||
@@ -61,9 +61,6 @@ spec:
|
||||
enabled: true
|
||||
storageClass: ceph-block
|
||||
size: 50Gi
|
||||
# A minAvailable:1 PDB on a single replica blocks every node drain.
|
||||
pdb:
|
||||
create: false
|
||||
# Bootstrap admin/org/bucket on the fresh PVC. Admin password + token come
|
||||
# from the influxdb-bootstrap Secret (OpenBao via ESO), deterministic across
|
||||
# rebuilds so the bootstrap token is the live admin token. `influx setup`
|
||||
|
||||
@@ -3,8 +3,8 @@ kind: Kustomization
|
||||
resources:
|
||||
- gatus/
|
||||
- external-dns/
|
||||
- gitea/
|
||||
- ical-proxy/
|
||||
# gitea/ has its own `gitea` Kustomization (clusters/homelab/gitea.yaml, #878).
|
||||
# woodpecker/ moved to its own dedicated `woodpecker` Kustomization
|
||||
# (clusters/homelab/woodpecker.yaml) so its wp-pg CNPG cluster's barman
|
||||
# dependency + Cluster health-gating don't contaminate this shared apps KS.
|
||||
|
||||
@@ -42,11 +42,9 @@ spec:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
# Seed the libation-config PVC from the bootstrap secret on
|
||||
# first run only — sentinel file gates re-runs so DB updates
|
||||
# from subsequent liberate passes are preserved. Refreshed
|
||||
# Audible tokens are preserved by the container args, not by
|
||||
# this sentinel: the entrypoint copies AccountsSettings.json
|
||||
# into ephemeral /config-internal rather than symlinking it.
|
||||
# first run only — sentinel file gates re-runs so token
|
||||
# refreshes in AccountsSettings.json and DB updates from
|
||||
# subsequent liberate passes are preserved.
|
||||
- name: seed-config
|
||||
image: busybox:1.37.0
|
||||
command: ["/bin/sh", "-eu", "-c"]
|
||||
@@ -68,11 +66,6 @@ spec:
|
||||
install -m 660 /bootstrap/AccountsSettings.json /libation-state/AccountsSettings.json
|
||||
install -m 660 /bootstrap/Settings.json /libation-state/Settings.json
|
||||
install -m 660 /bootstrap/LibationContext.db /libation-state/LibationContext.db
|
||||
# Without this key the seeded tokens cannot be decrypted:
|
||||
# TokenStorageMethod is Encrypted and the container has no
|
||||
# OS secret store, so a missing key file makes the
|
||||
# entrypoint mint a throwaway one on every run.
|
||||
install -m 660 /bootstrap/libation-master.key /libation-state/libation-master.key
|
||||
touch /libation-state/.bootstrapped
|
||||
ls -la /libation-state
|
||||
volumeMounts:
|
||||
@@ -92,23 +85,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: libation
|
||||
# Pinned by digest: Spegel caches mutable tag resolutions, so
|
||||
# :latest stayed on 13.7.11 despite imagePullPolicy Always.
|
||||
image: docker.io/rmcrackan/libation:14.2.1@sha256:6f9d8ba19dc3803dd6ba295ab5a7abfb6ff3bc06286eebf4ecd8ccbe43245888
|
||||
# The entrypoint stages config into ephemeral /config-internal
|
||||
# by copy (only the DB is symlinked to the PVC), so Audible
|
||||
# tokens refreshed during a run are lost at container exit.
|
||||
# Passing args makes the entrypoint set up the environment and
|
||||
# then exec this instead of its own loop; the trap copies the
|
||||
# refreshed file back to the PVC on every exit path.
|
||||
args:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
trap 'cp /config-internal/AccountsSettings.json /libation-state/AccountsSettings.json' EXIT
|
||||
set -e
|
||||
/libation/LibationCli scan
|
||||
/libation/LibationCli liberate
|
||||
image: docker.io/rmcrackan/libation:latest
|
||||
env:
|
||||
- name: LIBATION_CONFIG_DIR
|
||||
value: /libation-state
|
||||
|
||||
@@ -6,5 +6,3 @@ resources:
|
||||
- embed.yaml
|
||||
- gate-service.yaml
|
||||
- ingress.yaml
|
||||
- wiki-embed.yaml
|
||||
- wiki-embed-ingress.yaml
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
---
|
||||
# wiki-embed.renner.uno — LAN-only (no dns.renner.uno/public label), consumed by
|
||||
# bin/wiki_rank.py on the workstation and the nightly wiki jobs. Machine client,
|
||||
# so Traefik BasicAuth instead of the browser-only Authentik gate; same pattern
|
||||
# as kubernetes/apps/monitoring/loki-ingress.yaml.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: wiki-embed-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
basicAuth:
|
||||
secret: wiki-embed-basic-auth
|
||||
realm: wiki-embed
|
||||
---
|
||||
# OpenBao key apps/ai/wiki-embed-basic-auth, field `users` = htpasswd line.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wiki-embed-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: wiki-embed-basic-auth
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/ai/wiki-embed-basic-auth
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
traefik.ingress.kubernetes.io/router.middlewares: ai-wiki-embed-basic-auth@kubernetescrd
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- wiki-embed.renner.uno
|
||||
secretName: wiki-embed-tls
|
||||
rules:
|
||||
- host: wiki-embed.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: wiki-embed
|
||||
port:
|
||||
number: 8080
|
||||
@@ -1,117 +0,0 @@
|
||||
---
|
||||
# Embedding endpoint for the ~/Notes wiki ranker (bin/wiki_rank.py). Own
|
||||
# Deployment so wiki traffic never queues behind Paperless on llama-embed.
|
||||
# Same model and PVC as llama-embed, so no second download.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
replicas: 1
|
||||
# Above the 600s startupProbe budget; see embed.yaml.
|
||||
progressDeadlineSeconds: 900
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: wiki-embed
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
||||
args:
|
||||
- --hf-repo
|
||||
- gpustack/bge-m3-GGUF:Q8_0
|
||||
- --embeddings
|
||||
- --alias
|
||||
- bge-m3
|
||||
# bge-m3 is trained for CLS pooling; the GGUF default is not.
|
||||
- --pooling
|
||||
- cls
|
||||
- --n-gpu-layers
|
||||
- "99"
|
||||
- --threads
|
||||
- "2"
|
||||
# wiki_rank.py caps each note at 1500 chars and sends 2 per request.
|
||||
- --batch-size
|
||||
- "1536"
|
||||
- --ubatch-size
|
||||
- "1536"
|
||||
- --ctx-size
|
||||
- "4096"
|
||||
- --no-webui
|
||||
env:
|
||||
- name: LLAMA_CACHE
|
||||
value: /models
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
# Same sizing as llama-embed: 1.6 GiB measured peak, headroom for slots.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 2Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llama-server-models
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -1,25 +0,0 @@
|
||||
# Both external-dns releases (Cloudflare and the UniFi webhook) carry
|
||||
# app.kubernetes.io/name: external-dns and differ only by instance, so one
|
||||
# monitor covers both. jobLabel keeps them apart as separate job values.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: external-dns
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobLabel: app.kubernetes.io/instance
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- apps
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
endpoints:
|
||||
# The controller's own metrics. Port 8080 on the unifi release is the
|
||||
# webhook sidecar and exports no external_dns_* series.
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: 60s
|
||||
@@ -1,20 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
endpoints:
|
||||
- port: http-metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
relabelings:
|
||||
# The series would otherwise carry only the exporter pod's IP.
|
||||
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
targetLabel: node
|
||||
@@ -21,14 +21,6 @@ resources:
|
||||
# alerted if that stopped — HetznerBackupEndpointDown only probes the S3
|
||||
# endpoint, which proves reachability, not that anything was written.
|
||||
- prometheusrule-cnpg-backup.yaml
|
||||
# DNS publication. Nothing scraped external-dns, so it deleted six LAN
|
||||
# records on 2026-09-12 and stayed silent for five days.
|
||||
- prometheusrule-external-dns.yaml
|
||||
- external-dns-servicemonitor.yaml
|
||||
# Outcome check for the same failure: the names themselves, asked of the
|
||||
# UDM. Catches a cause external-dns cannot see, e.g. records removed there.
|
||||
- prometheusrule-dns-resolution.yaml
|
||||
- probes-dns.yaml
|
||||
- alloy-servicemonitor.yaml
|
||||
- cilium-servicemonitor.yaml
|
||||
- cnpg-podmonitors.yaml
|
||||
@@ -47,5 +39,4 @@ resources:
|
||||
- immich-servicemonitor.yaml
|
||||
- loki-servicemonitor.yaml
|
||||
- smartctl-exporter-servicemonitor.yaml
|
||||
- intel-gpu-exporter-servicemonitor.yaml
|
||||
- probes-blackbox.yaml
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
# DNS probes for the names external-dns publishes to the UDM from
|
||||
# LoadBalancer Services. One Probe per name because the dns prober's
|
||||
# query_name is module-scoped; the target is the resolver, queried on both
|
||||
# families so a single-stack blip does not read as a missing record.
|
||||
#
|
||||
# The UDM is queried directly rather than through the cluster resolver:
|
||||
# k8s-gateway caches these with `serve_stale 1h immediate`, which would keep
|
||||
# a deleted record green for an hour.
|
||||
#
|
||||
# `release` is load-bearing — probeSelector matches on it.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ha
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ha
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
# instance is the resolver and identical across these Probes, so the
|
||||
# queried name has to come from a label.
|
||||
labels:
|
||||
dns_name: ha.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-mqtt
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_mqtt
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: mqtt.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ldap
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ldap
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: ldap.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-syslog
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_syslog
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: syslog.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ftp-scanner
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ftp_scanner
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: ftp.paperless.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-syncthing
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_syncthing
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: syncthing.paperless.renner.uno
|
||||
@@ -34,9 +34,7 @@ spec:
|
||||
# query parameter, and an exact match would break silently if the
|
||||
# instance label ever encodes it differently.
|
||||
expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
|
||||
# Two consecutive failed probes: enough to skip a single lost probe
|
||||
# through Traefik, not enough to sit on a sealed store.
|
||||
for: 1m
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: dns-resolution
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# ruleSelector matches on this label; without it the rule is created but
|
||||
# never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: dns-resolution.rules
|
||||
rules:
|
||||
# min by dns_name: both resolver families are probed, and the record
|
||||
# is only gone when neither answers. A single-family failure is the
|
||||
# UDM's dnsmasq refusing connections in bursts (k8s-gateway carries
|
||||
# the same note), which is why the window is 30m rather than the 10m
|
||||
# used for HTTP probes.
|
||||
- alert: LanDnsRecordMissing
|
||||
expr: min by (dns_name) (probe_success{job="blackbox-dns"}) == 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.dns_name }} no longer resolves to its LAN address"
|
||||
description: >-
|
||||
The UDM stopped answering for a name external-dns publishes from
|
||||
a LoadBalancer Service, or answered with the public
|
||||
*.renner.uno wildcard instead of the record. Clients that use
|
||||
the name rather than the address lose the service while
|
||||
everything in the cluster stays healthy — nothing else in the
|
||||
stack reports this. Check the external-dns alerts first, then
|
||||
the record on the UDM.
|
||||
|
||||
# Dead-man's-switch, mirroring BlackboxProbesAbsent: without probe
|
||||
# results the rule above cannot fire.
|
||||
- alert: LanDnsProbesAbsent
|
||||
expr: absent(probe_success{job="blackbox-dns"})
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No LAN DNS probe results are reaching Prometheus"
|
||||
description: >-
|
||||
The blackbox-dns Probes stopped being discovered or the exporter
|
||||
rejects the modules. A 400 from /probe yields no probe_* series
|
||||
at all, which looks the same as no Probe — check that every
|
||||
dns_lan_* module the Probes reference exists in the
|
||||
blackbox-exporter config.
|
||||
@@ -1,87 +0,0 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: external-dns
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# ruleSelector matches on this label; without it the rule is created but
|
||||
# never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: external-dns.rules
|
||||
rules:
|
||||
# The signal that was missing on 2026-09-12: the sources stopped
|
||||
# producing endpoints they had produced before, and policy=sync
|
||||
# deleted the records. 10% below the 24h peak clears a single
|
||||
# retired Ingress (2 of ~48) and catches a whole class of names.
|
||||
- alert: ExternalDNSSourceEndpointsDropped
|
||||
expr: >-
|
||||
external_dns_source_endpoints_total
|
||||
< 0.9 * max_over_time(external_dns_source_endpoints_total[24h])
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is publishing {{ $value }} endpoints, below its 24h peak"
|
||||
description: >-
|
||||
external-dns stopped seeing endpoints it had before. With
|
||||
policy=sync the matching DNS records are deleted, so names stop
|
||||
resolving without anything else failing. Compare
|
||||
external_dns_source_endpoints_total against the last 24h, then
|
||||
check whether the resources still carry a hostname annotation
|
||||
the running version understands — a changed annotation prefix
|
||||
default empties the source silently. Deliberate removals also
|
||||
fire this and resolve themselves 24h later.
|
||||
|
||||
# Source errors mean the Kubernetes side failed; registry errors mean
|
||||
# the provider did. Either way desired state is computed from partial
|
||||
# input, which policy=sync acts on.
|
||||
- alert: ExternalDNSSourceErrors
|
||||
expr: increase(external_dns_source_errors_total[15m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is failing to read its sources"
|
||||
description: >-
|
||||
external-dns could not list the Kubernetes resources it derives
|
||||
records from. Check the pod log and its RBAC.
|
||||
|
||||
- alert: ExternalDNSRegistryErrors
|
||||
expr: increase(external_dns_registry_errors_total[15m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is failing to reach its DNS provider"
|
||||
description: >-
|
||||
external-dns could not read or write records at the provider.
|
||||
For external-dns-unifi that is the UniFi webhook sidecar and the
|
||||
controller behind it; for external-dns it is the Cloudflare API.
|
||||
|
||||
- alert: ExternalDNSDown
|
||||
expr: up{job=~"external-dns.*"} == 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is not being scraped"
|
||||
description: >-
|
||||
The external-dns instance is down or unreachable. Records already
|
||||
at the provider keep resolving, but nothing reconciles them any
|
||||
more and every other alert in this group is blind.
|
||||
|
||||
# Dead-man's-switch for the group, mirroring BlackboxProbesAbsent: a
|
||||
# dropped ServiceMonitor makes every rule above evaluate to nothing.
|
||||
- alert: ExternalDNSMetricsMissing
|
||||
expr: absent(external_dns_source_endpoints_total)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No external-dns metrics are reaching Prometheus"
|
||||
description: >-
|
||||
Either both external-dns instances are gone or the ServiceMonitor
|
||||
stopped being discovered. Verify it still carries
|
||||
release=kube-prometheus-stack.
|
||||
@@ -28,28 +28,24 @@ spec:
|
||||
# scheduling would also fire this — but both want a human too, so the
|
||||
# imprecision costs nothing.
|
||||
- alert: OpenBaoReplicaNotReady
|
||||
# A replica unseals itself from the static seal, so a restart is
|
||||
# Ready again in well under a minute; the duration only ignores that
|
||||
# ordinary rescheduling. A replica still sealed after it never
|
||||
# recovers on its own: the seal failed (openbao-seal-key Secret
|
||||
# missing or wrong).
|
||||
# max by: kube-state-metrics stamps instance/pod on the series, so a
|
||||
# rescheduled exporter starts a new series and resets the timer.
|
||||
expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 3
|
||||
# A replica that is sealed never recovers on its own, so the
|
||||
# duration is only there to ignore ordinary rescheduling: a healthy
|
||||
# pod moving between nodes is Ready again in well under a minute.
|
||||
expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "OpenBao has {{ $value }} of 3 replicas ready"
|
||||
description: >-
|
||||
An OpenBao replica has been unavailable for 10 minutes. A restart
|
||||
unseals itself from the static seal, so this is not the ordinary
|
||||
drain case. The service is still up on the remaining replicas,
|
||||
so nothing is broken yet, but redundancy is reduced and a second
|
||||
failure would cost quorum. Check `kubectl -n openbao get pods`,
|
||||
the pod log and `bao status` in the not-ready pod. If it is
|
||||
sealed, the static seal failed: verify the openbao-seal-key
|
||||
Secret matches `pass show homelab/openbao/seal-key`.
|
||||
An OpenBao replica has been unavailable for 10 minutes. The usual
|
||||
cause is a sealed replica after a node drain — every restart
|
||||
seals it, and it stays out of the raft cluster until unsealed by
|
||||
hand. The service is still up on the remaining replicas, so
|
||||
nothing is broken yet, but redundancy is reduced and a second
|
||||
drain would cost quorum. Check `kubectl -n openbao get pods` and
|
||||
`bao status` in the not-ready pod; unseal with the key from
|
||||
`pass` (homelab/openbao/unseal-key).
|
||||
|
||||
- alert: OpenBaoQuorumLost
|
||||
# Raft quorum of a three-node cluster is two. At one ready replica
|
||||
@@ -57,14 +53,11 @@ spec:
|
||||
# the ClusterSecretStore goes InvalidProviderConfig and no
|
||||
# ExternalSecret can refresh.
|
||||
#
|
||||
# Short window: readiness is only a proxy for quorum. A one-second
|
||||
# node NotReady flap makes the node lifecycle controller mark that
|
||||
# node's pods not-ready for ~30s while raft keeps its leader and
|
||||
# serves (issue #920, during a rolling reboot with one replica
|
||||
# already evicted). A really sealed or crashed majority persists,
|
||||
# and existing Secrets keep their last values, so 2m costs little.
|
||||
expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
|
||||
for: 2m
|
||||
# Shorter window than the warning above because this is an outage,
|
||||
# not a degradation — but not zero, so a rolling restart that
|
||||
# briefly dips does not page.
|
||||
expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
@@ -74,11 +67,9 @@ spec:
|
||||
quorum and the secret store is unavailable. ExternalSecrets stop
|
||||
refreshing; existing Secrets keep their last values, so failures
|
||||
surface only when something rotates or a new secret is needed.
|
||||
Replicas unseal themselves from the static seal; one that stays
|
||||
sealed could not. Check its log and the openbao-seal-key Secret
|
||||
against `pass show homelab/openbao/seal-key`, fix the Secret and
|
||||
restart the pod. The recovery key cannot unseal an auto-sealed
|
||||
server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
|
||||
Unseal the sealed replicas (key in `pass`,
|
||||
homelab/openbao/unseal-key) — each one individually, since a
|
||||
sealed server cannot be a standby.
|
||||
|
||||
- alert: OpenBaoMetricsMissing
|
||||
# Guards the two alerts above. Both are threshold comparisons, so if
|
||||
|
||||
@@ -1,368 +0,0 @@
|
||||
{
|
||||
"title": "Intel iGPU",
|
||||
"uid": "intel-gpu",
|
||||
"tags": ["gpu", "intel", "i915"],
|
||||
"schemaVersion": 39,
|
||||
"editable": true,
|
||||
"timezone": "",
|
||||
"refresh": "1m",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "node",
|
||||
"label": "Node",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": {
|
||||
"query": "label_values(gpumon_power, node)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"definition": "label_values(gpumon_power, node)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"current": { "selected": true, "text": ["All"], "value": ["$__all"] }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Render/3D busy now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "GPU power now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"decimals": 1,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "GT frequency now",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "suffix: MHz",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "fixed", "fixedColor": "text" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "value_and_name",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Render/3D busy",
|
||||
"description": "Compute and 3D. llama.cpp's Vulkan backend runs here; 0 % during an inference means the model is on the CPU.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Render/3D\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Video busy",
|
||||
"description": "Fixed-function decode/encode (QuickSync).",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Video\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "VideoEnhance busy",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"VideoEnhance\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Blitter busy",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_engine_usage{attrib=\"busy\", engine=\"Blitter\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GT frequency (actual)",
|
||||
"description": "0 while the GT sits in RC6.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "suffix: MHz",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_frequency{type=\"actual\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GPU power",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"gpu\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Package power",
|
||||
"description": "Whole SoC (CPU + GPU + uncore), from the same PMU.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_power{type=\"pkg\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory controller bandwidth",
|
||||
"description": "Reads solid, writes dashed. The iGPU shares DDR5 with the CPU, so this is the whole package's traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "MiBs",
|
||||
"min": 0,
|
||||
"custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": false }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-1/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#3987e5" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-2/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#d95926" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/k8s-3/" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#199e70" } }] },
|
||||
{ "matcher": { "id": "byRegexp", "options": "/write/" }, "properties": [{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] }
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "gpumon_imc_read{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} read",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "gpumon_imc_write{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} write",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -77,9 +77,6 @@ configMapGenerator:
|
||||
- name: dashboard-infrastructure-overview
|
||||
files:
|
||||
- infrastructure-overview.json
|
||||
- name: dashboard-intel-gpu
|
||||
files:
|
||||
- intel-gpu.json
|
||||
- name: dashboard-netbird-overview
|
||||
files:
|
||||
- netbird-overview.json
|
||||
|
||||
@@ -232,7 +232,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
io.cilium/lb-ipam-ips: "10.2.99.84,fd99::84"
|
||||
external-dns.kubernetes.io/hostname: syslog.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: syslog.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -72,73 +72,3 @@ spec:
|
||||
tcp:
|
||||
preferred_ip_protocol: ip6
|
||||
ip_protocol_fallback: true
|
||||
# One module per name: the dns prober takes the RESOLVER as its
|
||||
# target and reads query_name from the module, so a name cannot be
|
||||
# passed per scrape. The Probes in monitoring-rules/probes-dns.yaml
|
||||
# point these at the UDM.
|
||||
#
|
||||
# Both answer checks are load-bearing. valid_rcodes alone passes a
|
||||
# deleted record two labels deep (NOERROR, no answer), and the public
|
||||
# *.renner.uno wildcard answers one label deep with a CNAME to
|
||||
# proxy.renner.uno — NOERROR with the WAN proxy behind it. Requiring
|
||||
# an A and rejecting a CNAME separates a real record from both.
|
||||
dns_lan_ha:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ha.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_mqtt:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: mqtt.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_ldap:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ldap.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_syslog:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: syslog.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_ftp_scanner:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ftp.paperless.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_syncthing:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: syncthing.paperless.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
|
||||
@@ -91,11 +91,12 @@ spec:
|
||||
# Additional scrape targets not covered by ServiceMonitors.
|
||||
additionalScrapeConfigs:
|
||||
# Gitea exposes /metrics anonymously on its main HTTP port. The
|
||||
# `gitea_*` metrics power the gitea-overview dashboard. Scrapes the
|
||||
# ClusterIP Service of the StatefulSet in the gitea namespace.
|
||||
# `gitea_*` metrics power the gitea-overview dashboard. Gitea now runs
|
||||
# in-cluster (StatefulSet in apps ns; LXC 114 retired), so scrape the
|
||||
# ClusterIP Service directly instead of the dead LXC hostname.
|
||||
- job_name: gitea
|
||||
static_configs:
|
||||
- targets: ["gitea.gitea.svc.cluster.local:3000"]
|
||||
- targets: ["gitea.apps.svc.cluster.local:3000"]
|
||||
metrics_path: /metrics
|
||||
|
||||
# Grafana
|
||||
@@ -304,10 +305,6 @@ spec:
|
||||
- matchers:
|
||||
- alertname = "KubeCPUOvercommit"
|
||||
receiver: 'null'
|
||||
# Helper alert that only drives the info inhibit rule below.
|
||||
- matchers:
|
||||
- alertname = "InfoInhibitor"
|
||||
receiver: 'null'
|
||||
# Flux notification-controller posts info-severity events
|
||||
# here (e.g. image-automation commits). Keep them visible
|
||||
# but don't repeat-spam every 12h on the same event.
|
||||
@@ -317,23 +314,6 @@ spec:
|
||||
group_wait: 1m
|
||||
group_interval: 10m
|
||||
repeat_interval: 24h
|
||||
# From kube-prometheus manifests/alertmanager-secret.yaml.
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- severity = "critical"
|
||||
target_matchers:
|
||||
- severity =~ "warning|info"
|
||||
equal: ['namespace', 'alertname']
|
||||
- source_matchers:
|
||||
- severity = "warning"
|
||||
target_matchers:
|
||||
- severity = "info"
|
||||
equal: ['namespace', 'alertname']
|
||||
- source_matchers:
|
||||
- alertname = "InfoInhibitor"
|
||||
target_matchers:
|
||||
- severity = "info"
|
||||
equal: ['namespace']
|
||||
receivers:
|
||||
- name: email
|
||||
email_configs:
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# Intel iGPU utilisation, frequency and power for every node (#752).
|
||||
#
|
||||
# Not privileged: /dev/dri comes from the device plugin's monitoring resource
|
||||
# (every GPU on the node, no hostPath), and the i915 PMU that intel_gpu_top
|
||||
# reads needs CAP_PERFMON — the one capability granted. No hostPID, no
|
||||
# hostNetwork.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
# Every node is a control plane; without this the exporter covers nothing.
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: intel-gpu-exporter
|
||||
image: ghcr.io/clambin/intel-gpu-exporter:0.7.4@sha256:d78761fef018834c759db154fe9dcf3dd9db5974703fa50ca21f6abc35221be1
|
||||
args:
|
||||
# No -device: intel_gpu_top picks the only GPU the plugin injects.
|
||||
- -interval=5s
|
||||
ports:
|
||||
- name: http-metrics
|
||||
containerPort: 9100
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
# Root so the PERFMON capability is effective without file
|
||||
# capabilities on the binary.
|
||||
runAsUser: 0
|
||||
runAsNonRoot: false
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["PERFMON"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
gpu.intel.com/monitoring: 1
|
||||
limits:
|
||||
memory: 128Mi
|
||||
gpu.intel.com/monitoring: 1
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: http-metrics
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: intel-gpu-exporter
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/name: intel-gpu-exporter
|
||||
ports:
|
||||
- name: http-metrics
|
||||
port: 9100
|
||||
targetPort: http-metrics
|
||||
protocol: TCP
|
||||
@@ -16,7 +16,6 @@ resources:
|
||||
- prometheus-gate-ingress.yaml
|
||||
- ci-diagnostics-rbac.yaml
|
||||
- smartctl-exporter.yaml
|
||||
- intel-gpu-exporter.yaml
|
||||
- dashboards/
|
||||
# prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a
|
||||
# separate Flux Kustomization that dependsOn `monitoring` — otherwise the
|
||||
|
||||
@@ -28,7 +28,7 @@ spec:
|
||||
# net1 instead. master: ens19 resolves on every k8s-{1,2,3}/edge
|
||||
# worker, though it is no longer the same kind of device everywhere:
|
||||
# on the bare-metal control-plane nodes it is a VLAN 30 sub-interface
|
||||
# of uplink rather than a NIC, which is why the datapath needs
|
||||
# of enp2s0f0np0 rather than a NIC, which is why the datapath needs
|
||||
# bpf.vlanBypass to carry it (issue #592).
|
||||
k8s.v1.cni.cncf.io/networks: music-assistant-iot
|
||||
spec:
|
||||
@@ -55,7 +55,7 @@ spec:
|
||||
# stdlib only; Flux keeps the tag in lockstep via the shared image
|
||||
# policy) — no extra dependency.
|
||||
- name: reconcile-publish-ip
|
||||
image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
@@ -65,7 +65,7 @@ spec:
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: music-assistant
|
||||
image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.10.2 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
env:
|
||||
- name: LOG_LEVEL
|
||||
value: info
|
||||
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
mountPath: /consume
|
||||
containers:
|
||||
- name: vsftpd
|
||||
image: gitea.renner.uno/k.renner/vsftpd:build.1789630783 # {"$imagepolicy": "flux-system:vsftpd"}
|
||||
image: gitea.renner.uno/k.renner/vsftpd:build.1784731469 # {"$imagepolicy": "flux-system:vsftpd"}
|
||||
ports:
|
||||
- name: ftp-ctrl
|
||||
containerPort: 21
|
||||
@@ -70,19 +70,24 @@ spec:
|
||||
- name: consume
|
||||
mountPath: /home/scanner
|
||||
subPath: scanner
|
||||
# Asserts the control socket is listening without opening an FTP
|
||||
# connection: vsftpd 3.0.5 segfaults its listener on a fraction of
|
||||
# the connections it accepts. The passive range is negotiated per
|
||||
# transfer and is not listening at rest, so it is not probed.
|
||||
# FTP has no health verb, so the control port accepting a connection
|
||||
# is what can be asserted. It is enough for the failure that matters:
|
||||
# if vsftpd dies the scanner cannot deposit anything and nothing else
|
||||
# in the chain notices, because paperless only ever sees an empty
|
||||
# consume directory — which is indistinguishable from nobody having
|
||||
# scanned.
|
||||
#
|
||||
# Only the control port is probed. The passive range is negotiated
|
||||
# per transfer and is not listening at rest.
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "netstat -lnt | grep -q ':21 '"]
|
||||
tcpSocket:
|
||||
port: ftp-ctrl
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "netstat -lnt | grep -q ':21 '"]
|
||||
tcpSocket:
|
||||
port: ftp-ctrl
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -119,7 +124,7 @@ metadata:
|
||||
# label — FTP must not be exposed via the cargo tunnel (clear-text
|
||||
# protocol, public exposure not appropriate). The brother scanner
|
||||
# uses this hostname instead of the LB IP.
|
||||
external-dns.kubernetes.io/hostname: ftp.paperless.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: ftp.paperless.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -22,7 +22,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: gotenberg
|
||||
image: gotenberg/gotenberg:8.37.0 # {"$imagepolicy": "flux-system:gotenberg"}
|
||||
image: gotenberg/gotenberg:8.36.0 # {"$imagepolicy": "flux-system:gotenberg"}
|
||||
args:
|
||||
- "gotenberg"
|
||||
- "--chromium-disable-javascript=true"
|
||||
|
||||
@@ -11,6 +11,7 @@ resources:
|
||||
- protonmail-bridge.yaml
|
||||
- ftp-scanner.yaml
|
||||
- syncthing.yaml
|
||||
- pdb.yaml
|
||||
- network-policy.yaml
|
||||
# pprl-pg CNPG Postgres + data/media/consume/redis PVCs (moved here from the
|
||||
# retired seed-paperless KS; each carries prune: disabled so ownership
|
||||
@@ -24,15 +25,18 @@ resources:
|
||||
configMapGenerator:
|
||||
# vsftpd config for the ftp-scanner deployment. The content-hash name
|
||||
# suffix is intentionally LEFT ENABLED (unlike home-assistant's nginx
|
||||
# config): vsftpd reads these into vsftpd.conf at container start, so a
|
||||
# changed value must roll the pod, which the renamed ConfigMap does.
|
||||
#
|
||||
# No PASV_ADDRESS: the LB address is declared once, on the Service
|
||||
# (io.cilium/lb-ipam-ips). The image default 0.0.0.0 tells PASV clients
|
||||
# to reuse the control-connection address.
|
||||
# config): vsftpd templates pasv_address from PASV_ADDRESS at container
|
||||
# start, so a changed value must roll the pod. With the hash suffix,
|
||||
# editing any value here renames the generated ConfigMap, which mutates
|
||||
# the Deployment's pod template and triggers an automatic rolling
|
||||
# restart. Without it, an IP change (e.g. an LB re-allocation) silently
|
||||
# leaves the running pod advertising the stale PASV address, which
|
||||
# breaks classic-PASV clients (older Brother scanners) while EPSV
|
||||
# clients keep working.
|
||||
- name: ftp-scanner-config
|
||||
namespace: paperless
|
||||
literals:
|
||||
- PASV_ADDRESS=10.2.99.82
|
||||
- PASV_MIN_PORT=21100
|
||||
- PASV_MAX_PORT=21104
|
||||
- FTP_USER=scanner
|
||||
|
||||
@@ -27,8 +27,6 @@ data:
|
||||
PAPERLESS_ALLOWED_HOSTS: "*"
|
||||
PAPERLESS_TIME_ZONE: "Europe/Berlin"
|
||||
PAPERLESS_OCR_LANGUAGE: "deu+eng"
|
||||
# Default is cpu_count / TASK_WORKERS, 20 on k8s-1; OCR memory scales with it.
|
||||
PAPERLESS_THREADS_PER_WORKER: "5"
|
||||
PAPERLESS_CONSUMER_RECURSIVE: "true"
|
||||
PAPERLESS_CONSUMER_DELETE_DUPLICATES: "true"
|
||||
PAPERLESS_CONSUMER_ENABLE_BARCODES: "true"
|
||||
@@ -185,7 +183,7 @@ spec:
|
||||
drop: ["ALL"]
|
||||
containers:
|
||||
- name: paperless
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:3.2.1 # {"$imagepolicy": "flux-system:paperless-ngx"}
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:3.1.3 # {"$imagepolicy": "flux-system:paperless-ngx"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
@@ -275,7 +273,7 @@ spec:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 4Gi
|
||||
memory: 2Gi
|
||||
securityContext:
|
||||
runAsNonRoot: false
|
||||
allowPrivilegeEscalation: true
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
# Eviction protection for the single-replica stateful paperless sidecars.
|
||||
#
|
||||
# protonmail-bridge and syncthing each hold state on a single RWO PVC and run
|
||||
# replicas:1. minAvailable:1 makes the eviction API refuse voluntary evictions,
|
||||
# so a node drain blocks and surfaces these pods instead of evicting them
|
||||
# abruptly. Involuntary disruptions (node crash) are not affected.
|
||||
#
|
||||
# NOTE: the other single-replica paperless pods (paperless, redis, gotenberg,
|
||||
# tika, ftp-scanner) are intentionally omitted — they are caches / stateless
|
||||
# converters where a blocking PDB would only wedge node drains with no
|
||||
# recoverability benefit.
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: protonmail-bridge
|
||||
namespace: paperless
|
||||
labels:
|
||||
app.kubernetes.io/name: protonmail-bridge
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: protonmail-bridge
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: syncthing
|
||||
namespace: paperless
|
||||
labels:
|
||||
app.kubernetes.io/name: syncthing
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: syncthing
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:8.10.2-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
image: redis:8.10.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
|
||||
@@ -145,7 +145,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
io.cilium/lb-ipam-ips: "10.2.99.85,fd99::85"
|
||||
external-dns.kubernetes.io/hostname: syncthing.paperless.renner.uno
|
||||
external-dns.alpha.kubernetes.io/hostname: syncthing.paperless.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -4,7 +4,7 @@ metadata:
|
||||
name: tr-pg
|
||||
namespace: trmnl
|
||||
spec:
|
||||
instances: 2
|
||||
instances: 1
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
# PG 18, NOT the repo-standard 16. Terminus ships a committed
|
||||
@@ -47,7 +47,8 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volumes are lost outright.
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
|
||||
@@ -134,7 +134,7 @@ spec:
|
||||
# is a device-bricking class of risk. Bumps are a reviewed, manual
|
||||
# act — see kubernetes/infrastructure/image-automation/README.md.
|
||||
- name: terminus
|
||||
image: ghcr.io/usetrmnl/terminus:0.74.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 2345
|
||||
@@ -197,7 +197,7 @@ spec:
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
- name: sidekiq
|
||||
image: ghcr.io/usetrmnl/terminus:0.74.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
image: ghcr.io/usetrmnl/terminus:0.72.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
command: ["bundle", "exec", "sidekiq", "-r", "./config/sidekiq.rb"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
@@ -18,7 +18,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: whisper
|
||||
image: rhasspy/wyoming-whisper:3.8.1 # {"$imagepolicy": "flux-system:wyoming-whisper"}
|
||||
image: rhasspy/wyoming-whisper:3.7.0 # {"$imagepolicy": "flux-system:wyoming-whisper"}
|
||||
args:
|
||||
- --model=base-int8
|
||||
- --language=de
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
---
|
||||
# ServiceAccount is the OpenBao identity (auth/kubernetes/role/wiki-index); no
|
||||
# Kubernetes API access needed.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
---
|
||||
# Index (~17 MB JSON for 1700 notes). Survives restarts so a pod start does not
|
||||
# re-embed the wiki; a lost volume only costs one full sync (~5 min).
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: wiki-index-data
|
||||
namespace: ai
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
# Embedding index over the Notes repo's wiki/ (ci/images/wiki-index). Syncs from
|
||||
# Gitea on push webhook and every RESYNC_SECONDS, embeds via wiki-embed, serves
|
||||
# POST /rank to ~/Notes/bin/wiki_rank.py through wiki-index.renner.uno.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
serviceAccountName: wiki-index
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: wiki-index
|
||||
# Per-commit tag (build-ci-images.yml): immutable, no same-day overwrite.
|
||||
image: gitea.renner.uno/gitea_admin/wiki-index:2026-09-20-4b529cd1
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: GITEA_URL
|
||||
value: https://gitea.renner.uno
|
||||
- name: NOTES_REPO
|
||||
value: k.renner/notes
|
||||
- name: NOTES_BRANCH
|
||||
value: main
|
||||
- name: EMBED_URL
|
||||
value: http://wiki-embed.ai.svc.cluster.local:8080
|
||||
- name: BAO_ADDR
|
||||
value: http://openbao.openbao.svc.cluster.local:8200
|
||||
- name: BAO_ROLE
|
||||
value: wiki-index
|
||||
- name: BAO_SECRET_PATH
|
||||
value: homelab/data/apps/ai/wiki-index
|
||||
- name: INDEX_PATH
|
||||
value: /data/index.json
|
||||
- name: RESYNC_SECONDS
|
||||
value: "1800"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
# Pure-Python cosine over 1700 x 1024 floats per query; the index itself is ~60 MB in memory.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
# /health: the process serves (liveness). /ready: an index exists; a cold
|
||||
# start on an empty PVC embeds every note (~5 min) before traffic arrives.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
failureThreshold: 12
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 60
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 3
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: wiki-index-data
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -1,56 +0,0 @@
|
||||
---
|
||||
# wiki-index.renner.uno — LAN-only (no dns.renner.uno/public label), machine
|
||||
# client (bin/wiki_rank.py), so Traefik BasicAuth like wiki-embed.renner.uno.
|
||||
# Gitea's webhook does not pass here; it targets the ClusterIP Service directly.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: wiki-index-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
basicAuth:
|
||||
secret: wiki-index-basic-auth
|
||||
realm: wiki-index
|
||||
---
|
||||
# OpenBao key apps/ai/wiki-index-basic-auth, field `users` = htpasswd line.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wiki-index-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: wiki-index-basic-auth
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/ai/wiki-index-basic-auth
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
traefik.ingress.kubernetes.io/router.middlewares: ai-wiki-index-basic-auth@kubernetescrd
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- wiki-index.renner.uno
|
||||
secretName: wiki-index-tls
|
||||
rules:
|
||||
- host: wiki-index.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: wiki-index
|
||||
port:
|
||||
number: 8080
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
@@ -34,7 +34,7 @@ spec:
|
||||
# Image automation, range 3.x (see image-automation/woodpecker.yaml).
|
||||
# MUST stay version-matched with woodpecker-server (gRPC parity) — both
|
||||
# track the same range/filter so they bump together on each release.
|
||||
image: woodpeckerci/woodpecker-agent:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
env:
|
||||
- name: WOODPECKER_SERVER
|
||||
value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
|
||||
|
||||
@@ -22,9 +22,7 @@ metadata:
|
||||
# migration deliberately withheld this while a rollback still existed.
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
instances: 2
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
@@ -93,7 +91,8 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volumes are lost outright.
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
archive_timeout: "30min"
|
||||
resources:
|
||||
requests:
|
||||
|
||||
@@ -3,6 +3,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- rbac.yaml
|
||||
- server.yaml
|
||||
- pdb.yaml
|
||||
- agent.yaml
|
||||
- service.yaml
|
||||
- ingress.yaml
|
||||
@@ -21,7 +22,7 @@ resources:
|
||||
# OpenBao access, no NetworkPolicy needed. server.yaml/agent.yaml keep their
|
||||
# Reloader annotations; externalsecret.yaml keeps the 10m refreshInterval.
|
||||
# --- namespace migration, stages 1-2 (issue #433) ---
|
||||
# Stage 2 has moved the workloads above (server, agent, service, ingress,
|
||||
# Stage 2 has moved the workloads above (server, agent, service, ingress, pdb,
|
||||
# rbac, secrets, policies) into the `woodpecker` namespace and pointed the
|
||||
# server's DSN at wp-pg-rw.woodpecker.svc. Stage 3 retired the `apps`
|
||||
# originals; these four now describe the surviving cluster. History and the
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
---
|
||||
# Eviction protection for the single-replica Woodpecker server StatefulSet.
|
||||
#
|
||||
# The server is SQLite-backed (single writer) on a single RWO PVC, so it is
|
||||
# deliberately replicas:1. minAvailable:1 makes the eviction API refuse
|
||||
# voluntary evictions so a node drain blocks and surfaces this pod rather
|
||||
# than yanking the live CI DB mid-write. Involuntary disruptions are not
|
||||
# affected.
|
||||
#
|
||||
# The selector MUST include app.kubernetes.io/component: server: the agent
|
||||
# Deployment in this same namespace shares app.kubernetes.io/name: woodpecker
|
||||
# (component: agent), and a name-only selector would wrongly also cover the
|
||||
# stateless, disposable agents.
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: woodpecker-server
|
||||
namespace: woodpecker
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: server
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: server
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
# minor + patch auto-update within 3.x; only the 4.0 major is held for
|
||||
# a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
|
||||
# -bump hazard is gone). Stays in lockstep with the agent.
|
||||
image: woodpeckerci/woodpecker-server:v3.18.1 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
env:
|
||||
- name: WOODPECKER_HOST
|
||||
value: https://ci.renner.uno
|
||||
|
||||
@@ -21,7 +21,7 @@ spec:
|
||||
# Service; nothing on this path needs the host's network namespace.
|
||||
containers:
|
||||
- name: zone-configurator
|
||||
image: everythingsmarthome/everything-presence-mmwave-configurator:2.2.5 # {"$imagepolicy": "flux-system:mmwave-configurator"}
|
||||
image: everythingsmarthome/everything-presence-mmwave-configurator:2.2.4 # {"$imagepolicy": "flux-system:mmwave-configurator"}
|
||||
env:
|
||||
- name: HA_BASE_URL
|
||||
value: "http://home-assistant:80"
|
||||
|
||||
+103
-123
@@ -179,18 +179,16 @@ phase_runs() {
|
||||
OPENBAO_NS="openbao"
|
||||
|
||||
# Where the secrets come from. On a workstation, the operator's pass store. In
|
||||
# the in-cluster Job there is no pass and no GPG key — and there should not be:
|
||||
# the pass store opens everything, and it stays on the operator's machine. Set
|
||||
# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
|
||||
# operator streams in when the run asks for them:
|
||||
# the in-cluster Job there is no pass and no GPG key — and there should not be,
|
||||
# because the key that protects OpenBao has no business living in the cluster
|
||||
# OpenBao secures. Set OPENBAO_SECRETS_DIR and the run reads them from files
|
||||
# instead, which the operator streams in when the run asks for them:
|
||||
#
|
||||
# pass show homelab/openbao/seal-key \
|
||||
# | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
|
||||
# pass show homelab/openbao/unseal-key \
|
||||
# | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/unseal-key'
|
||||
#
|
||||
# They exist in the pod's filesystem for the length of the run and are never
|
||||
# written to a Secret in this cluster, so they do not land in its etcd or in
|
||||
# any backup of it. The seal key alone becomes a Secret in the cluster being
|
||||
# built, which is what lets that OpenBao unseal itself.
|
||||
# written to a Secret, so they do not land in etcd or in any backup of it.
|
||||
OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}"
|
||||
|
||||
# Every secret the run needs, named once, in one place. The list is here and
|
||||
@@ -245,9 +243,7 @@ OPENBAO_REQUIRED_SECRETS=(
|
||||
"git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
|
||||
"s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
|
||||
"s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
|
||||
# The static seal key. The snapshots are encrypted under it, so it has to
|
||||
# be the one the backed-up cluster ran with — a fresh key restores nothing.
|
||||
"seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
|
||||
"unseal-key|OpenBao unseal key|pass show homelab/openbao/unseal-key"
|
||||
# Read by the node registry auth (see registry_auth_patch): the images an app
|
||||
# needs while the cluster is still being built are mirrored to a private
|
||||
# ghcr.io namespace, and a node has to be able to pull them before anything
|
||||
@@ -281,7 +277,7 @@ OPENBAO_REQUIRED_SECRETS=(
|
||||
# That phase used to need nothing but the machines, and `bootstrap.sh talos` is
|
||||
# documented as the part that can run unattended. Node registry auth changed
|
||||
# that: the machine config now carries a pull credential, so the phase needs
|
||||
# exactly one secret. Demanding all six there would drag in the OpenBao seal
|
||||
# exactly one secret. Demanding all six there would drag in the OpenBao unseal
|
||||
# key, which deliberately never reaches CI — and demanding none, as before,
|
||||
# left the phase to discover the missing file halfway through `talosctl gen`.
|
||||
require_secrets() {
|
||||
@@ -412,7 +408,7 @@ read_secret() {
|
||||
# the kernel link name enp2s0f1np1, and that name cannot be manufactured here
|
||||
# without assuming a Hetzner private network passes VLAN tags. See
|
||||
# talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase
|
||||
# needs the OpenBao seal key, which deliberately never reaches CI, and this
|
||||
# needs the OpenBao unseal key, which deliberately never reaches CI, and this
|
||||
# test is meant to run unattended.
|
||||
# Talos version and Image Factory schematic, read out of the installer image in
|
||||
# cluster.yaml. Both throwaway-cluster phases need them and neither may state
|
||||
@@ -610,8 +606,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
||||
# Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which
|
||||
# would not match the amd64 schematic this cluster installs. That leaves the
|
||||
# cx*/cpx*2 families in the German locations.
|
||||
# Two segments only, unlike the homelab's split in
|
||||
# NETWORK-traffic-classes.md: here the node address carries management,
|
||||
# Today's traffic-class split, not the target one in
|
||||
# PROPOSAL-traffic-class-separation.md: the node address carries management,
|
||||
# cluster-internal and external alike, and Ceph replication has its own.
|
||||
#
|
||||
# The order is load-bearing: it decides which kernel interface each segment
|
||||
@@ -1125,7 +1121,7 @@ EOF
|
||||
# nothing has demanded the secrets yet — the full require_secrets is gated
|
||||
# on the flux phase. Ask for the one this needs, and only that one: pulling
|
||||
# in the whole list would make an unattended talos run depend on the OpenBao
|
||||
# seal key, which never reaches CI.
|
||||
# unseal key, which never reaches CI.
|
||||
require_secrets ghcr-pull
|
||||
REGISTRY_PATCH=$(registry_auth_patch)
|
||||
[[ -s "$REGISTRY_PATCH" ]] \
|
||||
@@ -1410,11 +1406,9 @@ fi
|
||||
# waits on infrastructure-crds. A rebuild could not reach a reconciled state
|
||||
# without someone at a keyboard.
|
||||
#
|
||||
# The seal key stays where it was — the operator's `pass` store — and is read
|
||||
# The unseal key stays where it was — the operator's `pass` store — and is read
|
||||
# here at run time. It is never written to the repo and never reaches CI, which
|
||||
# is the reason this was left manual in the first place. It reaches the cluster
|
||||
# as the openbao-seal-key Secret, created by the flux phase before Flux starts
|
||||
# OpenBao, and from then on every replica unseals itself.
|
||||
# is the reason this was left manual in the first place.
|
||||
#
|
||||
# Everything below is idempotent: it is safe to re-run against an OpenBao that
|
||||
# is already unsealed and configured, which is what makes it usable as part of
|
||||
@@ -1504,63 +1498,61 @@ restore_openbao() {
|
||||
echo " extracted $(stat -c%s "$work/restore.snap") bytes"
|
||||
|
||||
# A snapshot can only be loaded into a running OpenBao, so the fresh one is
|
||||
# initialised first. Under the static seal, init hands out a recovery key
|
||||
# rather than an unseal key and the server unseals itself; the only thing
|
||||
# used from its output is the root token, for the next few seconds, and
|
||||
# then it stops existing: restoring replaces the keyring with the
|
||||
# snapshot's, after which the root token and recovery key from pass apply.
|
||||
#
|
||||
# No -force. Without it OpenBao checks that the snapshot was written under
|
||||
# the seal key this server runs with, and refuses a mismatch by name
|
||||
# ("possibly the snapshot is using a different autoseal key"). That check
|
||||
# is the one guard against restoring under the wrong seal-key Secret;
|
||||
# -force would skip it and leave a server that can never unseal.
|
||||
#
|
||||
# The init output is not echoed on a parse failure: it carries the root
|
||||
# token and the recovery key of a server that is about to be overwritten,
|
||||
# but a log line is a log line.
|
||||
echo -n " initialising the empty server (throwaway recovery key)... "
|
||||
local init_json root
|
||||
init_json=$(bao_exec "$pod" bao operator init -recovery-shares=1 -recovery-threshold=1 -format=json 2>&1) \
|
||||
# initialised and unsealed with a throwaway key first. That key is used for
|
||||
# the next few seconds and then stops existing: restoring replaces the
|
||||
# keyring with the snapshot's, after which the real key from pass applies.
|
||||
# -force is required for exactly that reason — without it OpenBao refuses a
|
||||
# snapshot whose keys do not match the cluster's own.
|
||||
echo -n " initialising the empty server with a throwaway key... "
|
||||
local init_json throwaway
|
||||
init_json=$(bao_exec "$pod" bao operator init -key-shares=1 -key-threshold=1 -format=json 2>&1) \
|
||||
|| { echo; echo "$init_json" >&2; fail "could not initialise OpenBao"; }
|
||||
root=$(jq -r '.root_token // empty' <<<"$init_json")
|
||||
[[ -n "$root" ]] || fail "init output carried no root token"
|
||||
throwaway=$(jq -r '.unseal_keys_b64[0]' <<<"$init_json")
|
||||
[[ -n "$throwaway" && "$throwaway" != "null" ]] || fail "could not read the throwaway unseal key from init output"
|
||||
ok
|
||||
|
||||
# Sealed after init means the static seal is not working — the Secret is
|
||||
# missing or malformed — and the restore below cannot run against it.
|
||||
echo -n " waiting for it to unseal itself"
|
||||
for i in $(seq 1 30); do
|
||||
if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
|
||||
| jq -e '.sealed == false' >/dev/null; then
|
||||
echo; break
|
||||
fi
|
||||
printf '.'
|
||||
sleep 2
|
||||
if [[ $i -eq 30 ]]; then
|
||||
echo
|
||||
cat "$OPENBAO_SECRETS_DIR/.baoerr" >&2
|
||||
fail "$pod did not unseal itself after init — check the openbao-seal-key Secret and the pod log"
|
||||
fi
|
||||
done
|
||||
echo -n " unsealing with it... "
|
||||
# The key goes as an argument, which OpenBao's own error message asks for:
|
||||
#
|
||||
# you are executing outside of a terminal (tty) ... If this is not an
|
||||
# option, the unseal key can be provided as the first argument
|
||||
#
|
||||
# Two earlier shapes failed. `unseal -` is Vault syntax; OpenBao takes
|
||||
# argv[1] literally, so the dash was sent as the key. Omitting the argument
|
||||
# makes it prompt, and the prompt needs a real tty — a pipe gives "file
|
||||
# descriptor 0 is not a terminal". There is no stdin path at all.
|
||||
#
|
||||
# This does put the key in the pod's process list for the length of the
|
||||
# call, which the code here previously went out of its way to avoid. It is
|
||||
# accepted rather than worked around: reading that list requires exec into
|
||||
# the very pod that holds the key in memory, and the alternative — a
|
||||
# port-forward plus curl against /v1/sys/unseal — puts a background process
|
||||
# in the most critical path of a recovery. The openbao image has no curl,
|
||||
# wget or nc, so it would have to run from here.
|
||||
bao_exec "$pod" bao operator unseal "$throwaway" >/dev/null \
|
||||
|| fail "could not unseal with the throwaway key"
|
||||
ok
|
||||
|
||||
echo -n " loading the snapshot... "
|
||||
kubectl -n "$OPENBAO_NS" cp "$work/restore.snap" "$pod:/tmp/restore.snap" >/dev/null \
|
||||
|| fail "could not copy the snapshot into $pod"
|
||||
local root
|
||||
root=$(jq -r '.root_token' <<<"$init_json")
|
||||
local out
|
||||
out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore /tmp/restore.snap 2>&1) \
|
||||
out=$(bao_exec "$pod" env BAO_TOKEN="$root" bao operator raft snapshot restore -force /tmp/restore.snap 2>&1) \
|
||||
|| { echo; echo "$out" >&2; fail "snapshot restore rejected"; }
|
||||
ok
|
||||
|
||||
rm -rf "$work"
|
||||
# The server seals itself against the keyring that came with the snapshot
|
||||
# and unseals again from the static seal — the same key the snapshot was
|
||||
# written under. bring_up_openbao waits for that on every replica.
|
||||
echo " restored — the server re-seals and unseals itself from the seal key"
|
||||
# The server is now sealed again, against the keyring that came with the
|
||||
# snapshot. unseal_openbao carries on from here with the real key.
|
||||
echo " restored — the server now expects the original unseal key"
|
||||
}
|
||||
|
||||
bring_up_openbao() {
|
||||
step 0 "Bringing OpenBao up..."
|
||||
unseal_openbao() {
|
||||
step 0 "Unsealing OpenBao..."
|
||||
[[ -n "$OPENBAO_SECRETS_DIR" ]] || command -v pass >/dev/null \
|
||||
|| fail "pass not found — the unseal key lives in the operator's pass store, or set OPENBAO_SECRETS_DIR and stream it in"
|
||||
command -v jq >/dev/null || fail "jq not found (needed to read OpenBao's seal status)"
|
||||
|
||||
# OpenBao is deployed by the infrastructure Kustomization. What this step
|
||||
@@ -1666,28 +1658,40 @@ bring_up_openbao() {
|
||||
restore_openbao "${pods[0]}"
|
||||
fi
|
||||
|
||||
# Wait for every replica the StatefulSet wants, not the ones that happen to
|
||||
# exist right now. On a cold start only openbao-0 exists when this runs:
|
||||
# podManagementPolicy is OrderedReady, so openbao-1 is created once
|
||||
# openbao-0 passes its readiness probe and openbao-2 after that. Each
|
||||
# unseals itself from the static seal; nothing here types a key.
|
||||
local key
|
||||
key=$(read_secret unseal-key) \
|
||||
|| fail "could not read the unseal key"
|
||||
|
||||
# Unseal every replica the StatefulSet wants, not the ones that happen to
|
||||
# exist right now. On a cold start only openbao-0 exists when this runs, and
|
||||
# unsealing just that one deadlocks the StatefulSet: podManagementPolicy is
|
||||
# OrderedReady, a sealed pod never passes its readiness probe, so openbao-1
|
||||
# is created only after openbao-0 is unsealed and openbao-2 is never created
|
||||
# at all. The cluster comes up at 1 of 3.
|
||||
#
|
||||
# Waiting for the whole chain rather than the first pod matters because the
|
||||
# openbao Service sets publishNotReadyAddresses — the chart does this so
|
||||
# replicas can reach each other to join raft before any of them is Ready —
|
||||
# so a replica that is still starting is a live backend on the service IP.
|
||||
# Clients balanced onto it get `dial tcp <svc>:8200: connect: no route to
|
||||
# host`, and the failure surfaces nowhere near here: external-secrets'
|
||||
# ClusterSecretStore flaps, every ExternalSecret reports "could not get
|
||||
# secret data from provider", and infrastructure-crds — which waits on those
|
||||
# — never goes Ready, taking all twenty app Kustomizations with it.
|
||||
# That alone would be survivable. What makes it fatal is that the openbao
|
||||
# Service sets publishNotReadyAddresses — the chart does this so replicas can
|
||||
# reach each other to join raft before any of them is unsealed — so the
|
||||
# sealed openbao-1 is a live backend on the service IP. Clients are balanced
|
||||
# onto a node that cannot answer, and the failure surfaces nowhere near here:
|
||||
# external-secrets' ClusterSecretStore flaps between "store validated" and
|
||||
# `dial tcp <svc>:8200: connect: no route to host`, every ExternalSecret in
|
||||
# the cluster reports "could not get secret data from provider", and
|
||||
# infrastructure-crds — which waits on those — never goes Ready, taking all
|
||||
# twenty app Kustomizations with it.
|
||||
#
|
||||
# So this converges instead of iterating once: re-list the pods each round,
|
||||
# unseal whatever is sealed, and finish when spec.replicas of them are
|
||||
# unsealed. A pod that is still starting cannot be unsealed yet and is not an
|
||||
# error — it is retried next round, which is exactly how the OrderedReady
|
||||
# chain is meant to be walked.
|
||||
local want
|
||||
want=$(kubectl -n "$OPENBAO_NS" get statefulset openbao -o jsonpath='{.spec.replicas}') \
|
||||
|| fail "could not read the openbao StatefulSet's replica count"
|
||||
[[ "$want" =~ ^[0-9]+$ ]] || fail "openbao StatefulSet reported a non-numeric replica count: '$want'"
|
||||
echo " waiting for $want replica(s) to unseal themselves"
|
||||
echo " unsealing $want replica(s)"
|
||||
|
||||
local unsealed=0 round pod
|
||||
local unsealed=0 round pod err
|
||||
local -A seen=()
|
||||
for round in $(seq 1 180); do
|
||||
mapfile -t pods < <(kubectl -n "$OPENBAO_NS" get pods -l app.kubernetes.io/name=openbao \
|
||||
@@ -1696,21 +1700,29 @@ bring_up_openbao() {
|
||||
for pod in "${pods[@]}"; do
|
||||
if bao_exec "$pod" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr" \
|
||||
| jq -e '.sealed == false' >/dev/null; then
|
||||
[[ -n "${seen[$pod]:-}" ]] || { echo -e " $pod: ${GREEN}unsealed${NC}"; seen[$pod]=1; }
|
||||
[[ -n "${seen[$pod]:-}" ]] || { echo " $pod: already unsealed"; seen[$pod]=1; }
|
||||
unsealed=$(( unsealed + 1 ))
|
||||
continue
|
||||
fi
|
||||
# Argument, not stdin — see the restore path above for why: OpenBao
|
||||
# has no stdin form, and its prompt requires a tty a Job does not
|
||||
# have. The key is exposed in the pod's process list for the length
|
||||
# of the call.
|
||||
if err=$(bao_exec "$pod" bao operator unseal "$key" 2>&1); then
|
||||
echo -e " $pod: ${GREEN}unsealed${NC}"
|
||||
seen[$pod]=1
|
||||
unsealed=$(( unsealed + 1 ))
|
||||
else
|
||||
# Not fatal on its own: a container still in Init or still
|
||||
# pulling answers this way, and the next round finds it ready.
|
||||
# Only running out of rounds is a failure.
|
||||
echo " $pod: not yet (${err##*$'\n'})"
|
||||
fi
|
||||
done
|
||||
(( unsealed >= want )) && break
|
||||
sleep 10
|
||||
if [[ $round -eq 180 ]]; then
|
||||
# A replica that is running and still sealed did not get a working
|
||||
# seal. Say which, with its own answer, rather than "timed out".
|
||||
for pod in "${pods[@]}"; do
|
||||
[[ -n "${seen[$pod]:-}" ]] && continue
|
||||
echo " $pod:" >&2
|
||||
bao_exec "$pod" bao status 2>&1 | sed 's/^/ /' >&2 || true
|
||||
done
|
||||
fail "only $unsealed of $want OpenBao replicas unsealed after 30 minutes — a sealed one means the static seal failed (openbao-seal-key Secret, pod log)"
|
||||
fail "only $unsealed of $want OpenBao replicas came unsealed after 30 minutes"
|
||||
fi
|
||||
done
|
||||
ok
|
||||
@@ -1735,7 +1747,7 @@ if phase_runs flux; then
|
||||
command -v flux >/dev/null || fail "flux not found"
|
||||
# In the in-cluster Job neither the SOPS key nor the Gitea token is present,
|
||||
# and neither should be baked in. Both are streamed into the secrets
|
||||
# directory when the run asks for them, the same way the seal key is.
|
||||
# directory when the run asks for them, the same way the unseal key is.
|
||||
# Unconditional: the same collection step on a laptop and in the Job, so the
|
||||
# recovery path is the one that gets exercised.
|
||||
require_secrets
|
||||
@@ -1758,37 +1770,6 @@ if phase_runs flux; then
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
ok
|
||||
|
||||
# The static seal key, before Flux brings OpenBao up: a pod whose env
|
||||
# references a Secret that does not exist is never started, and OpenBao
|
||||
# reads the key at start. Left alone when the cluster already holds the
|
||||
# same value. A DIFFERENT value is refused rather than applied: the data
|
||||
# is encrypted under whichever key OpenBao started with, and swapping the
|
||||
# Secret would leave every replica sealed for good at its next restart.
|
||||
# That is a migration, not a bootstrap step.
|
||||
#
|
||||
# --from-literal, not --from-file: `pass show` ends its output with a
|
||||
# newline, the substitution strips it, and OpenBao rejects a key that is
|
||||
# not exactly 32 bytes of base64. The key sits in this process's argument
|
||||
# list for the length of the call, as it did in the pod's before.
|
||||
echo -n "Creating OpenBao seal key secret... "
|
||||
seal_key=$(read_secret seal-key) || fail "could not read the seal key"
|
||||
kubectl create namespace "$OPENBAO_NS" --dry-run=client -o yaml | kubectl apply -f -
|
||||
if existing=$(kubectl -n "$OPENBAO_NS" get secret openbao-seal-key \
|
||||
-o go-template='{{index .data "key" | base64decode}}' 2>"$OPENBAO_SECRETS_DIR/.err"); then
|
||||
[[ "$existing" == "$seal_key" ]] \
|
||||
|| fail "openbao-seal-key in $OPENBAO_NS holds a different key than pass — not replacing it (OpenBao's data is written under the key it started with)"
|
||||
echo "already present"
|
||||
elif grep -q NotFound "$OPENBAO_SECRETS_DIR/.err"; then
|
||||
kubectl -n "$OPENBAO_NS" create secret generic openbao-seal-key \
|
||||
--from-literal=key="$seal_key" >/dev/null \
|
||||
|| fail "could not create the openbao-seal-key Secret"
|
||||
ok
|
||||
else
|
||||
echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
|
||||
fail "could not read the openbao-seal-key Secret"
|
||||
fi
|
||||
unset seal_key existing
|
||||
|
||||
if [[ "$FLUX_SOURCE" == "github" ]]; then
|
||||
step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..."
|
||||
echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}"
|
||||
@@ -1839,7 +1820,7 @@ if phase_runs flux; then
|
||||
--token-auth --timeout 15m 2>&1
|
||||
fi
|
||||
|
||||
bring_up_openbao
|
||||
unseal_openbao
|
||||
|
||||
step 0 "Waiting for infrastructure..."
|
||||
for i in $(seq 1 60); do
|
||||
@@ -1946,7 +1927,6 @@ if phase_runs wait-apps; then
|
||||
# apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up
|
||||
# is genuinely 5–10 min on a fresh OSD-wiped cluster).
|
||||
wait_kustomization apps "apps" 90
|
||||
wait_kustomization gitea "Gitea (incl. data-gitea-0 PVC)" 90
|
||||
wait_kustomization influxdb "InfluxDB (HR install + token Jobs)" 600
|
||||
wait_kustomization woodpecker "Woodpecker CI (incl. wp-pg CNPG backend)" 600
|
||||
wait_kustomization authentik "Authentik (incl. ak-pg CNPG backend)" 600
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,20 +0,0 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: gitea
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# Gitea in its own `gitea` namespace (#878), formerly under the shared `apps` KS.
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 15m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/apps/gitea
|
||||
prune: true
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user