diff --git a/.woodpecker/build-ci-images.yml b/.woodpecker/build-ci-images.yml index 9791979c..0d4626e8 100644 --- a/.woodpecker/build-ci-images.yml +++ b/.woodpecker/build-ci-images.yml @@ -377,37 +377,6 @@ steps: --destination gitea.renner.uno/gitea_admin/netbird-service-sync:$TAG \ --destination gitea.renner.uno/gitea_admin/netbird-service-sync:latest - # wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea and - # queried by ~/Notes/bin/wiki_rank.py. Runtime artifact like netbird-service-sync. - - name: build-wiki-index - image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug - directory: ci/images/wiki-index - depends_on: [] - when: - - event: deployment - - event: push - path: ["ci/images/wiki-index/**", ".woodpecker/build-ci-images.yml"] - environment: - GITEA_USER: gitea_admin - GITEA_TOKEN: - from_secret: ci_images_token - commands: - - mkdir -p /kaniko/.docker - - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n') - - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json - - TAG=$(date +%Y-%m-%d) - # Also a per-commit tag: several builds a day (branch pushes and the - # merge) all rewrite the dated tag, and the deployment pin raced them - # on 2026-09-20. The commit tag is immutable and needs no digest lookup. - - SHORT=$(printf '%s' "$CI_COMMIT_SHA" | cut -c1-8) - - | - /kaniko/executor \ - --context . \ - --dockerfile Containerfile \ - --destination gitea.renner.uno/gitea_admin/wiki-index:$TAG \ - --destination gitea.renner.uno/gitea_admin/wiki-index:$TAG-$SHORT \ - --destination gitea.renner.uno/gitea_admin/wiki-index:latest - # bird: BGP daemon for the pod-network mesh over pod0 (#576). Runtime # artifact like heating-can; same registry so the audit tracks it. - name: build-bird diff --git a/.woodpecker/wiki-maintenance.yml b/.woodpecker/wiki-maintenance.yml index f7541068..6634135a 100644 --- a/.woodpecker/wiki-maintenance.yml +++ b/.woodpecker/wiki-maintenance.yml @@ -4,7 +4,7 @@ # script (structural checks, no model), then consolidate (wiki-consolidate # agent: verifies a cohort of notes against sources, repos and the live # cluster and edits them). The pipeline commits and pushes as Claude Bot; the -# push webhook then refreshes the wiki-index service. +# push webhook then refreshes the wiki service. # # The agents address the vault as ~/Notes and the repos as ~/source/, so # each agent step links those into the workspace instead of rewriting them. diff --git a/ci/images/wiki-index/Containerfile b/ci/images/wiki-index/Containerfile deleted file mode 100644 index 4948cefb..00000000 --- a/ci/images/wiki-index/Containerfile +++ /dev/null @@ -1,10 +0,0 @@ -# wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea, -# queried by ~/Notes/bin/wiki_rank.py. Standard library only; runs unprivileged. -FROM python:3.13-slim - -RUN useradd --system --no-create-home --shell /usr/sbin/nologin wikiindex -COPY wiki_index.py /opt/wiki_index.py - -ENV PYTHONUNBUFFERED=1 -USER wikiindex -ENTRYPOINT ["python3", "/opt/wiki_index.py"] diff --git a/ci/images/wiki-index/test_wiki_index.py b/ci/images/wiki-index/test_wiki_index.py deleted file mode 100644 index 5e301067..00000000 --- a/ci/images/wiki-index/test_wiki_index.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 -"""Unit tests for wiki_index. Run: python3 -m unittest test_wiki_index""" - -import hashlib -import hmac -import json -import tempfile -import unittest -from pathlib import Path - -import wiki_index as wi - -NOTE = """--- -title: 'UDM intercepts DNS' -type: wiki -tags: [dns, unifi] -aliases: [udm dns interception, dig 1.1.1.1 wrong] ---- - -# UDM intercepts DNS - -The UDM answers every UDP/53 query itself. - -## Details - -More. -""" - - -class ParseTests(unittest.TestCase): - def test_note_text(self): - title, text = wi.note_text("udm-dns", NOTE) - self.assertEqual(title, "UDM intercepts DNS") - self.assertIn("udm dns UDM intercepts DNS udm dns interception", text) - self.assertIn("dns unifi", text) - self.assertTrue(text.endswith("The UDM answers every UDP/53 query itself.")) - self.assertLessEqual(len(text), wi.EMBED_CHARS) - - def test_credentials_are_stripped(self): - original = wi.http - wi.http = lambda method, url, body=None, headers=None, timeout=60.0: ( - {"auth": {"client_token": "t"}} if url.endswith("/login") - else {"data": {"data": {"gitea_token": "abc\n", "webhook_secret": " s3cret\n"}}}) - original_sa = wi.SA_TOKEN - try: - with tempfile.TemporaryDirectory() as d: - wi.SA_TOKEN = Path(d) / "token" - wi.SA_TOKEN.write_text("jwt\n") - self.assertEqual(wi.Credentials().fetch(), {"gitea_token": "abc", "webhook_secret": "s3cret"}) - finally: - wi.http, wi.SA_TOKEN = original, original_sa - - def test_tree_paging_stops_on_short_last_page(self): - # Gitea: truncated=true on every page including the last, "tree": null past it. - pages = { - 1: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000)], "truncated": True, "total_count": 1192}, - 2: {"tree": [{"path": f"wiki/n{i}.md", "type": "blob", "sha": f"s{i}"} for i in range(1000, 1192)] + [{"path": "raw/x.md", "type": "blob", "sha": "r"}, {"path": "wiki/sub/y.md", "type": "blob", "sha": "y"}], "truncated": True, "total_count": 1192}, - 3: {"tree": None, "truncated": False, "total_count": 1192}, - } - calls = [] - original = wi.http - - def fake_http(method, url, body=None, headers=None, timeout=60.0): - page = int(url.rsplit("page=", 1)[1]) - calls.append(page) - return pages[page] - - wi.http = fake_http - try: - tree = wi.Gitea("t").tree("abc") - finally: - wi.http = original - self.assertEqual(calls, [1, 2]) - self.assertEqual(len(tree), 1192) - self.assertNotIn("raw/x.md", tree) - self.assertNotIn("wiki/sub/y.md", tree) - self.assertEqual(tree["wiki/n1191.md"], "s1191") - - def test_signature(self): - body = b'{"ref":"refs/heads/master"}' - sig = hmac.new(b"s3cret", body, hashlib.sha256).hexdigest() - self.assertTrue(wi.verify_signature("s3cret", body, sig)) - self.assertFalse(wi.verify_signature("other", body, sig)) - self.assertFalse(wi.verify_signature("s3cret", body, "")) - - -class FakeGitea: - def __init__(self, head, tree, files): - self._head, self._tree, self.files = head, tree, files - self.raw_calls = [] - - def head(self): - return self._head - - def tree(self, commit): - return self._tree - - def raw(self, path, ref): - self.raw_calls.append(path) - return self.files[path] - - -class IndexTests(unittest.TestCase): - def setUp(self): - self.tmp = tempfile.TemporaryDirectory() - self.path = Path(self.tmp.name) / "index.json" - self.original_embed = wi.embed - self.embed_calls = [] - - def fake_embed(texts, batch=2): - self.embed_calls.append(list(texts)) - return [wi.unit([1.0, float(len(t) % 7)]) for t in texts] - - wi.embed = fake_embed - - def tearDown(self): - wi.embed = self.original_embed - self.tmp.cleanup() - - def test_sync_is_incremental_and_persists(self): - files = {"wiki/udm-dns.md": NOTE, "wiki/ceph.md": "---\ntitle: Ceph\n---\n# Ceph\n\nOSD.\n", "wiki/_index.md": "---\ntitle: Wiki\n---\n"} - tree = {"wiki/udm-dns.md": "aaa", "wiki/ceph.md": "bbb", "wiki/_index.md": "ccc"} - g = FakeGitea("c1", tree, files) - idx = wi.Index(self.path) - s = idx.sync(g) - self.assertEqual((s["changed"], s["removed"], s["notes"]), (2, 0, 2)) - self.assertEqual(set(idx.notes), {"udm-dns", "ceph"}) - self.assertEqual(idx.notes["udm-dns"]["title"], "UDM intercepts DNS") - self.assertTrue(self.path.exists()) - - # head unchanged: nothing fetched - s = idx.sync(g) - self.assertEqual(s.get("skipped"), "head unchanged") - self.assertEqual(len(g.raw_calls), 2) - - # one note changed, one removed, new head - g2 = FakeGitea("c2", {"wiki/udm-dns.md": "aaa2"}, {"wiki/udm-dns.md": NOTE.replace("itself", "itself!")}) - s = idx.sync(g2) - self.assertEqual((s["changed"], s["removed"], s["notes"]), (1, 1, 1)) - self.assertEqual(g2.raw_calls, ["wiki/udm-dns.md"]) - - # restart loads the persisted index - idx2 = wi.Index(self.path) - self.assertEqual(idx2.commit, "c2") - self.assertEqual(set(idx2.notes), {"udm-dns"}) - - def test_rank_orders_by_cosine(self): - idx = wi.Index(self.path) - idx.notes = {"a": {"sha": "1", "title": "A", "vector": wi.unit([1.0, 0.0])}, - "b": {"sha": "2", "title": "B", "vector": wi.unit([0.0, 1.0])}} - wi.embed = lambda texts, batch=2: [wi.unit([0.2, 1.0])] - rows = idx.rank("q", k=5) - self.assertEqual([r["slug"] for r in rows], ["b", "a"]) - self.assertGreater(rows[0]["score"], rows[1]["score"]) - - def test_status_reports_age_and_errors(self): - idx = wi.Index(self.path) - self.assertFalse(idx.status()["ready"]) - g = FakeGitea("c1", {"wiki/a.md": "x"}, {"wiki/a.md": "---\ntitle: A\n---\n# A\n\nt\n"}) - idx.sync(g) - st = idx.status() - self.assertTrue(st["ready"]) - self.assertEqual(st["notes"], 1) - self.assertEqual(st["commit"], "c1") - - class Broken(FakeGitea): - def head(self): - raise wi.ApiError("gitea down") - - with self.assertRaises(wi.ApiError): - idx.sync(Broken("c1", {}, {})) - self.assertIn("gitea down", idx.status()["last_error"]) - self.assertEqual(idx.status()["notes"], 1) # last good index kept - self.assertTrue(idx.status()["ready"]) # and still served - - -if __name__ == "__main__": - unittest.main() diff --git a/ci/images/wiki-index/wiki_index.py b/ci/images/wiki-index/wiki_index.py deleted file mode 100644 index dc125ebc..00000000 --- a/ci/images/wiki-index/wiki_index.py +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env python3 -"""wiki-index: embedding index over the Notes repo's wiki/, fed from Gitea. - -Source of truth is the notes repository on Gitea. The index holds, per wiki -note, the git blob sha, title and a bge-m3 vector of slug/title/aliases/tags/ -first paragraph. A sync fetches the repo tree, embeds every note whose blob sha -changed (via wiki-embed, in-cluster) and drops notes that left the tree; it -runs at start, on every push webhook from Gitea, and every RESYNC_SECONDS. - -HTTP: - POST /rank {"query": str, "k": int} -> {"rows": [{"slug","title","score"}], "index": {...}} - GET /health -> 200 always (liveness) with {"ready", "notes", "commit", "synced_at", "age_s", "syncing", "last_error"} - GET /ready -> 200 when an index exists, else 503 (readiness) - POST /webhook Gitea push event, HMAC-SHA256 in X-Gitea-Signature -> {"queued": true} - -Credentials (Gitea token, webhook secret) are read from OpenBao through the -Kubernetes auth role on every use; they never land in a Secret or a log line. -Standard library only. -""" - -from __future__ import annotations - -import hashlib -import hmac -import json -import logging -import os -import re -import sys -import threading -import time -import urllib.error -import urllib.parse -import urllib.request -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path - -LOG = logging.getLogger("wiki-index") - -GITEA_URL = os.environ.get("GITEA_URL", "https://gitea.renner.uno").rstrip("/") -NOTES_REPO = os.environ.get("NOTES_REPO", "k.renner/notes") -NOTES_BRANCH = os.environ.get("NOTES_BRANCH", "master") -WIKI_DIR = os.environ.get("WIKI_DIR", "wiki").strip("/") -EMBED_URL = os.environ.get("EMBED_URL", "http://wiki-embed.ai.svc.cluster.local:8080").rstrip("/") -EMBED_MODEL = os.environ.get("EMBED_MODEL", "bge-m3") -EMBED_CHARS = int(os.environ.get("EMBED_CHARS", "1500")) -INDEX_PATH = Path(os.environ.get("INDEX_PATH", "/data/index.json")) -RESYNC_SECONDS = int(os.environ.get("RESYNC_SECONDS", "1800")) -LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080")) -BAO_ADDR = os.environ.get("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200").rstrip("/") -BAO_ROLE = os.environ.get("BAO_ROLE", "wiki-index") -BAO_SECRET_PATH = os.environ.get("BAO_SECRET_PATH", "homelab/data/apps/ai/wiki-index") -SA_TOKEN = Path(os.environ.get("SA_TOKEN", "/var/run/secrets/kubernetes.io/serviceaccount/token")) - - -class ApiError(Exception): - pass - - -# ---------------------------------------------------------------- http helpers - - -def http(method: str, url: str, body=None, headers=None, timeout: float = 60.0): - data = None - hdrs = dict(headers or {}) - if body is not None: - data = json.dumps(body).encode("utf-8") - hdrs.setdefault("Content-Type", "application/json") - req = urllib.request.Request(url, data=data, method=method, headers=hdrs) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - raw = resp.read() - ctype = resp.headers.get("Content-Type", "") - except urllib.error.HTTPError as e: - raise ApiError(f"{method} {url}: HTTP {e.code}: {e.read().decode('utf-8', 'replace')[:300]}") from e - except urllib.error.URLError as e: - raise ApiError(f"{method} {url}: {e.reason}") from e - if "json" in ctype: - return json.loads(raw.decode("utf-8")) - return raw - - -# ---------------------------------------------------------------- credentials - - -class Credentials: - """Gitea token and webhook secret from OpenBao, fetched per use.""" - - def __init__(self, static: dict | None = None): - self.static = static # tests / local runs - - def fetch(self) -> dict: - if self.static is not None: - return self.static - jwt = SA_TOKEN.read_text(encoding="utf-8").strip() - login = http("POST", f"{BAO_ADDR}/v1/auth/kubernetes/login", {"role": BAO_ROLE, "jwt": jwt}) - token = login["auth"]["client_token"] - kv = http("GET", f"{BAO_ADDR}/v1/{BAO_SECRET_PATH}", headers={"X-Vault-Token": token}) - data = {k: str(v).strip() for k, v in kv["data"]["data"].items()} # `bao kv put key=@file` keeps the file's newline - for key in ("gitea_token", "webhook_secret"): - if not data.get(key): - raise ApiError(f"OpenBao {BAO_SECRET_PATH} has no key {key}") - return data - - -# ---------------------------------------------------------------- gitea - - -class Gitea: - def __init__(self, token: str, base: str = GITEA_URL, repo: str = NOTES_REPO, branch: str = NOTES_BRANCH): - self.base, self.repo, self.branch = base, repo, branch - self.headers = {"Authorization": f"token {token}"} - - def api(self, path: str, **params): - q = ("?" + urllib.parse.urlencode(params)) if params else "" - return http("GET", f"{self.base}/api/v1/repos/{self.repo}{path}{q}", headers=self.headers) - - def head(self) -> str: - return self.api(f"/branches/{urllib.parse.quote(self.branch, safe='')}")["commit"]["id"] - - def tree(self, commit: str) -> dict[str, str]: - """path -> blob sha for every wiki note at that commit.""" - out: dict[str, str] = {} - page, per_page, seen = 1, 1000, 0 - while True: - t = self.api(f"/git/trees/{commit}", recursive="true", per_page=per_page, page=page) - entries = t.get("tree") or [] # Gitea answers "tree": null past the last page - for e in entries: - p = e.get("path", "") - if e.get("type") == "blob" and p.startswith(WIKI_DIR + "/") and p.endswith(".md") and "/" not in p[len(WIKI_DIR) + 1:]: - out[p] = e["sha"] - seen += len(entries) - total = int(t.get("total_count") or 0) - if not entries or len(entries) < per_page or (total and seen >= total): - break - page += 1 - if page > 50: - raise ApiError("tree paging runaway") - return out - - def raw(self, path: str, ref: str) -> str: - data = http("GET", f"{self.base}/api/v1/repos/{self.repo}/raw/{urllib.parse.quote(path)}?ref={urllib.parse.quote(ref, safe='')}", - headers=self.headers) - return data.decode("utf-8") if isinstance(data, bytes) else json.dumps(data) - - -# ---------------------------------------------------------------- notes - - -def parse_frontmatter(text: str) -> tuple[dict, str]: - m = re.match(r"^---[ \t]*\n(.*?)\n---[ \t]*(?:\n|$)", text, re.DOTALL) - if not m: - return {}, text - fm: dict = {} - for line in m.group(1).splitlines(): - mm = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line) - if mm: - fm[mm.group(1)] = mm.group(2).strip() - return fm, text[m.end():] - - -def as_list(value: str) -> list[str]: - s = (value or "").strip() - if s.startswith("[") and s.endswith("]"): - s = s[1:-1] - return [p.strip().strip("'\"") for p in s.split(",") if p.strip()] - - -def first_paragraph(body: str) -> str: - para: list[str] = [] - seen_h1 = False - for line in body.splitlines(): - if line.startswith("# "): - seen_h1 = True - continue - if not seen_h1: - continue - if line.strip() == "" or line.startswith("#"): - if para: - break - continue - para.append(line.strip()) - return " ".join(para) - - -def note_text(slug: str, text: str) -> tuple[str, str]: - """(title, embedding text) for one note file.""" - fm, body = parse_frontmatter(text) - title = (fm.get("title") or slug).strip().strip("'\"") - parts = [slug.replace("-", " "), title, " ".join(as_list(fm.get("aliases", ""))), - " ".join(as_list(fm.get("tags", ""))), first_paragraph(body)] - return title, " ".join(parts)[:EMBED_CHARS] - - -# ---------------------------------------------------------------- embeddings - - -def embed(texts: list[str], batch: int = 2) -> list[list[float]]: - out: list[list[float]] = [] - for i in range(0, len(texts), batch): - resp = http("POST", f"{EMBED_URL}/v1/embeddings", {"model": EMBED_MODEL, "input": texts[i:i + batch]}, timeout=120) - rows = sorted(resp["data"], key=lambda d: d["index"]) - out.extend(unit(d["embedding"]) for d in rows) - return out - - -def unit(v: list[float]) -> list[float]: - n = sum(x * x for x in v) ** 0.5 or 1.0 - return [round(x / n, 5) for x in v] - - -# ---------------------------------------------------------------- index - - -class Index: - def __init__(self, path: Path): - self.path = path - self.lock = threading.Lock() - self.notes: dict[str, dict] = {} # slug -> {sha, title, vector} - self.commit = "" - self.synced_at = 0.0 - self.syncing = False - self.last_error = "" - if path.exists(): - try: - d = json.loads(path.read_text(encoding="utf-8")) - self.notes, self.commit, self.synced_at = d.get("notes", {}), d.get("commit", ""), d.get("synced_at", 0.0) - LOG.info("loaded index: %d notes at %s", len(self.notes), self.commit[:8]) - except (OSError, ValueError) as e: - LOG.warning("index unreadable (%s); starting empty", e) - - def save(self) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - tmp = self.path.with_suffix(".tmp") - tmp.write_text(json.dumps({"notes": self.notes, "commit": self.commit, "synced_at": self.synced_at}), encoding="utf-8") - tmp.replace(self.path) - - def sync(self, gitea: Gitea) -> dict: - """Bring the index to the branch head; returns a summary.""" - with self.lock: - self.syncing = True - try: - head = gitea.head() - if head == self.commit and self.notes: - summary = {"commit": head, "changed": 0, "removed": 0, "notes": len(self.notes), "skipped": "head unchanged"} - self.synced_at = time.time() - self.save() - return summary - tree = gitea.tree(head) - slugs = {p[len(WIKI_DIR) + 1:-3]: (p, sha) for p, sha in tree.items() if not p.endswith("/_index.md")} - todo = [s for s, (p, sha) in slugs.items() if self.notes.get(s, {}).get("sha") != sha] - removed = [s for s in list(self.notes) if s not in slugs] - LOG.info("sync %s: %d notes, %d to embed, %d removed", head[:8], len(slugs), len(todo), len(removed)) - fresh: dict[str, dict] = {} - for i in range(0, len(todo), 20): - chunk = todo[i:i + 20] - texts, titles = [], {} - for s in chunk: - title, text = note_text(s, gitea.raw(slugs[s][0], head)) - titles[s] = title - texts.append(text) - for s, vec in zip(chunk, embed(texts)): - fresh[s] = {"sha": slugs[s][1], "title": titles[s], "vector": vec} - with self.lock: - for s in removed: - self.notes.pop(s, None) - self.notes.update(fresh) - self.commit = head - self.synced_at = time.time() - self.last_error = "" - self.save() - return {"commit": head, "changed": len(todo), "removed": len(removed), "notes": len(self.notes)} - except Exception as e: - with self.lock: - self.last_error = str(e)[:300] - raise - finally: - with self.lock: - self.syncing = False - - def rank(self, query: str, k: int = 60) -> list[dict]: - q = embed([query])[0] - with self.lock: - rows = [(s, n["title"], sum(a * b for a, b in zip(q, n["vector"]))) for s, n in self.notes.items()] - rows.sort(key=lambda r: -r[2]) - return [{"slug": s, "title": t, "score": round(sc, 4)} for s, t, sc in rows[:k]] - - def status(self) -> dict: - """`ready` = an index exists and can answer /rank; a failed sync keeps the last good index.""" - with self.lock: - return {"ready": bool(self.notes), "notes": len(self.notes), "commit": self.commit, - "synced_at": self.synced_at, "age_s": round(time.time() - self.synced_at) if self.synced_at else None, - "syncing": self.syncing, "last_error": self.last_error or None} - - -# ---------------------------------------------------------------- syncer - - -class Syncer(threading.Thread): - """One worker: syncs on demand (webhook) and every RESYNC_SECONDS.""" - - def __init__(self, index: Index, creds: Credentials, interval: int): - super().__init__(daemon=True, name="syncer") - self.index, self.creds, self.interval = index, creds, interval - self.wake = threading.Event() - - def trigger(self) -> None: - self.wake.set() - - RETRY_SECONDS = 60 - - def run(self) -> None: - while True: - try: - data = self.creds.fetch() - summary = self.index.sync(Gitea(data["gitea_token"])) - LOG.info("sync done: %s", summary) - wait = self.interval - except Exception as e: # keep serving the last good index; retry soon, not next interval - LOG.error("sync failed: %s", e) - wait = self.RETRY_SECONDS - self.wake.wait(wait) - self.wake.clear() - - -def verify_signature(secret: str, body: bytes, signature: str) -> bool: - digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() - return hmac.compare_digest(digest, (signature or "").strip()) - - -# ---------------------------------------------------------------- server - - -def make_handler(index: Index, syncer: Syncer, creds: Credentials): - class Handler(BaseHTTPRequestHandler): - def log_message(self, fmt, *args): - LOG.info("%s %s", self.address_string(), fmt % args) - - def send_json(self, code: int, obj) -> None: - data = json.dumps(obj).encode("utf-8") - self.send_response(code) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def body(self) -> bytes: - n = int(self.headers.get("Content-Length") or 0) - return self.rfile.read(n) if n else b"" - - def do_GET(self): - if self.path == "/health": # liveness: the process serves - return self.send_json(200, index.status()) - if self.path == "/ready": # readiness: an index exists - st = index.status() - return self.send_json(200 if st["ready"] else 503, st) - self.send_json(404, {"error": "not found"}) - - def do_POST(self): - if self.path == "/rank": - try: - req = json.loads(self.body() or b"{}") - query = str(req.get("query", "")).strip() - k = max(1, min(int(req.get("k", 60)), 500)) - if not query: - return self.send_json(400, {"error": "query required"}) - rows = index.rank(query, k) - return self.send_json(200, {"rows": rows, "index": index.status()}) - except ApiError as e: - return self.send_json(502, {"error": str(e)}) - except (ValueError, TypeError) as e: - return self.send_json(400, {"error": str(e)}) - if self.path == "/webhook": - raw = self.body() - try: - secret = creds.fetch()["webhook_secret"] - except ApiError as e: - return self.send_json(503, {"error": str(e)}) - if not verify_signature(secret, raw, self.headers.get("X-Gitea-Signature", "")): - return self.send_json(403, {"error": "bad signature"}) - try: - event = json.loads(raw or b"{}") - except ValueError: - return self.send_json(400, {"error": "bad json"}) - ref = event.get("ref", "") - if ref and ref != f"refs/heads/{NOTES_BRANCH}": - return self.send_json(200, {"queued": False, "ref": ref}) - syncer.trigger() - return self.send_json(200, {"queued": True}) - self.send_json(404, {"error": "not found"}) - - return Handler - - -def main() -> int: - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) - index = Index(INDEX_PATH) - creds = Credentials() - syncer = Syncer(index, creds, RESYNC_SECONDS) - syncer.start() - server = ThreadingHTTPServer(("", LISTEN_PORT), make_handler(index, syncer, creds)) - LOG.info("listening on :%d, repo %s@%s, embed %s", LISTEN_PORT, NOTES_REPO, NOTES_BRANCH, EMBED_URL) - server.serve_forever() - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/kubernetes/apps/wiki-index/deployment.yaml b/kubernetes/apps/wiki-index/deployment.yaml deleted file mode 100644 index 27540bb3..00000000 --- a/kubernetes/apps/wiki-index/deployment.yaml +++ /dev/null @@ -1,137 +0,0 @@ ---- -# ServiceAccount is the OpenBao identity (auth/kubernetes/role/wiki-index); no -# Kubernetes API access needed. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: wiki-index - namespace: ai - labels: - app.kubernetes.io/name: wiki-index ---- -# Index (~17 MB JSON for 1700 notes). Survives restarts so a pod start does not -# re-embed the wiki; a lost volume only costs one full sync (~5 min). -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: wiki-index-data - namespace: ai -spec: - accessModes: - - ReadWriteOnce - storageClassName: ceph-block - resources: - requests: - storage: 1Gi ---- -# Embedding index over the Notes repo's wiki/ (ci/images/wiki-index). Syncs from -# Gitea on push webhook and every RESYNC_SECONDS, embeds via wiki-embed, serves -# POST /rank to ~/Notes/bin/wiki_rank.py through wiki-index.renner.uno. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: wiki-index - namespace: ai - labels: - app.kubernetes.io/name: wiki-index -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: wiki-index - template: - metadata: - labels: - app.kubernetes.io/name: wiki-index - spec: - serviceAccountName: wiki-index - securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - fsGroup: 65534 - seccompProfile: - type: RuntimeDefault - containers: - - name: wiki-index - # Per-commit tag (build-ci-images.yml): immutable, no same-day overwrite. - image: gitea.renner.uno/gitea_admin/wiki-index:2026-09-20-4b529cd1 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - env: - - name: GITEA_URL - value: https://gitea.renner.uno - - name: NOTES_REPO - value: k.renner/notes - - name: NOTES_BRANCH - value: main - - name: EMBED_URL - value: http://wiki-embed.ai.svc.cluster.local:8080 - - name: BAO_ADDR - value: http://openbao.openbao.svc.cluster.local:8200 - - name: BAO_ROLE - value: wiki-index - - name: BAO_SECRET_PATH - value: homelab/data/apps/ai/wiki-index - - name: INDEX_PATH - value: /data/index.json - - name: RESYNC_SECONDS - value: "1800" - ports: - - name: http - containerPort: 8080 - # Pure-Python cosine over 1700 x 1024 floats per query; the index itself is ~60 MB in memory. - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: "1" - memory: 512Mi - # /health: the process serves (liveness). /ready: an index exists; a cold - # start on an empty PVC embeds every note (~5 min) before traffic arrives. - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 5 - failureThreshold: 12 - readinessProbe: - httpGet: - path: /ready - port: http - periodSeconds: 10 - livenessProbe: - httpGet: - path: /health - port: http - periodSeconds: 60 - timeoutSeconds: 10 - failureThreshold: 3 - volumeMounts: - - name: data - mountPath: /data - volumes: - - name: data - persistentVolumeClaim: - claimName: wiki-index-data ---- -apiVersion: v1 -kind: Service -metadata: - name: wiki-index - namespace: ai - labels: - app.kubernetes.io/name: wiki-index -spec: - selector: - app.kubernetes.io/name: wiki-index - ports: - - name: http - port: 8080 - targetPort: http diff --git a/kubernetes/apps/wiki-index/ingress.yaml b/kubernetes/apps/wiki-index/ingress.yaml deleted file mode 100644 index 9c01a9ed..00000000 --- a/kubernetes/apps/wiki-index/ingress.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -# wiki-index.renner.uno — LAN-only (no dns.renner.uno/public label), machine -# client (bin/wiki_rank.py), so Traefik BasicAuth like wiki-embed.renner.uno. -# Gitea's webhook does not pass here; it targets the ClusterIP Service directly. -apiVersion: traefik.io/v1alpha1 -kind: Middleware -metadata: - name: wiki-index-basic-auth - namespace: ai -spec: - basicAuth: - secret: wiki-index-basic-auth - realm: wiki-index ---- -# OpenBao key apps/ai/wiki-index-basic-auth, field `users` = htpasswd line. -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: wiki-index-basic-auth - namespace: ai -spec: - refreshInterval: 1h - secretStoreRef: - kind: ClusterSecretStore - name: openbao - target: - name: wiki-index-basic-auth - dataFrom: - - extract: - key: apps/ai/wiki-index-basic-auth diff --git a/kubernetes/apps/wiki-index/kustomization.yaml b/kubernetes/apps/wiki-index/kustomization.yaml deleted file mode 100644 index ab78bd62..00000000 --- a/kubernetes/apps/wiki-index/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - deployment.yaml - - ingress.yaml diff --git a/kubernetes/clusters/homelab/wiki-index.yaml b/kubernetes/clusters/homelab/wiki-index.yaml deleted file mode 100644 index 4af75951..00000000 --- a/kubernetes/clusters/homelab/wiki-index.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: wiki-index - namespace: flux-system -spec: - interval: 10m - retryInterval: 1m - # First start on an empty PVC embeds every note (~5 min) before /health is 200. - timeout: 20m - sourceRef: - kind: GitRepository - name: flux-system - path: ./kubernetes/apps/wiki-index - prune: true - wait: true - dependsOn: - - name: infrastructure - - name: infrastructure-crds - - name: infrastructure-ceph-cluster - # wiki-embed (the embedding backend) lives in the llama-server app. - - name: llama-server diff --git a/kubernetes/infrastructure/network-policies/ai-ns.yaml b/kubernetes/infrastructure/network-policies/ai-ns.yaml index fae41b92..e7abb479 100644 --- a/kubernetes/infrastructure/network-policies/ai-ns.yaml +++ b/kubernetes/infrastructure/network-policies/ai-ns.yaml @@ -177,37 +177,3 @@ spec: - ports: - port: "8080" protocol: TCP ---- -# wiki-index: Traefik (LAN clients behind BasicAuth) and Gitea (push webhook) -# may reach it; it reaches OpenBao for its credentials. Gitea itself is reached -# through Traefik, which the namespace egress above already allows. -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: wiki-index - namespace: ai -spec: - endpointSelector: - matchLabels: - app.kubernetes.io/name: wiki-index - ingress: - - fromEndpoints: - - matchLabels: - "k8s:io.kubernetes.pod.namespace": ingress - app.kubernetes.io/name: traefik - - matchLabels: - "k8s:io.kubernetes.pod.namespace": gitea - app.kubernetes.io/name: gitea - toPorts: - - ports: - - port: "8080" - protocol: TCP - egress: - - toEndpoints: - - matchLabels: - "k8s:io.kubernetes.pod.namespace": openbao - app.kubernetes.io/name: openbao - toPorts: - - ports: - - port: "8200" - protocol: TCP diff --git a/kubernetes/infrastructure/network-policies/gitea-ns.yaml b/kubernetes/infrastructure/network-policies/gitea-ns.yaml index 12cebc7b..c5e3dc06 100644 --- a/kubernetes/infrastructure/network-policies/gitea-ns.yaml +++ b/kubernetes/infrastructure/network-policies/gitea-ns.yaml @@ -65,11 +65,8 @@ spec: protocol: UDP - port: "53" protocol: TCP - # Push webhook of the notes repo to wiki-index and the wiki service. + # Push webhook of the notes repo to the wiki service. - toEndpoints: - - matchLabels: - "k8s:io.kubernetes.pod.namespace": ai - app.kubernetes.io/name: wiki-index - matchLabels: "k8s:io.kubernetes.pod.namespace": ai app.kubernetes.io/name: wiki diff --git a/kubernetes/infrastructure/network-policies/openbao-ns.yaml b/kubernetes/infrastructure/network-policies/openbao-ns.yaml index d146a7c7..b9ecd9d5 100644 --- a/kubernetes/infrastructure/network-policies/openbao-ns.yaml +++ b/kubernetes/infrastructure/network-policies/openbao-ns.yaml @@ -31,16 +31,6 @@ spec: - ports: - port: "8200" protocol: TCP - # wiki-index (ns ai) logs in via auth/kubernetes/role/wiki-index and reads - # its Gitea token and webhook secret on every sync. - - fromEndpoints: - - matchLabels: - "k8s:io.kubernetes.pod.namespace": ai - app.kubernetes.io/name: wiki-index - toPorts: - - ports: - - port: "8200" - protocol: TCP # netbird-service-sync logs in via auth/kubernetes/role/netbird-service-sync # and reads the NetBird PAT on every pass. - fromEndpoints: diff --git a/kubernetes/infrastructure/openbao/policies/wiki-index.hcl b/kubernetes/infrastructure/openbao/policies/wiki-index.hcl deleted file mode 100644 index e4b7596c..00000000 --- a/kubernetes/infrastructure/openbao/policies/wiki-index.hcl +++ /dev/null @@ -1,21 +0,0 @@ -# OpenBao policy: wiki-index -# -# The wiki-index Deployment (namespace ai) reads its Gitea token and the -# webhook HMAC secret on every use. Read-only on that one path. -# -# NOT applied by Flux. Apply ONCE with an admin token: -# -# bao policy write wiki-index kubernetes/infrastructure/openbao/policies/wiki-index.hcl -# bao write auth/kubernetes/role/wiki-index \ -# bound_service_account_names=wiki-index \ -# bound_service_account_namespaces=ai \ -# policies=wiki-index ttl=15m -# -# Secret: homelab/apps/ai/wiki-index, keys `gitea_token` (read:repository on -# the notes repo) and `webhook_secret` (the Gitea webhook's secret). -# -# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy). - -path "homelab/data/apps/ai/wiki-index" { - capabilities = ["read"] -}