From 8d51f1958d7122328ad3d8299e871897d1e38594 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 08:55:20 +0100 Subject: [PATCH 01/16] docs: bounded encyclopedia planning design Co-Authored-By: Claude Opus 5.5 --- ...26-encyclopedia-bounded-planning-design.md | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-26-encyclopedia-bounded-planning-design.md diff --git a/docs/superpowers/specs/2026-10-26-encyclopedia-bounded-planning-design.md b/docs/superpowers/specs/2026-10-26-encyclopedia-bounded-planning-design.md new file mode 100644 index 0000000..4d01f5b --- /dev/null +++ b/docs/superpowers/specs/2026-10-26-encyclopedia-bounded-planning-design.md @@ -0,0 +1,112 @@ +# House encyclopedia: bounded planning + +Supersedes the planning part of `2026-10-24-house-encyclopedia-design.md` (one model call that sees every note and returns the whole plan). Everything else in that spec stays: article shape, public/private split, leak layers, translation, rendering, auth. + +## Why + +The first production run failed: the plan reply for 1977 notes hit the 32k output cap. Raising the cap only moves the wall; the wiki grows. The owner's rule: **no model call may need the whole wiki in its context.** Two calls broke it: + +1. Planning: input and output grow with the note count. +2. Writing: the prompt lists every planned article as a link target (`WriteArticleUseCase.cs:257`), growing with the article count. + +## Goal + +Every model call has an input and output bounded by constants, independent of the size of the wiki. The cost of a nightly run depends on how much changed, not on how big the wiki is. + +Non-goals: changing article content rules, prompts for writing/translation/leak check beyond the link list, the renderer's leak guards, or the UI beyond redirects. + +Anti-goals: anything that passes all notes, all articles or all titles to one call; a plan that is thrown away and rebuilt from scratch each night; renaming or renumbering addresses people may have bookmarked. + +## The plan becomes state + +`articles/_plan.json` is no longer the output of one call; it is a document every run updates in place. + +``` +{ + "articles": [{ "slug", "titleDe", "titleEn", "category", "scope", "sources": ["note-slug", ...] }], + "assignments": { "": ["", ...] | [] }, // [] = deliberately skipped + "redirects": { "": "" } +} +``` + +- `scope`: one plain sentence saying what the article covers. Written by the model when the article is created or split; it is what later assignment calls see instead of the article's notes. +- `assignments` is the source of truth for which notes an article uses; `sources` is derived from it, ordered most relevant first (order from the assignment call, new notes appended). +- `related` is removed from the plan (it was only fed to the write prompt; see Links). +- A slug that appears in `redirects` is never used again for a new article. +- The old format (`coveredNotes`, `related`, no `assignments`) is read as "no plan": the first run bootstraps. Production has no article files yet (the only run so far committed `_state.json` alone), so bootstrap starts from empty; no orphan or slug migration is needed. The old `failedPlan` entry in `_state.json` is ignored. + +## A run + +Same entry point (`POST /api/v1/encyclopedia/run` from Wiki#18, called by the `wiki-encyclopedia` CronJob; curl times out after 10 800 s, the Job after 11 400 s), same freshness gate, same single commit at the end. A bootstrap night (~100 assign calls plus up to `MaxArticlesPerRun` writes) fits that window; if it does not, the per-step caps leave the rest for the next night. Steps, each with its own per-run cap: + +1. **Diff.** Notes in the catalog but not in `assignments` are *new*. Notes in `assignments` but no longer in the catalog are *removed*: dropped from every article's sources. An article left without sources is deleted (its files removed; no redirect, the subject is gone). A note whose content changed keeps its assignment; the existing staleness check rewrites its articles. +2. **Assign** new notes, in batches (below). Cap: `MaxAssignBatchesPerRun` (default 120 — enough to bootstrap the current wiki in one night; ~100 batches). +3. **Split** articles that are too big. Cap: `MaxSplitsPerRun` (default 5). +4. **Fold** articles that are too thin. Cap: `MaxFoldsPerRun` (default 5). +5. **Write** stale articles, as today (`MaxArticlesPerRun`, PilotSlugs, MaxArticles unchanged). + +A step that hits its cap leaves the rest for the next night; nothing done so far is lost. + +## Finding nearby articles + +The vector index already holds one vector per wiki note (`EmbeddingText.For(doc, 1500)`, active embedding model) and answers `QueryAsync(model, vector, k)` for any vector. Articles get no vectors of their own. + +- **Note vector:** re-embedded from `EmbeddingText.For(note, 1500)` with the active `IEmbeddingModel` (local, cheap); no new port method. +- **Nearby articles of a note:** query the note's `NeighbourNotes` (default 30) nearest notes, map each hit through `assignments` to its articles, score an article by the sum of its hits' scores, keep the top `CandidateArticles` (default 10). +- **Nearby articles of an article:** the same over its first `NeighbourSources` (default 10) sources, excluding itself. + +## Assign + +Batching: take the first unassigned new note (slug order) as seed, add its nearest still-unassigned new notes until `AssignBatchSize` (default 20). Similar notes land in one batch, so a new subject is proposed once, not twenty times. + +One call per batch. Input: for each note `slug | title | tags | body length | first paragraph (≤300 chars)`; the union of the batch's nearby articles (≤ `CandidateArticles` × 2, deduplicated) as `slug | titleDe | category | scope | source count`. Bounded by the constants above. + +Reply (JSON): for each note exactly one of + +- `{"note": "...", "articles": ["slug", ...]}` — existing or new-in-this-reply articles, most relevant first; +- `{"note": "...", "skip": true}` — meta notes and notes about the wiki's own tooling only; + +plus `"new": [{"slug","titleDe","titleEn","category","scope"}]` for articles this batch creates. The prompt carries today's granularity rule: a subject gets its own article only when there is enough material; otherwise the note goes into the article of its parent system. + +Validation (per entry): slug pattern, slug unused and not in `redirects`, references only to candidates or this reply's `new`. Invalid entries are dropped; their notes stay unassigned and come back next run. Leaks in a new article's slug or title are not judged here: as today, the write step's leak layers check title and slug and make the article private when they leak, so nothing is dropped and no extra call is spent at assign time. (Private articles' slugs and titles never reach anonymous readers.) A batch whose reply is unusable as a whole is remembered by the fingerprint of its note set (like `FailedPlan` today, but per batch) so the same batch is not paid for again until one of its notes changes or the prompt version changes. + +## Split + +Candidate: an article whose sources exceed `MaxSourceCharsPerArticle` in body (unchanged, 300 000) or `MaxSourcesPerArticle` (default 120) notes. + +Assignment never refuses a note because an article is full, so an article waiting for its split can grow past the limit. The split input is therefore the article's first `MaxSourcesPerArticle` sources (most relevant first); sources beyond that stay with the overview and are reconsidered by the next split. Biggest articles are split first. Until an article is split, the write step keeps today's `CapSources` (sources past `MaxSourceCharsPerArticle` are left out). + +One call. Input: the article (slug, titles, category, scope) and those sources as `slug | title | tags | body length`. Reply: the article keeps its slug and becomes the overview (new scope, the general notes); 2–6 new narrower articles with their notes. Every source of the original ends up in at least one of them. The overview's write prompt gets its children as link targets. + +Addresses stay: the original slug is never renamed. + +## Fold + +Candidate: an article with fewer than `FoldMinSources` (default 3) sources and less than `FoldMinChars` (default 4 000) body in total, that has at least one nearby article. + +One call. Input: the thin article (slug, titles, scope, its sources' titles) and its nearby articles (≤ `CandidateArticles`, slug | titleDe | scope). Reply: `{"into": "slug"}` or `{"keep": true}`. On fold: the sources move into the target's assignments, the thin article's files are deleted, `redirects[thin] = target`. A kept article is remembered by its subject fingerprint so it is not asked again until its sources change. + +## Links + +The write prompt's "Articles you may link to" becomes the article's nearby articles (≤ `CandidateArticles`), plus for an overview its split children. Bounded. + +A change in an article's link targets does not make it stale: only its own sources and title do (as today), so one split or fold does not rewrite its neighbours. Links stay correct anyway: the renderer resolves `[[slug]]` through `redirects` (a link to a folded article points at its target), and still drops links to unknown or invisible articles. + +## Redirects + +`/house/{lang}/{slug}` for a slug in `redirects` answers 301 to `/house/{lang}/{target}`, following chains (a fold into an article that was later folded) up to 5 hops. The redirect happens only after the visibility check of the final target: an anonymous visitor asking for a redirect whose target is private gets the same 404 as for a missing article, so redirects reveal nothing. + +## Run outcome + +`ReconcileSummary` gains `Assigned`, `Split`, `Folded`. The endpoint answers 502 when the model stopped the run (as today) and when the run ends with an empty plan; a run that made partial progress answers 200. + +## Cost + +Bootstrap of the current wiki: ~100 assign calls of roughly 6k input / 2k output tokens, plus the first splits and folds — a few dollars once. A normal night: the notes distilled that day, a handful of calls. + +## Left to the builder + +- Exact prompt wording (the rules above are binding, the phrasing is not); `Prompts.Version` bump. +- The defaults above may be tuned if tests or the first run show a reason; record the reason. +- How the reconciler's tests are restructured. +- Whether `scope` is also shown on the article page (not required). -- 2.54.0 From fa45a96a818c18d0521ee2f0d0fff975944c722b Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:00:17 +0100 Subject: [PATCH 02/16] docs: bounded encyclopedia planning implementation plan Co-Authored-By: Claude Opus 5.5 --- ...026-10-26-encyclopedia-bounded-planning.md | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-26-encyclopedia-bounded-planning.md diff --git a/docs/superpowers/plans/2026-10-26-encyclopedia-bounded-planning.md b/docs/superpowers/plans/2026-10-26-encyclopedia-bounded-planning.md new file mode 100644 index 0000000..01bc210 --- /dev/null +++ b/docs/superpowers/plans/2026-10-26-encyclopedia-bounded-planning.md @@ -0,0 +1,205 @@ +# Encyclopedia Bounded Planning Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the one-call article planner with a stored plan that every run updates through bounded model calls (assign, split, fold), and bound the writer's link list. + +**Architecture:** `_plan.json` becomes state (articles with scope, note→article assignments, redirects). A run diffs notes against assignments, assigns new notes in batches of similar notes against nearby articles found through the existing per-note vector index, splits oversized and folds thin articles, then writes stale articles as today. The host follows redirects with 301s after the visibility check. + +**Tech Stack:** .NET 10, xunit, hand-written fakes (`test/Wiki.Application.Tests/Fakes.cs`), Markdig, Blazor static SSR. + +**Spec:** `docs/superpowers/specs/2026-10-26-encyclopedia-bounded-planning-design.md` (planning), `docs/superpowers/specs/2026-10-24-house-encyclopedia-design.md` (everything else). + +## Global Constraints + +- No model call's input or output may grow with the number of notes or articles. Every list passed to a prompt is capped by a setting. +- No new NuGet packages, no mocking libraries; tests use the fakes in `Fakes.cs` (`FakeTextModel`, `FakeIndex`, `FakeEmbeddings`, `FakeStore`, `FakeCatalog`). +- `Prompts.Version` goes from 5 to 6. +- Defaults (in `EncyclopediaOptions` and `EncyclopediaSettings`): `AssignBatchSize` 20, `CandidateArticles` 10, `NeighbourNotes` 30, `NeighbourSources` 10, `MaxAssignBatchesPerRun` 120, `MaxSplitsPerRun` 5, `MaxFoldsPerRun` 5, `MaxSourcesPerArticle` 120, `FoldMinSources` 3, `FoldMinChars` 4000, `AssignMaxTokens` 8000, `SplitMaxTokens` 8000, `FoldMaxTokens` 1000. Unchanged: `ArticleMaxTokens` 16000, `MaxArticlesPerRun` 10, `MaxSourceCharsPerArticle` 300000, `PilotSlugs`, `MaxArticles`. `PlanMaxTokens` is removed. +- Slugs match `^[a-z0-9]+(-[a-z0-9]+)*$`. A slug in `redirects` is never reused. +- Comments state facts, terse; match surrounding code style (primary constructors, records, `sealed`). +- One commit per run, as today. Model-unavailable stops the run with what is staged committed. +- Build and test: `env --chdir= dotnet test Wiki.slnx` — all green before each commit. + +## Review Focus + +1. **Bootstrap from an empty plan:** no candidates exist, the first batches only create articles; later batches must see those as candidates *within the same run*. Pinned in Task 8 (`Run_Bootstrap_LaterBatchesSeeEarlierArticles`). +2. **A reply naming notes outside its batch or articles outside its candidates:** those entries are dropped, the batch's other entries kept. Pinned in Task 4. +3. **Removing the last source of an article that is a fold target or split parent:** article deleted, redirects pointing at it are removed so no chain dangles. Pinned in Task 1 (`RemoveNote_LastSource_DeletesArticleAndRedirectsToIt`). +4. **Redirect cycles and long chains:** resolution stops after 5 hops and treats the slug as not found. Pinned in Task 1 and Task 9. +5. **Embedding service down mid-run:** neighbour lookups throw `HttpRequestException`; the run stops assigning, keeps what is staged, still writes stale articles, answers 200. Pinned in Task 8. + +--- + +### Task 1: Plan model v2 + +**Files:** +- Modify: `src/Wiki.Application/Encyclopedia/ArticlePlan.cs` +- Test: `test/Wiki.Application.Tests/ArticlePlanTests.cs` (create) + +**Interfaces:** +- Produces: + - `record PlannedArticle(string Slug, string TitleDe, string TitleEn, ArticleCategory Category, string Scope, IReadOnlyList Sources)` (`Related` removed). + - `sealed class ArticlePlan` — mutable, the only writer of plan state: + - `IReadOnlyList Articles`, `IReadOnlyDictionary> Assignments` (note → article slugs; empty list = skipped), `IReadOnlyDictionary Redirects`. + - `static ArticlePlan Empty()`. + - `bool IsAssigned(string note)`; `PlannedArticle? Find(string slug)`; `bool SlugTaken(string slug)` (article or redirect). + - `void AddArticle(PlannedArticle a)` (throws `InvalidOperationException` when `SlugTaken`). + - `void Assign(string note, IReadOnlyList articles)` — appends the note to each article's `Sources` (no duplicates), records the assignment; `[]` records a skip. + - `IReadOnlyList RemoveNote(string note)` — drops it from every article and from assignments; returns the slugs of articles deleted because they lost their last source. Redirects pointing at a deleted article are deleted too. + - `void ReplaceArticle(PlannedArticle a)` (same slug, new scope/sources; keeps assignments consistent with the new `Sources`). + - `void Fold(string thin, string into)` — moves `thin`'s sources to the end of `into`'s, rewrites assignments, deletes `thin`, sets `Redirects[thin] = into`, and repoints existing redirects whose target was `thin`. + - `string? Resolve(string slug)` — follows `Redirects` up to 5 hops; returns the final article slug, `null` for unknown, cycles or > 5 hops. + - `static class ArticlePlanJson`: `string Write(ArticlePlan)` (indented, camelCase, shape from the spec), `ArticlePlan? Read(string json)` — returns `null` for malformed JSON and for the old format (no `assignments` key); drops articles with bad slug/category; drops assignments to unknown articles. + +- [ ] **Step 1: Write the failing tests** — `ArticlePlanTests`: `AddArticle_TakenSlug_Throws` (incl. a slug that is a redirect), `Assign_AppendsSourcesInOrderWithoutDuplicates`, `Assign_Empty_RecordsSkip` (`IsAssigned` true, no article gains it), `RemoveNote_LastSource_DeletesArticleAndRedirectsToIt` (redirect `a→b`, remove b's only note ⇒ returns `["b"]`, `Redirects` no longer has `a`), `Fold_MovesSourcesAndRedirects` (also repoints `x→thin` to `x→into`), `Resolve_FollowsChain_StopsAtCycleAndFiveHops`, `Json_RoundTrips`, `Json_OldFormat_IsNull` (input `{"articles":[...],"coveredNotes":[...]}`), `Json_AssignmentToUnknownArticle_Dropped`. +- [ ] **Step 2: Run, verify they fail** — `dotnet test Wiki.slnx --filter ArticlePlanTests` → compile errors / failures. +- [ ] **Step 3: Implement** in `ArticlePlan.cs`. Existing callers (`PlanArticlesUseCase`, reconciler, writer, their tests) use `Related`/`CoveredNotes`/the old constructor: adapt them minimally so the solution builds and their tests pass (e.g. the old planner builds an `ArticlePlan` through `AddArticle`/`Assign`); Tasks 7–8 replace them. +- [ ] **Step 4: Run tests** — the new tests pass; the whole solution builds. +- [ ] **Step 5: Commit** `feat(encyclopedia): plan v2 model with assignments and redirects`. + +### Task 2: State v2 + +**Files:** +- Modify: `src/Wiki.Application/Encyclopedia/EncyclopediaState.cs` +- Test: `test/Wiki.Application.Tests/EncyclopediaStateTests.cs` + +**Interfaces:** +- Produces: `record EncyclopediaState(IReadOnlyDictionary Subjects, IReadOnlyDictionary FailedBatches, IReadOnlyDictionary KeptArticles)`. + - `FailedBatches`: fingerprint of a batch whose reply was unusable → when. + - `KeptArticles`: article slug → `SubjectFingerprint` at which the fold call answered keep. + - `static string BatchFingerprint(IReadOnlyList notes)` — SHA256 over `Prompts.Version` and sorted `slug@contentHash`. + - `PlanAttempt` and `NoteSetFingerprint` are removed; JSON with an old `failedPlan` field still reads (field ignored). + - `SubjectFingerprint(PlannedArticle, notes)` keeps its formula minus `Related` (which no longer exists), plus `Scope`. + +- [ ] **Step 1: Failing tests** — `BatchFingerprint_ChangesWhenANoteChanges`, `Read_OldFailedPlanField_Ignored`, `RoundTrip_FailedBatchesAndKept`. +- [ ] **Step 2: Run, verify fail.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests pass; solution builds.** +- [ ] **Step 5: Commit** `feat(encyclopedia): state v2 with per-batch and keep memory`. + +### Task 3: Neighbours + +**Files:** +- Create: `src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs` +- Test: `test/Wiki.Application.Tests/ArticleNeighboursTests.cs` + +**Interfaces:** +- Consumes: `IEmbeddingModel` (active), `IVectorIndex.QueryAsync(modelId, vector, k, ct)`, `EmbeddingText.For(doc, 1500)`, `ArticlePlan` (Task 1). +- Produces: `sealed class ArticleNeighbours(IEmbeddingModel model, IVectorIndex index, EncyclopediaSettings settings)`: + - `Task VectorAsync(Document note, ct)` — embeds `EmbeddingText.For(note, 1500)`; caches per (slug, ContentHash) for the lifetime of the instance. + - `Task> NearbyArticlesAsync(Document note, ArticlePlan plan, ct)` — `NeighbourNotes` hits, mapped through `plan.Assignments`, summed scores, top `CandidateArticles`, ties by slug. + - `Task> NearbyArticlesAsync(PlannedArticle article, IReadOnlyDictionary notes, ArticlePlan plan, ct)` — over its first `NeighbourSources` sources, excluding itself. + - `Task> NearestAsync(Document seed, IReadOnlySet among, int n, IReadOnlyDictionary notes, ct)` — the seed plus its nearest hits that are in `among`, up to `n`; queries `max(n * 3, NeighbourNotes)` hits. +- Exceptions from the embedding model or index propagate. + +- [ ] **Step 1: Failing tests** with `FakeIndex` seeded by hand-made vectors: `NearbyArticles_SumsScoresPerArticle`, `NearbyArticles_CapsAtCandidateArticles`, `NearbyArticles_OfArticle_ExcludesItself`, `Nearest_OnlyAmongAndCapped`, `Vector_CachedPerContentHash` (`FakeEmbeddings.Batches` count). +- [ ] **Step 2–4: Run fail, implement, run pass.** +- [ ] **Step 5: Commit** `feat(encyclopedia): nearby articles through the note index`. + +### Task 4: Assign + +**Files:** +- Create: `src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs` +- Modify: `src/Wiki.Application/Encyclopedia/Prompts.cs` (add `Assign`; remove `Plan`; `Version = 6`) +- Test: `test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs` + +**Interfaces:** +- Consumes: `ITextModel.CompleteAsync(system, user, maxTokens, ct)`, `ArticlePlan`, `EncyclopediaSettings`. +- Produces: + - `record AssignOutcome(IReadOnlyList New, IReadOnlyDictionary> Assignments, bool Unusable)` — `Assignments` keys ⊆ the batch; `[]` = skip; `Unusable` when the reply was truncated, not JSON, or had no valid entry. + - `sealed class AssignNotesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log)` with `Task AssignAsync(IReadOnlyList batch, IReadOnlyList candidates, ArticlePlan plan, ct)`. +- User prompt lines: notes as `slug | title | tags | body length | first paragraph` (first paragraph ≤ 300 chars, flattened like today's `Flatten`); candidates as `slug | titleDe | category | scope | source count`. +- Reply shape: `{"new":[{"slug","titleDe","titleEn","category","scope"}],"notes":[{"note","articles":[...]}|{"note","skip":true}]}`. +- Validation: drop `new` entries with bad slug, taken slug (`plan.SlugTaken`), duplicate, unknown category, empty scope; drop note entries for notes not in the batch, duplicates, or referencing slugs that are neither candidates nor valid `new`; drop `new` articles no note references. `HttpRequestException` 400 → `Unusable`; `ModelUnavailableException` propagates. +- Prompt rules (binding content, wording free): readers and scope as in today's `Plan`; granularity rule (own article only with enough material, else parent system's article); skip only meta/tooling notes; prefer existing candidates; titles/slugs name nothing concrete (no hostnames, IDs, addresses); JSON only, compact. + +- [ ] **Step 1: Failing tests** with `FakeTextModel`: `Assign_ExistingAndNew`, `Assign_NoteOutsideBatch_Dropped`, `Assign_ArticleOutsideCandidates_Dropped`, `Assign_NewWithTakenSlug_DroppedAndItsNotesUnassigned`, `Assign_UnreferencedNew_Dropped`, `Assign_Truncated_IsUnusable`, `Assign_PromptIsBounded` (batch 20, candidates 20 ⇒ user prompt has exactly 20 note lines and ≤ 20 candidate lines regardless of `plan` size — seed the plan with 500 articles), `Assign_ModelUnavailable_Propagates`. +- [ ] **Step 2–4.** +- [ ] **Step 5: Commit** `feat(encyclopedia): assign notes in bounded batches`. + +### Task 5: Split + +**Files:** +- Create: `src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs` +- Modify: `Prompts.cs` (add `Split`) +- Test: `test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs` + +**Interfaces:** +- Produces: + - `record SplitOutcome(PlannedArticle Overview, IReadOnlyList Children)`; `null` = unusable. + - `sealed class SplitArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log)`: `Task SplitAsync(PlannedArticle article, IReadOnlyDictionary notes, ArticlePlan plan, ct)`. + - `static bool NeedsSplit(PlannedArticle a, IReadOnlyDictionary notes, EncyclopediaSettings s)` — body chars of existing sources > `MaxSourceCharsPerArticle` or source count > `MaxSourcesPerArticle`. +- Input: the article and its first `MaxSourcesPerArticle` sources as `slug | title | tags | body length`. +- Reply: `{"overview":{"scope","sources":[...]},"children":[{"slug","titleDe","titleEn","category","scope","sources":[...]}]}`. Valid: 2–6 children, child slugs valid and not taken, every input source in overview or ≥ 1 child, sources ⊆ input. Overview keeps slug, titles, category; its sources = model's overview sources + the sources beyond the input window (in original order). + +- [ ] **Step 1: Failing tests:** `Split_Valid`, `Split_LostSource_IsNull`, `Split_OneChild_IsNull`, `Split_InputCappedAtMaxSources_RestStaysWithOverview`, `NeedsSplit_ByCharsAndByCount`. +- [ ] **Step 2–4.** +- [ ] **Step 5: Commit** `feat(encyclopedia): split oversized articles`. + +### Task 6: Fold + +**Files:** +- Create: `src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs` +- Modify: `Prompts.cs` (add `Fold`) +- Test: `test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs` + +**Interfaces:** +- Produces: + - `sealed class FoldArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log)`: `Task FoldAsync(PlannedArticle thin, IReadOnlyDictionary notes, IReadOnlyList nearby, ct)` — target slug, or `null` for keep / unusable. + - `static bool IsThin(PlannedArticle a, IReadOnlyDictionary notes, EncyclopediaSettings s)` — sources < `FoldMinSources` and total body < `FoldMinChars`. +- Reply: `{"into":"slug"}` (must be one of `nearby`) or `{"keep":true}`. + +- [ ] **Step 1: Failing tests:** `Fold_Into`, `Fold_Keep_IsNull`, `Fold_TargetNotNearby_IsNull`, `IsThin_Thresholds`. +- [ ] **Step 2–4.** +- [ ] **Step 5: Commit** `feat(encyclopedia): fold thin articles into a neighbour`. + +### Task 7: Writer link targets + +**Files:** +- Modify: `src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs` (`IArticleWriter`, user prompt ~l.249-274) +- Test: `test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs` + +**Interfaces:** +- Produces: `IArticleWriter.WriteAsync(PlannedArticle planned, IReadOnlyList sources, IReadOnlyList linkTargets, ct)` — the parameter is renamed and now carries only the bounded link targets; the "Related articles" line is removed; the "Articles you may link to" section lists `linkTargets` minus the article itself. The subject line adds `Scope`. + +- [ ] **Step 1: Failing test:** `Write_LinkListIsExactlyTheTargets` (pass 3 targets, assert the prompt lists those 3 and no other slug), update existing tests for the signature. +- [ ] **Step 2–4.** +- [ ] **Step 5: Commit** `feat(encyclopedia): writer links only to bounded targets`. + +### Task 8: Reconciler + +**Files:** +- Modify: `src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs`, `src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs` (delete; move `EncyclopediaSettings` to `EncyclopediaSettings.cs`), `src/Wiki.Host/Composition/WikiOptions.cs`, `src/Wiki.Host/Program.cs`, `src/Wiki.Host/Api/ApiV1.cs` +- Test: `test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs` (rewrite planning tests; delete `PlanArticlesUseCaseTests.cs`), `test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs` + +**Interfaces:** +- Consumes: Tasks 1–7. +- Produces: + - `EncyclopediaSettings` with the fields of Global Constraints (positional record; update every `new EncyclopediaSettings(...)`). + - `ReconcileSummary(int Planned, int Assigned, int Split, int Folded, int Written, int Deleted, int Rejected, int Remembered, bool StoppedByModel, string? Skipped, string? Commit)`; `Planned` = article count after the run. + - Reconciler takes `ArticleNeighbours`, `AssignNotesUseCase`, `SplitArticleUseCase`, `FoldArticleUseCase` in place of `PlanArticlesUseCase`. DI: `ArticleNeighbours` gets the active `IEmbeddingModel`. +- Run order (spec "A run"): freshness gate → load plan (`ArticlePlanJson.Read`, `null` ⇒ `ArticlePlan.Empty()`) and state → diff (removed notes: `plan.RemoveNote`, collect deleted articles) → assign loop: while new notes remain and batches < `MaxAssignBatchesPerRun`: seed = first new note (ordinal slug order) not yet assigned or tried this run; batch = `NearestAsync(seed, untried, AssignBatchSize)`; if `BatchFingerprint(batch)` is in `FailedBatches`, mark the batch's notes tried and continue without a call (does not count against the cap); candidates = union of `NearbyArticlesAsync(note)` over the batch (dedupe, cap `CandidateArticles * 2`) resolved against the *current, already-updated* plan; apply outcome (`AddArticle`, `Assign`) or remember `FailedBatches[fp]` and mark the batch's notes as tried this run → split loop over `NeedsSplit` articles, biggest first, ≤ `MaxSplitsPerRun` → fold loop over `IsThin` articles not in `KeptArticles` at their current fingerprint, ≤ `MaxFoldsPerRun` → stale/due/write as today, with `linkTargets` = `NearbyArticlesAsync(article)` resolved to articles, plus split children for an overview → delete files of articles no longer in the plan (removed or folded) → stage `_plan.json` if changed → `FinishAsync`. + - `HttpRequestException` from neighbours/embeddings: log, stop the assign/split/fold steps, continue with writing. `ModelUnavailableException` anywhere: stop, commit staged, `StoppedByModel = true`. + - Endpoint (`ApiV1.cs`): 502 also when `summary.Planned == 0`. + - `WikiOptions.EncyclopediaOptions`: add the new fields with defaults, remove `PlanMaxTokens`. + +- [ ] **Step 1: Failing tests** (`EncyclopediaReconcilerTests`, `FakeTextModel` scripted per call; `FakeIndex`/`FakeEmbeddings` seeded through the real `SyncUseCase` as today): `Run_Bootstrap_LaterBatchesSeeEarlierArticles` (40 notes, batch size 20: second assign prompt lists the article the first created), `Run_OnlyNewNotesAreAssigned` (second run with one new note ⇒ one assign call), `Run_RemovedNote_DropsSourceAndDeletesEmptyArticle`, `Run_UnusableBatch_RememberedAndNotRetried` (second run, unchanged notes ⇒ no assign call), `Run_SplitThenWriteOverviewWithChildrenAsLinks`, `Run_FoldDeletesFilesAndAddsRedirect`, `Run_KeptArticle_NotAskedAgain`, `Run_EmbeddingsDown_StillWritesStale` (Review Focus 5), `Run_StepCapsRespected` (`MaxAssignBatchesPerRun` 1 ⇒ one call), `Run_ModelUnavailable_CommitsStaged`. Endpoint: `Run_EmptyPlan_Is502`. +- [ ] **Step 2–4.** +- [ ] **Step 5: Commit** `feat(encyclopedia): reconcile through bounded assign, split and fold`. + +### Task 9: Redirects in the host + +**Files:** +- Modify: `src/Wiki.Application/Reading/ArticleQueries.cs`, `src/Wiki.Host/Components/Pages/HouseArticle.razor`, `src/Wiki.Host/Rendering/ArticleMarkdown.cs`, `src/Wiki.Host/Program.cs` (if DI changes) +- Test: `test/Wiki.Host.Tests/HouseTests.cs`, `test/Wiki.Host.Tests/ArticleMarkdownTests.cs` + +**Interfaces:** +- Produces: + - `ArticleQueries(IDocumentCatalog catalog, INoteStore store)` gains `Task> RedirectsAsync(ct)` — reads `articles/_plan.json` at the catalog's commit via `store.ReadTextAsync`, parses with `ArticlePlanJson.Read`, caches per commit; empty when missing or unreadable. And `Task ResolveAsync(string slug, ct)` = the plan's `Resolve` semantics over those redirects (≤ 5 hops, cycles ⇒ `null`); a slug that is itself an article resolves to itself. + - `HouseArticle.razor`: when `GetAsync(Lang, Slug)` is null, resolve; if the final target exists in `Lang` and is visible to the requester (same checks as a direct request), respond 301 to `/house/{Lang}/{target}`; otherwise the usual 404. + - `ArticleMarkdown.Render(..., IReadOnlyDictionary visibleTitles, IReadOnlyDictionary redirects, ...)`: a `[[slug]]` not in `visibleTitles` is resolved through `redirects` (≤ 5 hops) before being dropped; the link `href` is the target. + +- [ ] **Step 1: Failing tests:** Host `Redirect_ToVisibleTarget_Is301`, `Redirect_ToPrivateTarget_Anonymous_Is404` (same body/status as a missing slug), `Redirect_Chain_Followed`, `Redirect_Cycle_Is404`; Markdown `WikiLink_ToFoldedSlug_PointsAtTarget`, `WikiLink_RedirectToInvisible_Dropped`. +- [ ] **Step 2–4.** +- [ ] **Step 5: Commit** `feat(house): follow article redirects`. -- 2.54.0 From 996ff182c0ac9142e29519f5e03d25e3347936a1 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:04:49 +0100 Subject: [PATCH 03/16] feat(encyclopedia): plan v2 model with assignments and redirects Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/ArticlePlan.cs | 209 ++++++++++++++++-- .../Encyclopedia/PlanArticlesUseCase.cs | 12 +- .../Encyclopedia/WriteArticleUseCase.cs | 1 - .../ArticlePlanTests.cs | 197 +++++++++++++++++ .../EncyclopediaReconcilerTests.cs | 13 +- .../EncyclopediaStateTests.cs | 2 +- .../PlanArticlesUseCaseTests.cs | 37 ++-- .../WriteArticleUseCaseTests.cs | 4 +- 8 files changed, 427 insertions(+), 48 deletions(-) create mode 100644 test/Wiki.Application.Tests/ArticlePlanTests.cs diff --git a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs index 21ec768..b81d638 100644 --- a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs +++ b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs @@ -1,28 +1,161 @@ using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.RegularExpressions; using Wiki.Core.Model; namespace Wiki.Application.Encyclopedia; +/// is one plain sentence on what the article covers; are note slugs, most relevant first. public sealed record PlannedArticle( string Slug, string TitleDe, string TitleEn, ArticleCategory Category, - IReadOnlyList Sources, - IReadOnlyList Related); + string Scope, + IReadOnlyList Sources); -/// The article set. are all wiki slugs seen when it was planned. -public sealed record ArticlePlan(IReadOnlyList Articles, IReadOnlyList CoveredNotes); +/// +/// The stored plan: the articles, which notes feed them, and where retired slugs point. +/// The only writer of plan state, so assignments, sources and redirects stay consistent. +/// +public sealed class ArticlePlan +{ + private const int MaxRedirectHops = 5; -public static class ArticlePlanJson + private readonly List _articles = []; + private readonly Dictionary> _assignments = new(StringComparer.Ordinal); + private readonly Dictionary _redirects = new(StringComparer.Ordinal); + + private ArticlePlan() { } + + public static ArticlePlan Empty() => new(); + + public IReadOnlyList Articles => _articles; + + /// Note slug to article slugs; an empty list marks a note deliberately skipped. + public IReadOnlyDictionary> Assignments => _assignments; + + /// Retired article slug to the slug that took over. + public IReadOnlyDictionary Redirects => _redirects; + + public bool IsAssigned(string note) => _assignments.ContainsKey(note); + + public PlannedArticle? Find(string slug) => _articles.Find(a => a.Slug == slug); + + public bool SlugTaken(string slug) => Find(slug) is not null || _redirects.ContainsKey(slug); + + public void AddArticle(PlannedArticle a) + { + if (SlugTaken(a.Slug)) throw new InvalidOperationException($"Article slug '{a.Slug}' is already taken"); + _articles.Add(a); + } + + /// Appends to each article's sources and records the assignment; no articles records a skip. + public void Assign(string note, IReadOnlyList articles) + { + foreach (var slug in articles) + { + var a = Find(slug) ?? throw new InvalidOperationException($"Unknown article '{slug}'"); + if (!a.Sources.Contains(note)) Replace(a with { Sources = [.. a.Sources, note] }); + } + _assignments[note] = [.. articles.Distinct(StringComparer.Ordinal)]; + } + + /// Drops everywhere. Returns the slugs of articles deleted for lack of sources; redirects to them go too. + public IReadOnlyList RemoveNote(string note) + { + _assignments.Remove(note); + var deleted = new List(); + foreach (var a in _articles.ToList()) + { + if (!a.Sources.Contains(note)) continue; + var sources = a.Sources.Where(s => s != note).ToList(); + if (sources.Count == 0) + { + _articles.Remove(a); + deleted.Add(a.Slug); + } + else Replace(a with { Sources = sources }); + } + foreach (var slug in deleted) + { + foreach (var key in _redirects.Where(r => r.Value == slug).Select(r => r.Key).ToList()) _redirects.Remove(key); + foreach (var n in _assignments.Keys.ToList()) + if (_assignments[n].Contains(slug)) _assignments[n] = [.. _assignments[n].Where(s => s != slug)]; + } + return deleted; + } + + /// Same slug, new content. Assignments follow the new : a note that left loses the article, one that joined gains it. + public void ReplaceArticle(PlannedArticle a) + { + var old = Find(a.Slug) ?? throw new InvalidOperationException($"Unknown article '{a.Slug}'"); + Replace(a); + foreach (var note in old.Sources.Except(a.Sources)) + { + if (!_assignments.TryGetValue(note, out var slugs)) continue; + var rest = slugs.Where(s => s != a.Slug).ToList(); + if (rest.Count == 0) _assignments.Remove(note); + else _assignments[note] = rest; + } + foreach (var note in a.Sources) + { + if (!_assignments.TryGetValue(note, out var slugs)) _assignments[note] = [a.Slug]; + else if (!slugs.Contains(a.Slug)) _assignments[note] = [.. slugs, a.Slug]; + } + } + + /// Merges into and leaves a redirect behind. + public void Fold(string thin, string into) + { + var from = Find(thin) ?? throw new InvalidOperationException($"Unknown article '{thin}'"); + var target = Find(into) ?? throw new InvalidOperationException($"Unknown article '{into}'"); + if (thin == into) throw new InvalidOperationException("Cannot fold an article into itself"); + + Replace(target with { Sources = [.. target.Sources, .. from.Sources.Where(s => !target.Sources.Contains(s))] }); + _articles.Remove(from); + foreach (var note in _assignments.Keys.ToList()) + { + if (!_assignments[note].Contains(thin)) continue; + _assignments[note] = [.. _assignments[note].Select(s => s == thin ? into : s).Distinct(StringComparer.Ordinal)]; + } + foreach (var key in _redirects.Where(r => r.Value == thin).Select(r => r.Key).ToList()) _redirects[key] = into; + _redirects[thin] = into; + } + + /// The article stands for, following redirects; null when unknown, cyclic or more than five hops. + public string? Resolve(string slug) + { + var current = slug; + for (var hops = 0; hops <= MaxRedirectHops; hops++) + { + if (Find(current) is not null) return current; + if (!_redirects.TryGetValue(current, out var next)) return null; + current = next; + } + return null; + } + + /// Plan-file loading only: trusts the caller to have checked slugs. + internal void SetRedirect(string from, string to) => _redirects[from] = to; + + /// Plan-file loading only. + internal void SetAssignment(string note, IReadOnlyList articles) => _assignments[note] = articles; + + private void Replace(PlannedArticle a) => _articles[_articles.FindIndex(x => x.Slug == a.Slug)] = a; +} + +public static partial class ArticlePlanJson { private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { WriteIndented = true }; private static readonly JsonSerializerOptions CompactOptions = new(JsonSerializerDefaults.Web); + [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] + private static partial Regex SlugPattern(); + public static string Write(ArticlePlan p) => - JsonSerializer.Serialize(new { Articles = Shape(p), p.CoveredNotes }, Options); + JsonSerializer.Serialize(new { Articles = Shape(p), p.Assignments, p.Redirects }, Options); /// Only the articles array, compact: the form the model sees and answers in. public static string WriteArticlesCompact(ArticlePlan p) => @@ -34,31 +167,38 @@ public static class ArticlePlanJson a.TitleDe, a.TitleEn, Category = a.Category.ToString().ToLowerInvariant(), + a.Scope, a.Sources, - a.Related, }); /// - /// The plan in , which may be wrapped in a code fence; null when it is not a plan. - /// Articles with a missing slug or an unknown category are skipped. Never throws. + /// The plan in , which may be wrapped in a code fence; null when it is not a plan, + /// including the old format without assignments. Articles with a bad slug or category are skipped, + /// as are assignments to unknown articles. Never throws. /// public static ArticlePlan? Read(string json) { try { - if (JsonNode.Parse(StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items) return null; - var articles = new List(); - foreach (var item in items) + if (JsonNode.Parse(StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items + || root["assignments"] is not JsonObject assignments) return null; + var plan = ArticlePlan.Empty(); + foreach (var a in Articles(items)) + if (!plan.SlugTaken(a.Slug)) plan.AddArticle(a); + if (root["redirects"] is JsonObject redirects) + foreach (var (from, to) in redirects) + { + var target = Text(to); + if (target is not null && SlugPattern().IsMatch(from) && plan.Find(from) is null) plan.SetRedirect(from, target); + } + foreach (var (note, value) in assignments) { - if (item is not JsonObject o) continue; - var slug = Text(o["slug"]); - var categoryText = Text(o["category"]); - if (slug is null || categoryText is null || !categoryText.All(char.IsAsciiLetter) - || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) continue; - articles.Add(new PlannedArticle(slug, Text(o["titleDe"]) ?? slug, Text(o["titleEn"]) ?? slug, category, - Strings(o["sources"]), Strings(o["related"]))); + if (value is not JsonArray slugs) continue; + var all = Strings(slugs); + var known = all.Where(s => plan.Find(s) is not null).Distinct(StringComparer.Ordinal).ToList(); + if (all.Count == 0 || known.Count > 0) plan.SetAssignment(note, known); } - return new ArticlePlan(articles, Strings(root["coveredNotes"])); + return plan; } catch (Exception e) when (e is JsonException or InvalidOperationException) { @@ -66,6 +206,35 @@ public static class ArticlePlanJson } } + /// The articles array of a model reply in the one-shot format (no assignments); null when it is not JSON with one. Never throws. + internal static IReadOnlyList? ReadArticles(string json) + { + try + { + return JsonNode.Parse(StripFence(json)) is JsonObject root && root["articles"] is JsonArray items ? Articles(items) : null; + } + catch (Exception e) when (e is JsonException or InvalidOperationException) + { + return null; + } + } + + private static List Articles(JsonArray items) + { + var articles = new List(); + foreach (var item in items) + { + if (item is not JsonObject o) continue; + var slug = Text(o["slug"]); + var categoryText = Text(o["category"]); + if (slug is null || !SlugPattern().IsMatch(slug) || categoryText is null || !categoryText.All(char.IsAsciiLetter) + || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) continue; + articles.Add(new PlannedArticle(slug, Text(o["titleDe"]) ?? slug, Text(o["titleEn"]) ?? slug, category, + Text(o["scope"]) ?? "", Strings(o["sources"]))); + } + return articles; + } + /// Trimmed, as frontmatter reads it back; null when blank. private static string? Text(JsonNode? n) => n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; diff --git a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs b/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs index b1aaf70..4ec8778 100644 --- a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs @@ -29,7 +29,7 @@ public sealed partial class PlanArticlesUseCase(ITextModel model, EncyclopediaSe public static bool NeedsReplan(ArticlePlan? current, PlanAttempt? failedPlan, IReadOnlyList notes) { var changed = current is null - || !current.CoveredNotes.ToHashSet(StringComparer.Ordinal).SetEquals(notes.Select(n => n.Slug)); + || !current.Assignments.Keys.ToHashSet(StringComparer.Ordinal).SetEquals(notes.Select(n => n.Slug)); return changed && failedPlan?.NoteSetFingerprint != EncyclopediaState.NoteSetFingerprint(notes); } @@ -55,20 +55,24 @@ public sealed partial class PlanArticlesUseCase(ITextModel model, EncyclopediaSe return null; } - var parsed = ArticlePlanJson.Read(reply.Text); + var parsed = ArticlePlanJson.ReadArticles(reply.Text); if (parsed is null) { log.LogWarning("Plan reply is not valid JSON; keeping the current plan"); return null; } - var articles = Validate(parsed.Articles, notes); + var articles = Validate(parsed, notes); if (articles.Count == 0) { log.LogWarning("Plan reply has no valid article; keeping the current plan"); return null; } - return new ArticlePlan(articles, notes.Select(n => n.Slug).Distinct(StringComparer.Ordinal).ToList()); + var plan = ArticlePlan.Empty(); + foreach (var a in articles) plan.AddArticle(a); + foreach (var slug in notes.Select(n => n.Slug).Distinct(StringComparer.Ordinal)) + plan.Assign(slug, articles.Where(a => a.Sources.Contains(slug)).Select(a => a.Slug).ToList()); + return plan; } private List Validate(IReadOnlyList proposed, IReadOnlyList notes) diff --git a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs index 1c87e93..02217de 100644 --- a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs @@ -252,7 +252,6 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe sb.Append("Subject: ").Append(planned.Slug).Append('\n'); sb.Append("German title: ").Append(Flatten(planned.TitleDe)).Append('\n'); sb.Append("Category: ").Append(planned.Category.ToString().ToLowerInvariant()).Append('\n'); - if (planned.Related.Count > 0) sb.Append("Related articles: ").AppendJoin(", ", planned.Related).Append('\n'); sb.Append("\nArticles you may link to (slug | German title):\n"); foreach (var a in allArticles.Where(a => a.Slug != planned.Slug)) diff --git a/test/Wiki.Application.Tests/ArticlePlanTests.cs b/test/Wiki.Application.Tests/ArticlePlanTests.cs new file mode 100644 index 0000000..f5fa602 --- /dev/null +++ b/test/Wiki.Application.Tests/ArticlePlanTests.cs @@ -0,0 +1,197 @@ +using Wiki.Application.Encyclopedia; +using Wiki.Core.Model; + +namespace Wiki.Application.Tests; + +public class ArticlePlanTests +{ + private static PlannedArticle Article(string slug, params string[] sources) => + new(slug, slug.ToUpperInvariant(), slug.ToUpperInvariant(), ArticleCategory.Topic, $"About {slug}.", sources); + + [Fact] + public void AddArticle_TakenSlug_Throws() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a", "n1")); + plan.AddArticle(Article("b", "n2")); + plan.Fold("b", "a"); + + Assert.Throws(() => plan.AddArticle(Article("a"))); + Assert.Throws(() => plan.AddArticle(Article("b"))); + } + + [Fact] + public void Assign_AppendsSourcesInOrderWithoutDuplicates() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a")); + plan.AddArticle(Article("b")); + + plan.Assign("n2", ["a", "b"]); + plan.Assign("n1", ["a"]); + plan.Assign("n1", ["a"]); + + Assert.Equal(["n2", "n1"], plan.Find("a")!.Sources); + Assert.Equal(["n2"], plan.Find("b")!.Sources); + Assert.Equal(["a", "b"], plan.Assignments["n2"]); + } + + [Fact] + public void Assign_Empty_RecordsSkip() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a")); + + plan.Assign("meta", []); + + Assert.True(plan.IsAssigned("meta")); + Assert.False(plan.IsAssigned("other")); + Assert.Empty(plan.Assignments["meta"]); + Assert.Empty(plan.Find("a")!.Sources); + } + + [Fact] + public void RemoveNote_LastSource_DeletesArticleAndRedirectsToIt() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a", "n1")); + plan.AddArticle(Article("b", "n2")); + plan.Assign("n1", ["a"]); + plan.Assign("n2", ["b"]); + plan.Fold("a", "b"); + plan.Assign("n3", ["b"]); + + Assert.Equal("b", plan.Redirects["a"]); + Assert.Empty(plan.RemoveNote("n3")); + Assert.Equal(["b"], plan.RemoveNote("n2").Concat(plan.RemoveNote("n1"))); + + Assert.Null(plan.Find("b")); + Assert.False(plan.Redirects.ContainsKey("a")); + Assert.False(plan.IsAssigned("n2")); + } + + [Fact] + public void Fold_MovesSourcesAndRedirects() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("thin")); + plan.AddArticle(Article("into")); + plan.AddArticle(Article("x")); + plan.Assign("n1", ["into"]); + plan.Assign("n2", ["thin"]); + plan.Assign("n3", ["thin", "into"]); + plan.Fold("x", "thin"); + + plan.Fold("thin", "into"); + + Assert.Null(plan.Find("thin")); + Assert.Equal(["n1", "n3", "n2"], plan.Find("into")!.Sources); + Assert.Equal(["into"], plan.Assignments["n2"]); + Assert.Equal(["into"], plan.Assignments["n3"]); + Assert.Equal("into", plan.Redirects["thin"]); + Assert.Equal("into", plan.Redirects["x"]); + } + + [Fact] + public void Resolve_FollowsChain_StopsAtCycleAndFiveHops() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("end", "n")); + plan.AddArticle(Article("a1", "n")); + plan.Fold("a1", "end"); + + Assert.Equal("end", plan.Resolve("end")); + Assert.Equal("end", plan.Resolve("a1")); + Assert.Null(plan.Resolve("a2")); + Assert.Null(plan.Resolve("unknown")); + + var sixHops = ArticlePlanJson.Read(""" + {"articles":[{"slug":"end","category":"topic"}],"assignments":{}, + "redirects":{"r1":"end","r2":"r1","r3":"r2","r4":"r3","r5":"r4","r6":"r5"}} + """)!; + Assert.Equal("end", sixHops.Resolve("r5")); + Assert.Null(sixHops.Resolve("r6")); + + var cycle = ArticlePlanJson.Read(""" + {"articles":[],"assignments":{},"redirects":{"p":"q","q":"p"}} + """)!; + Assert.Null(cycle.Resolve("p")); + } + + [Fact] + public void ReplaceArticle_KeepsAssignmentsConsistent() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a")); + plan.Assign("n1", ["a"]); + plan.Assign("n2", ["a"]); + + plan.ReplaceArticle(Article("a", "n2", "n3") with { Scope = "New scope." }); + + Assert.Equal("New scope.", plan.Find("a")!.Scope); + Assert.False(plan.IsAssigned("n1")); + Assert.Equal(["a"], plan.Assignments["n3"]); + Assert.Equal(["a"], plan.Assignments["n2"]); + } + + [Fact] + public void Json_RoundTrips() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, "Heating system.", [])); + plan.AddArticle(Article("old")); + plan.Assign("n1", ["heating"]); + plan.Assign("n2", ["heating", "old"]); + plan.Assign("meta", []); + plan.Fold("old", "heating"); + + var json = ArticlePlanJson.Write(plan); + var read = ArticlePlanJson.Read(json); + + Assert.Contains("\"assignments\"", json); + Assert.Contains("\"category\": \"system\"", json); + Assert.NotNull(read); + Assert.Equal(plan.Articles, read.Articles, new PlannedArticleComparer()); + Assert.Equal(["n1", "n2", "meta"], read.Assignments.Keys); + Assert.Equal(["heating"], read.Assignments["n2"]); + Assert.Empty(read.Assignments["meta"]); + Assert.Equal("heating", read.Redirects["old"]); + } + + [Fact] + public void Json_OldFormat_IsNull() => + Assert.Null(ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"topic","sources":["x"],"related":[]}],"coveredNotes":["x"]}""")); + + [Fact] + public void Json_Malformed_IsNull() => Assert.Null(ArticlePlanJson.Read("not json")); + + [Fact] + public void Json_BadSlugOrCategory_ArticleDropped() + { + var plan = ArticlePlanJson.Read(""" + {"articles":[{"slug":"Bad Slug","category":"topic"},{"slug":"a","category":"1"},{"slug":"b","category":"room"}],"assignments":{}} + """); + Assert.Equal(["b"], plan!.Articles.Select(a => a.Slug)); + } + + [Fact] + public void Json_AssignmentToUnknownArticle_Dropped() + { + var plan = ArticlePlanJson.Read(""" + {"articles":[{"slug":"a","category":"topic","sources":["n1"]}], + "assignments":{"n1":["a","ghost"],"n2":["ghost"],"n3":[]}} + """)!; + + Assert.Equal(["a"], plan.Assignments["n1"]); + Assert.False(plan.IsAssigned("n2")); + Assert.True(plan.IsAssigned("n3")); + } + + private sealed class PlannedArticleComparer : IEqualityComparer + { + public bool Equals(PlannedArticle? x, PlannedArticle? y) => + x!.Slug == y!.Slug && x.TitleDe == y.TitleDe && x.TitleEn == y.TitleEn && x.Category == y.Category + && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources); + public int GetHashCode(PlannedArticle obj) => obj.Slug.GetHashCode(); + } +} diff --git a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs index fcf1902..b0d4346 100644 --- a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs @@ -399,9 +399,11 @@ public class EncyclopediaReconcilerTests public void Due_MaxArticles_KeepsOnlyFirstOfPlan() { var notes = new[] { PlanArticlesUseCaseTests.Note("a"), PlanArticlesUseCaseTests.Note("b") }; - var a = new PlannedArticle("a", "A", "A", ArticleCategory.Topic, ["a"], []); - var b = new PlannedArticle("b", "B", "B", ArticleCategory.Topic, ["b"], []); - var plan = new ArticlePlan([a, b], ["a", "b"]); + var a = new PlannedArticle("a", "A", "A", ArticleCategory.Topic, "", ["a"]); + var b = new PlannedArticle("b", "B", "B", ArticleCategory.Topic, "", ["b"]); + var plan = ArticlePlan.Empty(); + plan.AddArticle(a); + plan.AddArticle(b); var due = EncyclopediaReconciler.Due([a, b], plan, notes, EncyclopediaState.Empty, Defaults with { MaxArticles = 1 }); @@ -471,10 +473,11 @@ public class EncyclopediaReconcilerTests public void Stale_TitleOnlyChange_IsStale() { var notes = new[] { PlanArticlesUseCaseTests.Note("a", hash: "h1") }; - var planned = new PlannedArticle("a", "Neu", "New", ArticleCategory.Topic, ["a"], []); + var planned = new PlannedArticle("a", "Neu", "New", ArticleCategory.Topic, "", ["a"]); Article Existing(string lang, string title) => new(lang, "a", title, ArticleCategory.Topic, ArticleVisibility.Public, [new ArticleSource("a", "h1")], new DateOnly(2026, 10, 1), "m", "x\n"); - var plan = new ArticlePlan([planned], ["a"]); + var plan = ArticlePlan.Empty(); + plan.AddArticle(planned); Assert.Empty(EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "New")])); Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Alt"), Existing("en", "New")])); diff --git a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs index e0561f4..5fe6c93 100644 --- a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs @@ -49,7 +49,7 @@ public class EncyclopediaStateTests [Fact] public void SubjectFingerprint_ChangesWithSourceHashOrTitle() { - var planned = new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, ["a", "b"], []); + var planned = new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, "", ["a", "b"]); var notes = new[] { PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "2") }; var baseline = EncyclopediaState.SubjectFingerprint(planned, notes); diff --git a/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs b/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs index eed782a..4e172f1 100644 --- a/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs +++ b/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs @@ -16,8 +16,16 @@ public class PlanArticlesUseCaseTests private static PlanArticlesUseCase UseCase(FakeTextModel model, EncyclopediaSettings? settings = null) => new(model, settings ?? Settings, NullLogger.Instance); + private static ArticlePlan PlanFrom(PlannedArticle article, params string[] notes) + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(article); + foreach (var n in notes) plan.Assign(n, article.Sources.Contains(n) ? [article.Slug] : []); + return plan; + } + private static ArticlePlan PlanOf(params string[] covered) => - new([new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, covered, [])], covered); + PlanFrom(new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, "", covered), covered); [Fact] public void NeedsReplan_NullPlan_True() => @@ -52,10 +60,10 @@ public class PlanArticlesUseCaseTests var model = new FakeTextModel().Reply(""" ```json {"articles":[ - {"slug":"heating","titleDe":"Heizung","titleEn":"Heating","category":"system","sources":["a","ghost","b"],"related":["kitchen"]}, - {"slug":"Bad Slug","titleDe":"x","titleEn":"x","category":"topic","sources":["a"],"related":[]}, - {"slug":"heating","titleDe":"dup","titleEn":"dup","category":"topic","sources":["a"],"related":[]}, - {"slug":"empty","titleDe":"x","titleEn":"x","category":"topic","sources":["ghost"],"related":[]} + {"slug":"heating","titleDe":"Heizung","titleEn":"Heating","category":"system","sources":["a","ghost","b"]}, + {"slug":"Bad Slug","titleDe":"x","titleEn":"x","category":"topic","sources":["a"]}, + {"slug":"heating","titleDe":"dup","titleEn":"dup","category":"topic","sources":["a"]}, + {"slug":"empty","titleDe":"x","titleEn":"x","category":"topic","sources":["ghost"]} ]} ``` """); @@ -67,8 +75,7 @@ public class PlanArticlesUseCaseTests Assert.Equal("heating", article.Slug); Assert.Equal(ArticleCategory.System, article.Category); Assert.Equal(["a", "b"], article.Sources); - Assert.Equal(["kitchen"], article.Related); - Assert.Equal(["a", "b"], plan.CoveredNotes.Order()); + Assert.Equal(["a", "b"], plan.Assignments.Keys.Order()); } [Fact] @@ -81,7 +88,7 @@ public class PlanArticlesUseCaseTests [Fact] public async Task Plan_Truncated_KeepsExistingPlan() { - var json = """{"articles":[{"slug":"heating","titleDe":"H","titleEn":"H","category":"system","sources":["a"],"related":[]}]}"""; + var json = """{"articles":[{"slug":"heating","titleDe":"H","titleEn":"H","category":"system","sources":["a"]}]}"""; var model = new FakeTextModel().Reply(json, truncated: true); Assert.Null(await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); } @@ -110,7 +117,7 @@ public class PlanArticlesUseCaseTests [Fact] public async Task Plan_SourcesOverCharCap_DropsLowestRanked() { - var model = new FakeTextModel().Reply("""{"articles":[{"slug":"big","titleDe":"G","titleEn":"B","category":"topic","sources":["a","b","c"],"related":[]}]}"""); + var model = new FakeTextModel().Reply("""{"articles":[{"slug":"big","titleDe":"G","titleEn":"B","category":"topic","sources":["a","b","c"]}]}"""); var notes = new[] { Note("a", 200_000), Note("b", 200_000), Note("c", 200_000) }; var plan = await UseCase(model).PlanAsync(null, notes, CancellationToken.None); @@ -122,7 +129,7 @@ public class PlanArticlesUseCaseTests public async Task Plan_PromptContainsExistingSlugs() { var model = new FakeTextModel().Reply("nope"); - var current = new ArticlePlan([new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, ["a"], [])], ["a"]); + var current = PlanFrom(new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, "", ["a"]), ["a"]); await UseCase(model).PlanAsync(current, [Note("a", 42, title: "Valve notes")], CancellationToken.None); @@ -149,7 +156,7 @@ public class PlanArticlesUseCaseTests public async Task Plan_PromptAsksForCompactJson_AndOmitsCoveredNotes() { var model = new FakeTextModel().Reply("nope"); - var current = new ArticlePlan([new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, ["a"], [])], ["a", "covered-only"]); + var current = PlanFrom(new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, "", ["a"]), ["a", "covered-only"]); await UseCase(model).PlanAsync(current, [Note("a")], CancellationToken.None); @@ -158,7 +165,7 @@ public class PlanArticlesUseCaseTests Assert.DoesNotContain("coveredNotes", call.User); Assert.DoesNotContain("covered-only", call.User); var jsonLine = call.User.Split('\n').Single(l => l.StartsWith("[{")); - Assert.Equal("""[{"slug":"heating-valves","titleDe":"H","titleEn":"H","category":"device","sources":["a"],"related":[]}]""", jsonLine); + Assert.Equal("""[{"slug":"heating-valves","titleDe":"H","titleEn":"H","category":"device","scope":"","sources":["a"]}]""", jsonLine); } [Fact] @@ -177,7 +184,7 @@ public class PlanArticlesUseCaseTests [Fact] public void ArticlePlanJson_NumericCategory_ArticleSkipped() { - var plan = ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"1","sources":["x"]},{"slug":"b","category":"room","sources":["x"]}]}"""); + var plan = ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"1","sources":["x"]},{"slug":"b","category":"room","sources":["x"]}],"assignments":{}}"""); Assert.Equal(["b"], plan!.Articles.Select(a => a.Slug)); } @@ -195,14 +202,14 @@ public class PlanArticlesUseCaseTests var read = ArticlePlanJson.Read(ArticlePlanJson.Write(plan)); Assert.NotNull(read); Assert.Equal(plan.Articles[0], read.Articles[0],new PlannedArticleComparer()); - Assert.Equal(plan.CoveredNotes, read.CoveredNotes); + Assert.Equal(plan.Assignments.Keys, read.Assignments.Keys); } private sealed class PlannedArticleComparer : IEqualityComparer { public bool Equals(PlannedArticle? x, PlannedArticle? y) => x!.Slug == y!.Slug && x.TitleDe == y.TitleDe && x.TitleEn == y.TitleEn && x.Category == y.Category - && x.Sources.SequenceEqual(y.Sources) && x.Related.SequenceEqual(y.Related); + && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources); public int GetHashCode(PlannedArticle obj) => obj.Slug.GetHashCode(); } } diff --git a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs index 71727d3..57fa72e 100644 --- a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs @@ -12,10 +12,10 @@ public class WriteArticleUseCaseTests private static readonly DateTimeOffset Now = new(2026, 10, 24, 3, 0, 0, TimeSpan.Zero); private static readonly PlannedArticle Planned = - new("wall-switches", "Wandschalter", "Wall switches", ArticleCategory.System, ["schalter"], ["heating"]); + new("wall-switches", "Wandschalter", "Wall switches", ArticleCategory.System, "", ["schalter"]); private static readonly IReadOnlyList All = - [Planned, new("heating", "Heizung", "Heating", ArticleCategory.System, ["heizung"], [])]; + [Planned, new("heating", "Heizung", "Heating", ArticleCategory.System, "", ["heizung"])]; private const string ThreeBlocks = "VISIBILITY: public\n\nAbsatz null.\n\nAbsatz eins.\n\nAbsatz zwei."; private const string WithPrivate = "VISIBILITY: public\n\nAbsatz null.\n\n::: private\nGEHEIM\n:::\n\nAbsatz zwei."; -- 2.54.0 From 13d5495899cfe080b1319f20e8d20beb5da48cce Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:07:19 +0100 Subject: [PATCH 04/16] feat(encyclopedia): state v2 with per-batch and keep memory Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/EncyclopediaReconciler.cs | 4 +- .../Encyclopedia/EncyclopediaState.cs | 22 ++++++++-- .../EncyclopediaStateTests.cs | 42 ++++++++++++++++++- 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs index ef0c25f..219b4af 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs @@ -85,7 +85,7 @@ public sealed class EncyclopediaReconciler( if (plan is null) { log.LogWarning("No usable article plan; no article is written or deleted"); - return await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan), stateText, + return await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText, new ReconcileSummary(0, 0, 0, 0, 0, false, "no-plan", null), ct); } @@ -162,7 +162,7 @@ public sealed class EncyclopediaReconciler( } } - var summary = await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan), stateText, + var summary = await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText, new ReconcileSummary(plan.Articles.Count, written, deleted, rejected, remembered, stopped, null, null), ct); failure?.Throw(); return summary; diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs index 4705da7..7c759cc 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs @@ -22,9 +22,21 @@ public sealed record PlanAttempt(string NoteSetFingerprint, DateTimeOffset At); /// What the nightly job remembers between runs (articles/_state.json), so it does not pay again /// for the same failure. An attempt is repeated only when its fingerprint changes. /// -public sealed record EncyclopediaState(IReadOnlyDictionary Subjects, PlanAttempt? FailedPlan) +/// +/// FailedBatches: fingerprint of an assignment batch whose reply was unusable → when. +/// KeptArticles: article slug → at which the fold call answered keep. +/// +public sealed record EncyclopediaState( + IReadOnlyDictionary Subjects, + PlanAttempt? FailedPlan, + IReadOnlyDictionary FailedBatches, + IReadOnlyDictionary KeptArticles) { - public static EncyclopediaState Empty => new(new Dictionary(), null); + public static EncyclopediaState Empty => + new(new Dictionary(), null, new Dictionary(), new Dictionary()); + + public static string BatchFingerprint(IReadOnlyList notes) => + Hash(Prompts.Version.ToString(), notes.Select(n => $"{n.Slug}@{n.ContentHash}").Order(StringComparer.Ordinal)); public static string NoteSetFingerprint(IReadOnlyList notes) => Hash(Prompts.Version.ToString(), notes.Select(n => n.Slug).Order(StringComparer.Ordinal)); @@ -35,7 +47,7 @@ public sealed record EncyclopediaState(IReadOnlyDictionary $"{s}@{hashes.GetValueOrDefault(s, "")}") .Order(StringComparer.Ordinal); - return Hash([Prompts.Version.ToString(), planned.TitleDe, planned.TitleEn, planned.Category.ToString(), .. sources]); + return Hash([Prompts.Version.ToString(), planned.TitleDe, planned.TitleEn, planned.Category.ToString(), planned.Scope, .. sources]); } private static string Hash(string first, IEnumerable rest) => Hash([first, .. rest]); @@ -62,7 +74,9 @@ public static class EncyclopediaStateJson var s = JsonSerializer.Deserialize(json, Options); if (s?.Subjects is null) return EncyclopediaState.Empty; var subjects = s.Subjects.Where(kv => kv.Value is not null).ToDictionary(kv => kv.Key, kv => kv.Value); - return new EncyclopediaState(subjects, s.FailedPlan); + var failedBatches = (s.FailedBatches ?? new Dictionary()).ToDictionary(); + var kept = (s.KeptArticles ?? new Dictionary()).Where(kv => kv.Value is not null).ToDictionary(); + return new EncyclopediaState(subjects, s.FailedPlan, failedBatches, kept); } catch (JsonException) { diff --git a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs index 5fe6c93..2a388db 100644 --- a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs @@ -16,7 +16,9 @@ public class EncyclopediaStateTests ["heating"] = new("abc", AttemptOutcome.Written, At), ["sauna"] = new("def", AttemptOutcome.Rejected, At), }, - new PlanAttempt("zzz", At)); + new PlanAttempt("zzz", At), + new Dictionary(), + new Dictionary()); var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); @@ -24,6 +26,43 @@ public class EncyclopediaStateTests Assert.Equal(state.FailedPlan, read.FailedPlan); } + [Fact] + public void RoundTrip_FailedBatchesAndKept() + { + var state = new EncyclopediaState( + new Dictionary(), + null, + new Dictionary { ["fp1"] = At }, + new Dictionary { ["heating"] = "fp2" }); + + var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); + + Assert.Equal(state.FailedBatches, read.FailedBatches); + Assert.Equal(state.KeptArticles, read.KeptArticles); + } + + [Fact] + public void Read_OldJsonWithoutNewFields_ReadsEmptyBatchesAndKept() + { + var read = EncyclopediaStateJson.Read("""{"subjects":{},"failedPlan":{"noteSetFingerprint":"x","at":"2026-10-24T03:00:00+00:00"}}"""); + Assert.NotNull(read.FailedPlan); + Assert.Empty(read.FailedBatches); + Assert.Empty(read.KeptArticles); + } + + [Fact] + public void BatchFingerprint_ChangesWhenANoteChanges() + { + var ab = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "2")]); + var ba = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("b", hash: "2"), PlanArticlesUseCaseTests.Note("a", hash: "1")]); + var changed = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "3")]); + var fewer = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1")]); + Assert.Equal(ab, ba); + Assert.NotEqual(ab, changed); + Assert.NotEqual(ab, fewer); + Assert.Matches("^[0-9a-f]{64}$", ab); + } + [Fact] public void EncyclopediaStateJson_Malformed_ReadsEmpty() { @@ -59,6 +98,7 @@ public class EncyclopediaStateTests var changedHash = new[] { notes[0], PlanArticlesUseCaseTests.Note("b", hash: "3") }; Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned, changedHash)); Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { TitleDe = "Heizungsanlage" }, notes)); + Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { Scope = "Only boilers" }, notes)); } [Fact] -- 2.54.0 From 45c76ac4baa01d09f0ac7a31e971b537f5e7c274 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:09:36 +0100 Subject: [PATCH 05/16] feat(encyclopedia): nearby articles through the note index Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/ArticleNeighbours.cs | 73 +++++++++++ .../Encyclopedia/PlanArticlesUseCase.cs | 17 ++- .../ArticleNeighboursTests.cs | 118 ++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs create mode 100644 test/Wiki.Application.Tests/ArticleNeighboursTests.cs diff --git a/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs new file mode 100644 index 0000000..501c4aa --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs @@ -0,0 +1,73 @@ +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Application.Encyclopedia; + +/// +/// Finds the articles and notes near a note through the note index, so prompts list a bounded candidate set +/// instead of every article. Embedding or index failures propagate. +/// +public sealed class ArticleNeighbours(IEmbeddingModel model, IVectorIndex index, EncyclopediaSettings settings) +{ + private const int EmbeddingChars = 1500; + + private readonly Dictionary<(string Slug, string Hash), float[]> _vectors = []; + + public async Task VectorAsync(Document note, CancellationToken ct) + { + var key = (note.Slug, note.ContentHash); + if (_vectors.TryGetValue(key, out var cached)) return cached; + var vector = (await model.EmbedAsync([EmbeddingText.For(note, EmbeddingChars)], ct))[0]; + _vectors[key] = vector; + return vector; + } + + /// Articles that the notes nearest to are assigned to, best first, at most . + public async Task> NearbyArticlesAsync(Document note, ArticlePlan plan, CancellationToken ct) + { + var scores = new Dictionary(StringComparer.Ordinal); + await Accumulate(note, plan, scores, ct); + return Top(scores); + } + + /// Articles near the first sources of , without the article itself. + public async Task> NearbyArticlesAsync( + PlannedArticle article, IReadOnlyDictionary notes, ArticlePlan plan, CancellationToken ct) + { + var scores = new Dictionary(StringComparer.Ordinal); + foreach (var slug in article.Sources.Take(settings.NeighbourSources)) + if (notes.TryGetValue(slug, out var source)) await Accumulate(source, plan, scores, ct); + scores.Remove(article.Slug); + return Top(scores); + } + + /// followed by its nearest notes that are in , at most in all. + public async Task> NearestAsync( + Document seed, IReadOnlySet among, int n, IReadOnlyDictionary notes, CancellationToken ct) + { + var result = new List { seed }; + foreach (var hit in await QueryAsync(seed, Math.Max(n * 3, settings.NeighbourNotes), ct)) + { + if (result.Count >= n) break; + if (hit.Slug != seed.Slug && among.Contains(hit.Slug) && notes.TryGetValue(hit.Slug, out var doc)) result.Add(doc); + } + return result; + } + + private async Task Accumulate(Document note, ArticlePlan plan, Dictionary scores, CancellationToken ct) + { + foreach (var hit in await QueryAsync(note, settings.NeighbourNotes, ct)) + { + if (hit.Slug == note.Slug || !plan.Assignments.TryGetValue(hit.Slug, out var articles)) continue; + foreach (var a in articles) scores[a] = scores.GetValueOrDefault(a) + hit.Score; + } + } + + private async Task> QueryAsync(Document note, int k, CancellationToken ct) => + await index.QueryAsync(model.ModelId, await VectorAsync(note, ct), k, ct); + + private List Top(Dictionary scores) => + scores.OrderByDescending(kv => kv.Value).ThenBy(kv => kv.Key, StringComparer.Ordinal) + .Take(settings.CandidateArticles).Select(kv => kv.Key).ToList(); +} diff --git a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs b/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs index 4ec8778..8d4b904 100644 --- a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs @@ -14,7 +14,22 @@ public sealed record EncyclopediaSettings( int MaxArticlesPerRun, int MaxSourceCharsPerArticle, IReadOnlyList PilotSlugs, - int? MaxArticles); + int? MaxArticles) +{ + public int AssignBatchSize { get; init; } = 20; + public int CandidateArticles { get; init; } = 10; + public int NeighbourNotes { get; init; } = 30; + public int NeighbourSources { get; init; } = 10; + public int MaxAssignBatchesPerRun { get; init; } = 120; + public int MaxSplitsPerRun { get; init; } = 5; + public int MaxFoldsPerRun { get; init; } = 5; + public int MaxSourcesPerArticle { get; init; } = 120; + public int FoldMinSources { get; init; } = 3; + public int FoldMinChars { get; init; } = 4000; + public int AssignMaxTokens { get; init; } = 8000; + public int SplitMaxTokens { get; init; } = 8000; + public int FoldMaxTokens { get; init; } = 1000; +} /// /// Asks the model which subjects get their own article and which notes feed each, and validates the answer. diff --git a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs new file mode 100644 index 0000000..f691e24 --- /dev/null +++ b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs @@ -0,0 +1,118 @@ +using Wiki.Application.Encyclopedia; +using Wiki.Core.Model; +using Wiki.Core.Parsing; + +namespace Wiki.Application.Tests; + +public class ArticleNeighboursTests +{ + private const string Model = "m"; + + private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + + private static Document Note(string slug, string hash = "h") => PlanArticlesUseCaseTests.Note(slug, hash: hash); + + private static PlannedArticle Article(string slug, params string[] sources) => + new(slug, slug, slug, ArticleCategory.Topic, "scope", sources); + + /// Index with the given note vectors; the query note's own vector is the embedding of its text, so tests seed that too. + private sealed record Rig(ArticleNeighbours Neighbours, FakeEmbeddings Embeddings, FakeIndex Index); + + private static async Task RigAsync(EncyclopediaSettings settings, Document query, params (string Slug, float[] Vector)[] vectors) + { + var embeddings = new FakeEmbeddings(Model); + var index = new FakeIndex(); + var q = (await embeddings.EmbedAsync([EmbeddingText.For(query, 1500)], default))[0]; + embeddings.Batches.Clear(); + index.Models[Model] = new(); + index.Models[Model][query.Slug] = ("h", q); + foreach (var (slug, v) in vectors) index.Models[Model][slug] = ("h", v); + return new Rig(new ArticleNeighbours(embeddings, index, settings), embeddings, index); + } + + private static float[] Along(float[] q, float factor) => q.Select(x => x * factor).ToArray(); + + [Fact] + public async Task NearbyArticles_SumsScoresPerArticle() + { + var note = Note("n"); + var q = (await new FakeEmbeddings(Model).EmbedAsync([EmbeddingText.For(note, 1500)], default))[0]; + var rig = await RigAsync(Settings, note, ("a1", Along(q, 0.9f)), ("a2", Along(q, 0.8f)), ("b1", Along(q, 1.5f)), ("skipped", Along(q, 2f))); + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("art-a", "a1", "a2")); + plan.AddArticle(Article("art-b", "b1")); + plan.Assign("a1", ["art-a"]); + plan.Assign("a2", ["art-a"]); + plan.Assign("b1", ["art-b"]); + plan.Assign("skipped", []); + + var nearby = await rig.Neighbours.NearbyArticlesAsync(note, plan, default); + + Assert.Equal(["art-a", "art-b"], nearby); + } + + [Fact] + public async Task NearbyArticles_CapsAtCandidateArticles() + { + var note = Note("n"); + var q = (await new FakeEmbeddings(Model).EmbedAsync([EmbeddingText.For(note, 1500)], default))[0]; + var rig = await RigAsync(Settings with { CandidateArticles = 2 }, note, + ("s1", Along(q, 3f)), ("s2", Along(q, 2f)), ("s3", Along(q, 1f))); + var plan = ArticlePlan.Empty(); + foreach (var i in new[] { "1", "2", "3" }) + { + plan.AddArticle(Article("art-" + i, "s" + i)); + plan.Assign("s" + i, ["art-" + i]); + } + + var nearby = await rig.Neighbours.NearbyArticlesAsync(note, plan, default); + + Assert.Equal(["art-1", "art-2"], nearby); + } + + [Fact] + public async Task NearbyArticles_OfArticle_ExcludesItself() + { + var s1 = Note("s1"); + var q = (await new FakeEmbeddings(Model).EmbedAsync([EmbeddingText.For(s1, 1500)], default))[0]; + var rig = await RigAsync(Settings, s1, ("s2", Along(q, 2f)), ("o1", Along(q, 1f))); + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("self", "s1", "s2")); + plan.AddArticle(Article("other", "o1")); + plan.Assign("s1", ["self"]); + plan.Assign("s2", ["self"]); + plan.Assign("o1", ["other"]); + var notes = new Dictionary { ["s1"] = s1, ["s2"] = Note("s2"), ["o1"] = Note("o1") }; + + var nearby = await rig.Neighbours.NearbyArticlesAsync(plan.Find("self")!, notes, plan, default); + + Assert.Equal(["other"], nearby); + } + + [Fact] + public async Task Nearest_OnlyAmongAndCapped() + { + var seed = Note("seed"); + var q = (await new FakeEmbeddings(Model).EmbedAsync([EmbeddingText.For(seed, 1500)], default))[0]; + var rig = await RigAsync(Settings, seed, ("a", Along(q, 4f)), ("outside", Along(q, 3f)), ("b", Along(q, 2f)), ("c", Along(q, 1f))); + var notes = new[] { "seed", "a", "outside", "b", "c" }.ToDictionary(s => s, s => Note(s)); + + var nearest = await rig.Neighbours.NearestAsync(seed, new HashSet { "seed", "a", "b", "c" }, 3, notes, default); + + Assert.Equal(["seed", "a", "b"], nearest.Select(d => d.Slug)); + } + + [Fact] + public async Task Vector_CachedPerContentHash() + { + var note = Note("n"); + var rig = await RigAsync(Settings, note); + + var first = await rig.Neighbours.VectorAsync(note, default); + var second = await rig.Neighbours.VectorAsync(note, default); + await rig.Neighbours.VectorAsync(Note("n", hash: "other"), default); + + Assert.Same(first, second); + Assert.Equal(2, rig.Embeddings.Batches.Count); + } +} -- 2.54.0 From 348f8e4f7522c3973964758ed757f9f31eb79464 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:11:24 +0100 Subject: [PATCH 06/16] test(encyclopedia): cover self-hit exclusion and ranking before cap Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/ArticleNeighbours.cs | 2 +- .../ArticleNeighboursTests.cs | 41 ++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs index 501c4aa..ffd1edc 100644 --- a/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs +++ b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs @@ -31,7 +31,7 @@ public sealed class ArticleNeighbours(IEmbeddingModel model, IVectorIndex index, return Top(scores); } - /// Articles near the first sources of , without the article itself. + /// Articles near the first sources of , without the article itself. Sources missing from are skipped. public async Task> NearbyArticlesAsync( PlannedArticle article, IReadOnlyDictionary notes, ArticlePlan plan, CancellationToken ct) { diff --git a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs index f691e24..ec56520 100644 --- a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs +++ b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs @@ -57,7 +57,7 @@ public class ArticleNeighboursTests var note = Note("n"); var q = (await new FakeEmbeddings(Model).EmbedAsync([EmbeddingText.For(note, 1500)], default))[0]; var rig = await RigAsync(Settings with { CandidateArticles = 2 }, note, - ("s1", Along(q, 3f)), ("s2", Along(q, 2f)), ("s3", Along(q, 1f))); + ("s1", Along(q, 1f)), ("s2", Along(q, 2f)), ("s3", Along(q, 3f))); var plan = ArticlePlan.Empty(); foreach (var i in new[] { "1", "2", "3" }) { @@ -67,7 +67,44 @@ public class ArticleNeighboursTests var nearby = await rig.Neighbours.NearbyArticlesAsync(note, plan, default); - Assert.Equal(["art-1", "art-2"], nearby); + Assert.Equal(["art-3", "art-2"], nearby); + } + + [Fact] + public async Task NearbyArticles_ExcludesTheNoteItself() + { + var note = Note("n"); + var q = (await new FakeEmbeddings(Model).EmbedAsync([EmbeddingText.For(note, 1500)], default))[0]; + var rig = await RigAsync(Settings, note, ("a1", Along(q, 0.5f))); + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("solo", "n")); + plan.AddArticle(Article("art-a", "a1")); + plan.Assign("n", ["solo"]); + plan.Assign("a1", ["art-a"]); + + var nearby = await rig.Neighbours.NearbyArticlesAsync(note, plan, default); + + Assert.Equal(["art-a"], nearby); + } + + [Fact] + public async Task NearbyArticles_OfArticle_SourceOwnHitDoesNotInflateOthers() + { + var s1 = Note("s1"); + var q = (await new FakeEmbeddings(Model).EmbedAsync([EmbeddingText.For(s1, 1500)], default))[0]; + var rig = await RigAsync(Settings, s1, ("o1", Along(q, 0.5f)), ("o2", Along(q, 0.9f))); + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("self", "s1")); + plan.AddArticle(Article("other", "o1")); + plan.AddArticle(Article("third", "o2")); + plan.Assign("s1", ["self", "other"]); + plan.Assign("o1", ["other"]); + plan.Assign("o2", ["third"]); + var notes = new Dictionary { ["s1"] = s1 }; + + var nearby = await rig.Neighbours.NearbyArticlesAsync(plan.Find("self")!, notes, plan, default); + + Assert.Equal(["third", "other"], nearby); } [Fact] -- 2.54.0 From e4384a2c54c9838b70074cf408b9ce63eb03566c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:13:40 +0100 Subject: [PATCH 07/16] feat(encyclopedia): assign notes in bounded batches Co-Authored-By: Claude Sonnet 5.5 --- .../Encyclopedia/AssignNotesUseCase.cs | 188 ++++++++++++++++++ src/Wiki.Application/Encyclopedia/Prompts.cs | 20 +- .../AssignNotesUseCaseTests.cs | 138 +++++++++++++ 3 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs create mode 100644 test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs diff --git a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs new file mode 100644 index 0000000..66794d8 --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs @@ -0,0 +1,188 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Encyclopedia; + +/// +/// are articles the batch introduces, with the batch notes that feed them as sources. +/// maps batch notes to article slugs; an empty list is a skip. +/// : the reply was truncated, not JSON, rejected, or held no valid entry. +/// +public sealed record AssignOutcome( + IReadOnlyList New, + IReadOnlyDictionary> Assignments, + bool Unusable); + +/// Assigns one batch of notes to nearby existing articles or to new ones, in one bounded model call. +public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +{ + private const int FirstParagraphChars = 300; + + [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] + private static partial Regex SlugPattern(); + + private static AssignOutcome Unusable() => + new([], new Dictionary>(), true); + + /// Model-unavailable and other errors propagate. + public async Task AssignAsync( + IReadOnlyList batch, IReadOnlyList candidates, ArticlePlan plan, CancellationToken ct) + { + var shown = candidates.Take(settings.CandidateArticles * 2).ToList(); + TextReply reply; + try + { + reply = await model.CompleteAsync(Prompts.Assign, UserPrompt(batch, shown), settings.AssignMaxTokens, ct); + } + catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + log.LogWarning(e, "Assign request was rejected"); + return Unusable(); + } + + if (reply.Truncated) + { + log.LogWarning("Assign reply hit the token cap ({Tokens})", settings.AssignMaxTokens); + return Unusable(); + } + + JsonObject? root; + try + { + root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; + } + catch (JsonException) + { + root = null; + } + if (root is null) + { + log.LogWarning("Assign reply is not a JSON object"); + return Unusable(); + } + + var outcome = Validate(root, batch, shown, plan); + if (outcome.Assignments.Count == 0) + { + log.LogWarning("Assign reply has no valid note entry"); + return Unusable(); + } + return outcome; + } + + private AssignOutcome Validate(JsonObject root, IReadOnlyList batch, List candidates, ArticlePlan plan) + { + var candidateSlugs = candidates.Select(c => c.Slug).ToHashSet(StringComparer.Ordinal); + var proposed = new Dictionary(StringComparer.Ordinal); + if (root["new"] is JsonArray items) + foreach (var item in items) + if (item is JsonObject o && ReadNew(o, candidateSlugs, proposed.Keys, plan) is { } a) proposed[a.Slug] = a; + + var inBatch = batch.Select(n => n.Slug).ToHashSet(StringComparer.Ordinal); + var assignments = new Dictionary>(StringComparer.Ordinal); + if (root["notes"] is JsonArray notes) + foreach (var entry in notes) + { + if (entry is not JsonObject o || Text(o["note"]) is not { } note) continue; + if (!inBatch.Contains(note) || assignments.ContainsKey(note)) + { + log.LogWarning("Dropping assignment entry for {Note}: not in the batch or duplicate", note); + continue; + } + if (o["skip"] is JsonValue s && s.TryGetValue(out var skip) && skip) + { + assignments[note] = []; + continue; + } + var all = Strings(o["articles"]); + var valid = all.Where(s => candidateSlugs.Contains(s) || proposed.ContainsKey(s)).Distinct(StringComparer.Ordinal).ToList(); + if (valid.Count == 0) + { + log.LogWarning("Dropping assignment entry for {Note}: no known article", note); + continue; + } + assignments[note] = valid; + } + + var created = new List(); + foreach (var a in proposed.Values) + { + var sources = assignments.Where(kv => kv.Value.Contains(a.Slug)).Select(kv => kv.Key).ToList(); + if (sources.Count == 0) + { + log.LogWarning("Dropping new article {Slug}: no note references it", a.Slug); + continue; + } + created.Add(a with { Sources = sources }); + } + return new AssignOutcome(created, assignments, false); + } + + private PlannedArticle? ReadNew(JsonObject o, HashSet candidateSlugs, IEnumerable seen, ArticlePlan plan) + { + var slug = Text(o["slug"]); + if (slug is null || !SlugPattern().IsMatch(slug)) + { + log.LogWarning("Dropping new article with invalid slug {Slug}", slug); + return null; + } + if (plan.SlugTaken(slug) || candidateSlugs.Contains(slug) || seen.Contains(slug)) + { + log.LogWarning("Dropping new article {Slug}: slug taken or duplicate", slug); + return null; + } + var categoryText = Text(o["category"]); + if (categoryText is null || !categoryText.All(char.IsAsciiLetter) + || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) + { + log.LogWarning("Dropping new article {Slug}: unknown category", slug); + return null; + } + var scope = Text(o["scope"]); + if (scope is null) + { + log.LogWarning("Dropping new article {Slug}: empty scope", slug); + return null; + } + return new PlannedArticle(slug, Text(o["titleDe"]) ?? slug, Text(o["titleEn"]) ?? slug, category, scope, []); + } + + private static string? Text(JsonNode? n) => + n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; + + private static List Strings(JsonNode? n) => + n is JsonArray a ? a.Select(Text).OfType().ToList() : []; + + private static string UserPrompt(IReadOnlyList batch, List candidates) + { + var sb = new StringBuilder(); + sb.Append("Notes (slug | title | tags | body length in chars | first paragraph):\n"); + foreach (var n in batch.OrderBy(n => n.Slug, StringComparer.Ordinal)) + sb.Append(n.Slug).Append(" | ").Append(Flatten(n.Title)).Append(" | ").Append(Flatten(string.Join(", ", n.Tags))) + .Append(" | ").Append(n.Body.Length).Append(" | ").Append(FirstParagraph(n.Body)).Append('\n'); + sb.Append("\nCandidate articles (slug | title | category | scope | source count):\n"); + foreach (var c in candidates) + sb.Append(c.Slug).Append(" | ").Append(Flatten(c.TitleDe)).Append(" | ").Append(c.Category.ToString().ToLowerInvariant()) + .Append(" | ").Append(Flatten(c.Scope)).Append(" | ").Append(c.Sources.Count).Append('\n'); + return sb.ToString(); + } + + /// First non-empty paragraph after leading heading lines, flattened and cut. + private static string FirstParagraph(string body) + { + var lines = body.Replace("\r", "").Split('\n') + .SkipWhile(l => l.Length == 0 || l.TrimStart().StartsWith('#')) + .TakeWhile(l => l.Trim().Length > 0); + var text = Flatten(string.Join(' ', lines)).Trim(); + return text.Length > FirstParagraphChars ? text[..FirstParagraphChars] : text; + } + + /// Keeps a field on its line and inside its column. + private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); +} diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index b27687d..1bbaeb8 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -6,7 +6,25 @@ namespace Wiki.Application.Encyclopedia; /// public static class Prompts { - public const int Version = 5; + public const int Version = 6; + + public const string Assign = """ + [assign] + You assign notes to the articles of an encyclopedia about one household: the house and its homelab IT. Its readers are everyone who needs to know things about the house, from the owner to guests. + You receive a batch of notes, one per line as `slug | title | tags | body length in chars | first paragraph`, and the nearby existing articles, one per line as `slug | title | category | scope | source count`. + + For each note decide which articles it feeds: existing articles from the list, or new articles you propose. + - Prefer an existing article from the list when the note fits its scope. + - Granularity follows the material, like notability on Wikipedia. A subject gets its own new article only when the notes hold enough on it. A subject that was set once and forgotten is not an article of its own; its note feeds the article of its parent system as a section. + - Skip a note only when it is a meta note or about the wiki's own tooling. + - A note may feed several articles. + - Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses. Name the subject by what it is or does, for example "Netzwerkspeicher" rather than the server's hostname. + - Slugs are lowercase ASCII words joined by single hyphens. Categories are room, system, device or topic. scope is one plain sentence on what the article covers. + - Every note of the batch gets exactly one entry. Reference only slugs from the candidate list or from your own new entries. + + Reply with JSON only, no prose and no code fence. Write it compact on a single line, with no indentation, no line breaks and no spaces between tokens, because the reply is size-limited. Shape: + {"new":[{"slug":"...","titleDe":"...","titleEn":"...","category":"room|system|device|topic","scope":"..."}],"notes":[{"note":"note-slug","articles":["article-slug"]},{"note":"note-slug","skip":true}]} + """; /// Placeholder in replaced by MaxSourceCharsPerArticle. public const string MaxSourceCharsToken = "{maxSourceChars}"; diff --git a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs new file mode 100644 index 0000000..9193922 --- /dev/null +++ b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs @@ -0,0 +1,138 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Encyclopedia; +using Wiki.Core.Model; + +namespace Wiki.Application.Tests; + +public class AssignNotesUseCaseTests +{ + private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + + private static AssignNotesUseCase UseCase(FakeTextModel model) => + new(model, Settings, NullLogger.Instance); + + private static PlannedArticle Article(string slug, params string[] sources) => + new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, sources); + + private static ArticlePlan PlanWith(params PlannedArticle[] articles) + { + var plan = ArticlePlan.Empty(); + foreach (var a in articles) plan.AddArticle(a); + return plan; + } + + private static Document[] Batch(params string[] slugs) => + slugs.Select(s => PlanArticlesUseCaseTests.Note(s)).ToArray(); + + [Fact] + public async Task Assign_ExistingAndNew() + { + var heating = Article("heating", "old"); + var model = new FakeTextModel().Reply(""" + {"new":[{"slug":"garden","titleDe":"Garten","titleEn":"Garden","category":"room","scope":"Der Garten."}], + "notes":[{"note":"a","articles":["heating","garden"]},{"note":"b","articles":["garden"]},{"note":"c","skip":true}]} + """); + + var outcome = await UseCase(model).AssignAsync(Batch("a", "b", "c"), [heating], PlanWith(heating), CancellationToken.None); + + Assert.False(outcome.Unusable); + var garden = Assert.Single(outcome.New); + Assert.Equal("garden", garden.Slug); + Assert.Equal(ArticleCategory.Room, garden.Category); + Assert.Equal(["a", "b"], garden.Sources); + Assert.Equal(["heating", "garden"], outcome.Assignments["a"]); + Assert.Empty(outcome.Assignments["c"]); + } + + [Fact] + public async Task Assign_NoteOutsideBatch_Dropped() + { + var heating = Article("heating"); + var model = new FakeTextModel().Reply("""{"new":[],"notes":[{"note":"a","articles":["heating"]},{"note":"zzz","articles":["heating"]}]}"""); + + var outcome = await UseCase(model).AssignAsync(Batch("a"), [heating], PlanWith(heating), CancellationToken.None); + + Assert.Equal(["a"], outcome.Assignments.Keys); + } + + [Fact] + public async Task Assign_ArticleOutsideCandidates_Dropped() + { + var heating = Article("heating"); + var other = Article("other"); + var model = new FakeTextModel().Reply("""{"new":[],"notes":[{"note":"a","articles":["heating","other"]},{"note":"b","articles":["other"]}]}"""); + + var outcome = await UseCase(model).AssignAsync(Batch("a", "b"), [heating], PlanWith(heating, other), CancellationToken.None); + + Assert.Equal(["heating"], outcome.Assignments["a"]); + Assert.False(outcome.Assignments.ContainsKey("b")); + } + + [Fact] + public async Task Assign_NewWithTakenSlug_DroppedAndItsNotesUnassigned() + { + var heating = Article("heating"); + var plan = PlanWith(heating, Article("garden")); + var model = new FakeTextModel().Reply(""" + {"new":[{"slug":"garden","titleDe":"G","titleEn":"G","category":"room","scope":"s"}], + "notes":[{"note":"a","articles":["garden"]},{"note":"b","articles":["heating"]}]} + """); + + var outcome = await UseCase(model).AssignAsync(Batch("a", "b"), [heating], plan, CancellationToken.None); + + Assert.Empty(outcome.New); + Assert.False(outcome.Assignments.ContainsKey("a")); + Assert.Equal(["heating"], outcome.Assignments["b"]); + } + + [Fact] + public async Task Assign_UnreferencedNew_Dropped() + { + var heating = Article("heating"); + var model = new FakeTextModel().Reply(""" + {"new":[{"slug":"garden","titleDe":"G","titleEn":"G","category":"room","scope":"s"}], + "notes":[{"note":"a","articles":["heating"]}]} + """); + + var outcome = await UseCase(model).AssignAsync(Batch("a"), [heating], PlanWith(heating), CancellationToken.None); + + Assert.False(outcome.Unusable); + Assert.Empty(outcome.New); + } + + [Fact] + public async Task Assign_Truncated_IsUnusable() + { + var model = new FakeTextModel().Reply("""{"new":[],"notes":[""", truncated: true); + + var outcome = await UseCase(model).AssignAsync(Batch("a"), [], ArticlePlan.Empty(), CancellationToken.None); + + Assert.True(outcome.Unusable); + Assert.Empty(outcome.Assignments); + } + + [Fact] + public async Task Assign_PromptIsBounded() + { + var articles = Enumerable.Range(0, 500).Select(i => Article($"article-{i}", "n")).ToArray(); + var plan = PlanWith(articles); + var batch = Batch(Enumerable.Range(0, 20).Select(i => $"note-{i:00}").ToArray()); + var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); + + await UseCase(model).AssignAsync(batch, articles.Take(20).ToArray(), plan, CancellationToken.None); + + var user = model.Calls[0].User; + Assert.Equal(20, user.Split('\n').Count(l => l.StartsWith("note-"))); + Assert.True(user.Split('\n').Count(l => l.StartsWith("article-")) <= 20); + Assert.DoesNotContain("article-20 ", user); + } + + [Fact] + public async Task Assign_ModelUnavailable_Propagates() + { + var model = new FakeTextModel().Throw(new ModelUnavailableException("down")); + + await Assert.ThrowsAsync(() => + UseCase(model).AssignAsync(Batch("a"), [], ArticlePlan.Empty(), CancellationToken.None)); + } +} -- 2.54.0 From a5322e71d26a6b3f2dac291e71acdca76afdb73e Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:16:24 +0100 Subject: [PATCH 08/16] fix(encyclopedia): tighten assign tests and first-paragraph handling Co-Authored-By: Claude Sonnet 5.5 --- .../Encyclopedia/AssignNotesUseCase.cs | 11 +++-- src/Wiki.Application/Encyclopedia/Prompts.cs | 2 +- .../AssignNotesUseCaseTests.cs | 45 +++++++++++++++++-- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs index 66794d8..108a6de 100644 --- a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs @@ -150,7 +150,12 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet log.LogWarning("Dropping new article {Slug}: empty scope", slug); return null; } - return new PlannedArticle(slug, Text(o["titleDe"]) ?? slug, Text(o["titleEn"]) ?? slug, category, scope, []); + if (Text(o["titleDe"]) is not { } titleDe || Text(o["titleEn"]) is not { } titleEn) + { + log.LogWarning("Dropping new article {Slug}: missing title", slug); + return null; + } + return new PlannedArticle(slug, titleDe, titleEn, category, scope, []); } private static string? Text(JsonNode? n) => @@ -177,8 +182,8 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet private static string FirstParagraph(string body) { var lines = body.Replace("\r", "").Split('\n') - .SkipWhile(l => l.Length == 0 || l.TrimStart().StartsWith('#')) - .TakeWhile(l => l.Trim().Length > 0); + .SkipWhile(l => string.IsNullOrWhiteSpace(l) || l.TrimStart().StartsWith('#')) + .TakeWhile(l => !string.IsNullOrWhiteSpace(l)); var text = Flatten(string.Join(' ', lines)).Trim(); return text.Length > FirstParagraphChars ? text[..FirstParagraphChars] : text; } diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index 1bbaeb8..8048a33 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -2,7 +2,7 @@ namespace Wiki.Application.Encyclopedia; /// /// Prompt texts of the encyclopedia. is bumped by hand whenever a prompt changes, -/// so that a prompt fix retries earlier rejections and failed plans. +/// so that a prompt fix retries earlier rejections and failed batches. /// public static class Prompts { diff --git a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs index 9193922..5c1c7a2 100644 --- a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs +++ b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs @@ -103,7 +103,7 @@ public class AssignNotesUseCaseTests [Fact] public async Task Assign_Truncated_IsUnusable() { - var model = new FakeTextModel().Reply("""{"new":[],"notes":[""", truncated: true); + var model = new FakeTextModel().Reply("""{"new":[],"notes":[{"note":"a","skip":true}]}""", truncated: true); var outcome = await UseCase(model).AssignAsync(Batch("a"), [], ArticlePlan.Empty(), CancellationToken.None); @@ -119,14 +119,53 @@ public class AssignNotesUseCaseTests var batch = Batch(Enumerable.Range(0, 20).Select(i => $"note-{i:00}").ToArray()); var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); - await UseCase(model).AssignAsync(batch, articles.Take(20).ToArray(), plan, CancellationToken.None); + await UseCase(model).AssignAsync(batch, articles.Take(100).ToArray(), plan, CancellationToken.None); var user = model.Calls[0].User; Assert.Equal(20, user.Split('\n').Count(l => l.StartsWith("note-"))); - Assert.True(user.Split('\n').Count(l => l.StartsWith("article-")) <= 20); + Assert.Equal(20, user.Split('\n').Count(l => l.StartsWith("article-"))); Assert.DoesNotContain("article-20 ", user); } + [Fact] + public async Task Assign_FirstParagraph_SkipsHeadingsAndBlankLines() + { + var note = PlanArticlesUseCaseTests.Note("a") with { Body = "# T\n\n \nfirst line\ncontinued\n\nsecond para" }; + var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); + + await UseCase(model).AssignAsync([note], [], ArticlePlan.Empty(), CancellationToken.None); + + var line = model.Calls[0].User.Split('\n').Single(l => l.StartsWith("a | ")); + Assert.EndsWith(" | first line continued", line); + } + + [Fact] + public async Task Assign_FirstParagraph_CutTo300() + { + var note = PlanArticlesUseCaseTests.Note("a") with { Body = new string('y', 500) }; + var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); + + await UseCase(model).AssignAsync([note], [], ArticlePlan.Empty(), CancellationToken.None); + + var line = model.Calls[0].User.Split('\n').Single(l => l.StartsWith("a | ")); + Assert.EndsWith(" | " + new string('y', 300), line); + } + + [Fact] + public async Task Assign_NewWithoutTitle_Dropped() + { + var heating = Article("heating"); + var model = new FakeTextModel().Reply(""" + {"new":[{"slug":"garden","titleDe":"G","category":"room","scope":"s"}], + "notes":[{"note":"a","articles":["garden","heating"]}]} + """); + + var outcome = await UseCase(model).AssignAsync(Batch("a"), [heating], PlanWith(heating), CancellationToken.None); + + Assert.Empty(outcome.New); + Assert.Equal(["heating"], outcome.Assignments["a"]); + } + [Fact] public async Task Assign_ModelUnavailable_Propagates() { -- 2.54.0 From ad45fe4ac220076aab4fbdc6b3e8b9895e5016a0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:21:16 +0100 Subject: [PATCH 09/16] feat(encyclopedia): split oversized articles Co-Authored-By: Claude Opus 5.5 --- src/Wiki.Application/Encyclopedia/Prompts.cs | 16 ++ .../Encyclopedia/SplitArticleUseCase.cs | 183 +++++++++++++++ .../SplitArticleUseCaseTests.cs | 222 ++++++++++++++++++ 3 files changed, 421 insertions(+) create mode 100644 src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs create mode 100644 test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index 8048a33..ed56930 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -26,6 +26,22 @@ public static class Prompts {"new":[{"slug":"...","titleDe":"...","titleEn":"...","category":"room|system|device|topic","scope":"..."}],"notes":[{"note":"note-slug","articles":["article-slug"]},{"note":"note-slug","skip":true}]} """; + public const string Split = """ + [split] + You split one oversized article of an encyclopedia about one household: the house and its homelab IT. Its readers are everyone who needs to know things about the house, from the owner to guests. + You receive the article as `slug | titleDe | titleEn | category | scope` and its source notes, one per line as `slug | title | tags | body length in chars`. + + The article keeps its slug and becomes the overview of its subject; the narrower subjects move into new child articles. + - The overview keeps the general notes and gets a new scope that fits the general part only. + - Children are narrower subjects of the same thing. Give 2 to 6 children; each has at least one source. + - Every source note goes to the overview or to at least one child. A note may feed several articles. Reference only note slugs from the list. + - Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses. Name the subject by what it is or does. + - Slugs are lowercase ASCII words joined by single hyphens. Categories are room, system, device or topic. scope is one plain sentence on what the article covers. + + Reply with JSON only, no prose and no code fence. Write it compact on a single line, with no indentation, no line breaks and no spaces between tokens, because the reply is size-limited. Shape: + {"overview":{"scope":"...","sources":["note-slug"]},"children":[{"slug":"...","titleDe":"...","titleEn":"...","category":"room|system|device|topic","scope":"...","sources":["note-slug"]}]} + """; + /// Placeholder in replaced by MaxSourceCharsPerArticle. public const string MaxSourceCharsToken = "{maxSourceChars}"; diff --git a/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs new file mode 100644 index 0000000..3a1de92 --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs @@ -0,0 +1,183 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Encyclopedia; + +/// The original article narrowed to an overview, and the 2-6 narrower articles split off it. +public sealed record SplitOutcome(PlannedArticle Overview, IReadOnlyList Children); + +/// Splits one oversized article into an overview and narrower children, in one bounded model call. +public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +{ + private const int MinChildren = 2; + private const int MaxChildren = 6; + + [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] + private static partial Regex SlugPattern(); + + /// Body chars of the sources that exist in exceed the char cap, or the source count exceeds the count cap. + public static bool NeedsSplit(PlannedArticle a, IReadOnlyDictionary notes, EncyclopediaSettings s) => + a.Sources.Count > s.MaxSourcesPerArticle + || a.Sources.Sum(n => notes.TryGetValue(n, out var d) ? (long)d.Body.Length : 0) > s.MaxSourceCharsPerArticle; + + /// Null when the reply is unusable. Model-unavailable and other errors propagate. + public async Task SplitAsync( + PlannedArticle article, IReadOnlyDictionary notes, ArticlePlan plan, CancellationToken ct) + { + var shown = article.Sources.Take(settings.MaxSourcesPerArticle).Where(notes.ContainsKey).Distinct(StringComparer.Ordinal).ToList(); + if (shown.Count == 0) + { + log.LogWarning("Split of {Slug} skipped: no source note is present", article.Slug); + return null; + } + + TextReply reply; + try + { + reply = await model.CompleteAsync(Prompts.Split, UserPrompt(article, shown, notes), settings.SplitMaxTokens, ct); + } + catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + log.LogWarning(e, "Split request for {Slug} was rejected", article.Slug); + return null; + } + + if (reply.Truncated) + { + log.LogWarning("Split reply for {Slug} hit the token cap ({Tokens})", article.Slug, settings.SplitMaxTokens); + return null; + } + + JsonObject? root; + try + { + root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; + } + catch (JsonException) + { + root = null; + } + if (root is null) + { + log.LogWarning("Split reply for {Slug} is not a JSON object", article.Slug); + return null; + } + return Validate(root, article, shown, plan); + } + + private SplitOutcome? Validate(JsonObject root, PlannedArticle article, List shown, ArticlePlan plan) + { + var input = shown.ToHashSet(StringComparer.Ordinal); + if (root["overview"] is not JsonObject ov || Text(ov["scope"]) is not { } overviewScope) + { + log.LogWarning("Split reply for {Slug} has no overview with a scope", article.Slug); + return null; + } + if (root["children"] is not JsonArray items || items.Count is < MinChildren or > MaxChildren) + { + log.LogWarning("Split reply for {Slug} has {Count} children, expected {Min} to {Max}", + article.Slug, (root["children"] as JsonArray)?.Count ?? 0, MinChildren, MaxChildren); + return null; + } + + var overviewSources = Strings(ov["sources"]); + if (overviewSources.Any(s => !input.Contains(s))) + { + log.LogWarning("Split reply for {Slug}: overview names a source outside the input", article.Slug); + return null; + } + + var children = new List(); + var seen = new HashSet(StringComparer.Ordinal) { article.Slug }; + foreach (var item in items) + { + if (item is not JsonObject o || ReadChild(o, article, input, plan, seen) is not { } child) + return null; + seen.Add(child.Slug); + children.Add(child); + } + + var covered = overviewSources.Concat(children.SelectMany(c => c.Sources)).ToHashSet(StringComparer.Ordinal); + if (shown.Any(s => !covered.Contains(s))) + { + log.LogWarning("Split reply for {Slug} loses source {Source}", article.Slug, shown.First(s => !covered.Contains(s))); + return null; + } + + var kept = overviewSources.Distinct(StringComparer.Ordinal) + .OrderBy(s => article.Sources.ToList().IndexOf(s)).ToList(); + var rest = article.Sources.Where(s => !input.Contains(s)).ToList(); + var overviewAll = kept.Concat(rest).ToList(); + if (overviewAll.Count == 0) + { + log.LogWarning("Split reply for {Slug} leaves the overview without sources", article.Slug); + return null; + } + return new SplitOutcome(article with { Scope = overviewScope, Sources = overviewAll }, children); + } + + private PlannedArticle? ReadChild(JsonObject o, PlannedArticle article, HashSet input, ArticlePlan plan, HashSet seen) + { + var slug = Text(o["slug"]); + if (slug is null || !SlugPattern().IsMatch(slug)) + { + log.LogWarning("Split of {Slug}: child with invalid slug {Child}", article.Slug, slug); + return null; + } + if (plan.SlugTaken(slug) || seen.Contains(slug)) + { + log.LogWarning("Split of {Slug}: child slug {Child} is taken or duplicate", article.Slug, slug); + return null; + } + var categoryText = Text(o["category"]); + if (categoryText is null || !categoryText.All(char.IsAsciiLetter) + || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) + { + log.LogWarning("Split of {Slug}: child {Child} has an unknown category", article.Slug, slug); + return null; + } + if (Text(o["scope"]) is not { } scope || Text(o["titleDe"]) is not { } titleDe || Text(o["titleEn"]) is not { } titleEn) + { + log.LogWarning("Split of {Slug}: child {Child} lacks scope or title", article.Slug, slug); + return null; + } + var sources = Strings(o["sources"]).Distinct(StringComparer.Ordinal).ToList(); + if (sources.Count == 0 || sources.Any(s => !input.Contains(s))) + { + log.LogWarning("Split of {Slug}: child {Child} has no sources or one outside the input", article.Slug, slug); + return null; + } + return new PlannedArticle(slug, titleDe, titleEn, category, scope, sources); + } + + private static string? Text(JsonNode? n) => + n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; + + private static List Strings(JsonNode? n) => + n is JsonArray a ? a.Select(Text).OfType().ToList() : []; + + private static string UserPrompt(PlannedArticle article, List shown, IReadOnlyDictionary notes) + { + var sb = new StringBuilder(); + sb.Append("Article to split: ").Append(article.Slug).Append(" | ").Append(Flatten(article.TitleDe)).Append(" | ") + .Append(Flatten(article.TitleEn)).Append(" | ").Append(article.Category.ToString().ToLowerInvariant()) + .Append(" | ").Append(Flatten(article.Scope)).Append('\n'); + sb.Append("\nSources (slug | title | tags | body length in chars):\n"); + foreach (var s in shown) + { + var n = notes[s]; + sb.Append(s).Append(" | ").Append(Flatten(n.Title)).Append(" | ").Append(Flatten(string.Join(", ", n.Tags))) + .Append(" | ").Append(n.Body.Length).Append('\n'); + } + return sb.ToString(); + } + + /// Keeps a field on its line and inside its column. + private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); +} diff --git a/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs b/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs new file mode 100644 index 0000000..094249c --- /dev/null +++ b/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs @@ -0,0 +1,222 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Encyclopedia; +using Wiki.Core.Model; + +namespace Wiki.Application.Tests; + +public class SplitArticleUseCaseTests +{ + private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null) { MaxSourcesPerArticle = 4 }; + + private static SplitArticleUseCase UseCase(FakeTextModel model) => + new(model, Settings, NullLogger.Instance); + + private static PlannedArticle Article(string slug, params string[] sources) => + new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, sources); + + private static Dictionary Notes(params string[] slugs) => + slugs.ToDictionary(s => s, s => PlanArticlesUseCaseTests.Note(s)); + + private static ArticlePlan PlanWith(params PlannedArticle[] articles) + { + var plan = ArticlePlan.Empty(); + foreach (var a in articles) plan.AddArticle(a); + return plan; + } + + private const string Valid = """ + {"overview":{"scope":"Allgemein.","sources":["a"]}, + "children":[{"slug":"x","titleDe":"X","titleEn":"X","category":"device","scope":"sx","sources":["b"]}, + {"slug":"y","titleDe":"Y","titleEn":"Y","category":"topic","scope":"sy","sources":["c","d"]}]} + """; + + [Fact] + public async Task Split_Valid() + { + var article = Article("heating", "a", "b", "c", "d"); + var model = new FakeTextModel().Reply(Valid); + + var outcome = await UseCase(model).SplitAsync(article, Notes("a", "b", "c", "d"), PlanWith(article), CancellationToken.None); + + Assert.NotNull(outcome); + Assert.Equal("heating", outcome.Overview.Slug); + Assert.Equal("Titel heating", outcome.Overview.TitleDe); + Assert.Equal(ArticleCategory.System, outcome.Overview.Category); + Assert.Equal("Allgemein.", outcome.Overview.Scope); + Assert.Equal(["a"], outcome.Overview.Sources); + Assert.Equal(["x", "y"], outcome.Children.Select(c => c.Slug)); + Assert.Equal(ArticleCategory.Device, outcome.Children[0].Category); + Assert.Equal(["c", "d"], outcome.Children[1].Sources); + Assert.Equal(Settings.SplitMaxTokens, model.Calls[0].MaxTokens); + Assert.StartsWith("[split]", model.Calls[0].System); + } + + [Fact] + public async Task Split_LostSource_IsNull() + { + var article = Article("heating", "a", "b", "c", "d"); + var model = new FakeTextModel().Reply(""" + {"overview":{"scope":"s","sources":["a"]}, + "children":[{"slug":"x","titleDe":"X","titleEn":"X","category":"device","scope":"sx","sources":["b"]}, + {"slug":"y","titleDe":"Y","titleEn":"Y","category":"topic","scope":"sy","sources":["c"]}]} + """); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b", "c", "d"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_OneChild_IsNull() + { + var article = Article("heating", "a", "b"); + var model = new FakeTextModel().Reply(""" + {"overview":{"scope":"s","sources":["a"]}, + "children":[{"slug":"x","titleDe":"X","titleEn":"X","category":"device","scope":"sx","sources":["b"]}]} + """); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_SevenChildren_IsNull() + { + var article = Article("heating", "a", "b"); + var children = string.Join(",", Enumerable.Range(0, 7).Select(i => + $$"""{"slug":"c{{i}}","titleDe":"X","titleEn":"X","category":"device","scope":"s","sources":["b"]}""")); + var model = new FakeTextModel().Reply($$"""{"overview":{"scope":"s","sources":["a"]},"children":[{{children}}]}"""); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_ChildSlugTaken_IsNull() + { + var article = Article("heating", "a", "b", "c", "d"); + var model = new FakeTextModel().Reply(Valid); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b", "c", "d"), PlanWith(article, Article("x", "z")), CancellationToken.None)); + } + + [Fact] + public async Task Split_ChildSlugDuplicate_IsNull() + { + var article = Article("heating", "a", "b"); + var model = new FakeTextModel().Reply(""" + {"overview":{"scope":"s","sources":["a"]}, + "children":[{"slug":"x","titleDe":"X","titleEn":"X","category":"device","scope":"sx","sources":["b"]}, + {"slug":"x","titleDe":"Y","titleEn":"Y","category":"topic","scope":"sy","sources":["b"]}]} + """); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_ChildWithoutSources_IsNull() + { + var article = Article("heating", "a", "b"); + var model = new FakeTextModel().Reply(""" + {"overview":{"scope":"s","sources":["a","b"]}, + "children":[{"slug":"x","titleDe":"X","titleEn":"X","category":"device","scope":"sx","sources":["b"]}, + {"slug":"y","titleDe":"Y","titleEn":"Y","category":"topic","scope":"sy","sources":[]}]} + """); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_SourceOutsideInput_IsNull() + { + var article = Article("heating", "a", "b"); + var model = new FakeTextModel().Reply(""" + {"overview":{"scope":"s","sources":["a"]}, + "children":[{"slug":"x","titleDe":"X","titleEn":"X","category":"device","scope":"sx","sources":["b","ghost"]}, + {"slug":"y","titleDe":"Y","titleEn":"Y","category":"topic","scope":"sy","sources":["b"]}]} + """); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_Truncated_IsNull() + { + var article = Article("heating", "a", "b", "c", "d"); + var model = new FakeTextModel().Reply(Valid, truncated: true); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b", "c", "d"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_NotJson_IsNull() + { + var article = Article("heating", "a", "b"); + var model = new FakeTextModel().Reply("sorry"); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_BadRequest_IsNull() + { + var article = Article("heating", "a", "b"); + var model = new FakeTextModel().Throw(new HttpRequestException("bad", null, System.Net.HttpStatusCode.BadRequest)); + + Assert.Null(await UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_ModelUnavailable_Propagates() + { + var article = Article("heating", "a", "b"); + var model = new FakeTextModel().Throw(new ModelUnavailableException("down")); + + await Assert.ThrowsAsync(() => + UseCase(model).SplitAsync(article, Notes("a", "b"), PlanWith(article), CancellationToken.None)); + } + + [Fact] + public async Task Split_InputCappedAtMaxSources_RestStaysWithOverview() + { + var article = Article("heating", "a", "b", "c", "d", "e", "f"); + var model = new FakeTextModel().Reply(Valid); + + var outcome = await UseCase(model).SplitAsync(article, Notes("a", "b", "c", "d", "e", "f"), PlanWith(article), CancellationToken.None); + + Assert.NotNull(outcome); + Assert.Equal(["a", "e", "f"], outcome.Overview.Sources); + var prompt = model.Calls[0].User; + Assert.Contains("\nd | ", prompt); + Assert.DoesNotContain("\ne | ", prompt); + Assert.DoesNotContain("\nf | ", prompt); + } + + [Fact] + public async Task Split_PromptFlattensTitles() + { + var article = Article("heating", "a", "b", "c", "d"); + var notes = Notes("a", "b", "c", "d"); + notes["a"] = PlanArticlesUseCaseTests.Note("a", title: "one | two\nthree"); + var model = new FakeTextModel().Reply(Valid); + + await UseCase(model).SplitAsync(article, notes, PlanWith(article), CancellationToken.None); + + Assert.Contains("\na | one two three | ", model.Calls[0].User); + } + + [Fact] + public void NeedsSplit_ByCharsAndByCount() + { + var s = new EncyclopediaSettings(8000, 16000, 10, 1000, [], null) { MaxSourcesPerArticle = 3 }; + var notes = new Dictionary + { + ["a"] = PlanArticlesUseCaseTests.Note("a", 600), + ["b"] = PlanArticlesUseCaseTests.Note("b", 600), + ["c"] = PlanArticlesUseCaseTests.Note("c", 100), + ["d"] = PlanArticlesUseCaseTests.Note("d", 1), + }; + + Assert.True(SplitArticleUseCase.NeedsSplit(Article("x", "a", "b"), notes, s)); + Assert.False(SplitArticleUseCase.NeedsSplit(Article("x", "a", "c"), notes, s)); + Assert.False(SplitArticleUseCase.NeedsSplit(Article("x", "a", "ghost"), notes, s)); + Assert.True(SplitArticleUseCase.NeedsSplit(Article("x", "c", "d", "c2", "d2"), notes, s)); + Assert.False(SplitArticleUseCase.NeedsSplit(Article("x", "c", "d", "ghost"), notes, s)); + } +} -- 2.54.0 From 2d94f9a8d8ee4b57b60561c0b4decac93c104d72 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:21:32 +0100 Subject: [PATCH 10/16] feat(encyclopedia): fold thin articles into a neighbour Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/FoldArticleUseCase.cs | 83 ++++++++++++ src/Wiki.Application/Encyclopedia/Prompts.cs | 12 ++ .../FoldArticleUseCaseTests.cs | 127 ++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs create mode 100644 test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs diff --git a/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs new file mode 100644 index 0000000..3d15791 --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs @@ -0,0 +1,83 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Encyclopedia; + +/// Asks whether a thin article reads as a section of a nearby one, in one bounded model call. +public sealed class FoldArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +{ + /// Fewer than FoldMinSources sources and less than FoldMinChars body in total; sources missing from count as zero chars. + public static bool IsThin(PlannedArticle a, IReadOnlyDictionary notes, EncyclopediaSettings s) => + a.Sources.Count < s.FoldMinSources + && a.Sources.Sum(n => notes.TryGetValue(n, out var d) ? (long)d.Body.Length : 0) < s.FoldMinChars; + + /// The slug of the nearby article to fold into; null for keep or an unusable reply. Model-unavailable and other errors propagate. + public async Task FoldAsync( + PlannedArticle thin, IReadOnlyDictionary notes, IReadOnlyList nearby, CancellationToken ct) + { + var shown = nearby.Where(n => n.Slug != thin.Slug).Take(settings.CandidateArticles).ToList(); + if (shown.Count == 0) return null; + + TextReply reply; + try + { + reply = await model.CompleteAsync(Prompts.Fold, UserPrompt(thin, notes, shown), settings.FoldMaxTokens, ct); + } + catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + log.LogWarning(e, "Fold request for {Slug} was rejected", thin.Slug); + return null; + } + + if (reply.Truncated) + { + log.LogWarning("Fold reply for {Slug} hit the token cap ({Tokens})", thin.Slug, settings.FoldMaxTokens); + return null; + } + + JsonObject? root; + try + { + root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; + } + catch (JsonException) + { + root = null; + } + if (root is null) + { + log.LogWarning("Fold reply for {Slug} is not a JSON object", thin.Slug); + return null; + } + + if (root["into"] is JsonValue v && v.TryGetValue(out var into) && !string.IsNullOrWhiteSpace(into)) + { + into = into.Trim(); + if (shown.Any(n => n.Slug == into)) return into; + log.LogWarning("Fold reply for {Slug} names {Target}, which is not a nearby article", thin.Slug, into); + } + return null; + } + + private static string UserPrompt(PlannedArticle thin, IReadOnlyDictionary notes, List nearby) + { + var sb = new StringBuilder(); + sb.Append("Thin article: ").Append(thin.Slug).Append(" | ").Append(Flatten(thin.TitleDe)).Append(" | ") + .Append(Flatten(thin.TitleEn)).Append(" | ").Append(Flatten(thin.Scope)).Append('\n'); + sb.Append("Its sources (title):\n"); + foreach (var s in thin.Sources) + sb.Append(notes.TryGetValue(s, out var d) ? Flatten(d.Title) : s).Append('\n'); + sb.Append("\nNearby articles (slug | title | scope):\n"); + foreach (var n in nearby) + sb.Append(n.Slug).Append(" | ").Append(Flatten(n.TitleDe)).Append(" | ").Append(Flatten(n.Scope)).Append('\n'); + return sb.ToString(); + } + + /// Keeps a field on its line and inside its column. + private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); +} diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index ed56930..1c3e871 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -42,6 +42,18 @@ public static class Prompts {"overview":{"scope":"...","sources":["note-slug"]},"children":[{"slug":"...","titleDe":"...","titleEn":"...","category":"room|system|device|topic","scope":"...","sources":["note-slug"]}]} """; + public const string Fold = """ + [fold] + You decide whether a thin article of an encyclopedia about one household (the house and its homelab IT) should become a section of a nearby article. + You receive the thin article as `slug | titleDe | titleEn | scope`, the titles of its source notes, and the nearby articles, one per line as `slug | title | scope`. + + - Fold only when the thin subject reads naturally as a section of one nearby article: a part of a parent system that was set once and forgotten. + - Otherwise keep it. When no nearby article is that parent, keep it. + - Choose the target only from the nearby articles. + + Reply with JSON only, no prose and no code fence: {"into":"article-slug"} to fold, or {"keep":true} to keep. + """; + /// Placeholder in replaced by MaxSourceCharsPerArticle. public const string MaxSourceCharsToken = "{maxSourceChars}"; diff --git a/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs b/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs new file mode 100644 index 0000000..ab6cdcc --- /dev/null +++ b/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs @@ -0,0 +1,127 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Encyclopedia; +using Wiki.Core.Model; + +namespace Wiki.Application.Tests; + +public class FoldArticleUseCaseTests +{ + private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null) { CandidateArticles = 2 }; + + private static FoldArticleUseCase UseCase(FakeTextModel model) => + new(model, Settings, NullLogger.Instance); + + private static PlannedArticle Article(string slug, params string[] sources) => + new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, sources); + + private static Dictionary Notes(params string[] slugs) => + slugs.ToDictionary(s => s, s => PlanArticlesUseCaseTests.Note(s)); + + [Fact] + public async Task Fold_Into() + { + var thin = Article("valve", "a"); + var model = new FakeTextModel().Reply("""{"into":"heating"}"""); + + var target = await UseCase(model).FoldAsync(thin, Notes("a"), [Article("heating", "h"), Article("garden", "g")], CancellationToken.None); + + Assert.Equal("heating", target); + Assert.Equal(Settings.FoldMaxTokens, model.Calls[0].MaxTokens); + Assert.StartsWith("[fold]", model.Calls[0].System); + } + + [Fact] + public async Task Fold_Keep_IsNull() + { + var model = new FakeTextModel().Reply("""{"keep":true}"""); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [Article("heating", "h")], CancellationToken.None)); + } + + [Fact] + public async Task Fold_TargetNotNearby_IsNull() + { + var model = new FakeTextModel().Reply("""{"into":"elsewhere"}"""); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [Article("heating", "h")], CancellationToken.None)); + } + + [Fact] + public async Task Fold_TargetBeyondCandidateCap_IsNull() + { + var nearby = new[] { Article("n1", "1"), Article("n2", "2"), Article("n3", "3") }; + var model = new FakeTextModel().Reply("""{"into":"n3"}"""); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), nearby, CancellationToken.None)); + Assert.DoesNotContain("n3 | ", model.Calls[0].User); + } + + [Fact] + public async Task Fold_IntoItself_IsNull() + { + var model = new FakeTextModel().Reply("""{"into":"valve"}"""); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [Article("valve", "a"), Article("heating", "h")], CancellationToken.None)); + } + + [Fact] + public async Task Fold_NoNearby_NoCall() + { + var model = new FakeTextModel(); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [], CancellationToken.None)); + Assert.Empty(model.Calls); + } + + [Fact] + public async Task Fold_Truncated_IsNull() + { + var model = new FakeTextModel().Reply("""{"into":"heating"}""", truncated: true); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [Article("heating", "h")], CancellationToken.None)); + } + + [Fact] + public async Task Fold_NotJson_IsNull() + { + var model = new FakeTextModel().Reply("hm"); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [Article("heating", "h")], CancellationToken.None)); + } + + [Fact] + public async Task Fold_BadRequest_IsNull() + { + var model = new FakeTextModel().Throw(new HttpRequestException("bad", null, System.Net.HttpStatusCode.BadRequest)); + + Assert.Null(await UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [Article("heating", "h")], CancellationToken.None)); + } + + [Fact] + public async Task Fold_ModelUnavailable_Propagates() + { + var model = new FakeTextModel().Throw(new ModelUnavailableException("down")); + + await Assert.ThrowsAsync(() => + UseCase(model).FoldAsync(Article("valve", "a"), Notes("a"), [Article("heating", "h")], CancellationToken.None)); + } + + [Fact] + public void IsThin_Thresholds() + { + var s = new EncyclopediaSettings(8000, 16000, 10, 300_000, [], null); + var notes = new Dictionary + { + ["a"] = PlanArticlesUseCaseTests.Note("a", 1000), + ["b"] = PlanArticlesUseCaseTests.Note("b", 1000), + ["c"] = PlanArticlesUseCaseTests.Note("c", 1000), + ["big"] = PlanArticlesUseCaseTests.Note("big", 4000), + }; + + Assert.True(FoldArticleUseCase.IsThin(Article("x", "a", "b"), notes, s)); + Assert.False(FoldArticleUseCase.IsThin(Article("x", "a", "b", "c"), notes, s)); + Assert.False(FoldArticleUseCase.IsThin(Article("x", "big"), notes, s)); + Assert.False(FoldArticleUseCase.IsThin(Article("x", "a", "b", "ghost"), notes, s)); + Assert.True(FoldArticleUseCase.IsThin(Article("x", "a"), notes, s)); + } +} -- 2.54.0 From 7a6cfc512453c2af58d8a1ccc0f9cbe921bb1c0c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:24:12 +0100 Subject: [PATCH 11/16] feat(encyclopedia): writer links only to bounded targets Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/WriteArticleUseCase.cs | 11 ++++++----- .../EncyclopediaReconcilerTests.cs | 2 +- .../WriteArticleUseCaseTests.cs | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs index 02217de..ec8dd9b 100644 --- a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs @@ -25,7 +25,7 @@ public sealed class ArticleRejectedException(string slug, string reason) : Excep public interface IArticleWriter { Task WriteAsync(PlannedArticle planned, IReadOnlyList sources, - IReadOnlyList allArticles, CancellationToken ct); + IReadOnlyList linkTargets, CancellationToken ct); } /// @@ -49,12 +49,12 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe /// and other s propagate. /// public async Task WriteAsync(PlannedArticle planned, IReadOnlyList sources, - IReadOnlyList allArticles, CancellationToken ct) + IReadOnlyList linkTargets, CancellationToken ct) { var slug = planned.Slug; var sourceTexts = sources.Select(FullText).ToList(); - var written = await CompleteAsync(slug, "write", Prompts.Write, WriteUserPrompt(planned, sources, allArticles), settings.ArticleMaxTokens, ct); + var written = await CompleteAsync(slug, "write", Prompts.Write, WriteUserPrompt(planned, sources, linkTargets), settings.ArticleMaxTokens, ct); var (visibility, de) = ParseWritten(written.Text); if (de.Length == 0) throw new ArticleRejectedException(slug, "the article body is empty"); @@ -246,15 +246,16 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe return sb.Append(d.Body).ToString(); } - private static string WriteUserPrompt(PlannedArticle planned, IReadOnlyList sources, IReadOnlyList allArticles) + private static string WriteUserPrompt(PlannedArticle planned, IReadOnlyList sources, IReadOnlyList linkTargets) { var sb = new StringBuilder(); sb.Append("Subject: ").Append(planned.Slug).Append('\n'); sb.Append("German title: ").Append(Flatten(planned.TitleDe)).Append('\n'); sb.Append("Category: ").Append(planned.Category.ToString().ToLowerInvariant()).Append('\n'); + sb.Append("Scope: ").Append(Flatten(planned.Scope)).Append('\n'); sb.Append("\nArticles you may link to (slug | German title):\n"); - foreach (var a in allArticles.Where(a => a.Slug != planned.Slug)) + foreach (var a in linkTargets.Where(a => a.Slug != planned.Slug)) sb.Append(a.Slug).Append(" | ").Append(Flatten(a.TitleDe)).Append('\n'); var images = sources.SelectMany(Attachments).Distinct(StringComparer.Ordinal).ToList(); diff --git a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs index b0d4346..58f9c4e 100644 --- a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs @@ -510,7 +510,7 @@ public class EncyclopediaReconcilerTests public Action? Before { get; set; } public Task WriteAsync(PlannedArticle planned, IReadOnlyList sources, - IReadOnlyList allArticles, CancellationToken ct) + IReadOnlyList linkTargets, CancellationToken ct) { Calls.Add(planned.Slug); Before?.Invoke(planned.Slug); diff --git a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs index 57fa72e..901ac26 100644 --- a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs @@ -44,6 +44,24 @@ public class WriteArticleUseCaseTests new WriteArticleUseCase(model, Settings, new FakeTimeProvider(Now)) .WriteAsync(planned, sources.Length == 0 ? [Source()] : sources, All, CancellationToken.None); + [Fact] + public async Task Write_LinkListIsExactlyTheTargets() + { + var targets = new[] { "heating", "lighting", "doors" } + .Select(s => new PlannedArticle(s, s + "-de", s + "-en", ArticleCategory.System, "", [s])).ToList(); + var planned = Planned with { Scope = "Alles zu Schaltern" }; + var model = Model(ThreeBlocks, Clean, """["Zero.","One.","Two."]""", Clean); + + await new WriteArticleUseCase(model, Settings, new FakeTimeProvider(Now)) + .WriteAsync(planned, [Source()], [.. targets, planned], CancellationToken.None); + + var user = model.Calls[0].User; + Assert.Contains("Scope: Alles zu Schaltern\n", user); + Assert.Contains("heating | heating-de\nlighting | lighting-de\ndoors | doors-de\n", user); + Assert.DoesNotContain("wall-switches |", user); + Assert.DoesNotContain("Related articles", user); + } + private static IReadOnlyList Public(string body) => ArticleBody.PublicBlocks(body).Select(b => b.Text).ToList(); [Fact] -- 2.54.0 From b41bee974330e249b43fdf92367de8392eeec0c8 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:42:32 +0100 Subject: [PATCH 12/16] feat(encyclopedia): reconcile through bounded assign, split and fold Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/ArticlePlan.cs | 86 +--- .../Encyclopedia/AssignNotesUseCase.cs | 31 +- .../Encyclopedia/EncyclopediaReconciler.cs | 379 ++++++++++---- .../Encyclopedia/EncyclopediaSettings.cs | 25 + .../Encyclopedia/EncyclopediaState.cs | 14 +- .../Encyclopedia/FoldArticleUseCase.cs | 23 +- .../Encyclopedia/PlanArticlesUseCase.cs | 152 ------ .../Encyclopedia/PlanParsing.cs | 53 ++ src/Wiki.Application/Encyclopedia/Prompts.cs | 23 - .../Encyclopedia/SplitArticleUseCase.cs | 35 +- .../Encyclopedia/WriteArticleUseCase.cs | 4 +- src/Wiki.Host/Api/ApiV1.cs | 4 +- .../Components/Shared/StatusPanel.razor | 2 +- src/Wiki.Host/Composition/WikiOptions.cs | 15 +- src/Wiki.Host/Program.cs | 23 +- .../ArticleNeighboursTests.cs | 4 +- .../ArticlePlanTests.cs | 23 +- .../AssignNotesUseCaseTests.cs | 8 +- .../EncyclopediaCoordinatorTests.cs | 2 +- .../EncyclopediaReconcilerTests.cs | 477 ++++++++++++++---- .../EncyclopediaStateTests.cs | 31 +- test/Wiki.Application.Tests/Fakes.cs | 8 + .../FoldArticleUseCaseTests.cs | 31 +- .../PlanArticlesUseCaseTests.cs | 215 -------- .../SplitArticleUseCaseTests.cs | 16 +- .../WriteArticleUseCaseTests.cs | 2 +- .../EncyclopediaEndpointTests.cs | 17 +- 27 files changed, 916 insertions(+), 787 deletions(-) create mode 100644 src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs delete mode 100644 src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs create mode 100644 src/Wiki.Application/Encyclopedia/PlanParsing.cs delete mode 100644 test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs diff --git a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs index b81d638..26ab426 100644 --- a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs +++ b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs @@ -1,6 +1,5 @@ using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.RegularExpressions; using Wiki.Core.Model; namespace Wiki.Application.Encyclopedia; @@ -145,31 +144,26 @@ public sealed class ArticlePlan private void Replace(PlannedArticle a) => _articles[_articles.FindIndex(x => x.Slug == a.Slug)] = a; } -public static partial class ArticlePlanJson +public static class ArticlePlanJson { private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { WriteIndented = true }; - private static readonly JsonSerializerOptions CompactOptions = new(JsonSerializerDefaults.Web); - - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - + /// Map keys in ordinal order, so an unchanged plan serializes identically. public static string Write(ArticlePlan p) => - JsonSerializer.Serialize(new { Articles = Shape(p), p.Assignments, p.Redirects }, Options); - - /// Only the articles array, compact: the form the model sees and answers in. - public static string WriteArticlesCompact(ArticlePlan p) => - JsonSerializer.Serialize(Shape(p), CompactOptions); - - private static IEnumerable Shape(ArticlePlan p) => p.Articles.Select(a => new - { - a.Slug, - a.TitleDe, - a.TitleEn, - Category = a.Category.ToString().ToLowerInvariant(), - a.Scope, - a.Sources, - }); + JsonSerializer.Serialize(new + { + Articles = p.Articles.Select(a => new + { + a.Slug, + a.TitleDe, + a.TitleEn, + Category = a.Category.ToString().ToLowerInvariant(), + a.Scope, + a.Sources, + }), + Assignments = new SortedDictionary>(p.Assignments.ToDictionary(), StringComparer.Ordinal), + Redirects = new SortedDictionary(p.Redirects.ToDictionary(), StringComparer.Ordinal), + }, Options); /// /// The plan in , which may be wrapped in a code fence; null when it is not a plan, @@ -180,7 +174,7 @@ public static partial class ArticlePlanJson { try { - if (JsonNode.Parse(StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items + if (JsonNode.Parse(PlanParsing.StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items || root["assignments"] is not JsonObject assignments) return null; var plan = ArticlePlan.Empty(); foreach (var a in Articles(items)) @@ -188,13 +182,13 @@ public static partial class ArticlePlanJson if (root["redirects"] is JsonObject redirects) foreach (var (from, to) in redirects) { - var target = Text(to); - if (target is not null && SlugPattern().IsMatch(from) && plan.Find(from) is null) plan.SetRedirect(from, target); + var target = PlanParsing.Text(to); + if (target is not null && PlanParsing.SlugPattern().IsMatch(from) && plan.Find(from) is null) plan.SetRedirect(from, target); } foreach (var (note, value) in assignments) { if (value is not JsonArray slugs) continue; - var all = Strings(slugs); + var all = PlanParsing.Strings(slugs); var known = all.Where(s => plan.Find(s) is not null).Distinct(StringComparer.Ordinal).ToList(); if (all.Count == 0 || known.Count > 0) plan.SetAssignment(note, known); } @@ -206,48 +200,18 @@ public static partial class ArticlePlanJson } } - /// The articles array of a model reply in the one-shot format (no assignments); null when it is not JSON with one. Never throws. - internal static IReadOnlyList? ReadArticles(string json) - { - try - { - return JsonNode.Parse(StripFence(json)) is JsonObject root && root["articles"] is JsonArray items ? Articles(items) : null; - } - catch (Exception e) when (e is JsonException or InvalidOperationException) - { - return null; - } - } - private static List Articles(JsonArray items) { var articles = new List(); foreach (var item in items) { if (item is not JsonObject o) continue; - var slug = Text(o["slug"]); - var categoryText = Text(o["category"]); - if (slug is null || !SlugPattern().IsMatch(slug) || categoryText is null || !categoryText.All(char.IsAsciiLetter) - || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) continue; - articles.Add(new PlannedArticle(slug, Text(o["titleDe"]) ?? slug, Text(o["titleEn"]) ?? slug, category, - Text(o["scope"]) ?? "", Strings(o["sources"]))); + var slug = PlanParsing.Text(o["slug"]); + if (slug is null || !PlanParsing.SlugPattern().IsMatch(slug) + || !PlanParsing.TryCategory(PlanParsing.Text(o["category"]), out var category)) continue; + articles.Add(new PlannedArticle(slug, PlanParsing.Text(o["titleDe"]) ?? slug, PlanParsing.Text(o["titleEn"]) ?? slug, category, + PlanParsing.Text(o["scope"]) ?? "", PlanParsing.Strings(o["sources"]))); } return articles; } - - /// Trimmed, as frontmatter reads it back; null when blank. - private static string? Text(JsonNode? n) => - n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; - - private static List Strings(JsonNode? n) => - n is JsonArray a ? a.Select(Text).OfType().ToList() : []; - - internal static string StripFence(string text) - { - var t = text.Trim(); - if (!t.StartsWith("```")) return t; - var start = t.IndexOf('\n'); - var end = t.LastIndexOf("```", StringComparison.Ordinal); - return start >= 0 && end > start ? t[(start + 1)..end].Trim() : t; - } } diff --git a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs index 108a6de..d38beff 100644 --- a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs @@ -1,11 +1,10 @@ using System.Net; using System.Text; -using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Wiki.Core.Model; using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; namespace Wiki.Application.Encyclopedia; @@ -20,13 +19,10 @@ public sealed record AssignOutcome( bool Unusable); /// Assigns one batch of notes to nearby existing articles or to new ones, in one bounded model call. -public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +public sealed class AssignNotesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) { private const int FirstParagraphChars = 300; - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - private static AssignOutcome Unusable() => new([], new Dictionary>(), true); @@ -52,15 +48,7 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet return Unusable(); } - JsonObject? root; - try - { - root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; - } - catch (JsonException) - { - root = null; - } + var root = PlanParsing.Object(reply.Text); if (root is null) { log.LogWarning("Assign reply is not a JSON object"); @@ -137,9 +125,7 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet log.LogWarning("Dropping new article {Slug}: slug taken or duplicate", slug); return null; } - var categoryText = Text(o["category"]); - if (categoryText is null || !categoryText.All(char.IsAsciiLetter) - || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) + if (!TryCategory(Text(o["category"]), out var category)) { log.LogWarning("Dropping new article {Slug}: unknown category", slug); return null; @@ -158,12 +144,6 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet return new PlannedArticle(slug, titleDe, titleEn, category, scope, []); } - private static string? Text(JsonNode? n) => - n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; - - private static List Strings(JsonNode? n) => - n is JsonArray a ? a.Select(Text).OfType().ToList() : []; - private static string UserPrompt(IReadOnlyList batch, List candidates) { var sb = new StringBuilder(); @@ -187,7 +167,4 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet var text = Flatten(string.Join(' ', lines)).Trim(); return text.Length > FirstParagraphChars ? text[..FirstParagraphChars] : text; } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); } diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs index 219b4af..72b6a60 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs @@ -7,11 +7,13 @@ using Wiki.Core.Ports; namespace Wiki.Application.Encyclopedia; /// -/// What one run did. Counts are subjects, each in both languages. counts subjects -/// rejected earlier whose inputs are unchanged, so they were not retried. is null when nothing was committed. +/// What one run did. is the article count after the run; counts notes, +/// and articles; the write counts are subjects, each in both languages. +/// counts subjects rejected earlier whose inputs are unchanged, so they were not retried. +/// is null when nothing was committed. /// -public sealed record ReconcileSummary(int Planned, int Written, int Deleted, int Rejected, int Remembered, - bool StoppedByModel, string? Skipped, string? Commit); +public sealed record ReconcileSummary(int Planned, int Assigned, int Split, int Folded, int Written, int Deleted, int Rejected, + int Remembered, bool StoppedByModel, string? Skipped, string? Commit); public interface IEncyclopediaRun { @@ -19,14 +21,18 @@ public interface IEncyclopediaRun } /// -/// Brings articles/ up to date with the notes: replans when the note set changed, rewrites the articles whose -/// inputs changed, deletes the subjects a new plan dropped, and commits everything as one commit. +/// Brings articles/ up to date with the notes: updates the stored plan in bounded steps (drop removed notes, assign new +/// ones, split oversized and fold thin articles), rewrites the articles whose inputs changed, deletes the files of articles +/// the plan dropped, and commits everything as one commit. /// Works only on a catalog that reflects the store's head, so what it compares is what it commits onto. /// public sealed class EncyclopediaReconciler( INoteStore store, IDocumentCatalog catalog, - PlanArticlesUseCase planner, + ArticleNeighbours neighbours, + AssignNotesUseCase assigner, + SplitArticleUseCase splitter, + FoldArticleUseCase folder, IArticleWriter writer, EncyclopediaSettings settings, TimeProvider clock, @@ -36,7 +42,27 @@ public sealed class EncyclopediaReconciler( private const string StatePath = "articles/_state.json"; private static readonly string[] Languages = ["de", "en"]; - /// An unexpected failure while writing is rethrown after what is staged is committed; caller cancellation commits nothing. + /// The inputs and mutable outputs of one run. + private sealed class Run(IReadOnlyList notes, ArticlePlan plan, EncyclopediaState state, DateTimeOffset now) + { + public IReadOnlyList Notes { get; } = notes; + public Dictionary NoteBySlug { get; } = + notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + public ArticlePlan Plan { get; } = plan; + public DateTimeOffset Now { get; } = now; + public SortedDictionary Subjects { get; } = new(state.Subjects.ToDictionary(), StringComparer.Ordinal); + public SortedDictionary FailedBatches { get; } = new(state.FailedBatches.ToDictionary(), StringComparer.Ordinal); + public SortedDictionary Kept { get; } = new(state.KeptArticles.ToDictionary(), StringComparer.Ordinal); + /// Children of the articles split this run: their overview's extra link targets. + public Dictionary> Children { get; } = new(StringComparer.Ordinal); + public int Assigned { get; set; } + public int Split { get; set; } + public int Folded { get; set; } + + public EncyclopediaState State => new(Subjects, FailedBatches, Kept); + } + + /// An unexpected failure is rethrown after what is staged is committed; caller cancellation commits nothing. public async Task RunAsync(CancellationToken ct) { var head = (await store.GetHeadAsync(ct)).Commit; @@ -44,79 +70,236 @@ public sealed class EncyclopediaReconciler( if (synced != head) { log.LogWarning("Catalog is at {Catalog}, the store at {Head}; skipping the encyclopedia run", synced, head); - return new ReconcileSummary(0, 0, 0, 0, 0, false, "catalog-behind", null); + return new ReconcileSummary(0, 0, 0, 0, 0, 0, 0, 0, false, "catalog-behind", null); } var notes = await catalog.ListDocumentsAsync(DocumentKind.Wiki, ct); var articleDocs = await catalog.ListDocumentsAsync(DocumentKind.Article, ct); var planText = await store.ReadTextAsync(PlanPath, head, ct); var stateText = await store.ReadTextAsync(StatePath, head, ct); - var current = ReadPlan(planText); - var state = ReadState(stateText); - var subjects = new SortedDictionary(state.Subjects.ToDictionary(), StringComparer.Ordinal); - var failedPlan = state.FailedPlan; - var now = clock.GetUtcNow(); + var run = new Run(notes, ReadPlan(planText), ReadState(stateText), clock.GetUtcNow()); + var plan = run.Plan; + var planned = plan.Articles.Select(a => a.Slug).ToList(); var changes = new List(); - ArticlePlan? replanned = null; - if (PlanArticlesUseCase.NeedsReplan(current, failedPlan, notes)) - { - try - { - replanned = await planner.PlanAsync(current, notes, ct); - } - catch (ModelUnavailableException e) - { - log.LogWarning(e, "Text model unavailable while planning; keeping the current plan"); - return new ReconcileSummary(current?.Articles.Count ?? 0, 0, 0, 0, 0, true, null, null); - } - if (replanned is null) - { - failedPlan = new PlanAttempt(EncyclopediaState.NoteSetFingerprint(notes), now); - } - else - { - failedPlan = null; - changes.Add(Put(PlanPath, ArticlePlanJson.Write(replanned), planText?.BlobSha)); - } - } + foreach (var note in plan.Assignments.Keys.Where(n => !run.NoteBySlug.ContainsKey(n)).Order(StringComparer.Ordinal).ToList()) + foreach (var slug in plan.RemoveNote(note)) + log.LogInformation("Article {Slug} lost its last source {Note}; deleting it", slug, note); - var plan = replanned ?? current; - if (plan is null) + var stopped = false; + ExceptionDispatchInfo? failure = null; + try { - log.LogWarning("No usable article plan; no article is written or deleted"); - return await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText, - new ReconcileSummary(0, 0, 0, 0, 0, false, "no-plan", null), ct); + await PlanStepsAsync(run, ct); + } + catch (ModelUnavailableException e) + { + log.LogWarning(e, "Text model unavailable while planning; committing the plan so far, writing nothing"); + stopped = true; + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + log.LogError(e, "Planning failed unexpectedly; committing the plan so far, then failing the run"); + failure = ExceptionDispatchInfo.Capture(e); } var hashByPath = articleDocs.ToDictionary(d => d.Path, d => d.ContentHash, StringComparer.Ordinal); - var stale = Stale(plan, notes, ParseArticles(articleDocs)); - var remembered = InScope(stale, plan, settings).Count(p => IsRemembered(p, notes, state)); - var due = Due(stale, plan, notes, state, settings); - var noteBySlug = notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); - + var stale = Stale(plan, notes, ParseArticles(articleDocs), settings); + var remembered = InScope(stale, plan, settings).Count(p => IsRemembered(p, notes, run.State)); int written = 0, rejected = 0; - var stopped = false; - ExceptionDispatchInfo? failure = null; + if (!stopped && failure is null) + (written, rejected, stopped, failure) = await WriteAsync(run, Due(stale, plan, notes, run.State, settings), hashByPath, changes, ct); + + var deleted = 0; + foreach (var slug in planned.Where(s => plan.Find(s) is null)) + { + var any = false; + foreach (var lang in Languages) + { + var path = ArticlePath(lang, slug); + if (!hashByPath.TryGetValue(path, out var hash)) continue; + changes.Add(new FileChange(FileChangeOp.Delete, path, null, hash)); + any = true; + } + run.Subjects.Remove(slug); + run.Kept.Remove(slug); + if (any) deleted++; + } + + var planJson = ArticlePlanJson.Write(plan); + if (planJson != (planText?.Text ?? ArticlePlanJson.Write(ArticlePlan.Empty()))) + changes.Add(Put(PlanPath, planJson, planText?.BlobSha)); + + var summary = await FinishAsync(changes, run.State, stateText, + new ReconcileSummary(plan.Articles.Count, run.Assigned, run.Split, run.Folded, written, deleted, rejected, remembered, + stopped, null, null), ct); + failure?.Throw(); + return summary; + } + + /// + /// Assign, split, fold, each within its cap. An unreachable embedding or index service () + /// ends the planning steps; what they changed stays. + /// + private async Task PlanStepsAsync(Run run, CancellationToken ct) + { + try + { + await AssignAsync(run, ct); + await SplitAsync(run, ct); + await FoldAsync(run, ct); + } + catch (HttpRequestException e) + { + log.LogWarning(e, "Planning stopped: a service did not answer; writing with the plan so far"); + } + } + + /// + /// Batches of similar new notes, seeded in slug order, each assigned in one call against the articles near its notes in the + /// plan as it stands, so later batches see the articles earlier ones created. A batch whose reply was unusable before, for + /// the same notes, is skipped without a call; a batch is tried once per run. + /// + private async Task AssignAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + var fresh = run.NoteBySlug.Values.Where(n => !plan.IsAssigned(n.Slug)).OrderBy(n => n.Slug, StringComparer.Ordinal).ToList(); + var tried = new HashSet(StringComparer.Ordinal); + var calls = 0; + while (calls < settings.MaxAssignBatchesPerRun) + { + var open = fresh.Where(n => !tried.Contains(n.Slug) && !plan.IsAssigned(n.Slug)).ToList(); + if (open.Count == 0) return; + var batch = await neighbours.NearestAsync(open[0], open.Select(n => n.Slug).ToHashSet(StringComparer.Ordinal), + settings.AssignBatchSize, run.NoteBySlug, ct); + tried.UnionWith(batch.Select(n => n.Slug)); + var fingerprint = EncyclopediaState.BatchFingerprint(batch); + if (run.FailedBatches.ContainsKey(fingerprint)) continue; + + var nearby = new List(); + foreach (var note in batch) + { + foreach (var slug in await neighbours.NearbyArticlesAsync(note, plan, ct)) + if (nearby.Count < settings.CandidateArticles * 2 && !nearby.Contains(slug)) nearby.Add(slug); + if (nearby.Count >= settings.CandidateArticles * 2) break; + } + + calls++; + var outcome = await assigner.AssignAsync(batch, Articles(plan, nearby), plan, ct); + if (outcome.Unusable) + { + run.FailedBatches[fingerprint] = run.Now; + continue; + } + foreach (var a in outcome.New) plan.AddArticle(a); + foreach (var (note, articles) in outcome.Assignments) plan.Assign(note, articles); + run.Assigned += outcome.Assignments.Count; + } + if (fresh.Any(n => !tried.Contains(n.Slug) && !plan.IsAssigned(n.Slug))) + log.LogInformation("Assign cap of {Cap} batches reached; the other new notes wait for the next run", settings.MaxAssignBatchesPerRun); + } + + /// Oversized articles, biggest first, at most calls. + private async Task SplitAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + var oversized = plan.Articles + .Where(a => SplitArticleUseCase.NeedsSplit(a, run.NoteBySlug, settings)) + .OrderByDescending(a => a.Sources.Sum(s => run.NoteBySlug.TryGetValue(s, out var d) ? (long)d.Body.Length : 0)) + .ThenByDescending(a => a.Sources.Count) + .ThenBy(a => a.Slug, StringComparer.Ordinal) + .Take(settings.MaxSplitsPerRun) + .ToList(); + foreach (var article in oversized) + { + var outcome = await splitter.SplitAsync(article, run.NoteBySlug, plan, ct); + if (outcome is null) continue; + // children first, so a note that moves wholly to a child never goes unassigned + foreach (var child in outcome.Children) + { + plan.AddArticle(child with { Sources = [] }); + plan.ReplaceArticle(child); + } + plan.ReplaceArticle(outcome.Overview); + run.Children[article.Slug] = outcome.Children; + run.Split++; + } + } + + /// + /// Thin articles with a nearby article, not kept before at the same inputs, at most + /// calls. An answer other than a usable fold is remembered like a keep, so the article is not asked again until its inputs change. + /// + private async Task FoldAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + var thin = plan.Articles.Where(a => FoldArticleUseCase.IsThin(a, run.NoteBySlug, settings)).Select(a => a.Slug).ToList(); + var calls = 0; + foreach (var slug in thin) + { + if (calls >= settings.MaxFoldsPerRun) return; + if (plan.Find(slug) is not { } article || !FoldArticleUseCase.IsThin(article, run.NoteBySlug, settings)) continue; + var fingerprint = EncyclopediaState.SubjectFingerprint(article, run.Notes); + if (run.Kept.GetValueOrDefault(slug) == fingerprint) continue; + var nearby = Articles(plan, await neighbours.NearbyArticlesAsync(article, run.NoteBySlug, plan, ct)); + if (nearby.Count == 0) continue; + + calls++; + var into = await folder.FoldAsync(article, run.NoteBySlug, nearby, ct); + if (into is null) + { + run.Kept[slug] = fingerprint; + continue; + } + plan.Fold(slug, into); + run.Folded++; + } + } + + private async Task<(int Written, int Rejected, bool Stopped, ExceptionDispatchInfo? Failure)> WriteAsync( + Run run, IReadOnlyList due, Dictionary hashByPath, List changes, CancellationToken ct) + { + int written = 0, rejected = 0; + var neighboursUp = true; foreach (var planned in due) { - var fingerprint = EncyclopediaState.SubjectFingerprint(planned, notes); - var sources = planned.Sources.Where(noteBySlug.ContainsKey).Select(s => noteBySlug[s]).ToList(); + var fingerprint = EncyclopediaState.SubjectFingerprint(planned, run.Notes); + var sources = WriteSources(planned, run.NoteBySlug, settings); try { - var article = await writer.WriteAsync(planned, sources, plan.Articles, ct); + IReadOnlyList targets = []; + if (neighboursUp) + { + try + { + targets = Articles(run.Plan, await neighbours.NearbyArticlesAsync(planned, run.NoteBySlug, run.Plan, ct)); + } + catch (HttpRequestException e) + { + log.LogWarning(e, "Nearby articles unavailable; writing without link targets"); + neighboursUp = false; + } + } + if (run.Children.TryGetValue(planned.Slug, out var children)) + targets = Articles(run.Plan, [.. targets.Select(t => t.Slug), .. children.Select(c => c.Slug)]); + + var article = await writer.WriteAsync(planned, sources, targets, ct); foreach (var a in new[] { article.De, article.En }) { var path = ArticlePath(a.Lang, a.Slug); changes.Add(Put(path, ArticleFile.Build(a), hashByPath.GetValueOrDefault(path))); } - subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Written, now); + run.Subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Written, run.Now); written++; } catch (ArticleRejectedException e) { log.LogWarning("Article {Slug} rejected: {Reason}; kept as it was, retried when its inputs change", e.Slug, e.Reason); - subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Rejected, now); + run.Subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Rejected, run.Now); rejected++; } catch (HttpRequestException e) @@ -127,8 +310,7 @@ public sealed class EncyclopediaReconciler( catch (ModelUnavailableException e) { log.LogWarning(e, "Text model unavailable; stopping after {Written} articles", written); - stopped = true; - break; + return (written, rejected, true, null); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { @@ -137,48 +319,29 @@ public sealed class EncyclopediaReconciler( catch (Exception e) { log.LogError(e, "Article {Slug} failed unexpectedly; committing what is done, then failing the run", planned.Slug); - failure = ExceptionDispatchInfo.Capture(e); - break; + return (written, rejected, false, ExceptionDispatchInfo.Capture(e)); } } - - // Deletes travel with the plan that drops the subjects: once that plan is committed, the difference is gone. - var deleted = 0; - if (replanned is not null && current is not null) - { - var kept = replanned.Articles.Select(a => a.Slug).ToHashSet(StringComparer.Ordinal); - foreach (var slug in current.Articles.Select(a => a.Slug).Where(s => !kept.Contains(s)).Distinct(StringComparer.Ordinal)) - { - var any = false; - foreach (var lang in Languages) - { - var path = ArticlePath(lang, slug); - if (!hashByPath.TryGetValue(path, out var hash)) continue; - changes.Add(new FileChange(FileChangeOp.Delete, path, null, hash)); - any = true; - } - subjects.Remove(slug); - if (any) deleted++; - } - } - - var summary = await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText, - new ReconcileSummary(plan.Articles.Count, written, deleted, rejected, remembered, stopped, null, null), ct); - failure?.Throw(); - return summary; + return (written, rejected, false, null); } /// /// Planned articles, in plan order, whose files no longer match the plan and the notes: missing in either language, - /// a different set of sources than the plan's sources that exist, a source hash other than the catalog's, or another title. + /// a different set of sources than the plan's sources the writer gets (), a source hash other + /// than the catalog's, or another title. /// - public static IReadOnlyList Stale(ArticlePlan plan, IReadOnlyList notes, IReadOnlyList
existing) + public static IReadOnlyList Stale(ArticlePlan plan, IReadOnlyList notes, IReadOnlyList
existing, + EncyclopediaSettings settings) { - var hashes = notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First().ContentHash, StringComparer.Ordinal); + var bySlug = notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); var articles = existing.GroupBy(a => (a.Lang, a.Slug)).ToDictionary(g => g.Key, g => g.First()); return plan.Articles - .Where(p => IsStale(p, hashes, articles.GetValueOrDefault(("de", p.Slug)), p.TitleDe) - || IsStale(p, hashes, articles.GetValueOrDefault(("en", p.Slug)), p.TitleEn)) + .Where(p => + { + var sources = WriteSources(p, bySlug, settings); + return IsStale(sources, articles.GetValueOrDefault(("de", p.Slug)), p.TitleDe) + || IsStale(sources, articles.GetValueOrDefault(("en", p.Slug)), p.TitleEn); + }) .ToList(); } @@ -194,6 +357,24 @@ public sealed class EncyclopediaReconciler( .Take(settings.MaxArticlesPerRun) .ToList(); + /// + /// The sources that exist, most relevant first, up to of body + /// (at least one): an article waiting for its split is written from its first sources. + /// + private static List WriteSources(PlannedArticle planned, IReadOnlyDictionary notes, EncyclopediaSettings settings) + { + var kept = new List(); + long total = 0; + foreach (var slug in planned.Sources) + { + if (!notes.TryGetValue(slug, out var note)) continue; + total += note.Body.Length; + if (kept.Count > 0 && total > settings.MaxSourceCharsPerArticle) break; + kept.Add(note); + } + return kept; + } + private static IEnumerable InScope(IReadOnlyList stale, ArticlePlan plan, EncyclopediaSettings settings) { if (settings.PilotSlugs.Count > 0) @@ -214,14 +395,18 @@ public sealed class EncyclopediaReconciler( && attempt.Outcome == AttemptOutcome.Rejected && attempt.Fingerprint == EncyclopediaState.SubjectFingerprint(planned, notes); - private static bool IsStale(PlannedArticle planned, Dictionary hashes, Article? article, string title) + private static bool IsStale(List sources, Article? article, string title) { if (article is null || article.Title != title) return true; - var sources = planned.Sources.Where(hashes.ContainsKey).ToHashSet(StringComparer.Ordinal); - return !article.Sources.Select(s => s.Slug).ToHashSet(StringComparer.Ordinal).SetEquals(sources) + var hashes = sources.ToDictionary(d => d.Slug, d => d.ContentHash, StringComparer.Ordinal); + return !article.Sources.Select(s => s.Slug).ToHashSet(StringComparer.Ordinal).SetEquals(hashes.Keys) || article.Sources.Any(s => hashes[s.Slug] != s.ContentHash); } + /// The live articles stand for, in order, each once. + private static List Articles(ArticlePlan plan, IEnumerable slugs) => + slugs.Select(plan.Resolve).OfType().Distinct(StringComparer.Ordinal).Select(s => plan.Find(s)!).ToList(); + private async Task FinishAsync(List changes, EncyclopediaState state, StoredText? stateText, ReconcileSummary summary, CancellationToken ct) { @@ -234,7 +419,8 @@ public sealed class EncyclopediaReconciler( return summary; } - var message = $"articles: {summary.Written} written, {summary.Deleted} deleted, {summary.Rejected} rejected"; + var message = $"articles: {summary.Assigned} assigned, {summary.Split} split, {summary.Folded} folded, " + + $"{summary.Written} written, {summary.Deleted} deleted, {summary.Rejected} rejected"; try { var commit = await store.CommitAsync(changes, message, ct); @@ -265,19 +451,20 @@ public sealed class EncyclopediaReconciler( return articles; } - private ArticlePlan? ReadPlan(StoredText? text) + private ArticlePlan ReadPlan(StoredText? text) { - if (text is null) return null; + if (text is null) return ArticlePlan.Empty(); var plan = ArticlePlanJson.Read(text.Text); - if (plan is null) log.LogWarning("{Path} is malformed; planning from scratch", PlanPath); - return plan; + if (plan is null) log.LogWarning("{Path} is malformed or in the old format; planning from scratch", PlanPath); + return plan ?? ArticlePlan.Empty(); } private EncyclopediaState ReadState(StoredText? text) { if (text is null) return EncyclopediaState.Empty; var state = EncyclopediaStateJson.Read(text.Text); - if (state.Subjects.Count == 0 && state.FailedPlan is null && text.Text.Trim() != EncyclopediaStateJson.Write(EncyclopediaState.Empty)) + if (state.Subjects.Count == 0 && state.FailedBatches.Count == 0 && state.KeptArticles.Count == 0 + && text.Text.Trim() != EncyclopediaStateJson.Write(EncyclopediaState.Empty)) log.LogWarning("{Path} is malformed or empty; starting without attempt memory", StatePath); return state; } diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs new file mode 100644 index 0000000..4fa2eab --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs @@ -0,0 +1,25 @@ +namespace Wiki.Application.Encyclopedia; + +/// +/// Limits of one encyclopedia run. and limit what is written, not what is planned. +/// The Max…PerRun fields cap each step's model calls; the rest bound what one call sees. +/// +public sealed record EncyclopediaSettings( + int ArticleMaxTokens, + int MaxArticlesPerRun, + int MaxSourceCharsPerArticle, + IReadOnlyList PilotSlugs, + int? MaxArticles, + int AssignBatchSize = 20, + int CandidateArticles = 10, + int NeighbourNotes = 30, + int NeighbourSources = 10, + int MaxAssignBatchesPerRun = 120, + int MaxSplitsPerRun = 5, + int MaxFoldsPerRun = 5, + int MaxSourcesPerArticle = 120, + int FoldMinSources = 3, + int FoldMinChars = 4000, + int AssignMaxTokens = 8000, + int SplitMaxTokens = 8000, + int FoldMaxTokens = 1000); diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs index 7c759cc..48ab77a 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs @@ -15,32 +15,26 @@ public enum AttemptOutcome /// The last attempt at one subject: the inputs it was made from and how it ended. public sealed record SubjectAttempt(string Fingerprint, AttemptOutcome Outcome, DateTimeOffset At); -/// A planning attempt that produced no usable plan, for the note set it was made with. -public sealed record PlanAttempt(string NoteSetFingerprint, DateTimeOffset At); - /// /// What the nightly job remembers between runs (articles/_state.json), so it does not pay again /// for the same failure. An attempt is repeated only when its fingerprint changes. /// /// /// FailedBatches: fingerprint of an assignment batch whose reply was unusable → when. -/// KeptArticles: article slug → at which the fold call answered keep. +/// KeptArticles: article slug → at which the fold call did not fold it. +/// The failedPlan entry of earlier versions is ignored on read. /// public sealed record EncyclopediaState( IReadOnlyDictionary Subjects, - PlanAttempt? FailedPlan, IReadOnlyDictionary FailedBatches, IReadOnlyDictionary KeptArticles) { public static EncyclopediaState Empty => - new(new Dictionary(), null, new Dictionary(), new Dictionary()); + new(new Dictionary(), new Dictionary(), new Dictionary()); public static string BatchFingerprint(IReadOnlyList notes) => Hash(Prompts.Version.ToString(), notes.Select(n => $"{n.Slug}@{n.ContentHash}").Order(StringComparer.Ordinal)); - public static string NoteSetFingerprint(IReadOnlyList notes) => - Hash(Prompts.Version.ToString(), notes.Select(n => n.Slug).Order(StringComparer.Ordinal)); - public static string SubjectFingerprint(PlannedArticle planned, IReadOnlyList notes) { var hashes = notes.GroupBy(n => n.Slug).ToDictionary(g => g.Key, g => g.First().ContentHash); @@ -76,7 +70,7 @@ public static class EncyclopediaStateJson var subjects = s.Subjects.Where(kv => kv.Value is not null).ToDictionary(kv => kv.Key, kv => kv.Value); var failedBatches = (s.FailedBatches ?? new Dictionary()).ToDictionary(); var kept = (s.KeptArticles ?? new Dictionary()).Where(kv => kv.Value is not null).ToDictionary(); - return new EncyclopediaState(subjects, s.FailedPlan, failedBatches, kept); + return new EncyclopediaState(subjects, failedBatches, kept); } catch (JsonException) { diff --git a/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs index 3d15791..ecdc117 100644 --- a/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs @@ -1,10 +1,9 @@ using System.Net; using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; using Microsoft.Extensions.Logging; using Wiki.Core.Model; using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; namespace Wiki.Application.Encyclopedia; @@ -40,44 +39,32 @@ public sealed class FoldArticleUseCase(ITextModel model, EncyclopediaSettings se return null; } - JsonObject? root; - try - { - root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; - } - catch (JsonException) - { - root = null; - } + var root = PlanParsing.Object(reply.Text); if (root is null) { log.LogWarning("Fold reply for {Slug} is not a JSON object", thin.Slug); return null; } - if (root["into"] is JsonValue v && v.TryGetValue(out var into) && !string.IsNullOrWhiteSpace(into)) + if (Text(root["into"]) is { } into) { - into = into.Trim(); if (shown.Any(n => n.Slug == into)) return into; log.LogWarning("Fold reply for {Slug} names {Target}, which is not a nearby article", thin.Slug, into); } return null; } - private static string UserPrompt(PlannedArticle thin, IReadOnlyDictionary notes, List nearby) + private string UserPrompt(PlannedArticle thin, IReadOnlyDictionary notes, List nearby) { var sb = new StringBuilder(); sb.Append("Thin article: ").Append(thin.Slug).Append(" | ").Append(Flatten(thin.TitleDe)).Append(" | ") .Append(Flatten(thin.TitleEn)).Append(" | ").Append(Flatten(thin.Scope)).Append('\n'); sb.Append("Its sources (title):\n"); - foreach (var s in thin.Sources) + foreach (var s in thin.Sources.Take(settings.FoldMinSources)) sb.Append(notes.TryGetValue(s, out var d) ? Flatten(d.Title) : s).Append('\n'); sb.Append("\nNearby articles (slug | title | scope):\n"); foreach (var n in nearby) sb.Append(n.Slug).Append(" | ").Append(Flatten(n.TitleDe)).Append(" | ").Append(Flatten(n.Scope)).Append('\n'); return sb.ToString(); } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); } diff --git a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs b/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs deleted file mode 100644 index 8d4b904..0000000 --- a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs +++ /dev/null @@ -1,152 +0,0 @@ -using System.Net; -using System.Text; -using System.Text.RegularExpressions; -using Microsoft.Extensions.Logging; -using Wiki.Core.Model; -using Wiki.Core.Ports; - -namespace Wiki.Application.Encyclopedia; - -/// Limits of one encyclopedia run. and limit what is written, not what is planned. -public sealed record EncyclopediaSettings( - int PlanMaxTokens, - int ArticleMaxTokens, - int MaxArticlesPerRun, - int MaxSourceCharsPerArticle, - IReadOnlyList PilotSlugs, - int? MaxArticles) -{ - public int AssignBatchSize { get; init; } = 20; - public int CandidateArticles { get; init; } = 10; - public int NeighbourNotes { get; init; } = 30; - public int NeighbourSources { get; init; } = 10; - public int MaxAssignBatchesPerRun { get; init; } = 120; - public int MaxSplitsPerRun { get; init; } = 5; - public int MaxFoldsPerRun { get; init; } = 5; - public int MaxSourcesPerArticle { get; init; } = 120; - public int FoldMinSources { get; init; } = 3; - public int FoldMinChars { get; init; } = 4000; - public int AssignMaxTokens { get; init; } = 8000; - public int SplitMaxTokens { get; init; } = 8000; - public int FoldMaxTokens { get; init; } = 1000; -} - -/// -/// Asks the model which subjects get their own article and which notes feed each, and validates the answer. -/// A reply that cannot be used leaves the current plan in place. -/// -public sealed partial class PlanArticlesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) -{ - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - - /// True when the notes changed since was planned, unless planning already failed for exactly this note set. - public static bool NeedsReplan(ArticlePlan? current, PlanAttempt? failedPlan, IReadOnlyList notes) - { - var changed = current is null - || !current.Assignments.Keys.ToHashSet(StringComparer.Ordinal).SetEquals(notes.Select(n => n.Slug)); - return changed && failedPlan?.NoteSetFingerprint != EncyclopediaState.NoteSetFingerprint(notes); - } - - /// The new plan, or null to keep . Model-unavailable and other errors propagate. - public async Task PlanAsync(ArticlePlan? current, IReadOnlyList notes, CancellationToken ct) - { - TextReply reply; - try - { - reply = await model.CompleteAsync( - Prompts.Plan.Replace(Prompts.MaxSourceCharsToken, settings.MaxSourceCharsPerArticle.ToString()), - UserPrompt(current, notes), settings.PlanMaxTokens, ct); - } - catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.BadRequest) - { - log.LogWarning(e, "Planning request was rejected; keeping the current plan"); - return null; - } - - if (reply.Truncated) - { - log.LogWarning("Plan reply hit the token cap ({Tokens}); keeping the current plan", settings.PlanMaxTokens); - return null; - } - - var parsed = ArticlePlanJson.ReadArticles(reply.Text); - if (parsed is null) - { - log.LogWarning("Plan reply is not valid JSON; keeping the current plan"); - return null; - } - - var articles = Validate(parsed, notes); - if (articles.Count == 0) - { - log.LogWarning("Plan reply has no valid article; keeping the current plan"); - return null; - } - var plan = ArticlePlan.Empty(); - foreach (var a in articles) plan.AddArticle(a); - foreach (var slug in notes.Select(n => n.Slug).Distinct(StringComparer.Ordinal)) - plan.Assign(slug, articles.Where(a => a.Sources.Contains(slug)).Select(a => a.Slug).ToList()); - return plan; - } - - private List Validate(IReadOnlyList proposed, IReadOnlyList notes) - { - var lengths = notes.GroupBy(n => n.Slug).ToDictionary(g => g.Key, g => g.First().Body.Length); - var seen = new HashSet(StringComparer.Ordinal); - var result = new List(); - foreach (var a in proposed) - { - if (!SlugPattern().IsMatch(a.Slug)) - { - log.LogWarning("Dropping planned article with invalid slug {Slug}", a.Slug); - continue; - } - if (!seen.Add(a.Slug)) - { - log.LogWarning("Dropping duplicate planned article {Slug}", a.Slug); - continue; - } - var sources = a.Sources.Where(lengths.ContainsKey).Distinct(StringComparer.Ordinal).ToList(); - if (sources.Count == 0) - { - log.LogWarning("Dropping planned article {Slug}: no known source", a.Slug); - continue; - } - sources = CapSources(a.Slug, sources, lengths); - result.Add(a with { Sources = sources }); - } - return result; - } - - private List CapSources(string slug, List sources, Dictionary lengths) - { - var kept = new List(); - long total = 0; - foreach (var s in sources) - { - total += lengths[s]; - if (kept.Count > 0 && total > settings.MaxSourceCharsPerArticle) break; - kept.Add(s); - } - if (kept.Count < sources.Count) - log.LogWarning("Planned article {Slug} exceeds {Cap} source chars; dropped {Dropped} lowest-ranked sources", - slug, settings.MaxSourceCharsPerArticle, sources.Count - kept.Count); - return kept; - } - - private static string UserPrompt(ArticlePlan? current, IReadOnlyList notes) - { - var sb = new StringBuilder(); - if (current is not null) - sb.Append("Current plan:\n").Append(ArticlePlanJson.WriteArticlesCompact(current)).Append("\n\n"); - sb.Append("Notes (slug | title | tags | body length in chars):\n"); - foreach (var n in notes.OrderBy(n => n.Slug, StringComparer.Ordinal)) - sb.Append(n.Slug).Append(" | ").Append(Flatten(n.Title)).Append(" | ").Append(Flatten(string.Join(", ", n.Tags))) - .Append(" | ").Append(n.Body.Length).Append('\n'); - return sb.ToString(); - } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); -} diff --git a/src/Wiki.Application/Encyclopedia/PlanParsing.cs b/src/Wiki.Application/Encyclopedia/PlanParsing.cs new file mode 100644 index 0000000..34a2e8d --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/PlanParsing.cs @@ -0,0 +1,53 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Wiki.Core.Model; + +namespace Wiki.Application.Encyclopedia; + +/// Reading of plan files and planning replies, and the prompt-line form of their fields. +internal static partial class PlanParsing +{ + [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] + public static partial Regex SlugPattern(); + + /// Trimmed, as frontmatter reads it back; null when blank or not a string. + public static string? Text(JsonNode? n) => + n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; + + /// The non-blank strings of an array; empty when is not an array. + public static List Strings(JsonNode? n) => + n is JsonArray a ? a.Select(Text).OfType().ToList() : []; + + /// A category name in letters only, any case. + public static bool TryCategory(string? text, out ArticleCategory category) + { + category = default; + return text is not null && text.All(char.IsAsciiLetter) && Enum.TryParse(text, ignoreCase: true, out category); + } + + /// The JSON object in a reply, which may be wrapped in a code fence; null when it is not one. + public static JsonObject? Object(string reply) + { + try + { + return JsonNode.Parse(StripFence(reply)) as JsonObject; + } + catch (JsonException) + { + return null; + } + } + + public static string StripFence(string text) + { + var t = text.Trim(); + if (!t.StartsWith("```")) return t; + var start = t.IndexOf('\n'); + var end = t.LastIndexOf("```", StringComparison.Ordinal); + return start >= 0 && end > start ? t[(start + 1)..end].Trim() : t; + } + + /// Keeps a field on its line and inside its column. + public static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); +} diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index 1c3e871..e0c680a 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -54,29 +54,6 @@ public static class Prompts Reply with JSON only, no prose and no code fence: {"into":"article-slug"} to fold, or {"keep":true} to keep. """; - /// Placeholder in replaced by MaxSourceCharsPerArticle. - public const string MaxSourceCharsToken = "{maxSourceChars}"; - - public const string Plan = """ - [plan] - You plan the article set of an encyclopedia about one household: the house and its homelab IT. Its readers are everyone who needs to know things about the house, from the owner to guests. - You receive the wiki's notes, one per line as `slug | title | tags | body length in chars`, and the current plan as JSON if there is one. - - Decide which subjects get their own article and which notes feed each article. - - Granularity follows the material, like notability on Wikipedia. A subject gets its own article when the wiki holds enough on it: for example a heating valve controller with 17 automations and a complicated history. - - A subject that was set once and forgotten is not an article of its own. Its notes go into the article of its parent system as a section. - - Every note is used by at least one article, or is deliberately skipped. Skip only meta notes and notes about the wiki's own tooling. - - Keep the existing slugs for existing subjects. Do not rename or renumber them. - - Keep the existing titles for existing subjects unless the subject changed. - - Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses. Name the subject by what it is or does, for example "Netzwerkspeicher" rather than the server's hostname. - - List each article's sources most important first. When the sources of an article add up to more than {maxSourceChars} characters of body, split the subject into narrower articles. - - Slugs are lowercase ASCII words joined by single hyphens. Categories are room, system, device or topic. - - related lists the slugs of other articles in this plan that a reader of the article should continue with. - - Reply with JSON only, no prose and no code fence. Write it compact on a single line, with no indentation, no line breaks and no spaces between tokens, because the reply is size-limited. Shape: - {"articles":[{"slug":"...","titleDe":"...","titleEn":"...","category":"room|system|device|topic","sources":["note-slug"],"related":["article-slug"]}]} - """; - /// The generic-examples rule, shared by and . public const string GenericExamples = "A generic example that explains a principle and names nothing concrete in this house, for example what an entity ID or an IP address looks like, is not a leak and stays public"; diff --git a/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs index 3a1de92..373e91c 100644 --- a/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs @@ -1,11 +1,10 @@ using System.Net; using System.Text; -using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Wiki.Core.Model; using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; namespace Wiki.Application.Encyclopedia; @@ -13,14 +12,11 @@ namespace Wiki.Application.Encyclopedia; public sealed record SplitOutcome(PlannedArticle Overview, IReadOnlyList Children); /// Splits one oversized article into an overview and narrower children, in one bounded model call. -public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +public sealed class SplitArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) { private const int MinChildren = 2; private const int MaxChildren = 6; - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - /// Body chars of the sources that exist in exceed the char cap, or the source count exceeds the count cap. public static bool NeedsSplit(PlannedArticle a, IReadOnlyDictionary notes, EncyclopediaSettings s) => a.Sources.Count > s.MaxSourcesPerArticle @@ -54,15 +50,7 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe return null; } - JsonObject? root; - try - { - root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; - } - catch (JsonException) - { - root = null; - } + var root = PlanParsing.Object(reply.Text); if (root is null) { log.LogWarning("Split reply for {Slug} is not a JSON object", article.Slug); @@ -110,8 +98,8 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe return null; } - var kept = overviewSources.Distinct(StringComparer.Ordinal) - .OrderBy(s => article.Sources.ToList().IndexOf(s)).ToList(); + var rank = article.Sources.Select((s, i) => (s, i)).DistinctBy(x => x.s).ToDictionary(x => x.s, x => x.i, StringComparer.Ordinal); + var kept = overviewSources.Distinct(StringComparer.Ordinal).OrderBy(s => rank[s]).ToList(); var rest = article.Sources.Where(s => !input.Contains(s)).ToList(); var overviewAll = kept.Concat(rest).ToList(); if (overviewAll.Count == 0) @@ -135,9 +123,7 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe log.LogWarning("Split of {Slug}: child slug {Child} is taken or duplicate", article.Slug, slug); return null; } - var categoryText = Text(o["category"]); - if (categoryText is null || !categoryText.All(char.IsAsciiLetter) - || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) + if (!TryCategory(Text(o["category"]), out var category)) { log.LogWarning("Split of {Slug}: child {Child} has an unknown category", article.Slug, slug); return null; @@ -156,12 +142,6 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe return new PlannedArticle(slug, titleDe, titleEn, category, scope, sources); } - private static string? Text(JsonNode? n) => - n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; - - private static List Strings(JsonNode? n) => - n is JsonArray a ? a.Select(Text).OfType().ToList() : []; - private static string UserPrompt(PlannedArticle article, List shown, IReadOnlyDictionary notes) { var sb = new StringBuilder(); @@ -177,7 +157,4 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe } return sb.ToString(); } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); } diff --git a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs index ec8dd9b..8380798 100644 --- a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs @@ -200,7 +200,7 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe { try { - if (JsonNode.Parse(ArticlePlanJson.StripFence(text)) is not JsonObject root || root["privatize"] is not JsonArray items) return null; + if (JsonNode.Parse(PlanParsing.StripFence(text)) is not JsonObject root || root["privatize"] is not JsonArray items) return null; var result = new List(); foreach (var item in items) { @@ -219,7 +219,7 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe { try { - if (JsonNode.Parse(ArticlePlanJson.StripFence(text)) is not JsonArray items) return null; + if (JsonNode.Parse(PlanParsing.StripFence(text)) is not JsonArray items) return null; var result = new List(); foreach (var item in items) { diff --git a/src/Wiki.Host/Api/ApiV1.cs b/src/Wiki.Host/Api/ApiV1.cs index dea0a65..e7ee6d9 100644 --- a/src/Wiki.Host/Api/ApiV1.cs +++ b/src/Wiki.Host/Api/ApiV1.cs @@ -28,7 +28,7 @@ public static class ApiV1 }).RequireAuthorization(Auth.Policies.Write); // Syncs first so the run plans from the current notes. The run outlives a caller that hangs up; - // only shutdown stops it. A run the text model stopped is a failure for the caller. + // only shutdown stops it. A run the text model stopped, or one that ends without a planned article, is a failure for the caller. v1.MapPost("/encyclopedia/run", async (IServiceProvider services, SyncCoordinator sync, IHostApplicationLifetime lifetime) => { var encyclopedia = services.GetService(); @@ -38,7 +38,7 @@ public static class ApiV1 await sync.RunAsync(force: false, ct); var summary = await encyclopedia.RunAsync(ct); return summary is null ? Results.Json(new { error = encyclopedia.LastError }, statusCode: StatusCodes.Status502BadGateway) - : summary.StoppedByModel ? Results.Json(summary, statusCode: StatusCodes.Status502BadGateway) + : summary.StoppedByModel || summary.Planned == 0 ? Results.Json(summary, statusCode: StatusCodes.Status502BadGateway) : Results.Ok(summary); }).RequireAuthorization(Auth.Policies.Write); } diff --git a/src/Wiki.Host/Components/Shared/StatusPanel.razor b/src/Wiki.Host/Components/Shared/StatusPanel.razor index d3c07cd..a2bd405 100644 --- a/src/Wiki.Host/Components/Shared/StatusPanel.razor +++ b/src/Wiki.Host/Components/Shared/StatusPanel.razor @@ -42,7 +42,7 @@ Last run@Status.LastEncyclopediaAttemptAt.Value.ToString("u") @if (Status.LastEncyclopediaRun is { } r) { - Result@r.Planned planned, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") + Result@r.Planned planned, @r.Assigned assigned, @r.Split split, @r.Folded folded, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") } @if (Status.LastEncyclopediaError is not null) { diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs index 0d60fd9..6bbcab6 100644 --- a/src/Wiki.Host/Composition/WikiOptions.cs +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -84,11 +84,24 @@ public sealed class EncyclopediaOptions public string AnswerModel { get; set; } = "claude-sonnet"; public string Model { get; set; } = "claude-sonnet-5"; public int ArticleMaxTokens { get; set; } = 16000; - public int PlanMaxTokens { get; set; } = 32000; public int MaxArticlesPerRun { get; set; } = 10; public int MaxSourceCharsPerArticle { get; set; } = 300_000; /// When non-empty, only these subjects are written. public List PilotSlugs { get; set; } = []; /// Fallback write cap: the first N planned subjects. public int? MaxArticles { get; set; } = null; + public int AssignBatchSize { get; set; } = 20; + /// Nearby articles per note or article; an assign call sees at most twice this. + public int CandidateArticles { get; set; } = 10; + public int NeighbourNotes { get; set; } = 30; + public int NeighbourSources { get; set; } = 10; + public int MaxAssignBatchesPerRun { get; set; } = 120; + public int MaxSplitsPerRun { get; set; } = 5; + public int MaxFoldsPerRun { get; set; } = 5; + public int MaxSourcesPerArticle { get; set; } = 120; + public int FoldMinSources { get; set; } = 3; + public int FoldMinChars { get; set; } = 4000; + public int AssignMaxTokens { get; set; } = 8000; + public int SplitMaxTokens { get; set; } = 8000; + public int FoldMaxTokens { get; set; } = 1000; } diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index 1fe9d15..b385f1b 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -114,9 +114,26 @@ if (encyclopedia.Enabled) o.ApiKey = builder.Configuration[$"Wiki:AnswerModels:Models:{encyclopedia.AnswerModel}:ApiKey"] ?? string.Empty; }); builder.Services.AddSingleton(new Wiki.Application.Encyclopedia.EncyclopediaSettings( - encyclopedia.PlanMaxTokens, encyclopedia.ArticleMaxTokens, encyclopedia.MaxArticlesPerRun, - encyclopedia.MaxSourceCharsPerArticle, encyclopedia.PilotSlugs, encyclopedia.MaxArticles)); - builder.Services.AddSingleton(); + encyclopedia.ArticleMaxTokens, encyclopedia.MaxArticlesPerRun, encyclopedia.MaxSourceCharsPerArticle, + encyclopedia.PilotSlugs, encyclopedia.MaxArticles, + AssignBatchSize: encyclopedia.AssignBatchSize, + CandidateArticles: encyclopedia.CandidateArticles, + NeighbourNotes: encyclopedia.NeighbourNotes, + NeighbourSources: encyclopedia.NeighbourSources, + MaxAssignBatchesPerRun: encyclopedia.MaxAssignBatchesPerRun, + MaxSplitsPerRun: encyclopedia.MaxSplitsPerRun, + MaxFoldsPerRun: encyclopedia.MaxFoldsPerRun, + MaxSourcesPerArticle: encyclopedia.MaxSourcesPerArticle, + FoldMinSources: encyclopedia.FoldMinSources, + FoldMinChars: encyclopedia.FoldMinChars, + AssignMaxTokens: encyclopedia.AssignMaxTokens, + SplitMaxTokens: encyclopedia.SplitMaxTokens, + FoldMaxTokens: encyclopedia.FoldMaxTokens)); + // ArticleNeighbours takes the active IEmbeddingModel: the index holds its vectors. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs index ec56520..a4d72ae 100644 --- a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs +++ b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs @@ -8,9 +8,9 @@ public class ArticleNeighboursTests { private const string Model = "m"; - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null); - private static Document Note(string slug, string hash = "h") => PlanArticlesUseCaseTests.Note(slug, hash: hash); + private static Document Note(string slug, string hash = "h") => TestNotes.Note(slug, hash: hash); private static PlannedArticle Article(string slug, params string[] sources) => new(slug, slug, slug, ArticleCategory.Topic, "scope", sources); diff --git a/test/Wiki.Application.Tests/ArticlePlanTests.cs b/test/Wiki.Application.Tests/ArticlePlanTests.cs index f5fa602..a925856 100644 --- a/test/Wiki.Application.Tests/ArticlePlanTests.cs +++ b/test/Wiki.Application.Tests/ArticlePlanTests.cs @@ -152,7 +152,7 @@ public class ArticlePlanTests Assert.Contains("\"category\": \"system\"", json); Assert.NotNull(read); Assert.Equal(plan.Articles, read.Articles, new PlannedArticleComparer()); - Assert.Equal(["n1", "n2", "meta"], read.Assignments.Keys); + Assert.Equal(["meta", "n1", "n2"], read.Assignments.Keys); Assert.Equal(["heating"], read.Assignments["n2"]); Assert.Empty(read.Assignments["meta"]); Assert.Equal("heating", read.Redirects["old"]); @@ -162,6 +162,27 @@ public class ArticlePlanTests public void Json_OldFormat_IsNull() => Assert.Null(ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"topic","sources":["x"],"related":[]}],"coveredNotes":["x"]}""")); + [Fact] + public void Json_Write_MapKeysOrdinal_SameForSameContent() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a")); + plan.AddArticle(Article("c", "n9")); + plan.AddArticle(Article("d", "n0")); + plan.Assign("n2", ["a"]); + plan.Assign("n1", ["a"]); + plan.Fold("d", "c"); + plan.AddArticle(Article("b", "n8")); + plan.Fold("b", "c"); + + var json = ArticlePlanJson.Write(plan); + + var maps = json[json.IndexOf("\"assignments\"", StringComparison.Ordinal)..]; + Assert.True(maps.IndexOf("\"n1\"", StringComparison.Ordinal) < maps.IndexOf("\"n2\"", StringComparison.Ordinal)); + Assert.True(maps.IndexOf("\"b\": \"c\"", StringComparison.Ordinal) < maps.IndexOf("\"d\": \"c\"", StringComparison.Ordinal)); + Assert.Equal(json, ArticlePlanJson.Write(ArticlePlanJson.Read(json)!)); + } + [Fact] public void Json_Malformed_IsNull() => Assert.Null(ArticlePlanJson.Read("not json")); diff --git a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs index 5c1c7a2..76e6208 100644 --- a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs +++ b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Tests; public class AssignNotesUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null); private static AssignNotesUseCase UseCase(FakeTextModel model) => new(model, Settings, NullLogger.Instance); @@ -22,7 +22,7 @@ public class AssignNotesUseCaseTests } private static Document[] Batch(params string[] slugs) => - slugs.Select(s => PlanArticlesUseCaseTests.Note(s)).ToArray(); + slugs.Select(s => TestNotes.Note(s)).ToArray(); [Fact] public async Task Assign_ExistingAndNew() @@ -130,7 +130,7 @@ public class AssignNotesUseCaseTests [Fact] public async Task Assign_FirstParagraph_SkipsHeadingsAndBlankLines() { - var note = PlanArticlesUseCaseTests.Note("a") with { Body = "# T\n\n \nfirst line\ncontinued\n\nsecond para" }; + var note = TestNotes.Note("a") with { Body = "# T\n\n \nfirst line\ncontinued\n\nsecond para" }; var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); await UseCase(model).AssignAsync([note], [], ArticlePlan.Empty(), CancellationToken.None); @@ -142,7 +142,7 @@ public class AssignNotesUseCaseTests [Fact] public async Task Assign_FirstParagraph_CutTo300() { - var note = PlanArticlesUseCaseTests.Note("a") with { Body = new string('y', 500) }; + var note = TestNotes.Note("a") with { Body = new string('y', 500) }; var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); await UseCase(model).AssignAsync([note], [], ArticlePlan.Empty(), CancellationToken.None); diff --git a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs index 0a62188..f2a4ab4 100644 --- a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs @@ -11,7 +11,7 @@ public class EncyclopediaCoordinatorTests public Task RunAsync(CancellationToken ct) => body(ct); } - private static readonly ReconcileSummary Done = new(3, 2, 0, 0, 0, false, null, "c1"); + private static readonly ReconcileSummary Done = new(3, 0, 0, 0, 2, 0, 0, 0, false, null, "c1"); private static EncyclopediaCoordinator Coordinator(Func> body) => new(new ScriptedRun(body), new FakeTimeProvider(new DateTimeOffset(2026, 10, 24, 4, 0, 0, TimeSpan.Zero)), diff --git a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs index 58f9c4e..67affb5 100644 --- a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs @@ -6,6 +6,7 @@ using Wiki.Application.Encyclopedia; using Wiki.Application.Sync; using Wiki.Core.Model; using Wiki.Core.Parsing; +using Wiki.Core.Ports; namespace Wiki.Application.Tests; @@ -14,39 +15,56 @@ public class EncyclopediaReconcilerTests private const string PlanPath = "articles/_plan.json"; private const string StatePath = "articles/_state.json"; private static readonly DateTimeOffset Now = new(2026, 10, 24, 3, 0, 0, TimeSpan.Zero); - private static readonly EncyclopediaSettings Defaults = new(8000, 16000, 10, 300_000, [], null); - private static readonly (string Slug, string[] Sources)[] HeatingAndKitchen = + /// Fold is off (no article has fewer than zero sources) unless a test turns it on. + private static readonly EncyclopediaSettings Defaults = new(16000, 10, 300_000, [], null) { FoldMinSources = 0 }; + + private static readonly (string Slug, string[] Notes)[] HeatingAndKitchen = [("heating", ["boiler", "radiators"]), ("kitchen", ["kitchen"])]; private readonly FakeStore _store = new(); private readonly FakeCatalog _catalog = new(); - private readonly FakeTextModel _planModel = new("plan"); + private readonly FakeIndex _index = new(); + private readonly SwitchableEmbeddings _embeddings = new(new FakeEmbeddings("m")); + private readonly FakeTextModel _model = new("plan"); private readonly FakeWriter _writer = new(); private readonly FakeTimeProvider _clock = new(Now); private static string Note(string title, string body = "Body.") => $"---\ntitle: {title}\ntype: wiki\n---\n\n{body}\n"; - private static string PlanReply(params (string Slug, string[] Sources)[] articles) => - "{\"articles\":[" + string.Join(",", articles.Select(a => - $"{{\"slug\":\"{a.Slug}\",\"titleDe\":\"{a.Slug} de\",\"titleEn\":\"{a.Slug} en\",\"category\":\"topic\"," + - $"\"sources\":[{string.Join(",", a.Sources.Select(s => $"\"{s}\""))}],\"related\":[]}}")) + "]}"; + /// An assign reply creating , each fed by its notes. + private static string AssignReply(params (string Slug, string[] Notes)[] articles) => + TitledAssignReply(articles.Select(a => (a.Slug, a.Slug + " de", a.Slug + " en", a.Notes)).ToArray()); - private static string TitledPlanReply(string titleDe, string titleEn) => + private static string TitledAssignReply(params (string Slug, string TitleDe, string TitleEn, string[] Notes)[] articles) => JsonSerializer.Serialize(new { - articles = new[] { new { slug = "heating", titleDe, titleEn, category = "system", sources = new[] { "boiler" }, related = Array.Empty() } }, + @new = articles.Select(a => new { slug = a.Slug, titleDe = a.TitleDe, titleEn = a.TitleEn, category = "topic", scope = "About " + a.Slug + "." }), + notes = articles.SelectMany(a => a.Notes.Select(n => (Note: n, a.Slug))) + .GroupBy(x => x.Note) + .Select(g => new { note = g.Key, articles = g.Select(x => x.Slug).ToArray() }), + }); + + /// An assign reply sending each note to existing articles. + private static string ExistingReply(params (string Note, string Article)[] notes) => + JsonSerializer.Serialize(new + { + @new = Array.Empty(), + notes = notes.Select(n => new { note = n.Note, articles = new[] { n.Article } }), }); private EncyclopediaReconciler Reconciler(EncyclopediaSettings? settings = null, IArticleWriter? writer = null) { var s = settings ?? Defaults; - return new(_store, _catalog, new PlanArticlesUseCase(_planModel, s, NullLogger.Instance), + return new(_store, _catalog, new ArticleNeighbours(_embeddings, _index, s), + new AssignNotesUseCase(_model, s, NullLogger.Instance), + new SplitArticleUseCase(_model, s, NullLogger.Instance), + new FoldArticleUseCase(_model, s, NullLogger.Instance), writer ?? _writer, s, _clock, NullLogger.Instance); } private Task Sync(bool force = true) => - new SyncUseCase(_store, _catalog, new FakeIndex(), [new FakeEmbeddings("m")], _clock, NullLogger.Instance) + new SyncUseCase(_store, _catalog, _index, [new FakeEmbeddings("m")], _clock, NullLogger.Instance) .RunAsync(force, CancellationToken.None); private async Task SeedAsync(params string[] notes) @@ -56,11 +74,13 @@ public class EncyclopediaReconcilerTests await Sync(); } - /// Seeds the default notes, runs once with and syncs the result. + private static string[] Numbered(int count) => Enumerable.Range(1, count).Select(i => $"note-{i:00}").ToArray(); + + /// Seeds the default notes, runs once assigning and syncs the result. private async Task FirstRunAsync(EncyclopediaSettings? settings = null) { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); await Reconciler(settings).RunAsync(CancellationToken.None); await Sync(); } @@ -69,23 +89,315 @@ public class EncyclopediaReconcilerTests private EncyclopediaState State => EncyclopediaStateJson.Read(_store.Files[StatePath]); + private ArticlePlan Plan => ArticlePlanJson.Read(_store.Files[PlanPath])!; + [Fact] - public async Task Reconcile_FirstRun_PlansWritesCommitsOnce() + public async Task Reconcile_FirstRun_AssignsWritesCommitsOnce() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); var summary = await Reconciler().RunAsync(CancellationToken.None); var commit = Assert.Single(_store.Commits); - Assert.Equal("articles: 2 written, 0 deleted, 0 rejected", commit.Message); + Assert.Equal("articles: 3 assigned, 0 split, 0 folded, 2 written, 0 deleted, 0 rejected", commit.Message); Assert.Equal( [PlanPath, StatePath, "articles/de/heating.md", "articles/de/kitchen.md", "articles/en/heating.md", "articles/en/kitchen.md"], commit.Changes.Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.All(commit.Changes, c => Assert.Equal(FileChangeOp.Create, c.Op)); - Assert.Equal(new ReconcileSummary(2, 2, 0, 0, 0, false, null, "c2"), summary); + Assert.Equal(new ReconcileSummary(2, 3, 0, 0, 2, 0, 0, 0, false, null, "c2"), summary); Assert.Equal(AttemptOutcome.Written, State.Subjects["heating"].Outcome); Assert.Equal(Now, State.Subjects["heating"].At); + Assert.Equal(["boiler", "radiators"], Plan.Find("heating")!.Sources); + Assert.StartsWith("[assign]", Assert.Single(_model.Calls).System); + } + + [Fact] + public async Task Run_Bootstrap_LaterBatchesSeeEarlierArticles() + { + var notes = Numbered(40); + await SeedAsync(notes); + _model.Reply(AssignReply(("alpha", notes))); + _model.Reply(AssignReply(("beta", notes))); + + var summary = await Reconciler(Defaults with { AssignBatchSize = 20 }).RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.DoesNotContain("alpha |", _model.Calls[0].User); + Assert.Contains("alpha |", _model.Calls[1].User); + Assert.Equal(40, summary.Assigned); + Assert.Equal(40, Plan.Assignments.Count); + Assert.Equal(20, Plan.Find("alpha")!.Sources.Count); + Assert.Equal(20, Plan.Find("beta")!.Sources.Count); + } + + [Fact] + public async Task Run_OnlyNewNotesAreAssigned() + { + await FirstRunAsync(); + _store.Files["wiki/oven.md"] = Note("oven"); + await Sync(); + _model.Reply(ExistingReply(("oven", "kitchen"))); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.Contains("oven |", _model.Calls[1].User); + Assert.DoesNotContain("boiler |", _model.Calls[1].User); + Assert.Equal(1, summary.Assigned); + Assert.Equal(["kitchen", "oven"], Plan.Find("kitchen")!.Sources); + Assert.Equal(["heating", "kitchen", "kitchen"], _writer.Calls); + } + + [Fact] + public async Task Run_RemovedNote_DropsSourceAndDeletesEmptyArticle() + { + await FirstRunAsync(); + _store.Files.Remove("wiki/kitchen.md"); + _store.Files.Remove("wiki/radiators.md"); + await Sync(); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Single(_model.Calls); + Assert.Equal(1, summary.Deleted); + Assert.Equal(1, summary.Written); + Assert.Equal(1, summary.Planned); + Assert.Equal(["articles/de/kitchen.md", "articles/en/kitchen.md"], + LastCommit.Where(c => c.Op == FileChangeOp.Delete).Select(c => c.Path).Order(StringComparer.Ordinal)); + Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("kitchen", State.Subjects.Keys); + Assert.Null(Plan.Find("kitchen")); + Assert.Equal(["boiler"], Plan.Find("heating")!.Sources); + Assert.Equal(["boiler"], Plan.Assignments.Keys); + Assert.Equal("heating", _writer.Calls[^1]); + } + + [Fact] + public async Task Run_UnusableBatch_RememberedAndNotRetried() + { + await SeedAsync(); + _model.Reply("I would rather not."); + + var first = await Reconciler().RunAsync(CancellationToken.None); + await Sync(); + var second = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(0, first.Planned); + Assert.Single(_model.Calls); + Assert.Single(State.FailedBatches); + Assert.Equal([StatePath], Assert.Single(_store.Commits).Changes.Select(c => c.Path)); + Assert.Null(second.Commit); + Assert.Empty(_writer.Calls); + } + + [Fact] + public async Task Run_UnusableBatch_RetriedWhenANoteChanges() + { + await SeedAsync(); + _model.Reply("I would rather not."); + await Reconciler().RunAsync(CancellationToken.None); + _store.Files["wiki/boiler.md"] = Note("boiler", "New boiler."); + await Sync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.Equal(2, summary.Planned); + } + + [Fact] + public async Task Run_SplitThenWriteOverviewWithChildrenAsLinks() + { + await SeedAsync("boiler", "radiators", "pump", "valve", "kitchen"); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "pump", "valve"]), ("kitchen", ["kitchen"]))); + _model.Reply(JsonSerializer.Serialize(new + { + overview = new { scope = "Heating in general.", sources = new[] { "boiler" } }, + children = new[] + { + new { slug = "heating-radiators", titleDe = "Heizkörper", titleEn = "Radiators", category = "device", scope = "The radiators.", sources = new[] { "radiators" } }, + new { slug = "heating-pump", titleDe = "Pumpe", titleEn = "Pump", category = "device", scope = "The pump.", sources = new[] { "pump" } }, + }, + })); + + // no nearby articles: the overview's link targets are its children alone + var summary = await Reconciler(Defaults with { MaxSourcesPerArticle = 3, CandidateArticles = 0 }).RunAsync(CancellationToken.None); + + Assert.Equal(1, summary.Split); + Assert.StartsWith("[split]", _model.Calls[1].System); + Assert.Equal(["boiler", "valve"], Plan.Find("heating")!.Sources); + Assert.Equal("Heating in general.", Plan.Find("heating")!.Scope); + Assert.Equal(["heating-pump"], Plan.Assignments["pump"]); + Assert.Equal(4, summary.Planned); + Assert.Equal(["heating-pump", "heating-radiators"], _writer.Links["heating"].Order(StringComparer.Ordinal)); + Assert.Empty(_writer.Links["kitchen"]); + Assert.Equal(["heating", "heating-pump", "heating-radiators", "kitchen"], _writer.Calls.Order(StringComparer.Ordinal)); + } + + [Fact] + public async Task Run_SplitChildrenAreLinksOnlyInTheSplittingRun() + { + await SeedAsync("boiler", "radiators", "pump", "valve"); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "pump", "valve"]))); + _model.Reply(JsonSerializer.Serialize(new + { + overview = new { scope = "Heating in general.", sources = new[] { "boiler" } }, + children = new[] + { + new { slug = "heating-radiators", titleDe = "Heizkörper", titleEn = "Radiators", category = "device", scope = "The radiators.", sources = new[] { "radiators" } }, + new { slug = "heating-pump", titleDe = "Pumpe", titleEn = "Pump", category = "device", scope = "The pump.", sources = new[] { "pump" } }, + }, + })); + var settings = Defaults with { MaxSourcesPerArticle = 3, CandidateArticles = 0 }; + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + _store.Files["wiki/boiler.md"] = Note("boiler", "New boiler."); + await Sync(); + + await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal("heating", _writer.Calls[^1]); + Assert.Empty(_writer.Links["heating"]); + } + + [Fact] + public async Task Run_FoldDeletesFilesAndAddsRedirect() + { + await FirstRunAsync(); + _model.Reply("""{"into":"heating"}"""); + + var summary = await Reconciler(Defaults with { FoldMinSources = 2 }).RunAsync(CancellationToken.None); + + Assert.Equal(1, summary.Folded); + Assert.Equal(1, summary.Deleted); + Assert.StartsWith("[fold]", _model.Calls[^1].System); + Assert.Contains("kitchen |", _model.Calls[^1].User); + Assert.Equal("heating", Plan.Redirects["kitchen"]); + Assert.Equal(["boiler", "radiators", "kitchen"], Plan.Find("heating")!.Sources); + Assert.Equal(["heating"], Plan.Assignments["kitchen"]); + Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("articles/en/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("kitchen", State.Subjects.Keys); + Assert.Equal("heating", _writer.Calls[^1]); + } + + [Fact] + public async Task Run_KeptArticle_NotAskedAgain() + { + var settings = Defaults with { FoldMinSources = 2 }; + await FirstRunAsync(); + _model.Reply("""{"keep":true}"""); + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + var calls = _model.Calls.Count; + + var summary = await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal(calls, _model.Calls.Count); + Assert.Equal(0, summary.Folded); + Assert.Equal(["kitchen"], State.KeptArticles.Keys); + Assert.Null(summary.Commit); + } + + [Fact] + public async Task Run_EmbeddingsDown_StillWritesStale() + { + await FirstRunAsync(); + _store.Files["wiki/boiler.md"] = Note("boiler", "New boiler."); + _store.Files["wiki/garage.md"] = Note("garage"); + await Sync(); + _embeddings.Down = true; + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Single(_model.Calls); + Assert.False(summary.StoppedByModel); + Assert.Equal(0, summary.Assigned); + Assert.Equal(1, summary.Written); + Assert.Equal("heating", _writer.Calls[^1]); + Assert.Empty(_writer.Links["heating"]); + Assert.NotNull(summary.Commit); + Assert.False(Plan.IsAssigned("garage")); + } + + [Fact] + public async Task Run_StepCapsRespected() + { + var notes = Numbered(40); + await SeedAsync(notes); + _model.Reply(AssignReply(("alpha", notes))); + + var summary = await Reconciler(Defaults with { AssignBatchSize = 20, MaxAssignBatchesPerRun = 1 }).RunAsync(CancellationToken.None); + + Assert.Single(_model.Calls); + Assert.Equal(20, summary.Assigned); + Assert.Equal(20, Plan.Assignments.Count); + } + + [Fact] + public async Task Run_SplitAndFoldCapsRespected() + { + await SeedAsync("a1", "a2", "b1", "b2", "c1", "c2"); + _model.Reply(AssignReply(("a", ["a1", "a2"]), ("b", ["b1", "b2"]), ("c", ["c1", "c2"]))); + _model.Reply("{}"); + _model.Reply("""{"keep":true}"""); + + await Reconciler(Defaults with { MaxSourcesPerArticle = 1, MaxSplitsPerRun = 1, FoldMinSources = 3, MaxFoldsPerRun = 1 }) + .RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[split]", "[fold]"], _model.Calls.Select(c => c.System.Split('\n')[0])); + } + + [Fact] + public async Task Run_ModelUnavailable_CommitsStaged() + { + var notes = Numbered(40); + await SeedAsync(notes); + _model.Reply(AssignReply(("alpha", notes))); + _model.Throw(new ModelUnavailableException("credit exhausted")); + + var summary = await Reconciler(Defaults with { AssignBatchSize = 20 }).RunAsync(CancellationToken.None); + + Assert.True(summary.StoppedByModel); + Assert.Equal(20, summary.Assigned); + Assert.Empty(_writer.Calls); + Assert.Equal([PlanPath], Assert.Single(_store.Commits).Changes.Select(c => c.Path)); + Assert.Equal(20, Plan.Assignments.Count); + } + + [Fact] + public async Task Run_WriteLinkTargetsAreBounded() + { + var notes = Numbered(15); + await SeedAsync(notes); + _model.Reply(AssignReply(notes.Select(n => ("a-" + n, new[] { n })).ToArray())); + var settings = Defaults with { CandidateArticles = 3, MaxArticlesPerRun = 20 }; + + await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal(15, _writer.Calls.Count); + Assert.All(_writer.Links.Values, l => Assert.InRange(l.Count, 0, 3)); + Assert.Contains(_writer.Links.Values, l => l.Count == 3); + } + + [Fact] + public async Task Run_SourcesBeyondCharCap_LeftOutAndNotStale() + { + _store.Files["wiki/a.md"] = Note("a", new string('x', 60)); + _store.Files["wiki/b.md"] = Note("b", new string('y', 60)); + await Sync(); + _model.Reply(AssignReply(("heating", ["a", "b"]))); + var settings = Defaults with { MaxSourceCharsPerArticle = 100, MaxSourcesPerArticle = 120, MaxSplitsPerRun = 0 }; + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + + var summary = await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal(["a"], _writer.Sources["heating"]); + Assert.Null(summary.Commit); + Assert.Single(_writer.Calls); } [Fact] @@ -97,7 +409,7 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); Assert.Equal("catalog-behind", summary.Skipped); - Assert.Empty(_planModel.Calls); + Assert.Empty(_model.Calls); Assert.Empty(_writer.Calls); Assert.Empty(_store.Commits); } @@ -106,7 +418,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_AfterOwnCommit_IsNoop() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); await Reconciler().RunAsync(CancellationToken.None); var sync = await Sync(force: false); Assert.False(sync.Skipped); @@ -116,7 +428,7 @@ public class EncyclopediaReconcilerTests Assert.Null(summary.Skipped); Assert.Null(summary.Commit); Assert.Single(_store.Commits); - Assert.Single(_planModel.Calls); + Assert.Single(_model.Calls); Assert.Equal(2, _writer.Calls.Count); } @@ -131,7 +443,7 @@ public class EncyclopediaReconcilerTests Assert.Equal(1, summary.Written); Assert.Equal(["heating", "kitchen", "heating"], _writer.Calls); - Assert.Single(_planModel.Calls); + Assert.Single(_model.Calls); Assert.Equal(["articles/_state.json", "articles/de/heating.md", "articles/en/heating.md"], LastCommit.Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.All(LastCommit, c => Assert.Equal(FileChangeOp.Update, c.Op)); @@ -150,10 +462,10 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Reconcile_ModelUnavailable_CommitsWhatIsDone() + public async Task Reconcile_WriterModelUnavailable_CommitsWhatIsDone() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); _writer.Throws["kitchen"] = new ModelUnavailableException("credit exhausted"); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -166,13 +478,13 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Reconcile_PlanModelUnavailable_StopsKeepsPlan() + public async Task Reconcile_AssignModelUnavailable_StopsKeepsPlan() { await FirstRunAsync(); var plan = _store.Files[PlanPath]; _store.Files["wiki/garage.md"] = Note("garage"); await Sync(); - _planModel.Throw(new ModelUnavailableException("credit exhausted")); + _model.Throw(new ModelUnavailableException("credit exhausted")); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -180,7 +492,6 @@ public class EncyclopediaReconcilerTests Assert.Single(_store.Commits); Assert.Equal(plan, _store.Files[PlanPath]); Assert.Equal(2, _writer.Calls.Count); - Assert.Null(State.FailedPlan); } [Fact] @@ -196,7 +507,7 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); Assert.Equal(1, summary.Rejected); - Assert.Equal("articles: 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); + Assert.Equal("articles: 0 assigned, 0 split, 0 folded, 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); Assert.Equal([StatePath], LastCommit.Select(c => c.Path)); Assert.Equal(de, _store.Files["articles/de/heating.md"]); Assert.Equal(en, _store.Files["articles/en/heating.md"]); @@ -247,7 +558,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_Http400_RememberedAsRejected() { await SeedAsync(); - _planModel.Reply(PlanReply(("heating", ["boiler"]))); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "kitchen"]))); var writeModel = new FakeTextModel("write") .Throw(new HttpRequestException("invalid request", null, HttpStatusCode.BadRequest)); var writer = new WriteArticleUseCase(writeModel, Defaults, _clock); @@ -267,7 +578,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_TransientHttpError_NotRemembered() { await SeedAsync(); - _planModel.Reply(PlanReply(("heating", ["boiler"]))); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "kitchen"]))); _writer.Throws["heating"] = new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway); var first = await Reconciler().RunAsync(CancellationToken.None); @@ -284,7 +595,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_StalledStream_CommitsOthersNotRemembered() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); _writer.Throws["heating"] = new HttpRequestException(HttpRequestError.ResponseEnded, "stream stalled", new TimeoutException()); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -295,73 +606,19 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Reconcile_PlanFailedSameNotes_DoesNotReplan() - { - await SeedAsync(); - _planModel.Reply("I would rather not."); - - var first = await Reconciler().RunAsync(CancellationToken.None); - await Sync(); - var second = await Reconciler().RunAsync(CancellationToken.None); - - Assert.Equal("no-plan", first.Skipped); - Assert.Equal("no-plan", second.Skipped); - Assert.Single(_planModel.Calls); - Assert.Single(_store.Commits); - Assert.NotNull(State.FailedPlan); - } - - [Fact] - public async Task Reconcile_NoPlan_WritesNothing() - { - var old = ArticleFile.Build(new Article("de", "old", "Alt", ArticleCategory.Topic, ArticleVisibility.Public, - [new ArticleSource("boiler", "h")], new DateOnly(2026, 1, 1), "m", "Alt.\n")); - _store.Files["articles/de/old.md"] = old; - await SeedAsync(); - _planModel.Reply("I would rather not."); - - var summary = await Reconciler().RunAsync(CancellationToken.None); - - Assert.Equal("no-plan", summary.Skipped); - Assert.Equal([StatePath], Assert.Single(_store.Commits).Changes.Select(c => c.Path)); - Assert.Equal(old, _store.Files["articles/de/old.md"]); - Assert.Empty(_writer.Calls); - } - - [Fact] - public async Task Reconcile_SubjectDropped_DeletesBothLanguages() - { - await FirstRunAsync(); - _store.Files["wiki/garage.md"] = Note("garage"); - await Sync(); - _planModel.Reply(PlanReply(("heating", ["boiler", "radiators"]), ("garage", ["garage"]))); - - var summary = await Reconciler().RunAsync(CancellationToken.None); - - Assert.Equal(1, summary.Deleted); - Assert.Equal(1, summary.Written); - Assert.Equal("articles: 1 written, 1 deleted, 0 rejected", _store.Commits[^1].Message); - Assert.Equal(["articles/de/kitchen.md", "articles/en/kitchen.md"], - LastCommit.Where(c => c.Op == FileChangeOp.Delete).Select(c => c.Path).Order(StringComparer.Ordinal)); - Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); - Assert.DoesNotContain("kitchen", State.Subjects.Keys); - Assert.Equal(["heating", "kitchen", "garage"], _writer.Calls); - } - - [Fact] - public async Task Reconcile_ArticleNotInAnyPlan_NotDeleted() + public async Task Reconcile_ArticleNotInPlan_NotDeleted() { var legacy = ArticleFile.Build(new Article("de", "legacy", "Alt", ArticleCategory.Topic, ArticleVisibility.Public, [new ArticleSource("boiler", "h")], new DateOnly(2026, 1, 1), "m", "Alt.\n")); _store.Files["articles/de/legacy.md"] = legacy; await FirstRunAsync(); - _store.Files["wiki/garage.md"] = Note("garage"); + _store.Files.Remove("wiki/kitchen.md"); await Sync(); - _planModel.Reply(PlanReply([.. HeatingAndKitchen, ("garage", ["garage"])])); await Reconciler().RunAsync(CancellationToken.None); - Assert.All(_store.Commits.SelectMany(c => c.Changes), c => Assert.NotEqual(FileChangeOp.Delete, c.Op)); + Assert.Equal(["articles/de/kitchen.md", "articles/en/kitchen.md"], + LastCommit.Where(c => c.Op == FileChangeOp.Delete).Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.Equal(legacy, _store.Files["articles/de/legacy.md"]); } @@ -369,7 +626,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_PilotCapShrinks_DeletesNothing() { await SeedAsync("boiler", "radiators", "kitchen", "garage"); - _planModel.Reply(PlanReply([.. HeatingAndKitchen, ("garage", ["garage"])])); + _model.Reply(AssignReply([.. HeatingAndKitchen, ("garage", ["garage"])])); await Reconciler().RunAsync(CancellationToken.None); await Sync(); Assert.Equal(6, _store.Files.Keys.Count(p => p.StartsWith("articles/d", StringComparison.Ordinal) @@ -386,7 +643,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_PilotSlugs_WritesOnlyListed() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); await Reconciler(Defaults with { PilotSlugs = ["kitchen"] }).RunAsync(CancellationToken.None); @@ -398,7 +655,7 @@ public class EncyclopediaReconcilerTests [Fact] public void Due_MaxArticles_KeepsOnlyFirstOfPlan() { - var notes = new[] { PlanArticlesUseCaseTests.Note("a"), PlanArticlesUseCaseTests.Note("b") }; + var notes = new[] { TestNotes.Note("a"), TestNotes.Note("b") }; var a = new PlannedArticle("a", "A", "A", ArticleCategory.Topic, "", ["a"]); var b = new PlannedArticle("b", "B", "B", ArticleCategory.Topic, "", ["b"]); var plan = ArticlePlan.Empty(); @@ -415,7 +672,7 @@ public class EncyclopediaReconcilerTests { await SeedAsync(); _store.Files["articles/de/heating.md"] = "written by someone else"; - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -430,7 +687,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_AmbiguousTitles_IsNoopOnSecondRun(string titleDe, string titleEn) { await SeedAsync(); - _planModel.Reply(TitledPlanReply(titleDe, titleEn)); + _model.Reply(TitledAssignReply(("heating", titleDe, titleEn, ["boiler", "radiators", "kitchen"]))); await Reconciler().RunAsync(CancellationToken.None); await Sync(); @@ -445,7 +702,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_WriterTimeout_CommitsStagedThenThrows() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); _writer.Throws["kitchen"] = new TaskCanceledException("HttpClient timeout"); await Assert.ThrowsAsync(() => Reconciler().RunAsync(CancellationToken.None)); @@ -459,7 +716,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_CallerCancelled_NoCommit() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); using var cts = new CancellationTokenSource(); _writer.Before = slug => { if (slug == "kitchen") cts.Cancel(); }; _writer.Throws["kitchen"] = new OperationCanceledException(cts.Token); @@ -472,16 +729,16 @@ public class EncyclopediaReconcilerTests [Fact] public void Stale_TitleOnlyChange_IsStale() { - var notes = new[] { PlanArticlesUseCaseTests.Note("a", hash: "h1") }; + var notes = new[] { TestNotes.Note("a", hash: "h1") }; var planned = new PlannedArticle("a", "Neu", "New", ArticleCategory.Topic, "", ["a"]); Article Existing(string lang, string title) => new(lang, "a", title, ArticleCategory.Topic, ArticleVisibility.Public, [new ArticleSource("a", "h1")], new DateOnly(2026, 10, 1), "m", "x\n"); var plan = ArticlePlan.Empty(); plan.AddArticle(planned); - Assert.Empty(EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "New")])); - Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Alt"), Existing("en", "New")])); - Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "Old")])); + Assert.Empty(EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "New")], Defaults)); + Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Alt"), Existing("en", "New")], Defaults)); + Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "Old")], Defaults)); } [Fact] @@ -490,7 +747,7 @@ public class EncyclopediaReconcilerTests await FirstRunAsync(); _store.Files["wiki/attic.md"] = Note("attic"); await Sync(); - _planModel.Reply(PlanReply([.. HeatingAndKitchen, ("attic", ["attic"])])); + _model.Reply(AssignReply(("attic", ["attic"]))); await Reconciler().RunAsync(CancellationToken.None); await Sync(); var commits = _store.Commits.Count; @@ -502,10 +759,28 @@ public class EncyclopediaReconcilerTests Assert.Equal(["attic", "heating", "kitchen"], State.Subjects.Keys); } + /// Embeddings that fail like an unreachable service while . + private sealed class SwitchableEmbeddings(IEmbeddingModel inner) : IEmbeddingModel + { + public bool Down { get; set; } + public string ModelId => inner.ModelId; + public int Dimension => inner.Dimension; + public string Name => inner.Name; + + public Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct) => + Down ? Task.FromException>(new HttpRequestException("connection refused")) : inner.EmbedAsync(texts, ct); + + public Task CheckAsync(CancellationToken ct) => inner.CheckAsync(ct); + } + /// Writes "Text {slug}" in both languages from the given sources; throws the scripted exception for a slug. private sealed class FakeWriter : IArticleWriter { public List Calls { get; } = []; + /// Link target slugs of the last write of each article. + public Dictionary> Links { get; } = new(); + /// Source slugs of the last write of each article. + public Dictionary> Sources { get; } = new(); public Dictionary Throws { get; } = new(); public Action? Before { get; set; } @@ -513,6 +788,8 @@ public class EncyclopediaReconcilerTests IReadOnlyList linkTargets, CancellationToken ct) { Calls.Add(planned.Slug); + Links[planned.Slug] = linkTargets.Select(a => a.Slug).ToList(); + Sources[planned.Slug] = sources.Select(d => d.Slug).ToList(); Before?.Invoke(planned.Slug); if (Throws.TryGetValue(planned.Slug, out var e)) return Task.FromException(e); var de = new Article("de", planned.Slug, planned.TitleDe, planned.Category, ArticleVisibility.Public, diff --git a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs index 2a388db..11b4977 100644 --- a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs @@ -16,14 +16,12 @@ public class EncyclopediaStateTests ["heating"] = new("abc", AttemptOutcome.Written, At), ["sauna"] = new("def", AttemptOutcome.Rejected, At), }, - new PlanAttempt("zzz", At), new Dictionary(), new Dictionary()); var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); Assert.Equal(state.Subjects, read.Subjects); - Assert.Equal(state.FailedPlan, read.FailedPlan); } [Fact] @@ -31,7 +29,6 @@ public class EncyclopediaStateTests { var state = new EncyclopediaState( new Dictionary(), - null, new Dictionary { ["fp1"] = At }, new Dictionary { ["heating"] = "fp2" }); @@ -42,21 +39,22 @@ public class EncyclopediaStateTests } [Fact] - public void Read_OldJsonWithoutNewFields_ReadsEmptyBatchesAndKept() + public void Read_OldJsonWithFailedPlan_IgnoresIt() { var read = EncyclopediaStateJson.Read("""{"subjects":{},"failedPlan":{"noteSetFingerprint":"x","at":"2026-10-24T03:00:00+00:00"}}"""); - Assert.NotNull(read.FailedPlan); + Assert.Empty(read.Subjects); Assert.Empty(read.FailedBatches); Assert.Empty(read.KeptArticles); + Assert.DoesNotContain("failedPlan", EncyclopediaStateJson.Write(read)); } [Fact] public void BatchFingerprint_ChangesWhenANoteChanges() { - var ab = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "2")]); - var ba = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("b", hash: "2"), PlanArticlesUseCaseTests.Note("a", hash: "1")]); - var changed = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "3")]); - var fewer = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1")]); + var ab = EncyclopediaState.BatchFingerprint([TestNotes.Note("a", hash: "1"), TestNotes.Note("b", hash: "2")]); + var ba = EncyclopediaState.BatchFingerprint([TestNotes.Note("b", hash: "2"), TestNotes.Note("a", hash: "1")]); + var changed = EncyclopediaState.BatchFingerprint([TestNotes.Note("a", hash: "1"), TestNotes.Note("b", hash: "3")]); + var fewer = EncyclopediaState.BatchFingerprint([TestNotes.Note("a", hash: "1")]); Assert.Equal(ab, ba); Assert.NotEqual(ab, changed); Assert.NotEqual(ab, fewer); @@ -68,7 +66,6 @@ public class EncyclopediaStateTests { var read = EncyclopediaStateJson.Read("{{{ nope"); Assert.Empty(read.Subjects); - Assert.Null(read.FailedPlan); } [Fact] @@ -89,25 +86,15 @@ public class EncyclopediaStateTests public void SubjectFingerprint_ChangesWithSourceHashOrTitle() { var planned = new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, "", ["a", "b"]); - var notes = new[] { PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "2") }; + var notes = new[] { TestNotes.Note("a", hash: "1"), TestNotes.Note("b", hash: "2") }; var baseline = EncyclopediaState.SubjectFingerprint(planned, notes); Assert.Equal(baseline, EncyclopediaState.SubjectFingerprint(planned, notes)); Assert.Matches("^[0-9a-f]{64}$", baseline); - var changedHash = new[] { notes[0], PlanArticlesUseCaseTests.Note("b", hash: "3") }; + var changedHash = new[] { notes[0], TestNotes.Note("b", hash: "3") }; Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned, changedHash)); Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { TitleDe = "Heizungsanlage" }, notes)); Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { Scope = "Only boilers" }, notes)); } - - [Fact] - public void NoteSetFingerprint_IgnoresOrder_ChangesWithSlugs() - { - var ab = EncyclopediaState.NoteSetFingerprint([PlanArticlesUseCaseTests.Note("a"), PlanArticlesUseCaseTests.Note("b")]); - var ba = EncyclopediaState.NoteSetFingerprint([PlanArticlesUseCaseTests.Note("b"), PlanArticlesUseCaseTests.Note("a")]); - var abc = EncyclopediaState.NoteSetFingerprint([PlanArticlesUseCaseTests.Note("a"), PlanArticlesUseCaseTests.Note("b"), PlanArticlesUseCaseTests.Note("c")]); - Assert.Equal(ab, ba); - Assert.NotEqual(ab, abc); - } } diff --git a/test/Wiki.Application.Tests/Fakes.cs b/test/Wiki.Application.Tests/Fakes.cs index c30c12a..bfb93b3 100644 --- a/test/Wiki.Application.Tests/Fakes.cs +++ b/test/Wiki.Application.Tests/Fakes.cs @@ -6,6 +6,14 @@ using Wiki.Core.Ports; namespace Wiki.Application.Tests; +public static class TestNotes +{ + /// A wiki note with a body of chars, tagged heating. + public static Document Note(string slug, int bodyLength = 100, string hash = "h", string title = "T") => + new(DocumentKind.Wiki, slug, $"wiki/{slug}.md", hash, title, null, ["heating"], [], [], [], null, + new Dictionary(), new string('x', bodyLength), 0); +} + /// In-memory ports for use-case tests. Content hash = SHA-1 of the text, like a git blob. public sealed class FakeStore : INoteStore { diff --git a/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs b/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs index ab6cdcc..b7e76b3 100644 --- a/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Tests; public class FoldArticleUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null) { CandidateArticles = 2 }; + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null) { CandidateArticles = 2 }; private static FoldArticleUseCase UseCase(FakeTextModel model) => new(model, Settings, NullLogger.Instance); @@ -15,7 +15,7 @@ public class FoldArticleUseCaseTests new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, sources); private static Dictionary Notes(params string[] slugs) => - slugs.ToDictionary(s => s, s => PlanArticlesUseCaseTests.Note(s)); + slugs.ToDictionary(s => s, s => TestNotes.Note(s)); [Fact] public async Task Fold_Into() @@ -30,6 +30,23 @@ public class FoldArticleUseCaseTests Assert.StartsWith("[fold]", model.Calls[0].System); } + [Fact] + public async Task Fold_ListsAtMostFoldMinSources() + { + var model = new FakeTextModel().Reply("""{"keep":true}"""); + var thin = Article("valve", "a", "b", "c", "d"); + var notes = new Dictionary + { + ["a"] = TestNotes.Note("a", title: "Na"), ["b"] = TestNotes.Note("b", title: "Nb"), + ["c"] = TestNotes.Note("c", title: "Nc"), ["d"] = TestNotes.Note("d", title: "Nd"), + }; + + await UseCase(model).FoldAsync(thin, notes, [Article("heating", "h")], CancellationToken.None); + + Assert.Contains("Nc", model.Calls[0].User); + Assert.DoesNotContain("Nd", model.Calls[0].User); + } + [Fact] public async Task Fold_Keep_IsNull() { @@ -109,13 +126,13 @@ public class FoldArticleUseCaseTests [Fact] public void IsThin_Thresholds() { - var s = new EncyclopediaSettings(8000, 16000, 10, 300_000, [], null); + var s = new EncyclopediaSettings(16000, 10, 300_000, [], null); var notes = new Dictionary { - ["a"] = PlanArticlesUseCaseTests.Note("a", 1000), - ["b"] = PlanArticlesUseCaseTests.Note("b", 1000), - ["c"] = PlanArticlesUseCaseTests.Note("c", 1000), - ["big"] = PlanArticlesUseCaseTests.Note("big", 4000), + ["a"] = TestNotes.Note("a", 1000), + ["b"] = TestNotes.Note("b", 1000), + ["c"] = TestNotes.Note("c", 1000), + ["big"] = TestNotes.Note("big", 4000), }; Assert.True(FoldArticleUseCase.IsThin(Article("x", "a", "b"), notes, s)); diff --git a/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs b/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs deleted file mode 100644 index 4e172f1..0000000 --- a/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs +++ /dev/null @@ -1,215 +0,0 @@ -using System.Net; -using Microsoft.Extensions.Logging.Abstractions; -using Wiki.Application.Encyclopedia; -using Wiki.Core.Model; - -namespace Wiki.Application.Tests; - -public class PlanArticlesUseCaseTests -{ - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); - - internal static Document Note(string slug, int bodyLength = 100, string hash = "h", string title = "T") => - new(DocumentKind.Wiki, slug, $"wiki/{slug}.md", hash, title, null, ["heating"], [], [], [], null, - new Dictionary(), new string('x', bodyLength), 0); - - private static PlanArticlesUseCase UseCase(FakeTextModel model, EncyclopediaSettings? settings = null) => - new(model, settings ?? Settings, NullLogger.Instance); - - private static ArticlePlan PlanFrom(PlannedArticle article, params string[] notes) - { - var plan = ArticlePlan.Empty(); - plan.AddArticle(article); - foreach (var n in notes) plan.Assign(n, article.Sources.Contains(n) ? [article.Slug] : []); - return plan; - } - - private static ArticlePlan PlanOf(params string[] covered) => - PlanFrom(new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, "", covered), covered); - - [Fact] - public void NeedsReplan_NullPlan_True() => - Assert.True(PlanArticlesUseCase.NeedsReplan(null, null, [Note("a")])); - - [Fact] - public void NeedsReplan_SameSlugs_False() => - Assert.False(PlanArticlesUseCase.NeedsReplan(PlanOf("b", "a"), null, [Note("a"), Note("b")])); - - [Fact] - public void NeedsReplan_NoteAdded_True() => - Assert.True(PlanArticlesUseCase.NeedsReplan(PlanOf("a"), null, [Note("a"), Note("b")])); - - [Fact] - public void NeedsReplan_FailedForSameNoteSet_False() - { - var notes = new[] { Note("a"), Note("b") }; - var failed = new PlanAttempt(EncyclopediaState.NoteSetFingerprint(notes), DateTimeOffset.UnixEpoch); - Assert.False(PlanArticlesUseCase.NeedsReplan(null, failed, notes)); - } - - [Fact] - public void NeedsReplan_FailedForOtherNoteSet_True() - { - var failed = new PlanAttempt(EncyclopediaState.NoteSetFingerprint([Note("a")]), DateTimeOffset.UnixEpoch); - Assert.True(PlanArticlesUseCase.NeedsReplan(null, failed, [Note("a"), Note("b")])); - } - - [Fact] - public async Task Plan_ParsesAndValidates() - { - var model = new FakeTextModel().Reply(""" - ```json - {"articles":[ - {"slug":"heating","titleDe":"Heizung","titleEn":"Heating","category":"system","sources":["a","ghost","b"]}, - {"slug":"Bad Slug","titleDe":"x","titleEn":"x","category":"topic","sources":["a"]}, - {"slug":"heating","titleDe":"dup","titleEn":"dup","category":"topic","sources":["a"]}, - {"slug":"empty","titleDe":"x","titleEn":"x","category":"topic","sources":["ghost"]} - ]} - ``` - """); - - var plan = await UseCase(model).PlanAsync(null, [Note("a"), Note("b")], CancellationToken.None); - - Assert.NotNull(plan); - var article = Assert.Single(plan.Articles); - Assert.Equal("heating", article.Slug); - Assert.Equal(ArticleCategory.System, article.Category); - Assert.Equal(["a", "b"], article.Sources); - Assert.Equal(["a", "b"], plan.Assignments.Keys.Order()); - } - - [Fact] - public async Task Plan_ModelGarbage_KeepsExistingPlan() - { - var model = new FakeTextModel().Reply("Sure! Here is"); - Assert.Null(await UseCase(model).PlanAsync(PlanOf("a"), [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_Truncated_KeepsExistingPlan() - { - var json = """{"articles":[{"slug":"heating","titleDe":"H","titleEn":"H","category":"system","sources":["a"]}]}"""; - var model = new FakeTextModel().Reply(json, truncated: true); - Assert.Null(await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_Http400_ReturnsNull() - { - var model = new FakeTextModel().Throw(new HttpRequestException("bad", null, HttpStatusCode.BadRequest)); - Assert.Null(await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_ModelUnavailable_Propagates() - { - var model = new FakeTextModel().Throw(new ModelUnavailableException("quota")); - await Assert.ThrowsAsync(() => UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_Http500_Propagates() - { - var model = new FakeTextModel().Throw(new HttpRequestException("boom", null, HttpStatusCode.InternalServerError)); - await Assert.ThrowsAsync(() => UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_SourcesOverCharCap_DropsLowestRanked() - { - var model = new FakeTextModel().Reply("""{"articles":[{"slug":"big","titleDe":"G","titleEn":"B","category":"topic","sources":["a","b","c"]}]}"""); - var notes = new[] { Note("a", 200_000), Note("b", 200_000), Note("c", 200_000) }; - - var plan = await UseCase(model).PlanAsync(null, notes, CancellationToken.None); - - Assert.Equal(["a"], Assert.Single(plan!.Articles).Sources); - } - - [Fact] - public async Task Plan_PromptContainsExistingSlugs() - { - var model = new FakeTextModel().Reply("nope"); - var current = PlanFrom(new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, "", ["a"]), ["a"]); - - await UseCase(model).PlanAsync(current, [Note("a", 42, title: "Valve notes")], CancellationToken.None); - - var call = Assert.Single(model.Calls); - Assert.Contains("heating-valves", call.User); - Assert.Contains("a | Valve notes | heating | 42", call.User); - Assert.StartsWith("[plan]", call.System); - Assert.Equal(Settings.PlanMaxTokens, call.MaxTokens); - } - - [Fact] - public async Task Plan_PromptForbidsConcreteTitles_AndKeepsTitles() - { - var model = new FakeTextModel().Reply("nope"); - - await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None); - - var system = Assert.Single(model.Calls).System; - Assert.Contains("Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses.", system); - Assert.Contains("Keep the existing titles for existing subjects unless the subject changed.", system); - } - - [Fact] - public async Task Plan_PromptAsksForCompactJson_AndOmitsCoveredNotes() - { - var model = new FakeTextModel().Reply("nope"); - var current = PlanFrom(new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, "", ["a"]), ["a", "covered-only"]); - - await UseCase(model).PlanAsync(current, [Note("a")], CancellationToken.None); - - var call = Assert.Single(model.Calls); - Assert.Contains("compact", call.System); - Assert.DoesNotContain("coveredNotes", call.User); - Assert.DoesNotContain("covered-only", call.User); - var jsonLine = call.User.Split('\n').Single(l => l.StartsWith("[{")); - Assert.Equal("""[{"slug":"heating-valves","titleDe":"H","titleEn":"H","category":"device","scope":"","sources":["a"]}]""", jsonLine); - } - - [Fact] - public async Task Plan_TitleWithPipeOrNewline_Sanitized() - { - var model = new FakeTextModel().Reply("nope"); - var note = Note("a", title: "Left | Right\nSecond line"); - - await UseCase(model).PlanAsync(null, [note], CancellationToken.None); - - var line = Assert.Single(model.Calls[0].User.Split('\n'), l => l.StartsWith("a | ")); - Assert.Equal(4, line.Split(" | ").Length); - Assert.Contains("Left Right Second line", line); - } - - [Fact] - public void ArticlePlanJson_NumericCategory_ArticleSkipped() - { - var plan = ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"1","sources":["x"]},{"slug":"b","category":"room","sources":["x"]}],"assignments":{}}"""); - Assert.Equal(["b"], plan!.Articles.Select(a => a.Slug)); - } - - [Fact] - public void ArticlePlanJson_Malformed_ReadsNull() - { - Assert.Null(ArticlePlanJson.Read("not json")); - Assert.Null(ArticlePlanJson.Read("""{"foo":1}""")); - } - - [Fact] - public void ArticlePlanJson_RoundTrip() - { - var plan = PlanOf("a", "b"); - var read = ArticlePlanJson.Read(ArticlePlanJson.Write(plan)); - Assert.NotNull(read); - Assert.Equal(plan.Articles[0], read.Articles[0],new PlannedArticleComparer()); - Assert.Equal(plan.Assignments.Keys, read.Assignments.Keys); - } - - private sealed class PlannedArticleComparer : IEqualityComparer - { - public bool Equals(PlannedArticle? x, PlannedArticle? y) => - x!.Slug == y!.Slug && x.TitleDe == y.TitleDe && x.TitleEn == y.TitleEn && x.Category == y.Category - && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources); - public int GetHashCode(PlannedArticle obj) => obj.Slug.GetHashCode(); - } -} diff --git a/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs b/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs index 094249c..ee1b387 100644 --- a/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Tests; public class SplitArticleUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null) { MaxSourcesPerArticle = 4 }; + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null) { MaxSourcesPerArticle = 4 }; private static SplitArticleUseCase UseCase(FakeTextModel model) => new(model, Settings, NullLogger.Instance); @@ -15,7 +15,7 @@ public class SplitArticleUseCaseTests new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, sources); private static Dictionary Notes(params string[] slugs) => - slugs.ToDictionary(s => s, s => PlanArticlesUseCaseTests.Note(s)); + slugs.ToDictionary(s => s, s => TestNotes.Note(s)); private static ArticlePlan PlanWith(params PlannedArticle[] articles) { @@ -193,7 +193,7 @@ public class SplitArticleUseCaseTests { var article = Article("heating", "a", "b", "c", "d"); var notes = Notes("a", "b", "c", "d"); - notes["a"] = PlanArticlesUseCaseTests.Note("a", title: "one | two\nthree"); + notes["a"] = TestNotes.Note("a", title: "one | two\nthree"); var model = new FakeTextModel().Reply(Valid); await UseCase(model).SplitAsync(article, notes, PlanWith(article), CancellationToken.None); @@ -204,13 +204,13 @@ public class SplitArticleUseCaseTests [Fact] public void NeedsSplit_ByCharsAndByCount() { - var s = new EncyclopediaSettings(8000, 16000, 10, 1000, [], null) { MaxSourcesPerArticle = 3 }; + var s = new EncyclopediaSettings(16000, 10, 1000, [], null) { MaxSourcesPerArticle = 3 }; var notes = new Dictionary { - ["a"] = PlanArticlesUseCaseTests.Note("a", 600), - ["b"] = PlanArticlesUseCaseTests.Note("b", 600), - ["c"] = PlanArticlesUseCaseTests.Note("c", 100), - ["d"] = PlanArticlesUseCaseTests.Note("d", 1), + ["a"] = TestNotes.Note("a", 600), + ["b"] = TestNotes.Note("b", 600), + ["c"] = TestNotes.Note("c", 100), + ["d"] = TestNotes.Note("d", 1), }; Assert.True(SplitArticleUseCase.NeedsSplit(Article("x", "a", "b"), notes, s)); diff --git a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs index 901ac26..6ea0ce3 100644 --- a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs @@ -8,7 +8,7 @@ namespace Wiki.Application.Tests; public class WriteArticleUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null); private static readonly DateTimeOffset Now = new(2026, 10, 24, 3, 0, 0, TimeSpan.Zero); private static readonly PlannedArticle Planned = diff --git a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs index af7c6c0..094f00a 100644 --- a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs +++ b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs @@ -18,12 +18,13 @@ public sealed class EncyclopediaEndpointTests public List CatalogCommits { get; } = []; public Exception? Failure { get; set; } public bool StoppedByModel { get; set; } + public int Planned { get; set; } = 3; public async Task RunAsync(CancellationToken ct) { CatalogCommits.Add((await catalog.GetStateAsync(ct)).Commit); if (Failure is not null) throw Failure; - return new ReconcileSummary(3, 0, 0, 0, 0, StoppedByModel, null, null); + return new ReconcileSummary(Planned, 0, 0, 0, 0, 0, 0, 0, StoppedByModel, null, null); } } @@ -135,4 +136,18 @@ public sealed class EncyclopediaEndpointTests Assert.Equal(HttpStatusCode.BadGateway, r.StatusCode); Assert.True((await r.Content.ReadFromJsonAsync()).GetProperty("stoppedByModel").GetBoolean()); } + + [Fact] + public async Task Run_EmptyPlan_Is502() + { + var (host, _, run) = Enabled(); + using var _h = host; + var client = host.CreateClient(); + run().Planned = 0; + + var r = await client.SendAsync(RunRequest()); + + Assert.Equal(HttpStatusCode.BadGateway, r.StatusCode); + Assert.Equal(0, (await r.Content.ReadFromJsonAsync()).GetProperty("planned").GetInt32()); + } } -- 2.54.0 From 41a54a58446a861a6e7914752cb1019a497f0b11 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 09:50:55 +0100 Subject: [PATCH 13/16] feat(house): follow article redirects Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/ArticlePlan.cs | 9 ++- .../Reading/ArticleQueries.cs | 30 +++++++- .../Components/Pages/HouseArticle.razor | 20 +++++- src/Wiki.Host/Rendering/ArticleMarkdown.cs | 26 ++++--- test/Wiki.Host.Tests/ArticleMarkdownTests.cs | 32 ++++++++- test/Wiki.Host.Tests/HouseTests.cs | 70 +++++++++++++++++++ 6 files changed, 170 insertions(+), 17 deletions(-) diff --git a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs index 26ab426..0044b3e 100644 --- a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs +++ b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs @@ -123,13 +123,16 @@ public sealed class ArticlePlan } /// The article stands for, following redirects; null when unknown, cyclic or more than five hops. - public string? Resolve(string slug) + public string? Resolve(string slug) => ResolveRedirect(slug, _redirects, s => Find(s) is not null); + + /// The slug accepts that stands for, following ; null when none, cyclic or more than five hops. + public static string? ResolveRedirect(string slug, IReadOnlyDictionary redirects, Func isArticle) { var current = slug; for (var hops = 0; hops <= MaxRedirectHops; hops++) { - if (Find(current) is not null) return current; - if (!_redirects.TryGetValue(current, out var next)) return null; + if (isArticle(current)) return current; + if (!redirects.TryGetValue(current, out var next)) return null; current = next; } return null; diff --git a/src/Wiki.Application/Reading/ArticleQueries.cs b/src/Wiki.Application/Reading/ArticleQueries.cs index 0c01a25..017d66f 100644 --- a/src/Wiki.Application/Reading/ArticleQueries.cs +++ b/src/Wiki.Application/Reading/ArticleQueries.cs @@ -1,3 +1,4 @@ +using Wiki.Application.Encyclopedia; using Wiki.Core.Model; using Wiki.Core.Parsing; using Wiki.Core.Ports; @@ -7,11 +8,38 @@ namespace Wiki.Application.Reading; /// /// Read side of the house encyclopedia, straight from the catalog. An article file that does not parse /// counts as missing. Anonymous visibility is per subject: both language files must exist and be public. +/// Redirects come from articles/_plan.json at the catalog's commit. /// -public sealed class ArticleQueries(IDocumentCatalog catalog) +public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store) { public static readonly IReadOnlyList Languages = ["de", "en"]; + private const string PlanPath = "articles/_plan.json"; + + private static readonly IReadOnlyDictionary NoRedirects = new Dictionary(); + + private (string Commit, IReadOnlyDictionary Redirects)? _redirects; + + /// Retired slug → slug that took over; empty when the plan is missing or unreadable. Cached per commit. + public async Task> RedirectsAsync(CancellationToken ct) + { + var commit = (await catalog.GetStateAsync(ct)).Commit; + if (commit is null) return NoRedirects; + if (_redirects is { } cached && cached.Commit == commit) return cached.Redirects; + var text = await store.ReadTextAsync(PlanPath, commit, ct); + var redirects = (text is null ? null : ArticlePlanJson.Read(text.Text))?.Redirects ?? NoRedirects; + _redirects = (commit, redirects); + return redirects; + } + + /// The article slug stands for (itself when it is an article), following redirects; null when none, cyclic or too long a chain. + public async Task ResolveAsync(string slug, CancellationToken ct) + { + var redirects = await RedirectsAsync(ct); + var slugs = (await AllAsync(ct)).Select(a => a.Slug).ToHashSet(StringComparer.Ordinal); + return ArticlePlan.ResolveRedirect(slug, redirects, slugs.Contains); + } + public async Task GetAsync(string lang, string slug, CancellationToken ct) { var doc = await catalog.GetAsync(DocumentKind.Article, $"{lang}/{slug}", ct); diff --git a/src/Wiki.Host/Components/Pages/HouseArticle.razor b/src/Wiki.Host/Components/Pages/HouseArticle.razor index ebd4b46..8754552 100644 --- a/src/Wiki.Host/Components/Pages/HouseArticle.razor +++ b/src/Wiki.Host/Components/Pages/HouseArticle.razor @@ -79,6 +79,15 @@ else if (_article is not null && _rendered is not null) { var ct = HttpContext.RequestAborted; _signedIn = HouseEndpoints.IsHuman(HttpContext.User); + if (Rest is null && HouseEndpoints.IsLanguage(Lang) && HouseEndpoints.IsSlug(Slug) + && await Articles.GetAsync(Lang, Slug, ct) is null + && await RedirectTargetAsync(ct) is { } target) + { + // NavigateTo answers 302 in static SSR; a retired slug moved for good. + HttpContext.Response.StatusCode = StatusCodes.Status301MovedPermanently; + HttpContext.Response.Headers.Location = $"/house/{Lang}/{target}"; + return; + } if (Rest is not null || !HouseEndpoints.IsLanguage(Lang) || !HouseEndpoints.IsSlug(Slug) || await Articles.GetAsync(Lang, Slug, ct) is not { } article || (!_signedIn && await Articles.IsPrivateAsync(Slug, ct))) @@ -88,12 +97,21 @@ else if (_article is not null && _rendered is not null) return; } var titles = await Articles.VisibleTitlesAsync(Lang, _signedIn, ct); + var redirects = await Articles.RedirectsAsync(ct); _article = article; - _rendered = ArticleMarkdown.Render(article.Body, Lang, Slug, _signedIn, titles, Log); + _rendered = ArticleMarkdown.Render(article.Body, Lang, Slug, _signedIn, titles, redirects, Log); _infobox = WithTitleBar(_rendered.InfoboxHtml, article.Title); _otherExists = await Articles.GetAsync(Other, Slug, ct) is not null; } + /// The slug a retired resolves to when that article exists in and this reader may see it; else null. + private async Task RedirectTargetAsync(CancellationToken ct) + { + var target = await Articles.ResolveAsync(Slug, ct); + if (target is null || target == Slug || await Articles.GetAsync(Lang, target, ct) is null) return null; + return !_signedIn && await Articles.IsPrivateAsync(target, ct) ? null : target; + } + /// Puts a title bar first into the infobox, after its opening tag. private static string? WithTitleBar(string? infoboxHtml, string title) { diff --git a/src/Wiki.Host/Rendering/ArticleMarkdown.cs b/src/Wiki.Host/Rendering/ArticleMarkdown.cs index c7bd936..7adc3a9 100644 --- a/src/Wiki.Host/Rendering/ArticleMarkdown.cs +++ b/src/Wiki.Host/Rendering/ArticleMarkdown.cs @@ -9,6 +9,7 @@ using Markdig.Renderers.Html; using Markdig.Syntax; using Markdig.Syntax.Inlines; using Microsoft.Extensions.Logging; +using Wiki.Application.Encyclopedia; using Wiki.Core.Model; using Wiki.Core.Parsing; @@ -51,6 +52,8 @@ public static partial class ArticleMarkdown private static readonly object WrappedKey = new(); + private static readonly IReadOnlyDictionary NoRedirects = new Dictionary(); + private static readonly Regex RawImage = new( $@"^raw/[A-Za-z0-9._-]+\.({string.Join('|', Attachment.FileExtensions.Select(Regex.Escape))})\z", RegexOptions.CultureInvariant); @@ -71,9 +74,10 @@ public static partial class ArticleMarkdown /// false: the body is redacted first. /// slug → title in of every article this reader may see. + /// retired slug → slug that took over; a link to a slug outside follows them. public static RenderedArticle Render(string body, string lang, string slug, bool signedIn, - IReadOnlyDictionary visibleTitles, ILogger? logger = null) => - RenderVisible(signedIn ? body : ArticleBody.RedactPrivate(body), lang, slug, signedIn, visibleTitles, logger); + IReadOnlyDictionary visibleTitles, IReadOnlyDictionary? redirects = null, ILogger? logger = null) => + RenderVisible(signedIn ? body : ArticleBody.RedactPrivate(body), lang, slug, signedIn, visibleTitles, redirects, logger); /// /// The raw/ image paths the media endpoint may serve for this reader: index n is the image @@ -81,15 +85,15 @@ public static partial class ArticleMarkdown /// public static IReadOnlyList VisibleImagePaths(string body, string lang, string slug, bool signedIn, IReadOnlyDictionary visibleTitles) => - Process(signedIn ? body : ArticleBody.RedactPrivate(body), lang, slug, signedIn, visibleTitles, logger: null).Images; + Process(signedIn ? body : ArticleBody.RedactPrivate(body), lang, slug, signedIn, visibleTitles, redirects: null, logger: null).Images; /// Renders a body that is already what this reader may see. internal static RenderedArticle RenderVisible(string visibleBody, string lang, string slug, bool signedIn, - IReadOnlyDictionary visibleTitles, ILogger? logger = null) => - Process(visibleBody, lang, slug, signedIn, visibleTitles, logger).Article; + IReadOnlyDictionary visibleTitles, IReadOnlyDictionary? redirects = null, ILogger? logger = null) => + Process(visibleBody, lang, slug, signedIn, visibleTitles, redirects, logger).Article; private static (RenderedArticle Article, IReadOnlyList Images) Process(string visibleBody, string lang, string slug, - bool signedIn, IReadOnlyDictionary visibleTitles, ILogger? logger) + bool signedIn, IReadOnlyDictionary visibleTitles, IReadOnlyDictionary? redirects, ILogger? logger) { var slugs = new List(); var doc = Parse(visibleBody, slugs); @@ -106,7 +110,7 @@ public static partial class ArticleMarkdown var images = new List(); SanitizeLinks(doc, lang, slug, visibleTitles, images); - var headings = ResolveWikiLinks(doc, lang, slugs, visibleTitles); + var headings = ResolveWikiLinks(doc, lang, slugs, visibleTitles, redirects ?? NoRedirects); var leftovers = doc.Descendants().Where(HasLeftover).ToList(); if (leftovers.Count > 0) @@ -304,7 +308,7 @@ public static partial class ArticleMarkdown /// that held a marker. /// private static HashSet ResolveWikiLinks(MarkdownDocument doc, string lang, List slugs, - IReadOnlyDictionary visibleTitles) + IReadOnlyDictionary visibleTitles, IReadOnlyDictionary redirects) { var headings = new HashSet(); foreach (var leaf in doc.Descendants().Where(b => b.Inline is not null).ToList()) @@ -315,7 +319,7 @@ public static partial class ArticleMarkdown var insideLink = false; for (Inline? p = container; p is not null; p = p.Parent) insideLink |= p is LinkInline; - changed |= RewriteLiteralRuns(container, text => ResolveMarkers(text, lang, slugs, visibleTitles, insideLink)); + changed |= RewriteLiteralRuns(container, text => ResolveMarkers(text, lang, slugs, visibleTitles, redirects, insideLink)); } if (changed && leaf is HeadingBlock heading) headings.Add(heading); } @@ -323,7 +327,7 @@ public static partial class ArticleMarkdown } private static List? ResolveMarkers(string text, string lang, List slugs, - IReadOnlyDictionary visibleTitles, bool insideLink) + IReadOnlyDictionary visibleTitles, IReadOnlyDictionary redirects, bool insideLink) { if (!text.Contains(Marker)) return null; var parts = new List(); @@ -336,6 +340,8 @@ public static partial class ArticleMarkdown pos = m.Index + m.Length; var index = int.Parse(m.Groups[1].Value, System.Globalization.CultureInfo.InvariantCulture); var slug = index < slugs.Count ? slugs[index] : null; + if (slug is not null && !visibleTitles.ContainsKey(slug)) + slug = ArticlePlan.ResolveRedirect(slug, redirects, visibleTitles.ContainsKey); if (slug is not null && visibleTitles.TryGetValue(slug, out var title)) { FlushText(); diff --git a/test/Wiki.Host.Tests/ArticleMarkdownTests.cs b/test/Wiki.Host.Tests/ArticleMarkdownTests.cs index b29a966..5f12441 100644 --- a/test/Wiki.Host.Tests/ArticleMarkdownTests.cs +++ b/test/Wiki.Host.Tests/ArticleMarkdownTests.cs @@ -11,6 +11,34 @@ public class ArticleMarkdownTests ["blinds"] = "Rollläden", }; + private static readonly Dictionary Redirects = new() + { + ["old-heating"] = "heating", + ["older-heating"] = "old-heating", + ["old-secret"] = "secret", + ["loop-a"] = "loop-b", + ["loop-b"] = "loop-a", + }; + + [Fact] + public void WikiLink_ToFoldedSlug_PointsAtTarget() + { + var html = ArticleMarkdown.Render("Siehe [[old-heating]] und [[older-heating|Alias]].", "de", "switches", signedIn: false, Titles, Redirects).Html; + + Assert.Equal(2, html.Split("Heizung").Length - 1); + Assert.DoesNotContain("old-heating", html); + } + + [Fact] + public void WikiLink_RedirectToInvisible_Dropped() + { + var html = ArticleMarkdown.Render("Siehe [[old-secret]], [[loop-a]] x.", "de", "switches", signedIn: false, Titles, Redirects).Html; + + Assert.DoesNotContain(" ArticleMarkdown.Render(body, "de", "switches", signedIn: false, Titles); private static RenderedArticle SignedIn(string body) => ArticleMarkdown.Render(body, "de", "switches", signedIn: true, Titles); @@ -69,7 +97,7 @@ public class ArticleMarkdownTests var logger = new ListLogger(); var article = ArticleMarkdown.RenderVisible( - "## Teil\n\nÖffentlich.\n\n::: warning\nGEHEIM\n:::", "de", "switches", signedIn: false, Titles, logger); + "## Teil\n\nÖffentlich.\n\n::: warning\nGEHEIM\n:::", "de", "switches", signedIn: false, Titles, logger: logger); Assert.DoesNotContain("GEHEIM", article.Html); Assert.DoesNotContain("Öffentlich", article.Html); @@ -418,7 +446,7 @@ public class ArticleMarkdownTests var body = "Öffentlich.\n\n\\[\\[network-internals]]"; var logger = new ListLogger(); - var anonymous = ArticleMarkdown.RenderVisible(body, "de", "switches", signedIn: false, Titles, logger); + var anonymous = ArticleMarkdown.RenderVisible(body, "de", "switches", signedIn: false, Titles, logger: logger); Assert.DoesNotContain("network", anonymous.Html); Assert.Contains("
", anonymous.Html); Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning); diff --git a/test/Wiki.Host.Tests/HouseTests.cs b/test/Wiki.Host.Tests/HouseTests.cs index d52bcf0..b20bdda 100644 --- a/test/Wiki.Host.Tests/HouseTests.cs +++ b/test/Wiki.Host.Tests/HouseTests.cs @@ -44,6 +44,12 @@ public class HouseTests(WikiHostFactory factory) : IClassFixture