From 1c8c0dc15af807b3fc1ce1dd2b2ca0a45a66293c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 14:23:22 +0100 Subject: [PATCH 1/5] docs(encyclopedia): consolidation, hierarchy and rooms design Co-Authored-By: Claude Opus 5.5 --- ...10-26-encyclopedia-consolidation-design.md | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md diff --git a/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md b/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md new file mode 100644 index 0000000..d684a8b --- /dev/null +++ b/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md @@ -0,0 +1,68 @@ +# House encyclopedia: consolidation, hierarchy and rooms + +Extends `2026-10-26-encyclopedia-bounded-planning-design.md`. Supersedes its sections *Finding nearby articles* (assign candidates), *A run* (step list), *Links*, and the plan format; everything else there stays. The binding rule stays: **no model call may need the whole wiki in its context**. + +## Why + +The first production plan (299 articles) had two faults: + +1. **One subject, many peers.** Heating became `heizungsanlage` (1 note), `waermepumpe`, `vitocal-heatpump`, `vitocal-heat-pump` (all the same heat pump), `open3e-integration`, `open3e-viessmann-integration`, … with overlapping scopes and no hierarchy. Assign candidates came only through the articles a note's neighbour notes were already assigned to, so early batches rarely saw existing articles; nothing compared articles with each other; fold only looks at thin articles. +2. **No room articles**, although the wiki has room notes (`eg-schlafzimmer-nord-tur-wall-switch`, `eg-schlafzimmer-rollos-toggle-script`, …). The assign prompt sent a note only to "its parent system". + +## Article vectors + +Each run embeds every planned article's `titleDe | titleEn | scope` with the active local `IEmbeddingModel`, in batches of 32 texts. Vectors are cached by slug and text hash, so an unchanged article is embedded once; the cache holds only the current plan's articles. Search is cosine similarity in memory. No model call sees the vectors or the article list: this is local embedding and arithmetic only. An embedding failure ends the planning steps like any other unreachable service. + +## Assign candidates + +The candidates of a batch are two lists, interleaved and deduplicated, capped at `CandidateArticles × 2` as before: + +- the note-neighbour articles (unchanged: articles the batch notes' nearest notes are assigned to); +- the `CandidateArticles` articles nearest to each batch note's vector by article vector. + +Interleaving keeps both kinds in the prompt when one alone would fill the cap. A batch now sees the articles earlier batches created even when none of its neighbour notes is assigned yet. + +## A run + +1. Diff. 2. Assign. 3. **Consolidate** (new). 4. Split. 5. Fold. 6. Write. Order and caps otherwise unchanged. + +## Consolidate + +**Groups.** A seed article and its nearest articles by article vector, `ConsolidateGroupSize` (default 8) articles in all, the seed included. Seeds are taken in slug order among the articles no group of this run has covered yet; every member of a group formed this run (asked or remembered) counts as covered. Slug order, not size: it does not depend on source counts, which change nightly, so an unchanged neighbourhood forms the same groups night after night and is not asked again. + +**Call.** One per group, at most `MaxConsolidationsPerRun` (default 60) per run. 60 groups of 8 cover ~300+ articles in one bootstrap night even with overlapping groups (299 articles need at least 38). Input, per member: `slug | titleDe | category | scope | source count | current parent` — bounded by the group size. `ConsolidateMaxTokens` 8000 (the reply is short; thinking tokens count, as for fold). + +**Reply** (JSON): `{"merges":[{"from","into"}],"parents":[{"child","parent"}]}`. A merge says both articles are about the same subject; a parent link says which article is the overview of which. + +**Validation.** Both slugs of an entry must be members of the group and differ. A `from` merges once (first wins); an article merged away is never a merge target. Parent entries are resolved through the merges of the same reply (a merged child or parent stands for its target); a link that would close a cycle anywhere in the plan is dropped. Invalid entries are dropped one by one; a reply that is truncated, not JSON, rejected with 400 or has neither array is unusable. + +**Merge** uses fold's mechanics (`ArticlePlan.Fold`): sources move into the target, `redirects[from] = into`, public when both language files of `from` were public at the run's start (`publicRedirects`), redirects to `from` move along. Additionally the children of `from` move to `into`; if `into` was a child of `from`, it takes `from`'s parent. A move that would close a cycle leaves the child without parent. + +**Memory.** `_state.json` gains `ConsolidatedGroups` (fingerprint → when) for usable replies and `FailedGroups` (fingerprint → when) for unusable ones; a group with a remembered fingerprint is not asked again. Fingerprint: `Prompts.Version` and, per member in slug order, slug, titles, category, scope and parent. Source counts are left out: they change whenever a note is added and say nothing about whether two articles share a subject; with them, most groups would be asked again every night. Both maps keep only the fingerprints of groups formed in the latest run that reached the step, so they do not grow without bound. A call that fails with `HttpRequestException` skips that group only and is not remembered. + +`ReconcileSummary` gains `Merged` (merges applied) and `Parented` (parent links set); the commit message, endpoint body and status panel report them. + +## Hierarchy + +`PlannedArticle` gets an optional `parent` slug. Sources of parent links: consolidation, and split (children of a split get the overview as parent). Fold and merge re-point children; an article deleted for losing its last source leaves its children without parent. + +**Write prompt.** Link targets are the nearby articles (≤ `CandidateArticles`), the parent and up to 20 children (`MaxChildLinks`), marked `main article` / `sub-article` in the list. Split children are linked through this rule in every run, not only the splitting one. + +**Not stale.** A change of parent or children does not make an article stale, like other link-target changes: staleness is still sources and title. + +**Article page.** Deterministic navigation from the plan at the catalog's commit: a "Hauptartikel / Main article" link to the parent above the body, and a "Unterartikel / Sub-articles" list below it, by title. Both are filtered exactly like links: only articles this reader may see in this language (anonymous: both language files public). A private parent or child is invisible, title and slug alike. + +## Rooms + +Category `room` exists; the prompts now use it: + +- **Assign:** a note about something in a specific room additionally feeds that room's article, titled floor + room (e.g. "EG Schlafzimmer"), created when none is among the candidates. The granularity rule still applies: a thin room article may be folded, e.g. into its floor. +- **Write:** a room article describes what is in the room and what it does there, and links to the system articles for how things work. + +## One-time re-bootstrap + +Nothing has been written or published yet; only `_plan.json` and `_state.json` exist. The plan file gains `"version": 3`; a plan without it (the current v2) or with another version reads as "no plan", so the next run bootstraps with the new rules. When the stored plan does not read, the stored state is dropped as well (its subjects, failed batches and kept articles belong to the old plan). `Prompts.Version` goes to 8. + +## Cost + +Bootstrap: up to 60 consolidation calls of ~1k input tokens each on top of the assign calls. A normal night: the groups around changed articles. -- 2.54.0 From 179bc3bcbcec7886ad85dc04bd423bb83fbba723 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 14:36:59 +0100 Subject: [PATCH 2/5] feat(encyclopedia): consolidate nearby articles, plan hierarchy, room articles Article vectors (titleDe | titleEn | scope, embedded locally, cosine in memory) feed the assign candidates alongside the note-neighbour articles and form the groups of a new consolidate step after assign: one bounded call per group of up to 8 nearby articles, at most 60 per run, returning merges (fold mechanics, redirects incl. public ones) and parent links (no cycles anywhere in the plan). Groups are remembered by fingerprint in _state.json. PlannedArticle gains a parent; split children get the overview. The write prompt marks main and sub-articles among its link targets (children capped at 20); the article page shows the main article and the visible sub-articles from the plan. Parent changes do not make articles stale. The assign prompt also sends room notes to the room article; the write prompt writes room articles from the room's angle. The plan file gets version 3: the v2 plan reads as no plan, its state is dropped, and the next run bootstraps. Prompts.Version 8. Co-Authored-By: Claude Opus 5.5 --- .../Encyclopedia/ArticleNeighbours.cs | 71 ++++- .../Encyclopedia/ArticlePlan.cs | 82 +++++- .../ConsolidateArticlesUseCase.cs | 114 ++++++++ .../Encyclopedia/EncyclopediaReconciler.cs | 169 ++++++++++-- .../Encyclopedia/EncyclopediaSettings.cs | 6 +- .../Encyclopedia/EncyclopediaState.cs | 19 +- src/Wiki.Application/Encyclopedia/Prompts.cs | 25 +- .../Encyclopedia/WriteArticleUseCase.cs | 9 +- .../Reading/ArticleQueries.cs | 56 +++- .../Components/Pages/HouseArticle.razor | 20 ++ .../Components/Shared/StatusPanel.razor | 2 +- src/Wiki.Host/Composition/WikiOptions.cs | 4 + src/Wiki.Host/Program.cs | 6 +- .../ArticleNeighboursTests.cs | 82 ++++++ .../ArticlePlanTests.cs | 143 +++++++++- .../AssignNotesUseCaseTests.cs | 7 + .../ConsolidateArticlesUseCaseTests.cs | 130 +++++++++ .../EncyclopediaCoordinatorTests.cs | 2 +- .../EncyclopediaReconcilerTests.cs | 255 +++++++++++++++++- .../EncyclopediaSettingsTests.cs | 12 +- .../EncyclopediaStateTests.cs | 37 ++- .../WriteArticleUseCaseTests.cs | 25 ++ .../EncyclopediaEndpointTests.cs | 2 +- test/Wiki.Host.Tests/HouseTests.cs | 71 ++++- 24 files changed, 1259 insertions(+), 90 deletions(-) create mode 100644 src/Wiki.Application/Encyclopedia/ConsolidateArticlesUseCase.cs create mode 100644 test/Wiki.Application.Tests/ConsolidateArticlesUseCaseTests.cs diff --git a/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs index 439d266..3aceaf2 100644 --- a/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs +++ b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs @@ -1,3 +1,5 @@ +using System.Security.Cryptography; +using System.Text; using Wiki.Core.Model; using Wiki.Core.Parsing; using Wiki.Core.Ports; @@ -5,19 +7,27 @@ using Wiki.Core.Ports; namespace Wiki.Application.Encyclopedia; /// -/// Finds the articles and notes near a note through the note index, so prompts list a bounded candidate set -/// instead of every article. Embedding or index failures propagate; a request timeout (not the caller's -/// cancellation) surfaces as , like an unreachable service. +/// Finds the articles and notes near a note or an article, so prompts list a bounded candidate set instead of every +/// article: through the note index, and through article vectors ( embedded locally, searched in +/// memory by cosine). Embedding or index failures propagate; a request timeout (not the caller's cancellation) surfaces as +/// , like an unreachable service. /// public sealed class ArticleNeighbours(IEmbeddingModel model, IVectorIndex index, EncyclopediaSettings settings) { private const int EmbeddingChars = 1500; + private const int ArticleEmbedBatch = 32; /// The widest note query makes, as a multiple of the batch size. public const int MaxWidening = 8; private readonly Dictionary<(string Slug, string Hash), float[]> _vectors = []; + /// Unit-length article vectors of the plan last searched, by slug, with the hash of the text they embed. + private Dictionary _articleVectors = new(StringComparer.Ordinal); + + /// What an article vector embeds. + public static string ArticleText(PlannedArticle a) => $"{a.TitleDe} | {a.TitleEn} | {a.Scope}"; + public async Task VectorAsync(Document note, CancellationToken ct) { var key = (note.Slug, note.ContentHash); @@ -78,6 +88,61 @@ public sealed class ArticleNeighbours(IEmbeddingModel model, IVectorIndex index, return result; } + /// The articles whose vectors are nearest to 's, best first, at most . + public async Task> NearestArticlesAsync(Document note, ArticlePlan plan, CancellationToken ct) + { + var vector = Unit(await VectorAsync(note, ct)); + return Rank(vector, await ArticleVectorsAsync(plan, ct), settings.CandidateArticles, null); + } + + /// The articles whose vectors are nearest to 's, best first, without itself. + public async Task> NearestArticlesAsync(PlannedArticle article, ArticlePlan plan, int k, CancellationToken ct) + { + var vectors = await ArticleVectorsAsync(plan, ct); + return Rank(vectors[article.Slug].Vector, vectors, k, article.Slug); + } + + /// Embeds the articles whose text is new or changed, in batches; drops the vectors of articles no longer planned. + private async Task> ArticleVectorsAsync(ArticlePlan plan, CancellationToken ct) + { + var current = new Dictionary(StringComparer.Ordinal); + var missing = new List<(string Slug, string Hash, string Text)>(); + foreach (var a in plan.Articles) + { + var text = ArticleText(a); + var hash = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text))); + if (_articleVectors.TryGetValue(a.Slug, out var cached) && cached.Hash == hash) current[a.Slug] = cached; + else missing.Add((a.Slug, hash, text)); + } + foreach (var chunk in missing.Chunk(ArticleEmbedBatch)) + { + var vectors = await TimeoutAsServiceDown(() => model.EmbedAsync(chunk.Select(m => m.Text).ToList(), ct), ct); + for (var i = 0; i < chunk.Length; i++) current[chunk[i].Slug] = (chunk[i].Hash, Unit(vectors[i])); + } + _articleVectors = current; + return current; + } + + private static List Rank(float[] unit, Dictionary vectors, int k, string? without) => + vectors.Where(kv => kv.Key != without) + .Select(kv => (Slug: kv.Key, Score: Dot(unit, kv.Value.Vector))) + .OrderByDescending(x => x.Score).ThenBy(x => x.Slug, StringComparer.Ordinal) + .Take(k).Select(x => x.Slug).ToList(); + + private static float Dot(float[] a, float[] b) + { + var sum = 0f; + for (var i = 0; i < Math.Min(a.Length, b.Length); i++) sum += a[i] * b[i]; + return sum; + } + + /// The vector scaled to length one; a zero vector stays zero. + private static float[] Unit(float[] v) + { + var norm = MathF.Sqrt(Dot(v, v)); + return norm == 0 ? v : v.Select(x => x / norm).ToArray(); + } + private async Task Accumulate(Document note, ArticlePlan plan, Dictionary scores, CancellationToken ct) { foreach (var hit in await QueryAsync(note, settings.NeighbourNotes, ct)) diff --git a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs index c241a4e..015e75e 100644 --- a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs +++ b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs @@ -4,14 +4,18 @@ using Wiki.Core.Model; namespace Wiki.Application.Encyclopedia; -/// is one plain sentence on what the article covers; are note slugs, most relevant first. +/// +/// is one plain sentence on what the article covers; are note slugs, most relevant first. +/// is the slug of the overview article this one is a sub-article of. +/// public sealed record PlannedArticle( string Slug, string TitleDe, string TitleEn, ArticleCategory Category, string Scope, - IReadOnlyList Sources); + IReadOnlyList Sources, + string? Parent = null); /// /// The stored plan: the articles, which notes feed them, and where retired slugs point. @@ -51,12 +55,39 @@ public sealed class ArticlePlan public bool SlugTaken(string slug) => Find(slug) is not null || _redirects.ContainsKey(slug) || _retired.Contains(slug); + /// The articles whose parent is , in plan order. + public IReadOnlyList ChildrenOf(string slug) => _articles.Where(a => a.Parent == slug).ToList(); + + /// A new article; its , when set, must be a live article. public void AddArticle(PlannedArticle a) { if (SlugTaken(a.Slug)) throw new InvalidOperationException($"Article slug '{a.Slug}' is already taken"); + if (a.Parent is { } parent && Find(parent) is null) throw new InvalidOperationException($"Unknown parent '{parent}'"); _articles.Add(a); } + /// + /// Makes a sub-article of , or of none when null. False, and nothing + /// changes, when that would make an article its own ancestor. + /// + public bool SetParent(string child, string? parent) + { + var a = Find(child) ?? throw new InvalidOperationException($"Unknown article '{child}'"); + if (parent is not null && Find(parent) is null) throw new InvalidOperationException($"Unknown parent '{parent}'"); + if (parent is not null && IsAncestorOrSelf(child, parent)) return false; + Replace(a with { Parent = parent }); + return true; + } + + /// True when is or on its parent chain. + private bool IsAncestorOrSelf(string ancestor, string slug) + { + var seen = new HashSet(StringComparer.Ordinal); + for (string? current = slug; current is not null && seen.Add(current); current = Find(current)?.Parent) + if (current == ancestor) return true; + return false; + } + /// Appends to each article's sources and records the assignment; no articles records a skip. public void Assign(string note, IReadOnlyList articles) { @@ -68,7 +99,10 @@ public sealed class ArticlePlan _assignments[note] = [.. articles.Distinct(StringComparer.Ordinal)]; } - /// Drops everywhere. Returns the slugs of articles deleted for lack of sources; redirects to them go too. + /// + /// Drops everywhere. Returns the slugs of articles deleted for lack of sources; redirects to them go + /// too, and their children lose their parent. + /// public IReadOnlyList RemoveNote(string note) { _assignments.Remove(note); @@ -87,6 +121,7 @@ public sealed class ArticlePlan foreach (var slug in deleted) { _retired.Add(slug); + foreach (var child in ChildrenOf(slug)) Replace(child with { Parent = null }); foreach (var key in _redirects.Where(r => r.Value == slug).Select(r => r.Key).ToList()) { _redirects.Remove(key); @@ -105,10 +140,14 @@ public sealed class ArticlePlan return deleted; } - /// Same slug, new content. Assignments follow the new : a note that left loses the article, one that joined gains it. + /// + /// Same slug, new content; the parent stays ( changes it). Assignments follow the new + /// : a note that left loses the article, one that joined gains it. + /// public void ReplaceArticle(PlannedArticle a) { var old = Find(a.Slug) ?? throw new InvalidOperationException($"Unknown article '{a.Slug}'"); + a = a with { Parent = old.Parent }; Replace(a); foreach (var note in old.Sources.Except(a.Sources)) { @@ -127,7 +166,9 @@ public sealed class ArticlePlan /// /// Merges into and leaves a redirect behind, public when /// (both language files of were public). Redirects to - /// move to and keep their flag. + /// move to and keep their flag. The children of + /// become children of , except itself, which takes the parent of + /// ; a move that would make an article its own ancestor leaves it without parent. /// public void Fold(string thin, string into, bool isPublic) { @@ -136,6 +177,11 @@ public sealed class ArticlePlan if (thin == into) throw new InvalidOperationException("Cannot fold an article into itself"); Replace(target with { Sources = [.. target.Sources, .. from.Sources.Where(s => !target.Sources.Contains(s))] }); + // children first leave thin, so no parent chain runs through an article that is gone + var children = ChildrenOf(thin).Select(c => c.Slug).ToList(); + foreach (var child in children) Replace(Find(child)! with { Parent = null }); + if (target.Parent == thin) SetParent(into, from.Parent); + foreach (var child in children.Where(c => c != into)) SetParent(child, into); _articles.Remove(from); foreach (var note in _assignments.Keys.ToList()) { @@ -180,12 +226,16 @@ public sealed class ArticlePlan public static class ArticlePlanJson { + /// The plan format; a file in any other format reads as no plan. + public const int FormatVersion = 3; + private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { WriteIndented = true }; /// Map keys in ordinal order, so an unchanged plan serializes identically. public static string Write(ArticlePlan p) => JsonSerializer.Serialize(new { + Version = FormatVersion, Articles = p.Articles.Select(a => new { a.Slug, @@ -194,6 +244,7 @@ public static class ArticlePlanJson Category = a.Category.ToString().ToLowerInvariant(), a.Scope, a.Sources, + a.Parent, }), Assignments = new SortedDictionary>(p.Assignments.ToDictionary(), StringComparer.Ordinal), Redirects = new SortedDictionary(p.Redirects.ToDictionary(), StringComparer.Ordinal), @@ -203,19 +254,24 @@ public static class ArticlePlanJson /// /// The plan in , which may be wrapped in a code fence; null when it is not a plan, - /// including the old format without assignments. Articles with a bad slug or category are skipped, - /// as are assignments to unknown articles and publicRedirects entries that are no redirect. - /// Missing publicRedirects or retired read as empty. Never throws. + /// including any version other than . Articles with a bad slug or category are skipped, + /// as are parents that are unknown or would close a cycle (in file order), assignments to unknown articles and + /// publicRedirects entries that are no redirect. Missing publicRedirects or retired read as empty. + /// Never throws. /// public static ArticlePlan? Read(string json) { try { - if (JsonNode.Parse(PlanParsing.StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items - || root["assignments"] is not JsonObject assignments) return null; + if (JsonNode.Parse(PlanParsing.StripFence(json)) is not JsonObject root + || root["version"] is not JsonValue version || version.GetValueKind() != JsonValueKind.Number + || !version.TryGetValue(out var v) || v != FormatVersion + || root["articles"] is not JsonArray items || root["assignments"] is not JsonObject assignments) return null; var plan = ArticlePlan.Empty(); - foreach (var a in Articles(items)) - if (!plan.SlugTaken(a.Slug)) plan.AddArticle(a); + var articles = Articles(items).DistinctBy(a => a.Slug).ToList(); + foreach (var a in articles) plan.AddArticle(a with { Parent = null }); + foreach (var a in articles) + if (a.Parent is { } parent && plan.Find(parent) is not null) plan.SetParent(a.Slug, parent); if (root["redirects"] is JsonObject redirects) foreach (var (from, to) in redirects) { @@ -251,7 +307,7 @@ public static class ArticlePlanJson if (slug is null || !PlanParsing.SlugPattern().IsMatch(slug) || !PlanParsing.TryCategory(PlanParsing.Text(o["category"]), out var category)) continue; articles.Add(new PlannedArticle(slug, PlanParsing.Text(o["titleDe"]) ?? slug, PlanParsing.Text(o["titleEn"]) ?? slug, category, - PlanParsing.Text(o["scope"]) ?? "", PlanParsing.Strings(o["sources"]))); + PlanParsing.Text(o["scope"]) ?? "", PlanParsing.Strings(o["sources"]), PlanParsing.Text(o["parent"]))); } return articles; } diff --git a/src/Wiki.Application/Encyclopedia/ConsolidateArticlesUseCase.cs b/src/Wiki.Application/Encyclopedia/ConsolidateArticlesUseCase.cs new file mode 100644 index 0000000..a6e6e48 --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/ConsolidateArticlesUseCase.cs @@ -0,0 +1,114 @@ +using System.Net; +using System.Text; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; +using Wiki.Core.Model; +using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; + +namespace Wiki.Application.Encyclopedia; + +/// and are about the same subject; goes. +public sealed record ArticleMerge(string From, string Into); + +/// is the overview article is a sub-article of. +public sealed record ArticleParent(string Child, string Parent); + +/// +/// What one consolidation call found in its group, in reply order. No article is merged twice or merged away after being a +/// target; parents name the merge targets, at most one per child. Parent cycles are not checked here: the plan refuses them. +/// +public sealed record ConsolidateOutcome(IReadOnlyList Merges, IReadOnlyList Parents); + +/// Compares a group of nearby articles: which share a subject, which is the overview of which, in one bounded model call. +public sealed class ConsolidateArticlesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +{ + /// + /// The first articles of are shown; entries + /// naming any other slug are dropped. Null when the reply is unusable. Model-unavailable and other errors propagate. + /// + public async Task ConsolidateAsync(IReadOnlyList group, CancellationToken ct) + { + var shown = group.Take(settings.ConsolidateGroupSize).ToList(); + if (shown.Count < 2) return new ConsolidateOutcome([], []); + TextReply reply; + try + { + reply = await model.CompleteAsync(Prompts.Consolidate, UserPrompt(shown), settings.ConsolidateMaxTokens, ct); + } + catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + log.LogWarning(e, "Consolidate request for the group of {Seed} was rejected", shown[0].Slug); + return null; + } + + if (reply.Truncated) + { + log.LogWarning("Consolidate reply for the group of {Seed} hit the token cap ({Tokens})", shown[0].Slug, settings.ConsolidateMaxTokens); + return null; + } + + var root = PlanParsing.Object(reply.Text); + if (root is null || (root["merges"] is not JsonArray && root["parents"] is not JsonArray)) + { + log.LogWarning("Consolidate reply for the group of {Seed} is not {{\"merges\":[…],\"parents\":[…]}}", shown[0].Slug); + return null; + } + return Validate(root, shown.Select(a => a.Slug).ToHashSet(StringComparer.Ordinal)); + } + + private ConsolidateOutcome Validate(JsonObject root, HashSet members) + { + var merges = new List(); + var into = new Dictionary(StringComparer.Ordinal); + foreach (var (from, to) in Pairs(root["merges"], "from", "into", members)) + { + if (into.ContainsKey(from) || into.ContainsValue(from) || into.ContainsKey(to)) + { + log.LogWarning("Dropping merge {From} into {Into}: one of them is merged already", from, to); + continue; + } + into[from] = to; + merges.Add(new ArticleMerge(from, to)); + } + + var parents = new List(); + foreach (var (rawChild, rawParent) in Pairs(root["parents"], "child", "parent", members)) + { + var child = into.GetValueOrDefault(rawChild, rawChild); + var parent = into.GetValueOrDefault(rawParent, rawParent); + if (child == parent || parents.Any(p => p.Child == child)) + { + log.LogWarning("Dropping parent {Parent} of {Child}: the same article after merges, or a second parent", rawParent, rawChild); + continue; + } + parents.Add(new ArticleParent(child, parent)); + } + return new ConsolidateOutcome(merges, parents); + } + + /// The entries of whose two fields are different members of the group. + private IEnumerable<(string, string)> Pairs(JsonNode? node, string first, string second, HashSet members) + { + if (node is not JsonArray items) yield break; + foreach (var item in items) + { + if (item is not JsonObject o || Text(o[first]) is not { } a || Text(o[second]) is not { } b) continue; + if (a == b || !members.Contains(a) || !members.Contains(b)) + { + log.LogWarning("Dropping {Field} entry {A} → {B}: not two different articles of the group", first, a, b); + continue; + } + yield return (a, b); + } + } + + private static string UserPrompt(List group) + { + var sb = new StringBuilder("Articles (slug | title | category | scope | source count | current parent):\n"); + foreach (var a in group) + sb.Append(a.Slug).Append(" | ").Append(Flatten(a.TitleDe)).Append(" | ").Append(a.Category.ToString().ToLowerInvariant()) + .Append(" | ").Append(Flatten(a.Scope)).Append(" | ").Append(a.Sources.Count).Append(" | ").Append(a.Parent ?? "-").Append('\n'); + return sb.ToString(); + } +} diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs index c09883b..99581ac 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs @@ -8,12 +8,13 @@ namespace Wiki.Application.Encyclopedia; /// /// What one run did. is the article count after the run; counts notes assigned to an article (not skipped ones), +/// articles merged into another by consolidation, parent links consolidation set, /// and articles; the write counts are subjects, each in both languages. /// counts subjects rejected earlier whose inputs are unchanged, so they were not retried. /// is null when nothing was committed. /// -public sealed record ReconcileSummary(int Planned, int Assigned, int Split, int Folded, int Written, int Deleted, int Rejected, - int Remembered, bool StoppedByModel, string? Skipped, string? Commit); +public sealed record ReconcileSummary(int Planned, int Assigned, int Merged, int Parented, int Split, int Folded, int Written, int Deleted, + int Rejected, int Remembered, bool StoppedByModel, string? Skipped, string? Commit); public interface IEncyclopediaRun { @@ -22,8 +23,8 @@ public interface IEncyclopediaRun /// /// Brings articles/ up to date with the notes: updates the stored plan in bounded steps (drop removed notes, assign new -/// ones, split oversized and fold thin articles), rewrites the articles whose inputs changed, deletes the files of articles -/// the plan dropped, and commits everything as one commit. +/// ones, consolidate nearby articles, split oversized and fold thin ones), rewrites the articles whose inputs changed, deletes +/// the files of articles the plan dropped, and commits everything as one commit. /// Works only on a catalog that reflects the store's head, so what it compares is what it commits onto. /// public sealed class EncyclopediaReconciler( @@ -31,6 +32,7 @@ public sealed class EncyclopediaReconciler( IDocumentCatalog catalog, ArticleNeighbours neighbours, AssignNotesUseCase assigner, + ConsolidateArticlesUseCase consolidator, SplitArticleUseCase splitter, FoldArticleUseCase folder, IArticleWriter writer, @@ -40,6 +42,8 @@ public sealed class EncyclopediaReconciler( { private const string PlanPath = "articles/_plan.json"; private const string StatePath = "articles/_state.json"; + /// Sub-articles an article's write prompt lists as link targets, at most. + private const int MaxChildLinks = 20; private static readonly string[] Languages = ["de", "en"]; /// The inputs and mutable outputs of one run. @@ -54,17 +58,21 @@ public sealed class EncyclopediaReconciler( public SortedDictionary Subjects { get; } = new(state.Subjects.ToDictionary(), StringComparer.Ordinal); public SortedDictionary FailedBatches { get; } = new(state.FailedBatches.ToDictionary(), StringComparer.Ordinal); public SortedDictionary Kept { get; } = new(state.KeptArticles.ToDictionary(), StringComparer.Ordinal); - /// Children of the articles split this run: their overview's extra link targets. - public Dictionary> Children { get; } = new(StringComparer.Ordinal); + public SortedDictionary ConsolidatedGroups { get; } = new(state.ConsolidatedGroups.ToDictionary(), StringComparer.Ordinal); + public SortedDictionary FailedGroups { get; } = new(state.FailedGroups.ToDictionary(), StringComparer.Ordinal); + /// Slugs of the children of the articles split this run. + public HashSet SplitOff { get; } = new(StringComparer.Ordinal); /// Subjects whose de and en files both exist and are public at the run's start. public IReadOnlySet PublicSlugs { get; } = publicSlugs; /// Thin articles found without a nearby article this run: no fold decision pending. public HashSet Unfoldable { get; } = new(StringComparer.Ordinal); public int Assigned { get; set; } + public int Merged { get; set; } + public int Parented { get; set; } public int Split { get; set; } public int Folded { get; set; } - public EncyclopediaState State => new(Subjects, FailedBatches, Kept); + public EncyclopediaState State => new(Subjects, FailedBatches, Kept, ConsolidatedGroups, FailedGroups); } /// An unexpected failure is rethrown after what is staged is committed; caller cancellation commits nothing. @@ -75,7 +83,7 @@ public sealed class EncyclopediaReconciler( if (synced != head) { log.LogWarning("Catalog is at {Catalog}, the store at {Head}; skipping the encyclopedia run", synced, head); - return new ReconcileSummary(0, 0, 0, 0, 0, 0, 0, 0, false, "catalog-behind", null); + return new ReconcileSummary(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, "catalog-behind", null); } var notes = await catalog.ListDocumentsAsync(DocumentKind.Wiki, ct); @@ -83,7 +91,10 @@ public sealed class EncyclopediaReconciler( var planText = await store.ReadTextAsync(PlanPath, head, ct); var stateText = await store.ReadTextAsync(StatePath, head, ct); var existing = ParseArticles(articleDocs); - var run = new Run(notes, ReadPlan(planText), ReadState(stateText), clock.GetUtcNow(), PublicSlugs(existing)); + var storedPlan = ReadPlan(planText); + // what the state remembers belongs to the plan it was made with + var state = planText is not null && storedPlan is null ? EncyclopediaState.Empty : ReadState(stateText); + var run = new Run(notes, storedPlan ?? ArticlePlan.Empty(), state, clock.GetUtcNow(), PublicSlugs(existing)); var plan = run.Plan; var planned = plan.Articles.Select(a => a.Slug).ToList(); var changes = new List(); @@ -142,21 +153,22 @@ public sealed class EncyclopediaReconciler( changes.Add(Put(PlanPath, planJson, planText?.BlobSha)); var summary = await FinishAsync(changes, run.State, stateText, - new ReconcileSummary(plan.Articles.Count, run.Assigned, run.Split, run.Folded, written, deleted, rejected, remembered, - stopped, null, null), ct); + new ReconcileSummary(plan.Articles.Count, run.Assigned, run.Merged, run.Parented, run.Split, run.Folded, written, deleted, + rejected, remembered, stopped, null, null), ct); failure?.Throw(); return summary; } /// - /// Assign, split, fold, each within its cap. A text-model call that fails with skips its - /// batch or article; an unreachable embedding or index service ends the planning steps; what they changed stays. + /// Assign, consolidate, split, fold, each within its cap. A text-model call that fails with + /// skips its batch, group or article; an unreachable embedding or index service ends the planning steps; what they changed stays. /// private async Task PlanStepsAsync(Run run, CancellationToken ct) { try { await AssignAsync(run, ct); + await ConsolidateAsync(run, ct); await SplitAsync(run, ct); await FoldAsync(run, ct); } @@ -168,8 +180,10 @@ public sealed class EncyclopediaReconciler( /// /// Batches of similar new notes, seeded in slug order, each assigned in one call against the articles near its notes in the - /// plan as it stands, so later batches see the articles earlier ones created. A batch whose reply was unusable before, for - /// the same notes, is skipped without a call; a batch is tried once per run. + /// plan as it stands, so later batches see the articles earlier ones created. Near means: assigned to a note near a batch + /// note, or with an article vector near a batch note's vector; the two lists are interleaved, so both are shown when either + /// alone would fill the cap. A batch whose reply was unusable before, for the same notes, is skipped without a call; a batch + /// is tried once per run. /// private async Task AssignAsync(Run run, CancellationToken ct) { @@ -187,13 +201,15 @@ public sealed class EncyclopediaReconciler( var fingerprint = EncyclopediaState.BatchFingerprint(batch); if (run.FailedBatches.ContainsKey(fingerprint)) continue; - var nearby = new List(); + var byNotes = new List(); + var byVectors = new List(); foreach (var note in batch) { - foreach (var slug in await neighbours.NearbyArticlesAsync(note, plan, ct)) - if (nearby.Count < settings.CandidateArticles * 2 && !nearby.Contains(slug)) nearby.Add(slug); - if (nearby.Count >= settings.CandidateArticles * 2) break; + byNotes.AddRange(await neighbours.NearbyArticlesAsync(note, plan, ct)); + byVectors.AddRange(await neighbours.NearestArticlesAsync(note, plan, ct)); } + var nearby = Interleave(byNotes.Distinct(StringComparer.Ordinal).ToList(), byVectors.Distinct(StringComparer.Ordinal).ToList()) + .Distinct(StringComparer.Ordinal).Take(settings.CandidateArticles * 2).ToList(); calls++; AssignOutcome outcome; @@ -219,7 +235,97 @@ public sealed class EncyclopediaReconciler( log.LogInformation("Assign cap of {Cap} batches reached; the other new notes wait for the next run", settings.MaxAssignBatchesPerRun); } - /// Oversized articles, biggest first, at most calls. + private static IEnumerable Interleave(List a, List b) + { + for (var i = 0; i < Math.Max(a.Count, b.Count); i++) + { + if (i < a.Count) yield return a[i]; + if (i < b.Count) yield return b[i]; + } + } + + /// + /// Groups of nearby articles: seeds in slug order among the articles no group of this run covered yet, each with its + /// nearest articles by article vector, in all. A group whose + /// fingerprint is remembered is not asked again; at most calls. + /// After a usable reply the group's surviving members, as the reply left them, are remembered, so the result is not asked + /// about again; an unusable reply is remembered as failed; a failed call is not remembered. Once every seed is done, only + /// the fingerprints of this run's groups are kept. + /// + private async Task ConsolidateAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + if (settings.MaxConsolidationsPerRun <= 0 || settings.ConsolidateGroupSize < 2 || plan.Articles.Count < 2) return; + var covered = new HashSet(StringComparer.Ordinal); + var seen = new HashSet(StringComparer.Ordinal); + var calls = 0; + foreach (var slug in plan.Articles.Select(a => a.Slug).Order(StringComparer.Ordinal).ToList()) + { + if (covered.Contains(slug) || plan.Find(slug) is not { } seed) continue; + var nearest = await neighbours.NearestArticlesAsync(seed, plan, settings.ConsolidateGroupSize - 1, ct); + List group = [seed, .. nearest.Select(s => plan.Find(s)!)]; + var members = group.Select(a => a.Slug).ToList(); + var fingerprint = EncyclopediaState.GroupFingerprint(group); + seen.Add(fingerprint); + if (run.ConsolidatedGroups.ContainsKey(fingerprint) || run.FailedGroups.ContainsKey(fingerprint)) + { + covered.UnionWith(members); + continue; + } + if (calls >= settings.MaxConsolidationsPerRun) continue; + + calls++; + covered.UnionWith(members); + ConsolidateOutcome? outcome; + try + { + outcome = await consolidator.ConsolidateAsync(group, ct); + } + catch (HttpRequestException e) + { + log.LogWarning(e, "Consolidation of the group of {Seed} failed; asked again next run", slug); + continue; + } + if (outcome is null) + { + run.FailedGroups[fingerprint] = run.Now; + continue; + } + Apply(run, outcome); + // the members as the reply left them; the same as asked when nothing changed + var after = members.Select(plan.Find).OfType().ToList(); + if (after.Count < 2) continue; + var judged = EncyclopediaState.GroupFingerprint(after); + run.ConsolidatedGroups[judged] = run.Now; + seen.Add(judged); + } + if (calls >= settings.MaxConsolidationsPerRun && plan.Articles.Any(a => !covered.Contains(a.Slug))) + log.LogInformation("Consolidation cap of {Cap} groups reached; the other articles wait for the next run", settings.MaxConsolidationsPerRun); + foreach (var memory in new[] { run.ConsolidatedGroups, run.FailedGroups }) + foreach (var key in memory.Keys.Where(k => !seen.Contains(k)).ToList()) memory.Remove(key); + } + + /// Merges as folds (public redirect when the merged article was public at the run's start), then the parent links that close no cycle. + private void Apply(Run run, ConsolidateOutcome outcome) + { + var plan = run.Plan; + foreach (var m in outcome.Merges) + { + if (plan.Find(m.From) is null || plan.Find(m.Into) is null) continue; + log.LogInformation("Merging article {From} into {Into}", m.From, m.Into); + plan.Fold(m.From, m.Into, run.PublicSlugs.Contains(m.From)); + run.Merged++; + } + foreach (var p in outcome.Parents) + { + if (plan.Resolve(p.Child) is not { } child || plan.Resolve(p.Parent) is not { } parent || child == parent + || plan.Find(child)!.Parent == parent) continue; + if (plan.SetParent(child, parent)) run.Parented++; + else log.LogWarning("Dropping parent {Parent} of {Child}: it would close a cycle", parent, child); + } + } + + /// Oversized articles, biggest first, at most calls. The children get the overview as parent. private async Task SplitAsync(Run run, CancellationToken ct) { var plan = run.Plan; @@ -246,11 +352,11 @@ public sealed class EncyclopediaReconciler( // children first, so a note that moves wholly to a child never goes unassigned foreach (var child in outcome.Children) { - plan.AddArticle(child with { Sources = [] }); + plan.AddArticle(child with { Sources = [], Parent = article.Slug }); plan.ReplaceArticle(child); + run.SplitOff.Add(child.Slug); } plan.ReplaceArticle(outcome.Overview); - run.Children[article.Slug] = outcome.Children; run.Split++; } } @@ -264,7 +370,7 @@ public sealed class EncyclopediaReconciler( private async Task FoldAsync(Run run, CancellationToken ct) { var plan = run.Plan; - var children = run.Children.Values.SelectMany(c => c).Select(c => c.Slug).ToHashSet(StringComparer.Ordinal); + var children = run.SplitOff; var thin = plan.Articles.Where(a => FoldArticleUseCase.IsThin(a, run.NoteBySlug, settings)).Select(a => a.Slug).ToList(); var calls = 0; foreach (var slug in thin) @@ -338,8 +444,9 @@ public sealed class EncyclopediaReconciler( neighboursUp = false; } } - if (run.Children.TryGetValue(planned.Slug, out var children)) - targets = Articles(run.Plan, [.. targets.Select(t => t.Slug), .. children.Select(c => c.Slug)]); + var family = run.Plan.ChildrenOf(planned.Slug).Take(MaxChildLinks).Select(c => c.Slug).ToList(); + if (planned.Parent is { } parent) family.Insert(0, parent); + targets = Articles(run.Plan, [.. targets.Select(t => t.Slug), .. family]); var article = await writer.WriteAsync(planned, sources, targets, ct); foreach (var a in new[] { article.De, article.En }) @@ -473,7 +580,8 @@ public sealed class EncyclopediaReconciler( return summary; } - var message = $"articles: {summary.Assigned} assigned, {summary.Split} split, {summary.Folded} folded, " + var message = $"articles: {summary.Assigned} assigned, {summary.Merged} merged, {summary.Parented} parented, " + + $"{summary.Split} split, {summary.Folded} folded, " + $"{summary.Written} written, {summary.Deleted} deleted, {summary.Rejected} rejected"; try { @@ -511,12 +619,14 @@ public sealed class EncyclopediaReconciler( return articles; } - private ArticlePlan ReadPlan(StoredText? text) + /// The stored plan; empty when there is none; null when there is one in another format or malformed. + private ArticlePlan? ReadPlan(StoredText? text) { if (text is null) return ArticlePlan.Empty(); var plan = ArticlePlanJson.Read(text.Text); - if (plan is null) log.LogWarning("{Path} is malformed or in the old format; planning from scratch", PlanPath); - return plan ?? ArticlePlan.Empty(); + if (plan is null) + log.LogWarning("{Path} is malformed or in an older format; planning from scratch, without the attempt memory of that plan", PlanPath); + return plan; } private EncyclopediaState ReadState(StoredText? text) @@ -524,6 +634,7 @@ public sealed class EncyclopediaReconciler( if (text is null) return EncyclopediaState.Empty; var state = EncyclopediaStateJson.Read(text.Text); if (state.Subjects.Count == 0 && state.FailedBatches.Count == 0 && state.KeptArticles.Count == 0 + && state.ConsolidatedGroups.Count == 0 && state.FailedGroups.Count == 0 && text.Text.Trim() != EncyclopediaStateJson.Write(EncyclopediaState.Empty)) log.LogWarning("{Path} is malformed or empty; starting without attempt memory", StatePath); return state; diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs index ba0f33a..378a8c5 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs @@ -20,7 +20,11 @@ public sealed record EncyclopediaSettings( int MaxSourcesPerArticle = 120, int FoldMinSources = 3, int FoldMinChars = 4000, + // articles per consolidation group, the seed included + int ConsolidateGroupSize = 8, + int MaxConsolidationsPerRun = 60, // thinking tokens count against max_tokens int AssignMaxTokens = 32000, int SplitMaxTokens = 32000, - int FoldMaxTokens = 8000); + int FoldMaxTokens = 8000, + int ConsolidateMaxTokens = 8000); diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs index 48ab77a..c41cbab 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs @@ -22,19 +22,30 @@ public sealed record SubjectAttempt(string Fingerprint, AttemptOutcome Outcome, /// /// FailedBatches: fingerprint of an assignment batch whose reply was unusable → when. /// KeptArticles: article slug → at which the fold call did not fold it. +/// ConsolidatedGroups and FailedGroups: of a consolidation group whose reply +/// was usable, or unusable, → when. /// The failedPlan entry of earlier versions is ignored on read. /// public sealed record EncyclopediaState( IReadOnlyDictionary Subjects, IReadOnlyDictionary FailedBatches, - IReadOnlyDictionary KeptArticles) + IReadOnlyDictionary KeptArticles, + IReadOnlyDictionary ConsolidatedGroups, + IReadOnlyDictionary FailedGroups) { public static EncyclopediaState Empty => - new(new Dictionary(), new Dictionary(), new Dictionary()); + new(new Dictionary(), new Dictionary(), new Dictionary(), + new Dictionary(), new Dictionary()); public static string BatchFingerprint(IReadOnlyList notes) => Hash(Prompts.Version.ToString(), notes.Select(n => $"{n.Slug}@{n.ContentHash}").Order(StringComparer.Ordinal)); + /// The members and what the consolidation call judges of them: titles, category, scope and parent, not sources. + public static string GroupFingerprint(IReadOnlyList group) => + Hash(Prompts.Version.ToString(), group + .Select(a => string.Join('\u001f', a.Slug, a.TitleDe, a.TitleEn, a.Category.ToString(), a.Scope, a.Parent ?? "")) + .Order(StringComparer.Ordinal)); + public static string SubjectFingerprint(PlannedArticle planned, IReadOnlyList notes) { var hashes = notes.GroupBy(n => n.Slug).ToDictionary(g => g.Key, g => g.First().ContentHash); @@ -70,7 +81,9 @@ public static class EncyclopediaStateJson var subjects = s.Subjects.Where(kv => kv.Value is not null).ToDictionary(kv => kv.Key, kv => kv.Value); var failedBatches = (s.FailedBatches ?? new Dictionary()).ToDictionary(); var kept = (s.KeptArticles ?? new Dictionary()).Where(kv => kv.Value is not null).ToDictionary(); - return new EncyclopediaState(subjects, failedBatches, kept); + var consolidated = (s.ConsolidatedGroups ?? new Dictionary()).ToDictionary(); + var failedGroups = (s.FailedGroups ?? new Dictionary()).ToDictionary(); + return new EncyclopediaState(subjects, failedBatches, kept, consolidated, failedGroups); } catch (JsonException) { diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index 9e66fb2..1e71279 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Encyclopedia; /// public static class Prompts { - public const int Version = 7; + public const int Version = 8; public const string Assign = """ [assign] @@ -16,6 +16,7 @@ public static class Prompts For each note decide which articles it feeds: existing articles from the list, or new articles you propose. - Prefer an existing article from the list when the note fits its scope. - Granularity follows the material, like notability on Wikipedia. A subject gets its own new article only when the notes hold enough on it. A subject that was set once and forgotten is not an article of its own; its note feeds the article of its parent system as a section. + - A note about something in a specific room additionally feeds that room's article (category room), titled by floor and room, for example "EG Schlafzimmer". Create the room article when none is in the list. - Skip a note only when it is a meta note or about the wiki's own tooling. - A note may feed several articles. - Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses. Name the subject by what it is or does, for example "Netzwerkspeicher" rather than the server's hostname. @@ -54,6 +55,21 @@ public static class Prompts Reply with JSON only, no prose and no code fence: {"into":"article-slug"} to fold, or {"keep":true} to keep. """; + public const string Consolidate = """ + [consolidate] + You tidy a group of nearby articles of an encyclopedia about one household: the house and its homelab IT. Its readers are everyone who needs to know things about the house, from the owner to guests. + You receive the articles, one per line as `slug | title | category | scope | source count | current parent` (`-` for none). + + - Merge two articles when they are about the same subject, for example the same device under two names. Merge the one with fewer sources into the other. An article you merge away is never the target of another merge. + - Give an article a parent when the parent is the overview of a broader subject it belongs to, for example a heat pump under the heating system, or a device under the system it is part of. A room is the parent of nothing but its own parts; a device in a room keeps its system as parent. + - Change a current parent only when another article of the group is clearly the better overview. + - Articles about different subjects, and siblings, stay as they are. When nothing needs to change, reply with empty lists. + - Use only slugs from the list. + + Reply with JSON only, no prose and no code fence. Shape: + {"merges":[{"from":"article-slug","into":"article-slug"}],"parents":[{"child":"article-slug","parent":"article-slug"}]} + """; + /// The generic-examples rule, shared by and . public const string GenericExamples = "A generic example that explains a principle and names nothing concrete in this house, for example what an entity ID or an IP address looks like, is not a leak and stays public"; @@ -63,7 +79,7 @@ public static class Prompts You write one article of an encyclopedia about one household: the house and its homelab IT (Kubernetes, servers, network). You write in German. The readers are everyone who needs to know something about the house: the owner, the household, a friend fixing a light switch while the owner is away, a guest who wants to know how the lighting automation works. Many of them are not technical. Open plainly, so that a non-technical reader understands the start, and get more technical further down. - You receive the article's subject, the articles you may link to, the images you may use, and the source notes, each as .... Inside a note, &, <, > and " stand for &, <, > and "; a note ends only at its own . + You receive the article's subject, the articles you may link to (its main article and sub-articles are marked), the images you may use, and the source notes, each as .... Inside a note, &, <, > and " stand for &, <, > and "; a note ends only at its own . Shape, in this order: 1. A lead paragraph of a few sentences, without a heading, that a non-technical reader understands. @@ -73,6 +89,11 @@ public static class Prompts 5. Optionally a short History section last (`## Geschichte`), a few lines at most. History never dominates. Do not repeat the article title as a heading; the page shows it. + Article kinds: + - A room article (category room) describes what is in the room and what it does there: lights, switches, blinds, sensors, heating. It explains how things work only as far as the room needs; for the system behind them it links to the system's article. + - An article with sub-articles is their overview: it gives the big picture and, for each sub-article, a sentence or two and a link. Details stay in the sub-articles. + - The page links the main article and the sub-articles on its own; link them in the text where they help the reader. + Body conventions: - The infobox is a line `::: infobox`, then a two-column pipe table, then a line `:::`. - A private block is a line `::: private`, its content, then a line `:::`. diff --git a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs index 8380798..22021fa 100644 --- a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs @@ -254,9 +254,14 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe sb.Append("Category: ").Append(planned.Category.ToString().ToLowerInvariant()).Append('\n'); sb.Append("Scope: ").Append(Flatten(planned.Scope)).Append('\n'); - sb.Append("\nArticles you may link to (slug | German title):\n"); + sb.Append("\nArticles you may link to (slug | German title | relation):\n"); foreach (var a in linkTargets.Where(a => a.Slug != planned.Slug)) - sb.Append(a.Slug).Append(" | ").Append(Flatten(a.TitleDe)).Append('\n'); + { + sb.Append(a.Slug).Append(" | ").Append(Flatten(a.TitleDe)); + if (a.Slug == planned.Parent) sb.Append(" | main article"); + else if (a.Parent == planned.Slug) sb.Append(" | sub-article"); + sb.Append('\n'); + } var images = sources.SelectMany(Attachments).Distinct(StringComparer.Ordinal).ToList(); sb.Append("\nImages you may use:\n"); diff --git a/src/Wiki.Application/Reading/ArticleQueries.cs b/src/Wiki.Application/Reading/ArticleQueries.cs index 23e32eb..20a048b 100644 --- a/src/Wiki.Application/Reading/ArticleQueries.cs +++ b/src/Wiki.Application/Reading/ArticleQueries.cs @@ -6,11 +6,17 @@ using Wiki.Core.Ports; namespace Wiki.Application.Reading; +/// A link to an article: its slug and its title in the reader's language. +public sealed record ArticleLink(string Slug, string Title); + +/// The main article of an article and its sub-articles, by title; only those the reader may see. +public sealed record ArticleFamily(ArticleLink? Parent, IReadOnlyList Children); + /// /// Read side of the house encyclopedia, straight from the catalog. An article file that does not parse /// counts as missing. Anonymous visibility is per subject: both language files must exist and be public. -/// Redirects come from articles/_plan.json at the catalog's commit; anonymous readers get only those whose -/// retired article was public when it was folded. +/// Redirects and the article hierarchy come from articles/_plan.json at the catalog's commit; anonymous readers get +/// only the redirects whose retired article was public when it was folded. /// public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, ILogger log) { @@ -18,11 +24,13 @@ public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, I private const string PlanPath = "articles/_plan.json"; - private static readonly IReadOnlyDictionary NoRedirects = new Dictionary(); + private static readonly IReadOnlyDictionary None = new Dictionary(); - private sealed record Cached(string Commit, IReadOnlyDictionary All, IReadOnlyDictionary Public); + /// : article slug → its parent's slug, from the plan. + private sealed record Cached(string Commit, IReadOnlyDictionary All, IReadOnlyDictionary Public, + IReadOnlyDictionary Parents); - private volatile Cached? _redirects; + private volatile Cached? _plan; /// /// Retired slug → slug that took over; anonymous ( false): only redirects whose retired article was @@ -30,15 +38,38 @@ public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, I /// unreadable or the store fails. Cached per commit, a failure too (no store request per page view until the commit changes). /// public async Task> RedirectsAsync(bool signedIn, CancellationToken ct) + { + var cached = await PlanAsync(ct); + return cached is null ? None : signedIn ? cached.All : cached.Public; + } + + /// + /// The plan's parent and children of that exist in and that this reader may + /// see, exactly as for links (). None when the plan is missing, unreadable or the store fails. + /// + public async Task FamilyAsync(string lang, string slug, bool signedIn, CancellationToken ct) + { + var parents = (await PlanAsync(ct))?.Parents ?? None; + var titles = await VisibleTitlesAsync(lang, signedIn, ct); + var parent = parents.TryGetValue(slug, out var p) && titles.TryGetValue(p, out var title) ? new ArticleLink(p, title) : null; + var children = parents.Where(kv => kv.Value == slug && titles.ContainsKey(kv.Key)) + .Select(kv => new ArticleLink(kv.Key, titles[kv.Key])) + .OrderBy(c => c.Title, StringComparer.InvariantCultureIgnoreCase) + .ThenBy(c => c.Slug, StringComparer.Ordinal) + .ToList(); + return new ArticleFamily(parent, children); + } + + private async Task PlanAsync(CancellationToken ct) { var commit = (await catalog.GetStateAsync(ct)).Commit; - if (commit is null) return NoRedirects; - if (_redirects is not { } cached || cached.Commit != commit) + if (commit is null) return null; + if (_plan is not { } cached || cached.Commit != commit) { cached = await LoadAsync(commit, ct); - _redirects = cached; + _plan = cached; } - return signedIn ? cached.All : cached.Public; + return cached; } private async Task LoadAsync(string commit, CancellationToken ct) @@ -48,13 +79,14 @@ public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, I var text = await store.ReadTextAsync(PlanPath, commit, ct); if (text is not null && ArticlePlanJson.Read(text.Text) is { } plan) return new Cached(commit, plan.Redirects, - plan.Redirects.Where(r => plan.PublicRedirects.Contains(r.Key)).ToDictionary(StringComparer.Ordinal)); + plan.Redirects.Where(r => plan.PublicRedirects.Contains(r.Key)).ToDictionary(StringComparer.Ordinal), + plan.Articles.Where(a => a.Parent is not null).ToDictionary(a => a.Slug, a => a.Parent!, StringComparer.Ordinal)); } catch (Exception e) when (e is not OperationCanceledException) { - log.LogWarning(e, "Cannot read {Path} at {Commit}; serving without redirects", PlanPath, commit); + log.LogWarning(e, "Cannot read {Path} at {Commit}; serving without redirects and hierarchy", PlanPath, commit); } - return new Cached(commit, NoRedirects, NoRedirects); + return new Cached(commit, None, None, None); } /// diff --git a/src/Wiki.Host/Components/Pages/HouseArticle.razor b/src/Wiki.Host/Components/Pages/HouseArticle.razor index 3ef688a..c471ae5 100644 --- a/src/Wiki.Host/Components/Pages/HouseArticle.razor +++ b/src/Wiki.Host/Components/Pages/HouseArticle.razor @@ -29,6 +29,11 @@ else if (_article is not null && _rendered is not null) } + @if (_family?.Parent is { } parent) + { +
@(English ? "Main article" : "Hauptartikel"): @parent.Title
+ } + @if (_infobox is not null) { @((MarkupString)_infobox) @@ -48,6 +53,19 @@ else if (_article is not null && _rendered is not null) } @((MarkupString)_rendered.Html) + + @if (_family?.Children.Count > 0) + { + + } } @code { @@ -69,6 +87,7 @@ else if (_article is not null && _rendered is not null) private Article? _article; private RenderedArticle? _rendered; private string? _infobox; + private ArticleFamily? _family; private bool English => Lang == "en"; private string Other => English ? "de" : "en"; @@ -100,6 +119,7 @@ else if (_article is not null && _rendered is not null) _rendered = ArticleMarkdown.Render(article.Body, Lang, Slug, _signedIn, titles, redirects, Log); _infobox = WithTitleBar(_rendered.InfoboxHtml, article.Title); _otherExists = await Articles.GetAsync(Other, Slug, ct) is not null; + _family = await Articles.FamilyAsync(Lang, Slug, _signedIn, ct); } /// diff --git a/src/Wiki.Host/Components/Shared/StatusPanel.razor b/src/Wiki.Host/Components/Shared/StatusPanel.razor index a2bd405..4fd4d03 100644 --- a/src/Wiki.Host/Components/Shared/StatusPanel.razor +++ b/src/Wiki.Host/Components/Shared/StatusPanel.razor @@ -42,7 +42,7 @@ Last run@Status.LastEncyclopediaAttemptAt.Value.ToString("u") @if (Status.LastEncyclopediaRun is { } r) { - Result@r.Planned planned, @r.Assigned assigned, @r.Split split, @r.Folded folded, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") + Result@r.Planned planned, @r.Assigned assigned, @r.Merged merged, @r.Parented parented, @r.Split split, @r.Folded folded, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") } @if (Status.LastEncyclopediaError is not null) { diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs index 64c00b4..3bb8366 100644 --- a/src/Wiki.Host/Composition/WikiOptions.cs +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -101,8 +101,12 @@ public sealed class EncyclopediaOptions public int MaxSourcesPerArticle { get; set; } = 120; public int FoldMinSources { get; set; } = 3; public int FoldMinChars { get; set; } = 4000; + /// Articles per consolidation group, the seed included. + public int ConsolidateGroupSize { get; set; } = 8; + public int MaxConsolidationsPerRun { get; set; } = 60; // thinking tokens count against max_tokens public int AssignMaxTokens { get; set; } = 32000; public int SplitMaxTokens { get; set; } = 32000; public int FoldMaxTokens { get; set; } = 8000; + public int ConsolidateMaxTokens { get; set; } = 8000; } diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index b385f1b..2098d32 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -126,12 +126,16 @@ if (encyclopedia.Enabled) MaxSourcesPerArticle: encyclopedia.MaxSourcesPerArticle, FoldMinSources: encyclopedia.FoldMinSources, FoldMinChars: encyclopedia.FoldMinChars, + ConsolidateGroupSize: encyclopedia.ConsolidateGroupSize, + MaxConsolidationsPerRun: encyclopedia.MaxConsolidationsPerRun, AssignMaxTokens: encyclopedia.AssignMaxTokens, SplitMaxTokens: encyclopedia.SplitMaxTokens, - FoldMaxTokens: encyclopedia.FoldMaxTokens)); + FoldMaxTokens: encyclopedia.FoldMaxTokens, + ConsolidateMaxTokens: encyclopedia.ConsolidateMaxTokens)); // ArticleNeighbours takes the active IEmbeddingModel: the index holds its vectors. builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs index a7759e6..6b3cc5b 100644 --- a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs +++ b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs @@ -171,6 +171,88 @@ public class ArticleNeighboursTests Assert.Equal(24, rig.Index.LastK); } + /// Embeddings with a fixed vector per text; any other text gets the fake's hash vector. + private sealed class FixedEmbeddings(Dictionary vectors) : Wiki.Core.Ports.IEmbeddingModel + { + private readonly FakeEmbeddings _fallback = new(Model); + public List> Batches { get; } = []; + public string ModelId => Model; + public int Dimension => 4; + public string Name => "fixed"; + + public async Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct) + { + Batches.Add(texts); + var fallback = await _fallback.EmbedAsync(texts, ct); + return texts.Select((t, i) => vectors.TryGetValue(t, out var v) ? v : fallback[i]).ToList(); + } + + public Task CheckAsync(CancellationToken ct) => _fallback.CheckAsync(ct); + } + + private static PlannedArticle Titled(string slug, string scope = "scope") => new(slug, slug + " de", slug + " en", ArticleCategory.Topic, scope, ["n"]); + + [Fact] + public async Task NearestArticles_OfNote_ByCosineOfArticleVectors_Capped() + { + var note = Note("n"); + var near = Titled("near"); + var nearer = Titled("nearer"); + var far = Titled("far"); + var embeddings = new FixedEmbeddings(new() + { + [EmbeddingText.For(note, 1500)] = [1, 0, 0, 0], + [ArticleNeighbours.ArticleText(near)] = [2, 1, 0, 0], + [ArticleNeighbours.ArticleText(nearer)] = [5, 1, 0, 0], + [ArticleNeighbours.ArticleText(far)] = [0, 0, 3, 0], + }); + var plan = ArticlePlan.Empty(); + foreach (var a in new[] { far, near, nearer }) plan.AddArticle(a); + var neighbours = new ArticleNeighbours(embeddings, new FakeIndex(), Settings with { CandidateArticles = 2 }); + + var nearest = await neighbours.NearestArticlesAsync(note, plan, default); + + Assert.Equal(["nearer", "near"], nearest); + } + + [Fact] + public async Task NearestArticles_OfArticle_ExcludesItself() + { + var self = Titled("self"); + var other = Titled("other"); + var third = Titled("third"); + var embeddings = new FixedEmbeddings(new() + { + [ArticleNeighbours.ArticleText(self)] = [1, 0, 0, 0], + [ArticleNeighbours.ArticleText(other)] = [1, 1, 0, 0], + [ArticleNeighbours.ArticleText(third)] = [0, 1, 0, 0], + }); + var plan = ArticlePlan.Empty(); + foreach (var a in new[] { self, other, third }) plan.AddArticle(a); + var neighbours = new ArticleNeighbours(embeddings, new FakeIndex(), Settings); + + var nearest = await neighbours.NearestArticlesAsync(self, plan, 5, default); + + Assert.Equal(["other", "third"], nearest); + } + + [Fact] + public async Task ArticleVectors_EmbeddedInBatchesOnce_AgainWhenTheTextChanges() + { + var embeddings = new FixedEmbeddings([]); + var plan = ArticlePlan.Empty(); + foreach (var i in Enumerable.Range(0, 40)) plan.AddArticle(Titled($"a-{i:00}")); + var neighbours = new ArticleNeighbours(embeddings, new FakeIndex(), Settings); + + await neighbours.NearestArticlesAsync(plan.Find("a-00")!, plan, 3, default); + await neighbours.NearestArticlesAsync(plan.Find("a-01")!, plan, 3, default); + plan.ReplaceArticle(plan.Find("a-05")! with { Scope = "new scope" }); + await neighbours.NearestArticlesAsync(plan.Find("a-01")!, plan, 3, default); + + Assert.Equal([32, 8, 1], embeddings.Batches.Select(b => b.Count)); + Assert.Equal("a-05 de | a-05 en | new scope", Assert.Single(embeddings.Batches[^1])); + } + [Fact] public async Task Vector_CachedPerContentHash() { diff --git a/test/Wiki.Application.Tests/ArticlePlanTests.cs b/test/Wiki.Application.Tests/ArticlePlanTests.cs index 0f9a4c6..8240c27 100644 --- a/test/Wiki.Application.Tests/ArticlePlanTests.cs +++ b/test/Wiki.Application.Tests/ArticlePlanTests.cs @@ -75,7 +75,7 @@ public class ArticlePlanTests { // n3 is assigned to b without being among its sources var plan = ArticlePlanJson.Read(""" - {"articles":[{"slug":"b","category":"topic","sources":["n2"]}], + {"version":3,"articles":[{"slug":"b","category":"topic","sources":["n2"]}], "assignments":{"n2":["b"],"n3":["b"],"meta":[]}} """)!; @@ -112,7 +112,7 @@ public class ArticlePlanTests plan.RemoveNote("n1"); var read = ArticlePlanJson.Read(ArticlePlanJson.Write(plan))!; - var old = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{}}""")!; + var old = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{}}""")!; Assert.Equal(["gone"], read.Retired); Assert.True(read.SlugTaken("gone")); @@ -148,8 +148,8 @@ public class ArticlePlanTests plan.Fold("priv", "end", isPublic: false); var read = ArticlePlanJson.Read(ArticlePlanJson.Write(plan))!; - var old = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{"x":"y"}}""")!; - var stray = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{"x":"y"},"publicRedirects":["x","not-a-redirect"]}""")!; + var old = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{"x":"y"}}""")!; + var stray = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{"x":"y"},"publicRedirects":["x","not-a-redirect"]}""")!; Assert.Equal(["pub"], read.PublicRedirects); Assert.Empty(old.PublicRedirects); @@ -192,14 +192,14 @@ public class ArticlePlanTests Assert.Null(plan.Resolve("unknown")); var sixHops = ArticlePlanJson.Read(""" - {"articles":[{"slug":"end","category":"topic"}],"assignments":{}, + {"version":3,"articles":[{"slug":"end","category":"topic"}],"assignments":{}, "redirects":{"r1":"end","r2":"r1","r3":"r2","r4":"r3","r5":"r4","r6":"r5"}} """)!; Assert.Equal("end", sixHops.Resolve("r5")); Assert.Null(sixHops.Resolve("r6")); var cycle = ArticlePlanJson.Read(""" - {"articles":[],"assignments":{},"redirects":{"p":"q","q":"p"}} + {"version":3,"articles":[],"assignments":{},"redirects":{"p":"q","q":"p"}} """)!; Assert.Null(cycle.Resolve("p")); } @@ -247,7 +247,7 @@ public class ArticlePlanTests [Fact] public void Json_RedirectWithBadTargetSlug_Dropped() { - var read = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{"ok":"good-slug","bad":"../x","bad2":"A b"}}"""); + var read = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{"ok":"good-slug","bad":"../x","bad2":"A b"}}"""); Assert.NotNull(read); Assert.Equal(["ok"], read.Redirects.Keys); @@ -257,6 +257,129 @@ public class ArticlePlanTests public void Json_OldFormat_IsNull() => Assert.Null(ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"topic","sources":["x"],"related":[]}],"coveredNotes":["x"]}""")); + [Theory] + [InlineData("""{"articles":[{"slug":"a","category":"topic","sources":["n1"]}],"assignments":{"n1":["a"]}}""")] + [InlineData("""{"version":2,"articles":[],"assignments":{}}""")] + [InlineData("""{"version":"3","articles":[],"assignments":{}}""")] + public void Json_EarlierFormatVersion_IsNull(string json) => Assert.Null(ArticlePlanJson.Read(json)); + + [Fact] + public void Json_Parent_RoundTrips() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("heating", "n1")); + plan.AddArticle(Article("heat-pump", "n2") with { Parent = "heating" }); + + var json = ArticlePlanJson.Write(plan); + var read = ArticlePlanJson.Read(json)!; + + Assert.StartsWith("{\n \"version\": 3,", json); + Assert.Null(read.Find("heating")!.Parent); + Assert.Equal("heating", read.Find("heat-pump")!.Parent); + Assert.Equal(json, ArticlePlanJson.Write(read)); + } + + [Fact] + public void Json_ParentUnknownOrCyclic_Dropped() + { + var plan = ArticlePlanJson.Read(""" + {"version":3,"articles":[ + {"slug":"a","category":"topic","parent":"ghost"}, + {"slug":"b","category":"topic","parent":"c"}, + {"slug":"c","category":"topic","parent":"b"}, + {"slug":"d","category":"topic","parent":"d"}, + {"slug":"e","category":"topic","parent":"b"}],"assignments":{}} + """)!; + + Assert.Null(plan.Find("a")!.Parent); + Assert.Null(plan.Find("d")!.Parent); + Assert.Equal("b", plan.Find("e")!.Parent); + // in file order: the link that would close the cycle is the one dropped + Assert.Equal("c", plan.Find("b")!.Parent); + Assert.Null(plan.Find("c")!.Parent); + } + + [Fact] + public void SetParent_RefusesCyclesAndSelf() + { + var plan = ArticlePlan.Empty(); + foreach (var s in new[] { "a", "b", "c" }) plan.AddArticle(Article(s)); + + Assert.True(plan.SetParent("b", "a")); + Assert.True(plan.SetParent("c", "b")); + Assert.False(plan.SetParent("a", "c")); + Assert.False(plan.SetParent("a", "a")); + Assert.True(plan.SetParent("c", "a")); + Assert.True(plan.SetParent("c", null)); + + Assert.Null(plan.Find("a")!.Parent); + Assert.Null(plan.Find("c")!.Parent); + Assert.Equal(["b"], plan.ChildrenOf("a").Select(c => c.Slug)); + Assert.Throws(() => plan.SetParent("ghost", "a")); + Assert.Throws(() => plan.SetParent("a", "ghost")); + } + + [Fact] + public void ReplaceArticle_KeepsParent() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a")); + plan.AddArticle(Article("b") with { Parent = "a" }); + + plan.ReplaceArticle(Article("b", "n1")); + + Assert.Equal("a", plan.Find("b")!.Parent); + } + + [Fact] + public void AddArticle_UnknownParent_Throws() + { + var plan = ArticlePlan.Empty(); + + Assert.Throws(() => plan.AddArticle(Article("b") with { Parent = "ghost" })); + } + + [Fact] + public void Fold_ChildrenMoveToTarget_TargetTakesFoldedParent() + { + var plan = ArticlePlan.Empty(); + foreach (var s in new[] { "top", "thin", "into", "kid" }) plan.AddArticle(Article(s, "n-" + s)); + plan.SetParent("thin", "top"); + plan.SetParent("into", "thin"); + plan.SetParent("kid", "thin"); + + plan.Fold("thin", "into", isPublic: false); + + Assert.Equal("top", plan.Find("into")!.Parent); + Assert.Equal("into", plan.Find("kid")!.Parent); + } + + [Fact] + public void Fold_ChildMoveWouldCycle_ChildLosesParent() + { + var plan = ArticlePlan.Empty(); + foreach (var s in new[] { "thin", "kid", "into" }) plan.AddArticle(Article(s, "n-" + s)); + plan.SetParent("kid", "thin"); + plan.SetParent("into", "kid"); + + plan.Fold("thin", "into", isPublic: false); + + Assert.Null(plan.Find("kid")!.Parent); + Assert.Equal("kid", plan.Find("into")!.Parent); + } + + [Fact] + public void RemoveNote_DeletedParent_ChildrenLoseParent() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a", "n1")); + plan.AddArticle(Article("b", "n2") with { Parent = "a" }); + + plan.RemoveNote("n1"); + + Assert.Null(plan.Find("b")!.Parent); + } + [Fact] public void Json_Write_MapKeysOrdinal_SameForSameContent() { @@ -285,7 +408,7 @@ public class ArticlePlanTests public void Json_BadSlugOrCategory_ArticleDropped() { var plan = ArticlePlanJson.Read(""" - {"articles":[{"slug":"Bad Slug","category":"topic"},{"slug":"a","category":"1"},{"slug":"b","category":"room"}],"assignments":{}} + {"version":3,"articles":[{"slug":"Bad Slug","category":"topic"},{"slug":"a","category":"1"},{"slug":"b","category":"room"}],"assignments":{}} """); Assert.Equal(["b"], plan!.Articles.Select(a => a.Slug)); } @@ -294,7 +417,7 @@ public class ArticlePlanTests public void Json_AssignmentToUnknownArticle_Dropped() { var plan = ArticlePlanJson.Read(""" - {"articles":[{"slug":"a","category":"topic","sources":["n1"]}], + {"version":3,"articles":[{"slug":"a","category":"topic","sources":["n1"]}], "assignments":{"n1":["a","ghost"],"n2":["ghost"],"n3":[]}} """)!; @@ -307,7 +430,7 @@ public class ArticlePlanTests { public bool Equals(PlannedArticle? x, PlannedArticle? y) => x!.Slug == y!.Slug && x.TitleDe == y.TitleDe && x.TitleEn == y.TitleEn && x.Category == y.Category - && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources); + && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources) && x.Parent == y.Parent; public int GetHashCode(PlannedArticle obj) => obj.Slug.GetHashCode(); } } diff --git a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs index ff153b0..5e18305 100644 --- a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs +++ b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs @@ -24,6 +24,13 @@ public class AssignNotesUseCaseTests private static Document[] Batch(params string[] slugs) => slugs.Select(s => TestNotes.Note(s)).ToArray(); + [Fact] + public void Prompt_RoomNotesAlsoFeedTheRoomArticle() + { + Assert.Contains("additionally feeds that room's article (category room)", Prompts.Assign); + Assert.Contains("\"EG Schlafzimmer\"", Prompts.Assign); + } + [Fact] public async Task Assign_ExistingAndNew() { diff --git a/test/Wiki.Application.Tests/ConsolidateArticlesUseCaseTests.cs b/test/Wiki.Application.Tests/ConsolidateArticlesUseCaseTests.cs new file mode 100644 index 0000000..d718c55 --- /dev/null +++ b/test/Wiki.Application.Tests/ConsolidateArticlesUseCaseTests.cs @@ -0,0 +1,130 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Encyclopedia; +using Wiki.Core.Model; + +namespace Wiki.Application.Tests; + +public class ConsolidateArticlesUseCaseTests +{ + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null) { ConsolidateGroupSize = 4 }; + + private static ConsolidateArticlesUseCase UseCase(FakeTextModel model) => + new(model, Settings, NullLogger.Instance); + + private static PlannedArticle Article(string slug, string? parent = null, int sources = 1) => + new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, + Enumerable.Range(0, sources).Select(i => $"{slug}-n{i}").ToList(), parent); + + private static readonly PlannedArticle[] Heating = + [Article("heating", sources: 2), Article("heat-pump", parent: "house"), Article("vitocal-heat-pump"), Article("radiators")]; + + [Fact] + public async Task Consolidate_MergesAndParents() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"vitocal-heat-pump","into":"heat-pump"}],"parents":[{"child":"heat-pump","parent":"heating"},{"child":"radiators","parent":"heating"}]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.NotNull(outcome); + Assert.Equal([new ArticleMerge("vitocal-heat-pump", "heat-pump")], outcome.Merges); + Assert.Equal([new ArticleParent("heat-pump", "heating"), new ArticleParent("radiators", "heating")], outcome.Parents); + var call = Assert.Single(model.Calls); + Assert.StartsWith("[consolidate]", call.System); + Assert.Equal(Settings.ConsolidateMaxTokens, call.MaxTokens); + Assert.Contains("heating | Titel heating | system | Scope heating | 2 | -\n", call.User); + Assert.Contains("heat-pump | Titel heat-pump | system | Scope heat-pump | 1 | house\n", call.User); + } + + [Fact] + public async Task Consolidate_ShowsAtMostTheGroupSize() + { + var model = new FakeTextModel().Reply("""{"merges":[],"parents":[]}"""); + + await UseCase(model).ConsolidateAsync([.. Heating, Article("garden")], CancellationToken.None); + + Assert.DoesNotContain("garden", model.Calls[0].User); + } + + [Fact] + public async Task Consolidate_EntriesOutsideTheGroupOrOnThemselves_Dropped() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"heating","into":"heating"},{"from":"garden","into":"heating"},{"from":"heating","into":"garden"},{"from":"radiators"}], + "parents":[{"child":"radiators","parent":"radiators"},{"child":"garden","parent":"heating"},{"child":"radiators","parent":"garden"}]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.Empty(outcome!.Merges); + Assert.Empty(outcome.Parents); + } + + [Fact] + public async Task Consolidate_MergedAwayArticleIsNeverATarget() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"vitocal-heat-pump","into":"heat-pump"},{"from":"heat-pump","into":"heating"}, + {"from":"radiators","into":"vitocal-heat-pump"},{"from":"vitocal-heat-pump","into":"heating"}],"parents":[]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.Equal([new ArticleMerge("vitocal-heat-pump", "heat-pump")], outcome!.Merges); + } + + [Fact] + public async Task Consolidate_ParentsFollowTheMergesAndAreUniquePerChild() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"vitocal-heat-pump","into":"heat-pump"}], + "parents":[{"child":"vitocal-heat-pump","parent":"heating"},{"child":"radiators","parent":"vitocal-heat-pump"}, + {"child":"heat-pump","parent":"radiators"},{"child":"heat-pump","parent":"vitocal-heat-pump"}]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.Equal([new ArticleParent("heat-pump", "heating"), new ArticleParent("radiators", "heat-pump")], outcome!.Parents); + } + + [Fact] + public async Task Consolidate_NothingToDo_IsUsable() + { + var model = new FakeTextModel().Reply("""{"merges":[]}"""); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.NotNull(outcome); + Assert.Empty(outcome.Merges); + Assert.Empty(outcome.Parents); + } + + [Theory] + [InlineData("not json", false)] + [InlineData("""{"keep":true}""", false)] + [InlineData("""{"merges":[],"parents":[]}""", true)] + public async Task Consolidate_UnusableReply_IsNull(string reply, bool truncated) + { + var model = new FakeTextModel().Reply(reply, truncated); + + Assert.Null(await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None)); + } + + [Fact] + public async Task Consolidate_BadRequest_IsNull() + { + var model = new FakeTextModel().Throw(new HttpRequestException("invalid", null, HttpStatusCode.BadRequest)); + + Assert.Null(await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None)); + } + + [Fact] + public async Task Consolidate_OtherHttpError_Propagates() + { + var model = new FakeTextModel().Throw(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway)); + + await Assert.ThrowsAsync(() => UseCase(model).ConsolidateAsync(Heating, CancellationToken.None)); + } +} diff --git a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs index f2a4ab4..8009069 100644 --- a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs @@ -11,7 +11,7 @@ public class EncyclopediaCoordinatorTests public Task RunAsync(CancellationToken ct) => body(ct); } - private static readonly ReconcileSummary Done = new(3, 0, 0, 0, 2, 0, 0, 0, false, null, "c1"); + private static readonly ReconcileSummary Done = new(3, 0, 0, 0, 0, 0, 2, 0, 0, 0, false, null, "c1"); private static EncyclopediaCoordinator Coordinator(Func> body) => new(new ScriptedRun(body), new FakeTimeProvider(new DateTimeOffset(2026, 10, 24, 4, 0, 0, TimeSpan.Zero)), diff --git a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs index 8d3ae3f..98c6748 100644 --- a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs @@ -16,8 +16,8 @@ public class EncyclopediaReconcilerTests private const string StatePath = "articles/_state.json"; private static readonly DateTimeOffset Now = new(2026, 10, 24, 3, 0, 0, TimeSpan.Zero); - /// Fold is off (no article has fewer than zero sources) unless a test turns it on. - private static readonly EncyclopediaSettings Defaults = new(16000, 10, 300_000, [], null) { FoldMinSources = 0 }; + /// Fold (no article has fewer than zero sources) and consolidation are off unless a test turns them on. + private static readonly EncyclopediaSettings Defaults = new(16000, 10, 300_000, [], null) { FoldMinSources = 0, MaxConsolidationsPerRun = 0 }; private static readonly (string Slug, string[] Notes)[] HeatingAndKitchen = [("heating", ["boiler", "radiators"]), ("kitchen", ["kitchen"])]; @@ -58,6 +58,7 @@ public class EncyclopediaReconcilerTests var s = settings ?? Defaults; return new(_store, _catalog, new ArticleNeighbours(_embeddings, _index, s), new AssignNotesUseCase(_model, s, NullLogger.Instance), + new ConsolidateArticlesUseCase(_model, s, NullLogger.Instance), new SplitArticleUseCase(_model, s, NullLogger.Instance), new FoldArticleUseCase(_model, s, NullLogger.Instance), writer ?? _writer, s, _clock, NullLogger.Instance); @@ -100,12 +101,12 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); var commit = Assert.Single(_store.Commits); - Assert.Equal("articles: 3 assigned, 0 split, 0 folded, 2 written, 0 deleted, 0 rejected", commit.Message); + Assert.Equal("articles: 3 assigned, 0 merged, 0 parented, 0 split, 0 folded, 2 written, 0 deleted, 0 rejected", commit.Message); Assert.Equal( [PlanPath, StatePath, "articles/de/heating.md", "articles/de/kitchen.md", "articles/en/heating.md", "articles/en/kitchen.md"], commit.Changes.Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.All(commit.Changes, c => Assert.Equal(FileChangeOp.Create, c.Op)); - Assert.Equal(new ReconcileSummary(2, 3, 0, 0, 2, 0, 0, 0, false, null, "c2"), summary); + Assert.Equal(new ReconcileSummary(2, 3, 0, 0, 0, 0, 2, 0, 0, 0, false, null, "c2"), summary); Assert.Equal(AttemptOutcome.Written, State.Subjects["heating"].Outcome); Assert.Equal(Now, State.Subjects["heating"].At); Assert.Equal(["boiler", "radiators"], Plan.Find("heating")!.Sources); @@ -237,7 +238,7 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Run_SplitChildrenAreLinksOnlyInTheSplittingRun() + public async Task Run_SplitChildrenAreSubArticles_LinkedInLaterRunsToo() { await SeedAsync("boiler", "radiators", "pump", "valve"); _model.Reply(AssignReply(("heating", ["boiler", "radiators", "pump", "valve"]))); @@ -259,7 +260,10 @@ public class EncyclopediaReconcilerTests await Reconciler(settings).RunAsync(CancellationToken.None); Assert.Equal("heating", _writer.Calls[^1]); - Assert.Empty(_writer.Links["heating"]); + Assert.Equal(["heating-pump", "heating-radiators"], _writer.Links["heating"].Order(StringComparer.Ordinal)); + Assert.Equal(["heating"], _writer.Links["heating-pump"]); + Assert.Equal("heating", Plan.Find("heating-pump")!.Parent); + Assert.Null(Plan.Find("heating")!.Parent); } [Fact] @@ -507,7 +511,7 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); Assert.Equal(1, summary.Rejected); - Assert.Equal("articles: 0 assigned, 0 split, 0 folded, 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); + Assert.Equal("articles: 0 assigned, 0 merged, 0 parented, 0 split, 0 folded, 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); Assert.Equal([StatePath], LastCommit.Select(c => c.Path)); Assert.Equal(de, _store.Files["articles/de/heating.md"]); Assert.Equal(en, _store.Files["articles/en/heating.md"]); @@ -946,6 +950,243 @@ public class EncyclopediaReconcilerTests Assert.Equal(["heating"], _writer.Calls); } + private static readonly EncyclopediaSettings Consolidating = Defaults with { MaxConsolidationsPerRun = 60 }; + + private static string Consolidation(object[]? merges = null, object[]? parents = null) => + JsonSerializer.Serialize(new { merges = merges ?? [], parents = parents ?? [] }); + + private string[] Steps => _model.Calls.Select(c => c.System.Split('\n')[0]).ToArray(); + + [Fact] + public async Task Run_Consolidate_MergesAndSetsParents() + { + await SeedAsync("boiler", "pump", "radiators", "valve"); + _model.Reply(AssignReply(("heat-pump", ["boiler"]), ("vitocal", ["pump"]), ("heating", ["radiators", "valve"]))); + _model.Reply(Consolidation( + [new { from = "vitocal", into = "heat-pump" }], + [new { child = "heat-pump", parent = "heating" }, new { child = "vitocal", parent = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + Assert.Equal((1, 1, 2), (summary.Merged, summary.Parented, summary.Planned)); + Assert.StartsWith("articles: 4 assigned, 1 merged, 1 parented,", _store.Commits[^1].Message); + Assert.Equal("heat-pump", Plan.Redirects["vitocal"]); + Assert.Empty(Plan.PublicRedirects); + Assert.Equal(["boiler", "pump"], Plan.Find("heat-pump")!.Sources); + Assert.Equal(["heat-pump"], Plan.Assignments["pump"]); + Assert.Equal("heating", Plan.Find("heat-pump")!.Parent); + Assert.Contains("heating", _writer.Links["heat-pump"]); + Assert.Contains("heat-pump", _writer.Links["heating"]); + Assert.DoesNotContain("vitocal", _writer.Calls); + } + + [Fact] + public async Task Run_Consolidate_GroupIsTheSeedAndItsNearestArticles() + { + await SeedAsync("boiler", "pump", "radiators"); + _model.Reply(AssignReply(("heat-pump", ["boiler"]), ("vitocal", ["pump"]), ("heating", ["radiators"]))); + _model.Reply(Consolidation()); + + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + var user = _model.Calls[1].User; + Assert.StartsWith("Articles (slug | title | category | scope | source count | current parent):\nheat-pump | heat-pump de | topic | About heat-pump. | 1 | -\n", user); + Assert.Contains("vitocal | vitocal de |", user); + Assert.Contains("heating | heating de |", user); + } + + [Fact] + public async Task Run_Consolidate_RememberedGroupNotAskedAgain() + { + await SeedAsync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + _model.Reply(Consolidation()); + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + await Sync(); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + Assert.Single(State.ConsolidatedGroups); + Assert.Null(summary.Commit); + } + + [Fact] + public async Task Run_Consolidate_ChangedGroupRememberedAsChanged() + { + await SeedAsync("boiler", "pump", "radiators"); + _model.Reply(AssignReply(("heat-pump", ["boiler"]), ("vitocal", ["pump"]), ("heating", ["radiators"]))); + _model.Reply(Consolidation([new { from = "vitocal", into = "heat-pump" }], [new { child = "heat-pump", parent = "heating" }])); + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + await Sync(); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.Single(State.ConsolidatedGroups); + Assert.Null(summary.Commit); + } + + [Fact] + public async Task Run_Consolidate_UnusableReplyRememberedAsFailed() + { + await SeedAsync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + _model.Reply("I would rather not."); + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + await Sync(); + + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + Assert.Single(State.FailedGroups); + Assert.Empty(State.ConsolidatedGroups); + } + + [Fact] + public async Task Run_Consolidate_HttpErrorSkipsOnlyThatGroup() + { + await SeedAsync("a1", "b1", "c1", "d1"); + _model.Reply(AssignReply(("a", ["a1"]), ("b", ["b1"]), ("c", ["c1"]), ("d", ["d1"]))); + _model.Throw(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway)); + foreach (var _ in Enumerable.Range(0, 3)) _model.Reply(Consolidation()); + + var summary = await Reconciler(Consolidating with { ConsolidateGroupSize = 2 }).RunAsync(CancellationToken.None); + + Assert.True(Steps.Count(s => s == "[consolidate]") >= 2); + Assert.Empty(State.FailedGroups); + Assert.NotEmpty(State.ConsolidatedGroups); + Assert.Equal(4, summary.Written); + } + + [Fact] + public async Task Run_Consolidate_CapRespected() + { + await SeedAsync("a1", "b1", "c1", "d1"); + _model.Reply(AssignReply(("a", ["a1"]), ("b", ["b1"]), ("c", ["c1"]), ("d", ["d1"]))); + _model.Reply(Consolidation()); + + await Reconciler(Consolidating with { ConsolidateGroupSize = 2, MaxConsolidationsPerRun = 1 }).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + } + + [Fact] + public async Task Run_Consolidate_MergeOfPublicArticle_LeavesPublicRedirectAndDeletesFiles() + { + await FirstRunAsync(); + _model.Reply(Consolidation([new { from = "kitchen", into = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal((1, 1), (summary.Merged, summary.Deleted)); + Assert.Equal("heating", Plan.Redirects["kitchen"]); + Assert.Equal(["kitchen"], Plan.PublicRedirects); + Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("kitchen", State.Subjects.Keys); + Assert.Equal("heating", _writer.Calls[^1]); + } + + [Fact] + public async Task Run_Consolidate_ParentLinkAloneRewritesNothing() + { + await FirstRunAsync(); + _model.Reply(Consolidation(parents: [new { child = "kitchen", parent = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(1, summary.Parented); + Assert.Equal(0, summary.Written); + Assert.Equal(2, _writer.Calls.Count); + Assert.Equal([PlanPath, StatePath], LastCommit.Select(c => c.Path).Order(StringComparer.Ordinal)); + Assert.Equal("heating", Plan.Find("kitchen")!.Parent); + } + + [Fact] + public async Task Run_Consolidate_ParentClosingACycle_Dropped() + { + await FirstRunAsync(); + var plan = Plan; + plan.SetParent("heating", "kitchen"); + _store.Files[PlanPath] = ArticlePlanJson.Write(plan); + await Sync(); + _model.Reply(Consolidation(parents: [new { child = "kitchen", parent = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(0, summary.Parented); + Assert.Null(Plan.Find("kitchen")!.Parent); + Assert.Equal("kitchen", Plan.Find("heating")!.Parent); + } + + [Fact] + public async Task Run_AssignCandidates_IncludeArticlesNearByVector() + { + var settings = Defaults with { NeighbourNotes = 0 }; + await SeedAsync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + _store.Files["wiki/oven.md"] = Note("oven"); + await Sync(); + _model.Reply(ExistingReply(("oven", "kitchen"))); + + await Reconciler(settings).RunAsync(CancellationToken.None); + + // no note neighbours at all: both candidates come from the article vectors + Assert.Contains("\nheating | heating de |", _model.Calls[1].User); + Assert.Contains("\nkitchen | kitchen de |", _model.Calls[1].User); + } + + [Fact] + public async Task Run_WriteLinksParentAndAtMost20Children() + { + var children = Numbered(25); + await SeedAsync(["hub-note", .. children]); + var plan = ArticlePlan.Empty(); + plan.AddArticle(new PlannedArticle("hub", "Hub", "Hub", ArticleCategory.System, "The hub.", [])); + plan.Assign("hub-note", ["hub"]); + foreach (var c in children) + { + plan.AddArticle(new PlannedArticle("a-" + c, c, c, ArticleCategory.Device, "One.", [], Parent: "hub")); + plan.Assign(c, ["a-" + c]); + } + _store.Files[PlanPath] = ArticlePlanJson.Write(plan); + await Sync(); + + await Reconciler(Defaults with { CandidateArticles = 0, MaxArticlesPerRun = 30 }).RunAsync(CancellationToken.None); + + Assert.Equal(20, _writer.Links["hub"].Count); + Assert.Equal(["hub"], _writer.Links["a-note-25"]); + } + + [Fact] + public async Task Run_PlanInOlderFormat_BootstrapsAndDropsItsState() + { + await SeedAsync(); + _store.Files[PlanPath] = """ + {"articles":[{"slug":"heizung","titleDe":"Heizung","titleEn":"Heating","category":"system","scope":"x","sources":["boiler"]}], + "assignments":{"boiler":["heizung"],"radiators":[],"kitchen":[]},"redirects":{}} + """; + _store.Files[StatePath] = EncyclopediaStateJson.Write(EncyclopediaState.Empty with + { + Subjects = new Dictionary { ["heizung"] = new("f", AttemptOutcome.Rejected, Now) }, + KeptArticles = new Dictionary { ["heizung"] = "f" }, + }); + await Sync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(3, summary.Assigned); + Assert.Contains("boiler |", _model.Calls[0].User); + Assert.Null(Plan.Find("heizung")); + Assert.Equal(["heating", "kitchen"], State.Subjects.Keys); + Assert.Empty(State.KeptArticles); + Assert.StartsWith("{\n \"version\": 3,", _store.Files[PlanPath]); + } + /// Embeddings that fail like an unreachable service while , or with when set. private sealed class SwitchableEmbeddings(IEmbeddingModel inner) : IEmbeddingModel { diff --git a/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs b/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs index 78a553f..f58fa36 100644 --- a/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs @@ -12,8 +12,18 @@ public class EncyclopediaSettingsTests Assert.Equal(32000, s.AssignMaxTokens); Assert.Equal(32000, s.SplitMaxTokens); Assert.Equal(8000, s.FoldMaxTokens); + Assert.Equal(8000, s.ConsolidateMaxTokens); } [Fact] - public void PromptsVersion_InvalidatesBatchesFailedAtTheOldCap() => Assert.Equal(7, Prompts.Version); + public void Defaults_ConsolidateABootstrapPlanInOneRun() + { + var s = new EncyclopediaSettings(16000, 10, 300_000, [], null); + + Assert.Equal(8, s.ConsolidateGroupSize); + Assert.Equal(60, s.MaxConsolidationsPerRun); + } + + [Fact] + public void PromptsVersion_InvalidatesMemoryOfThePreviousPlan() => Assert.Equal(8, Prompts.Version); } diff --git a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs index 11b4977..909d999 100644 --- a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs @@ -17,7 +17,9 @@ public class EncyclopediaStateTests ["sauna"] = new("def", AttemptOutcome.Rejected, At), }, new Dictionary(), - new Dictionary()); + new Dictionary(), + new Dictionary(), + new Dictionary()); var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); @@ -30,12 +32,42 @@ public class EncyclopediaStateTests var state = new EncyclopediaState( new Dictionary(), new Dictionary { ["fp1"] = At }, - new Dictionary { ["heating"] = "fp2" }); + new Dictionary { ["heating"] = "fp2" }, + new Dictionary { ["g1"] = At }, + new Dictionary { ["g2"] = At }); var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); Assert.Equal(state.FailedBatches, read.FailedBatches); Assert.Equal(state.KeptArticles, read.KeptArticles); + Assert.Equal(state.ConsolidatedGroups, read.ConsolidatedGroups); + Assert.Equal(state.FailedGroups, read.FailedGroups); + } + + [Fact] + public void Read_WithoutGroups_ReadsEmpty() + { + var read = EncyclopediaStateJson.Read("""{"subjects":{},"failedBatches":{},"keptArticles":{}}"""); + Assert.Empty(read.ConsolidatedGroups); + Assert.Empty(read.FailedGroups); + } + + [Fact] + public void GroupFingerprint_MembersAndWhatTheModelJudges_NotSources() + { + var a = new PlannedArticle("a", "A", "A", ArticleCategory.System, "About a.", ["n1"]); + var b = new PlannedArticle("b", "B", "B", ArticleCategory.Device, "About b.", ["n2"], Parent: "a"); + + var baseline = EncyclopediaState.GroupFingerprint([a, b]); + + Assert.Matches("^[0-9a-f]{64}$", baseline); + Assert.Equal(baseline, EncyclopediaState.GroupFingerprint([b, a])); + Assert.Equal(baseline, EncyclopediaState.GroupFingerprint([a, b with { Sources = ["n2", "n3"] }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { Parent = null }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { Scope = "Other." }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { TitleEn = "Bee" }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { Category = ArticleCategory.Room }])); } [Fact] @@ -45,6 +77,7 @@ public class EncyclopediaStateTests Assert.Empty(read.Subjects); Assert.Empty(read.FailedBatches); Assert.Empty(read.KeptArticles); + Assert.Empty(read.ConsolidatedGroups); Assert.DoesNotContain("failedPlan", EncyclopediaStateJson.Write(read)); } diff --git a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs index 6ea0ce3..0454141 100644 --- a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs @@ -62,6 +62,31 @@ public class WriteArticleUseCaseTests Assert.DoesNotContain("Related articles", user); } + [Fact] + public void Prompt_RoomArticlesDescribeTheRoomAndLinkTheSystems() + { + Assert.Contains("A room article (category room) describes what is in the room", Prompts.Write); + Assert.Contains("links to the system's article", Prompts.Write); + } + + [Fact] + public async Task Write_MainArticleAndSubArticlesMarked() + { + var planned = Planned with { Parent = "house" }; + var targets = new PlannedArticle[] + { + new("house", "Haus", "House", ArticleCategory.Topic, "", ["h"]), + new("lighting", "Licht", "Lighting", ArticleCategory.System, "", ["l"]), + new("dimmers", "Dimmer", "Dimmers", ArticleCategory.Device, "", ["d"], Parent: "wall-switches"), + }; + var model = Model(ThreeBlocks, Clean, """["Zero.","One.","Two."]""", Clean); + + await new WriteArticleUseCase(model, Settings, new FakeTimeProvider(Now)).WriteAsync(planned, [Source()], targets, CancellationToken.None); + + Assert.Contains("Articles you may link to (slug | German title | relation):\nhouse | Haus | main article\nlighting | Licht\ndimmers | Dimmer | sub-article\n", + model.Calls[0].User); + } + private static IReadOnlyList Public(string body) => ArticleBody.PublicBlocks(body).Select(b => b.Text).ToList(); [Fact] diff --git a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs index 094f00a..1a19d67 100644 --- a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs +++ b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs @@ -24,7 +24,7 @@ public sealed class EncyclopediaEndpointTests { CatalogCommits.Add((await catalog.GetStateAsync(ct)).Commit); if (Failure is not null) throw Failure; - return new ReconcileSummary(Planned, 0, 0, 0, 0, 0, 0, 0, StoppedByModel, null, null); + return new ReconcileSummary(Planned, 0, 0, 0, 0, 0, 0, 0, 0, 0, StoppedByModel, null, null); } } diff --git a/test/Wiki.Host.Tests/HouseTests.cs b/test/Wiki.Host.Tests/HouseTests.cs index 9ef63fa..062e753 100644 --- a/test/Wiki.Host.Tests/HouseTests.cs +++ b/test/Wiki.Host.Tests/HouseTests.cs @@ -44,9 +44,17 @@ public class HouseTests(WikiHostFactory factory) : IClassFixtureDas Haus", child); + Assert.DoesNotContain("Unterartikel", child); + Assert.Contains("

Unterartikel

", parent); + Assert.Contains("
  • Pfad
    • ", parent); + Assert.Contains("
  • Wandschalter
    • ", parent); + Assert.True(parent.IndexOf("/house/de/x\"", StringComparison.Ordinal) < parent.IndexOf("/house/de/schalter\"", StringComparison.Ordinal)); + Assert.DoesNotContain("network-internals", parent); + Assert.DoesNotContain("Netzwerk-Interna", parent); + Assert.DoesNotContain("heizung", parent); + Assert.DoesNotContain("geplant", parent); + Assert.DoesNotContain("Hauptartikel", parent); + } + + [Fact] + public async Task Navigation_Anonymous_PrivateMainArticleInvisible() + { + await SeedAsync(); + + var html = Text(await Html(await Browser().GetAsync("/house/de/y"))); + + Assert.DoesNotContain("Hauptartikel", html); + Assert.DoesNotContain("network-internals", html); + Assert.DoesNotContain("Netzwerk-Interna", html); + } + + [Fact] + public async Task Navigation_SignedIn_SeesPrivateRelatives() + { + await SeedAsync(); + var browser = await SignedInAsync(); + + var parent = Text(await Html(await browser.GetAsync("/house/de/haus"))); + var child = Text(await Html(await browser.GetAsync("/house/de/y"))); + + Assert.Contains("
  • Netzwerk-Interna
    • ", parent); + Assert.Contains("
  • Heizung
    • ", parent); + Assert.DoesNotContain("geplant", parent); + Assert.Contains("Hauptartikel: Netzwerk-Interna", child); + } + + [Fact] + public async Task Navigation_English_Labels() + { + await SeedAsync(); + + var child = Text(await Html(await Browser().GetAsync("/house/en/schalter"))); + var parent = Text(await Html(await Browser().GetAsync("/house/en/haus"))); + + Assert.Contains("Main article: The house", child); + Assert.Contains("

    Sub-articles

    ", parent); + Assert.Contains("
  • Wall switches
    • ", parent); + } + [Fact] public async Task FrontPage_Anonymous_ListsOnlyPublic() { -- 2.54.0 From d9ac615b67530c3dbb7b4dbd3bd38e247d81e1b5 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 14:37:08 +0100 Subject: [PATCH 3/5] docs(encyclopedia): which group fingerprint consolidation remembers Co-Authored-By: Claude Opus 5.5 --- .../specs/2026-10-26-encyclopedia-consolidation-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md b/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md index d684a8b..3c4edc5 100644 --- a/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md +++ b/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md @@ -38,7 +38,7 @@ Interleaving keeps both kinds in the prompt when one alone would fill the cap. A **Merge** uses fold's mechanics (`ArticlePlan.Fold`): sources move into the target, `redirects[from] = into`, public when both language files of `from` were public at the run's start (`publicRedirects`), redirects to `from` move along. Additionally the children of `from` move to `into`; if `into` was a child of `from`, it takes `from`'s parent. A move that would close a cycle leaves the child without parent. -**Memory.** `_state.json` gains `ConsolidatedGroups` (fingerprint → when) for usable replies and `FailedGroups` (fingerprint → when) for unusable ones; a group with a remembered fingerprint is not asked again. Fingerprint: `Prompts.Version` and, per member in slug order, slug, titles, category, scope and parent. Source counts are left out: they change whenever a note is added and say nothing about whether two articles share a subject; with them, most groups would be asked again every night. Both maps keep only the fingerprints of groups formed in the latest run that reached the step, so they do not grow without bound. A call that fails with `HttpRequestException` skips that group only and is not remembered. +**Memory.** `_state.json` gains `ConsolidatedGroups` (fingerprint → when) and `FailedGroups` (fingerprint → when); a group with a remembered fingerprint is not asked again. After a usable reply the fingerprint remembered is that of the group's surviving members as the reply left them (merged ones gone, new parents set; the same as asked when nothing changed), so the next night's identical group is not asked about its own result. After an unusable reply the fingerprint as asked goes to `FailedGroups`. Fingerprint: `Prompts.Version` and, per member in slug order, slug, titles, category, scope and parent. Source counts are left out: they change whenever a note is added and say nothing about whether two articles share a subject; with them, most groups would be asked again every night. Both maps keep only the fingerprints of groups formed in the latest run that reached the step, so they do not grow without bound. A call that fails with `HttpRequestException` skips that group only and is not remembered. `ReconcileSummary` gains `Merged` (merges applied) and `Parented` (parent links set); the commit message, endpoint body and status panel report them. -- 2.54.0 From d7de9988216183034d6f56c86f8d9d61fda19226 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 26 Oct 2026 14:51:47 +0100 Subject: [PATCH 4/5] fix(encyclopedia): stable consolidation groups, redirected slugs in assign, reset cleanup Consolidation runs passes until one changes nothing; a changing reply ends the pass and the next starts from the first seed, so the last pass's groups are the ones the next run forms. Memory is pruned to that pass. A stable neighbourhood after a merge is no longer asked again the next night. Assign treats a redirected slug, proposed as new or referenced directly, as its live target instead of dropping the note. A plan reset deletes article files whose slug the new plan lacks. The sub-articles heading has no id, so it cannot collide with a body heading. Co-Authored-By: Claude Opus 5.5 --- ...10-26-encyclopedia-consolidation-design.md | 10 +- .../Encyclopedia/AssignNotesUseCase.cs | 22 ++- .../Encyclopedia/EncyclopediaReconciler.cs | 136 +++++++++++------- .../Components/Pages/HouseArticle.razor | 3 +- .../AssignNotesUseCaseTests.cs | 25 +++- .../EncyclopediaReconcilerTests.cs | 119 +++++++++++++-- test/Wiki.Host.Tests/HouseTests.cs | 8 +- 7 files changed, 245 insertions(+), 78 deletions(-) diff --git a/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md b/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md index 3c4edc5..842fe98 100644 --- a/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md +++ b/docs/superpowers/specs/2026-10-26-encyclopedia-consolidation-design.md @@ -22,13 +22,17 @@ The candidates of a batch are two lists, interleaved and deduplicated, capped at Interleaving keeps both kinds in the prompt when one alone would fill the cap. A batch now sees the articles earlier batches created even when none of its neighbour notes is assigned yet. +Consolidation leaves many redirects with natural names, which the model may propose again. A slug in the reply that is a redirect, whether proposed as `new` or named in a note's `articles`, stands for the live article it resolves to; that article becomes a valid reference, as a proposed `new` slug that names a live article already did. + ## A run 1. Diff. 2. Assign. 3. **Consolidate** (new). 4. Split. 5. Fold. 6. Write. Order and caps otherwise unchanged. ## Consolidate -**Groups.** A seed article and its nearest articles by article vector, `ConsolidateGroupSize` (default 8) articles in all, the seed included. Seeds are taken in slug order among the articles no group of this run has covered yet; every member of a group formed this run (asked or remembered) counts as covered. Slug order, not size: it does not depend on source counts, which change nightly, so an unchanged neighbourhood forms the same groups night after night and is not asked again. +**Groups.** A seed article and its nearest articles by article vector, `ConsolidateGroupSize` (default 8) articles in all, the seed included. Seeds are taken in slug order among the articles no group of the current pass has covered yet; every member of a group formed in the pass (asked or remembered) counts as covered. Slug order, not size: it does not depend on source counts, which change nightly, so an unchanged neighbourhood forms the same groups night after night and is not asked again. + +**Passes.** The step runs passes until one changes nothing. A reply that changes the plan (a merge, or a parent link that was not there) ends the pass, and the next pass starts again from the first seed. A merge shifts every later group in slug order, since the merged article's place in the groups goes to the next-nearest article; restarting means the groups of the last pass are exactly the groups the next run forms from an unchanged plan. Re-forming the groups is local vector arithmetic only. Every pass counts its calls towards the same cap, so the loop ends: each extra pass needs a call that changed something. **Call.** One per group, at most `MaxConsolidationsPerRun` (default 60) per run. 60 groups of 8 cover ~300+ articles in one bootstrap night even with overlapping groups (299 articles need at least 38). Input, per member: `slug | titleDe | category | scope | source count | current parent` — bounded by the group size. `ConsolidateMaxTokens` 8000 (the reply is short; thinking tokens count, as for fold). @@ -38,7 +42,7 @@ Interleaving keeps both kinds in the prompt when one alone would fill the cap. A **Merge** uses fold's mechanics (`ArticlePlan.Fold`): sources move into the target, `redirects[from] = into`, public when both language files of `from` were public at the run's start (`publicRedirects`), redirects to `from` move along. Additionally the children of `from` move to `into`; if `into` was a child of `from`, it takes `from`'s parent. A move that would close a cycle leaves the child without parent. -**Memory.** `_state.json` gains `ConsolidatedGroups` (fingerprint → when) and `FailedGroups` (fingerprint → when); a group with a remembered fingerprint is not asked again. After a usable reply the fingerprint remembered is that of the group's surviving members as the reply left them (merged ones gone, new parents set; the same as asked when nothing changed), so the next night's identical group is not asked about its own result. After an unusable reply the fingerprint as asked goes to `FailedGroups`. Fingerprint: `Prompts.Version` and, per member in slug order, slug, titles, category, scope and parent. Source counts are left out: they change whenever a note is added and say nothing about whether two articles share a subject; with them, most groups would be asked again every night. Both maps keep only the fingerprints of groups formed in the latest run that reached the step, so they do not grow without bound. A call that fails with `HttpRequestException` skips that group only and is not remembered. +**Memory.** `_state.json` gains `ConsolidatedGroups` (fingerprint → when) and `FailedGroups` (fingerprint → when); a group with a remembered fingerprint is not asked again. A reply that changes nothing remembers the group as asked. A reply that changes the plan remembers the group's surviving members as the reply left them (merged ones gone, new parents set): if the next pass forms exactly that group, it is not asked about its own result; if the group gained an article it is asked, because the model has not judged the newcomer with the others. After an unusable reply the fingerprint as asked goes to `FailedGroups`. Fingerprint: `Prompts.Version` and, per member in slug order, slug, titles, category, scope and parent. Source counts are left out: they change whenever a note is added and say nothing about whether two articles share a subject; with them, most groups would be asked again every night. When the last pass formed every group without hitting the cap, both maps keep only that pass's fingerprints, so they do not grow without bound and an unchanged plan forms, finds and keeps the same set the next night (no calls, no state change). When the cap cut the step short, every fingerprint formed this run is kept. A call that fails with `HttpRequestException` skips that group only and is not remembered. `ReconcileSummary` gains `Merged` (merges applied) and `Parented` (parent links set); the commit message, endpoint body and status panel report them. @@ -61,7 +65,7 @@ Category `room` exists; the prompts now use it: ## One-time re-bootstrap -Nothing has been written or published yet; only `_plan.json` and `_state.json` exist. The plan file gains `"version": 3`; a plan without it (the current v2) or with another version reads as "no plan", so the next run bootstraps with the new rules. When the stored plan does not read, the stored state is dropped as well (its subjects, failed batches and kept articles belong to the old plan). `Prompts.Version` goes to 8. +Nothing has been written or published yet; only `_plan.json` and `_state.json` exist. The plan file gains `"version": 3`; a plan without it (the current v2) or with another version reads as "no plan", so the next run bootstraps with the new rules. When the stored plan does not read, the stored state is dropped as well (its subjects, failed batches and kept articles belong to the old plan). On the same reset, every article file whose slug the new plan does not contain is deleted, like the files of an article the plan dropped, so that no file outlives the plan that made it. That covers this switch and any later format change or corruption. `Prompts.Version` goes to 8. ## Cost diff --git a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs index be11071..848c68c 100644 --- a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs @@ -72,11 +72,11 @@ public sealed class AssignNotesUseCase(ITextModel model, EncyclopediaSettings se foreach (var item in items) { if (item is not JsonObject o) continue; - // a "new" slug that is a live article is a reference to it, not a new article - if (Text(o["slug"]) is { } existing && plan.Find(existing) is not null) + // a "new" slug that is a live article, or a redirect to one, is a reference to that article, not a new article + if (Text(o["slug"]) is { } existing && plan.Resolve(existing) is { } live) { - log.LogInformation("Merged proposed new article {Slug} into the existing article", existing); - candidateSlugs.Add(existing); + log.LogInformation("Merged proposed new article {Slug} into the existing article {Live}", existing, live); + candidateSlugs.Add(live); continue; } if (ReadNew(o, candidateSlugs, proposed.Keys, plan) is { } a) proposed[a.Slug] = a; @@ -99,7 +99,8 @@ public sealed class AssignNotesUseCase(ITextModel model, EncyclopediaSettings se continue; } var all = Strings(o["articles"]); - var valid = all.Where(s => candidateSlugs.Contains(s) || proposed.ContainsKey(s)).Distinct(StringComparer.Ordinal).ToList(); + var valid = all.Select(s => Referenced(s, candidateSlugs, plan)) + .Where(s => candidateSlugs.Contains(s) || proposed.ContainsKey(s)).Distinct(StringComparer.Ordinal).ToList(); if (valid.Count == 0) { log.LogWarning("Dropping assignment entry for {Note}: no known article", note); @@ -122,6 +123,17 @@ public sealed class AssignNotesUseCase(ITextModel model, EncyclopediaSettings se return new AssignOutcome(created, assignments, false); } + /// + /// What a reference to means: for a redirect to a live article that article, which becomes a valid + /// reference; otherwise . + /// + private static string Referenced(string slug, HashSet candidateSlugs, ArticlePlan plan) + { + if (!plan.Redirects.ContainsKey(slug) || plan.Resolve(slug) is not { } live) return slug; + candidateSlugs.Add(live); + return live; + } + private PlannedArticle? ReadNew(JsonObject o, HashSet candidateSlugs, IEnumerable seen, ArticlePlan plan) { var slug = Text(o["slug"]); diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs index 99581ac..66de66e 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs @@ -96,7 +96,10 @@ public sealed class EncyclopediaReconciler( var state = planText is not null && storedPlan is null ? EncyclopediaState.Empty : ReadState(stateText); var run = new Run(notes, storedPlan ?? ArticlePlan.Empty(), state, clock.GetUtcNow(), PublicSlugs(existing)); var plan = run.Plan; - var planned = plan.Articles.Select(a => a.Slug).ToList(); + // after a plan reset, every article file stands for a subject of the plan that was dropped + var planned = storedPlan is null + ? existing.Select(a => a.Slug).Distinct(StringComparer.Ordinal).ToList() + : plan.Articles.Select(a => a.Slug).ToList(); var changes = new List(); foreach (var note in plan.Assignments.Keys.Where(n => !run.NoteBySlug.ContainsKey(n)).Order(StringComparer.Ordinal).ToList()) @@ -245,84 +248,115 @@ public sealed class EncyclopediaReconciler( } /// - /// Groups of nearby articles: seeds in slug order among the articles no group of this run covered yet, each with its - /// nearest articles by article vector, in all. A group whose - /// fingerprint is remembered is not asked again; at most calls. - /// After a usable reply the group's surviving members, as the reply left them, are remembered, so the result is not asked - /// about again; an unusable reply is remembered as failed; a failed call is not remembered. Once every seed is done, only - /// the fingerprints of this run's groups are kept. + /// Passes over groups of nearby articles until one changes nothing: seeds in slug order among the articles no group of the + /// pass covered yet, each with its nearest articles by article vector, + /// in all. A group whose fingerprint is remembered is not asked again; at most + /// calls in all passes. A reply that changes the plan ends the + /// pass and the next one starts over, so the groups the last pass forms are the ones the next run forms from an unchanged + /// plan. A reply that changes nothing is remembered; after one that changes the plan, the group's surviving members as the + /// reply left them are, so they are not asked about again if they form the same group; an unusable reply is remembered as + /// failed; a failed call is not remembered. When the last pass asked every group it formed, only its fingerprints are kept; + /// otherwise every fingerprint this run formed. /// private async Task ConsolidateAsync(Run run, CancellationToken ct) { var plan = run.Plan; if (settings.MaxConsolidationsPerRun <= 0 || settings.ConsolidateGroupSize < 2 || plan.Articles.Count < 2) return; - var covered = new HashSet(StringComparer.Ordinal); - var seen = new HashSet(StringComparer.Ordinal); var calls = 0; - foreach (var slug in plan.Articles.Select(a => a.Slug).Order(StringComparer.Ordinal).ToList()) + var formed = new HashSet(StringComparer.Ordinal); + HashSet pass; + bool changed, capped; + do { - if (covered.Contains(slug) || plan.Find(slug) is not { } seed) continue; - var nearest = await neighbours.NearestArticlesAsync(seed, plan, settings.ConsolidateGroupSize - 1, ct); - List group = [seed, .. nearest.Select(s => plan.Find(s)!)]; - var members = group.Select(a => a.Slug).ToList(); - var fingerprint = EncyclopediaState.GroupFingerprint(group); - seen.Add(fingerprint); - if (run.ConsolidatedGroups.ContainsKey(fingerprint) || run.FailedGroups.ContainsKey(fingerprint)) + pass = new HashSet(StringComparer.Ordinal); + var covered = new HashSet(StringComparer.Ordinal); + changed = capped = false; + foreach (var slug in plan.Articles.Select(a => a.Slug).Order(StringComparer.Ordinal).ToList()) { - covered.UnionWith(members); - continue; - } - if (calls >= settings.MaxConsolidationsPerRun) continue; + if (covered.Contains(slug) || plan.Find(slug) is not { } seed) continue; + var nearest = await neighbours.NearestArticlesAsync(seed, plan, settings.ConsolidateGroupSize - 1, ct); + List group = [seed, .. nearest.Select(s => plan.Find(s)!)]; + if (group.Count < 2) continue; + var fingerprint = EncyclopediaState.GroupFingerprint(group); + pass.Add(fingerprint); + if (run.ConsolidatedGroups.ContainsKey(fingerprint) || run.FailedGroups.ContainsKey(fingerprint)) + { + covered.UnionWith(group.Select(a => a.Slug)); + continue; + } + if (calls >= settings.MaxConsolidationsPerRun) + { + capped = true; + continue; + } - calls++; - covered.UnionWith(members); - ConsolidateOutcome? outcome; - try - { - outcome = await consolidator.ConsolidateAsync(group, ct); + calls++; + covered.UnionWith(group.Select(a => a.Slug)); + ConsolidateOutcome? outcome; + try + { + outcome = await consolidator.ConsolidateAsync(group, ct); + } + catch (HttpRequestException e) + { + log.LogWarning(e, "Consolidation of the group of {Seed} failed; asked again next run", slug); + continue; + } + if (outcome is null) + { + run.FailedGroups[fingerprint] = run.Now; + continue; + } + var members = group.Select(a => a.Slug).ToList(); + if (!Apply(run, outcome)) + { + run.ConsolidatedGroups[fingerprint] = run.Now; + continue; + } + // the members as the reply left them are judged too: the next pass does not ask about them when they form a group again + var after = members.Select(plan.Find).OfType().ToList(); + if (after.Count >= 2) run.ConsolidatedGroups[EncyclopediaState.GroupFingerprint(after)] = run.Now; + changed = true; + break; } - catch (HttpRequestException e) - { - log.LogWarning(e, "Consolidation of the group of {Seed} failed; asked again next run", slug); - continue; - } - if (outcome is null) - { - run.FailedGroups[fingerprint] = run.Now; - continue; - } - Apply(run, outcome); - // the members as the reply left them; the same as asked when nothing changed - var after = members.Select(plan.Find).OfType().ToList(); - if (after.Count < 2) continue; - var judged = EncyclopediaState.GroupFingerprint(after); - run.ConsolidatedGroups[judged] = run.Now; - seen.Add(judged); - } - if (calls >= settings.MaxConsolidationsPerRun && plan.Articles.Any(a => !covered.Contains(a.Slug))) - log.LogInformation("Consolidation cap of {Cap} groups reached; the other articles wait for the next run", settings.MaxConsolidationsPerRun); + formed.UnionWith(pass); + } while (changed); + + if (capped) + log.LogInformation("Consolidation cap of {Cap} groups reached; the other groups wait for the next run", settings.MaxConsolidationsPerRun); + var keep = capped ? formed : pass; foreach (var memory in new[] { run.ConsolidatedGroups, run.FailedGroups }) - foreach (var key in memory.Keys.Where(k => !seen.Contains(k)).ToList()) memory.Remove(key); + foreach (var key in memory.Keys.Where(k => !keep.Contains(k)).ToList()) memory.Remove(key); } - /// Merges as folds (public redirect when the merged article was public at the run's start), then the parent links that close no cycle. - private void Apply(Run run, ConsolidateOutcome outcome) + /// + /// Merges as folds (public redirect when the merged article was public at the run's start), then the parent links that + /// close no cycle. True when the plan changed. + /// + private bool Apply(Run run, ConsolidateOutcome outcome) { var plan = run.Plan; + var changed = false; foreach (var m in outcome.Merges) { if (plan.Find(m.From) is null || plan.Find(m.Into) is null) continue; log.LogInformation("Merging article {From} into {Into}", m.From, m.Into); plan.Fold(m.From, m.Into, run.PublicSlugs.Contains(m.From)); run.Merged++; + changed = true; } foreach (var p in outcome.Parents) { if (plan.Resolve(p.Child) is not { } child || plan.Resolve(p.Parent) is not { } parent || child == parent || plan.Find(child)!.Parent == parent) continue; - if (plan.SetParent(child, parent)) run.Parented++; + if (plan.SetParent(child, parent)) + { + run.Parented++; + changed = true; + } else log.LogWarning("Dropping parent {Parent} of {Child}: it would close a cycle", parent, child); } + return changed; } /// Oversized articles, biggest first, at most calls. The children get the overview as parent. diff --git a/src/Wiki.Host/Components/Pages/HouseArticle.razor b/src/Wiki.Host/Components/Pages/HouseArticle.razor index c471ae5..b5d6401 100644 --- a/src/Wiki.Host/Components/Pages/HouseArticle.razor +++ b/src/Wiki.Host/Components/Pages/HouseArticle.razor @@ -57,7 +57,8 @@ else if (_article is not null && _rendered is not null) @if (_family?.Children.Count > 0) {     private async Task ConsolidateAsync(Run run, CancellationToken ct) @@ -264,6 +264,8 @@ public sealed class EncyclopediaReconciler( if (settings.MaxConsolidationsPerRun <= 0 || settings.ConsolidateGroupSize < 2 || plan.Articles.Count < 2) return; var calls = 0; var formed = new HashSet(StringComparer.Ordinal); + // groups whose call failed this run: not asked again in a later pass, asked again next run + var failedCalls = new HashSet(StringComparer.Ordinal); HashSet pass; bool changed, capped; do @@ -279,7 +281,8 @@ public sealed class EncyclopediaReconciler( if (group.Count < 2) continue; var fingerprint = EncyclopediaState.GroupFingerprint(group); pass.Add(fingerprint); - if (run.ConsolidatedGroups.ContainsKey(fingerprint) || run.FailedGroups.ContainsKey(fingerprint)) + if (run.ConsolidatedGroups.ContainsKey(fingerprint) || run.FailedGroups.ContainsKey(fingerprint) + || failedCalls.Contains(fingerprint)) { covered.UnionWith(group.Select(a => a.Slug)); continue; @@ -300,6 +303,7 @@ public sealed class EncyclopediaReconciler( catch (HttpRequestException e) { log.LogWarning(e, "Consolidation of the group of {Seed} failed; asked again next run", slug); + failedCalls.Add(fingerprint); continue; } if (outcome is null) diff --git a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs index 258a5a8..5a60e53 100644 --- a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs @@ -1074,6 +1074,27 @@ public class EncyclopediaReconcilerTests Assert.Equal(4, first.Written); } + [Fact] + public async Task Run_Consolidate_FailedGroupNotAskedAgainInLaterPassesOfTheRun() + { + await SeedAsync("a1", "b1", "c1", "d1"); + _model.Reply(AssignReply(("a", ["a1"]), ("b", ["b1"]), ("c", ["c1"]), ("d", ["d1"]))); + ArticleVector("a", 1, 0, 0, 0); + ArticleVector("b", 1, 0.1f, 0, 0); + ArticleVector("c", 0, 0, 0, 1); + ArticleVector("d", 0, 0, 0.1f, 1); + _model.Throw(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway)); + // the merge in c's group ends the pass; the next pass forms a's group again + _model.Reply(Consolidation([new { from = "d", into = "c" }])); + foreach (var _ in Enumerable.Range(0, 3)) _model.Reply(Consolidation()); + + await Reconciler(Consolidating with { ConsolidateGroupSize = 2 }).RunAsync(CancellationToken.None); + + const string header = "Articles (slug | title | category | scope | source count | current parent):\n"; + Assert.Equal(1, _model.Calls.Count(c => c.User.StartsWith(header + "a |", StringComparison.Ordinal))); + Assert.Empty(State.FailedGroups); + } + [Fact] public async Task Run_Consolidate_StableNeighbourhoodAfterAMerge_NotAskedNextRun() { -- 2.54.0