diff --git a/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs index 439d266..3aceaf2 100644 --- a/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs +++ b/src/Wiki.Application/Encyclopedia/ArticleNeighbours.cs @@ -1,3 +1,5 @@ +using System.Security.Cryptography; +using System.Text; using Wiki.Core.Model; using Wiki.Core.Parsing; using Wiki.Core.Ports; @@ -5,19 +7,27 @@ using Wiki.Core.Ports; namespace Wiki.Application.Encyclopedia; /// -/// Finds the articles and notes near a note through the note index, so prompts list a bounded candidate set -/// instead of every article. Embedding or index failures propagate; a request timeout (not the caller's -/// cancellation) surfaces as , like an unreachable service. +/// Finds the articles and notes near a note or an article, so prompts list a bounded candidate set instead of every +/// article: through the note index, and through article vectors ( embedded locally, searched in +/// memory by cosine). Embedding or index failures propagate; a request timeout (not the caller's cancellation) surfaces as +/// , like an unreachable service. /// public sealed class ArticleNeighbours(IEmbeddingModel model, IVectorIndex index, EncyclopediaSettings settings) { private const int EmbeddingChars = 1500; + private const int ArticleEmbedBatch = 32; /// The widest note query makes, as a multiple of the batch size. public const int MaxWidening = 8; private readonly Dictionary<(string Slug, string Hash), float[]> _vectors = []; + /// Unit-length article vectors of the plan last searched, by slug, with the hash of the text they embed. + private Dictionary _articleVectors = new(StringComparer.Ordinal); + + /// What an article vector embeds. + public static string ArticleText(PlannedArticle a) => $"{a.TitleDe} | {a.TitleEn} | {a.Scope}"; + public async Task VectorAsync(Document note, CancellationToken ct) { var key = (note.Slug, note.ContentHash); @@ -78,6 +88,61 @@ public sealed class ArticleNeighbours(IEmbeddingModel model, IVectorIndex index, return result; } + /// The articles whose vectors are nearest to 's, best first, at most . + public async Task> NearestArticlesAsync(Document note, ArticlePlan plan, CancellationToken ct) + { + var vector = Unit(await VectorAsync(note, ct)); + return Rank(vector, await ArticleVectorsAsync(plan, ct), settings.CandidateArticles, null); + } + + /// The articles whose vectors are nearest to 's, best first, without itself. + public async Task> NearestArticlesAsync(PlannedArticle article, ArticlePlan plan, int k, CancellationToken ct) + { + var vectors = await ArticleVectorsAsync(plan, ct); + return Rank(vectors[article.Slug].Vector, vectors, k, article.Slug); + } + + /// Embeds the articles whose text is new or changed, in batches; drops the vectors of articles no longer planned. + private async Task> ArticleVectorsAsync(ArticlePlan plan, CancellationToken ct) + { + var current = new Dictionary(StringComparer.Ordinal); + var missing = new List<(string Slug, string Hash, string Text)>(); + foreach (var a in plan.Articles) + { + var text = ArticleText(a); + var hash = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text))); + if (_articleVectors.TryGetValue(a.Slug, out var cached) && cached.Hash == hash) current[a.Slug] = cached; + else missing.Add((a.Slug, hash, text)); + } + foreach (var chunk in missing.Chunk(ArticleEmbedBatch)) + { + var vectors = await TimeoutAsServiceDown(() => model.EmbedAsync(chunk.Select(m => m.Text).ToList(), ct), ct); + for (var i = 0; i < chunk.Length; i++) current[chunk[i].Slug] = (chunk[i].Hash, Unit(vectors[i])); + } + _articleVectors = current; + return current; + } + + private static List Rank(float[] unit, Dictionary vectors, int k, string? without) => + vectors.Where(kv => kv.Key != without) + .Select(kv => (Slug: kv.Key, Score: Dot(unit, kv.Value.Vector))) + .OrderByDescending(x => x.Score).ThenBy(x => x.Slug, StringComparer.Ordinal) + .Take(k).Select(x => x.Slug).ToList(); + + private static float Dot(float[] a, float[] b) + { + var sum = 0f; + for (var i = 0; i < Math.Min(a.Length, b.Length); i++) sum += a[i] * b[i]; + return sum; + } + + /// The vector scaled to length one; a zero vector stays zero. + private static float[] Unit(float[] v) + { + var norm = MathF.Sqrt(Dot(v, v)); + return norm == 0 ? v : v.Select(x => x / norm).ToArray(); + } + private async Task Accumulate(Document note, ArticlePlan plan, Dictionary scores, CancellationToken ct) { foreach (var hit in await QueryAsync(note, settings.NeighbourNotes, ct)) diff --git a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs index c241a4e..015e75e 100644 --- a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs +++ b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs @@ -4,14 +4,18 @@ using Wiki.Core.Model; namespace Wiki.Application.Encyclopedia; -/// is one plain sentence on what the article covers; are note slugs, most relevant first. +/// +/// is one plain sentence on what the article covers; are note slugs, most relevant first. +/// is the slug of the overview article this one is a sub-article of. +/// public sealed record PlannedArticle( string Slug, string TitleDe, string TitleEn, ArticleCategory Category, string Scope, - IReadOnlyList Sources); + IReadOnlyList Sources, + string? Parent = null); /// /// The stored plan: the articles, which notes feed them, and where retired slugs point. @@ -51,12 +55,39 @@ public sealed class ArticlePlan public bool SlugTaken(string slug) => Find(slug) is not null || _redirects.ContainsKey(slug) || _retired.Contains(slug); + /// The articles whose parent is , in plan order. + public IReadOnlyList ChildrenOf(string slug) => _articles.Where(a => a.Parent == slug).ToList(); + + /// A new article; its , when set, must be a live article. public void AddArticle(PlannedArticle a) { if (SlugTaken(a.Slug)) throw new InvalidOperationException($"Article slug '{a.Slug}' is already taken"); + if (a.Parent is { } parent && Find(parent) is null) throw new InvalidOperationException($"Unknown parent '{parent}'"); _articles.Add(a); } + /// + /// Makes a sub-article of , or of none when null. False, and nothing + /// changes, when that would make an article its own ancestor. + /// + public bool SetParent(string child, string? parent) + { + var a = Find(child) ?? throw new InvalidOperationException($"Unknown article '{child}'"); + if (parent is not null && Find(parent) is null) throw new InvalidOperationException($"Unknown parent '{parent}'"); + if (parent is not null && IsAncestorOrSelf(child, parent)) return false; + Replace(a with { Parent = parent }); + return true; + } + + /// True when is or on its parent chain. + private bool IsAncestorOrSelf(string ancestor, string slug) + { + var seen = new HashSet(StringComparer.Ordinal); + for (string? current = slug; current is not null && seen.Add(current); current = Find(current)?.Parent) + if (current == ancestor) return true; + return false; + } + /// Appends to each article's sources and records the assignment; no articles records a skip. public void Assign(string note, IReadOnlyList articles) { @@ -68,7 +99,10 @@ public sealed class ArticlePlan _assignments[note] = [.. articles.Distinct(StringComparer.Ordinal)]; } - /// Drops everywhere. Returns the slugs of articles deleted for lack of sources; redirects to them go too. + /// + /// Drops everywhere. Returns the slugs of articles deleted for lack of sources; redirects to them go + /// too, and their children lose their parent. + /// public IReadOnlyList RemoveNote(string note) { _assignments.Remove(note); @@ -87,6 +121,7 @@ public sealed class ArticlePlan foreach (var slug in deleted) { _retired.Add(slug); + foreach (var child in ChildrenOf(slug)) Replace(child with { Parent = null }); foreach (var key in _redirects.Where(r => r.Value == slug).Select(r => r.Key).ToList()) { _redirects.Remove(key); @@ -105,10 +140,14 @@ public sealed class ArticlePlan return deleted; } - /// Same slug, new content. Assignments follow the new : a note that left loses the article, one that joined gains it. + /// + /// Same slug, new content; the parent stays ( changes it). Assignments follow the new + /// : a note that left loses the article, one that joined gains it. + /// public void ReplaceArticle(PlannedArticle a) { var old = Find(a.Slug) ?? throw new InvalidOperationException($"Unknown article '{a.Slug}'"); + a = a with { Parent = old.Parent }; Replace(a); foreach (var note in old.Sources.Except(a.Sources)) { @@ -127,7 +166,9 @@ public sealed class ArticlePlan /// /// Merges into and leaves a redirect behind, public when /// (both language files of were public). Redirects to - /// move to and keep their flag. + /// move to and keep their flag. The children of + /// become children of , except itself, which takes the parent of + /// ; a move that would make an article its own ancestor leaves it without parent. /// public void Fold(string thin, string into, bool isPublic) { @@ -136,6 +177,11 @@ public sealed class ArticlePlan if (thin == into) throw new InvalidOperationException("Cannot fold an article into itself"); Replace(target with { Sources = [.. target.Sources, .. from.Sources.Where(s => !target.Sources.Contains(s))] }); + // children first leave thin, so no parent chain runs through an article that is gone + var children = ChildrenOf(thin).Select(c => c.Slug).ToList(); + foreach (var child in children) Replace(Find(child)! with { Parent = null }); + if (target.Parent == thin) SetParent(into, from.Parent); + foreach (var child in children.Where(c => c != into)) SetParent(child, into); _articles.Remove(from); foreach (var note in _assignments.Keys.ToList()) { @@ -180,12 +226,16 @@ public sealed class ArticlePlan public static class ArticlePlanJson { + /// The plan format; a file in any other format reads as no plan. + public const int FormatVersion = 3; + private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { WriteIndented = true }; /// Map keys in ordinal order, so an unchanged plan serializes identically. public static string Write(ArticlePlan p) => JsonSerializer.Serialize(new { + Version = FormatVersion, Articles = p.Articles.Select(a => new { a.Slug, @@ -194,6 +244,7 @@ public static class ArticlePlanJson Category = a.Category.ToString().ToLowerInvariant(), a.Scope, a.Sources, + a.Parent, }), Assignments = new SortedDictionary>(p.Assignments.ToDictionary(), StringComparer.Ordinal), Redirects = new SortedDictionary(p.Redirects.ToDictionary(), StringComparer.Ordinal), @@ -203,19 +254,24 @@ public static class ArticlePlanJson /// /// The plan in , which may be wrapped in a code fence; null when it is not a plan, - /// including the old format without assignments. Articles with a bad slug or category are skipped, - /// as are assignments to unknown articles and publicRedirects entries that are no redirect. - /// Missing publicRedirects or retired read as empty. Never throws. + /// including any version other than . Articles with a bad slug or category are skipped, + /// as are parents that are unknown or would close a cycle (in file order), assignments to unknown articles and + /// publicRedirects entries that are no redirect. Missing publicRedirects or retired read as empty. + /// Never throws. /// public static ArticlePlan? Read(string json) { try { - if (JsonNode.Parse(PlanParsing.StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items - || root["assignments"] is not JsonObject assignments) return null; + if (JsonNode.Parse(PlanParsing.StripFence(json)) is not JsonObject root + || root["version"] is not JsonValue version || version.GetValueKind() != JsonValueKind.Number + || !version.TryGetValue(out var v) || v != FormatVersion + || root["articles"] is not JsonArray items || root["assignments"] is not JsonObject assignments) return null; var plan = ArticlePlan.Empty(); - foreach (var a in Articles(items)) - if (!plan.SlugTaken(a.Slug)) plan.AddArticle(a); + var articles = Articles(items).DistinctBy(a => a.Slug).ToList(); + foreach (var a in articles) plan.AddArticle(a with { Parent = null }); + foreach (var a in articles) + if (a.Parent is { } parent && plan.Find(parent) is not null) plan.SetParent(a.Slug, parent); if (root["redirects"] is JsonObject redirects) foreach (var (from, to) in redirects) { @@ -251,7 +307,7 @@ public static class ArticlePlanJson if (slug is null || !PlanParsing.SlugPattern().IsMatch(slug) || !PlanParsing.TryCategory(PlanParsing.Text(o["category"]), out var category)) continue; articles.Add(new PlannedArticle(slug, PlanParsing.Text(o["titleDe"]) ?? slug, PlanParsing.Text(o["titleEn"]) ?? slug, category, - PlanParsing.Text(o["scope"]) ?? "", PlanParsing.Strings(o["sources"]))); + PlanParsing.Text(o["scope"]) ?? "", PlanParsing.Strings(o["sources"]), PlanParsing.Text(o["parent"]))); } return articles; } diff --git a/src/Wiki.Application/Encyclopedia/ConsolidateArticlesUseCase.cs b/src/Wiki.Application/Encyclopedia/ConsolidateArticlesUseCase.cs new file mode 100644 index 0000000..a6e6e48 --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/ConsolidateArticlesUseCase.cs @@ -0,0 +1,114 @@ +using System.Net; +using System.Text; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; +using Wiki.Core.Model; +using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; + +namespace Wiki.Application.Encyclopedia; + +/// and are about the same subject; goes. +public sealed record ArticleMerge(string From, string Into); + +/// is the overview article is a sub-article of. +public sealed record ArticleParent(string Child, string Parent); + +/// +/// What one consolidation call found in its group, in reply order. No article is merged twice or merged away after being a +/// target; parents name the merge targets, at most one per child. Parent cycles are not checked here: the plan refuses them. +/// +public sealed record ConsolidateOutcome(IReadOnlyList Merges, IReadOnlyList Parents); + +/// Compares a group of nearby articles: which share a subject, which is the overview of which, in one bounded model call. +public sealed class ConsolidateArticlesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +{ + /// + /// The first articles of are shown; entries + /// naming any other slug are dropped. Null when the reply is unusable. Model-unavailable and other errors propagate. + /// + public async Task ConsolidateAsync(IReadOnlyList group, CancellationToken ct) + { + var shown = group.Take(settings.ConsolidateGroupSize).ToList(); + if (shown.Count < 2) return new ConsolidateOutcome([], []); + TextReply reply; + try + { + reply = await model.CompleteAsync(Prompts.Consolidate, UserPrompt(shown), settings.ConsolidateMaxTokens, ct); + } + catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + log.LogWarning(e, "Consolidate request for the group of {Seed} was rejected", shown[0].Slug); + return null; + } + + if (reply.Truncated) + { + log.LogWarning("Consolidate reply for the group of {Seed} hit the token cap ({Tokens})", shown[0].Slug, settings.ConsolidateMaxTokens); + return null; + } + + var root = PlanParsing.Object(reply.Text); + if (root is null || (root["merges"] is not JsonArray && root["parents"] is not JsonArray)) + { + log.LogWarning("Consolidate reply for the group of {Seed} is not {{\"merges\":[…],\"parents\":[…]}}", shown[0].Slug); + return null; + } + return Validate(root, shown.Select(a => a.Slug).ToHashSet(StringComparer.Ordinal)); + } + + private ConsolidateOutcome Validate(JsonObject root, HashSet members) + { + var merges = new List(); + var into = new Dictionary(StringComparer.Ordinal); + foreach (var (from, to) in Pairs(root["merges"], "from", "into", members)) + { + if (into.ContainsKey(from) || into.ContainsValue(from) || into.ContainsKey(to)) + { + log.LogWarning("Dropping merge {From} into {Into}: one of them is merged already", from, to); + continue; + } + into[from] = to; + merges.Add(new ArticleMerge(from, to)); + } + + var parents = new List(); + foreach (var (rawChild, rawParent) in Pairs(root["parents"], "child", "parent", members)) + { + var child = into.GetValueOrDefault(rawChild, rawChild); + var parent = into.GetValueOrDefault(rawParent, rawParent); + if (child == parent || parents.Any(p => p.Child == child)) + { + log.LogWarning("Dropping parent {Parent} of {Child}: the same article after merges, or a second parent", rawParent, rawChild); + continue; + } + parents.Add(new ArticleParent(child, parent)); + } + return new ConsolidateOutcome(merges, parents); + } + + /// The entries of whose two fields are different members of the group. + private IEnumerable<(string, string)> Pairs(JsonNode? node, string first, string second, HashSet members) + { + if (node is not JsonArray items) yield break; + foreach (var item in items) + { + if (item is not JsonObject o || Text(o[first]) is not { } a || Text(o[second]) is not { } b) continue; + if (a == b || !members.Contains(a) || !members.Contains(b)) + { + log.LogWarning("Dropping {Field} entry {A} → {B}: not two different articles of the group", first, a, b); + continue; + } + yield return (a, b); + } + } + + private static string UserPrompt(List group) + { + var sb = new StringBuilder("Articles (slug | title | category | scope | source count | current parent):\n"); + foreach (var a in group) + sb.Append(a.Slug).Append(" | ").Append(Flatten(a.TitleDe)).Append(" | ").Append(a.Category.ToString().ToLowerInvariant()) + .Append(" | ").Append(Flatten(a.Scope)).Append(" | ").Append(a.Sources.Count).Append(" | ").Append(a.Parent ?? "-").Append('\n'); + return sb.ToString(); + } +} diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs index c09883b..99581ac 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs @@ -8,12 +8,13 @@ namespace Wiki.Application.Encyclopedia; /// /// What one run did. is the article count after the run; counts notes assigned to an article (not skipped ones), +/// articles merged into another by consolidation, parent links consolidation set, /// and articles; the write counts are subjects, each in both languages. /// counts subjects rejected earlier whose inputs are unchanged, so they were not retried. /// is null when nothing was committed. /// -public sealed record ReconcileSummary(int Planned, int Assigned, int Split, int Folded, int Written, int Deleted, int Rejected, - int Remembered, bool StoppedByModel, string? Skipped, string? Commit); +public sealed record ReconcileSummary(int Planned, int Assigned, int Merged, int Parented, int Split, int Folded, int Written, int Deleted, + int Rejected, int Remembered, bool StoppedByModel, string? Skipped, string? Commit); public interface IEncyclopediaRun { @@ -22,8 +23,8 @@ public interface IEncyclopediaRun /// /// Brings articles/ up to date with the notes: updates the stored plan in bounded steps (drop removed notes, assign new -/// ones, split oversized and fold thin articles), rewrites the articles whose inputs changed, deletes the files of articles -/// the plan dropped, and commits everything as one commit. +/// ones, consolidate nearby articles, split oversized and fold thin ones), rewrites the articles whose inputs changed, deletes +/// the files of articles the plan dropped, and commits everything as one commit. /// Works only on a catalog that reflects the store's head, so what it compares is what it commits onto. /// public sealed class EncyclopediaReconciler( @@ -31,6 +32,7 @@ public sealed class EncyclopediaReconciler( IDocumentCatalog catalog, ArticleNeighbours neighbours, AssignNotesUseCase assigner, + ConsolidateArticlesUseCase consolidator, SplitArticleUseCase splitter, FoldArticleUseCase folder, IArticleWriter writer, @@ -40,6 +42,8 @@ public sealed class EncyclopediaReconciler( { private const string PlanPath = "articles/_plan.json"; private const string StatePath = "articles/_state.json"; + /// Sub-articles an article's write prompt lists as link targets, at most. + private const int MaxChildLinks = 20; private static readonly string[] Languages = ["de", "en"]; /// The inputs and mutable outputs of one run. @@ -54,17 +58,21 @@ public sealed class EncyclopediaReconciler( public SortedDictionary Subjects { get; } = new(state.Subjects.ToDictionary(), StringComparer.Ordinal); public SortedDictionary FailedBatches { get; } = new(state.FailedBatches.ToDictionary(), StringComparer.Ordinal); public SortedDictionary Kept { get; } = new(state.KeptArticles.ToDictionary(), StringComparer.Ordinal); - /// Children of the articles split this run: their overview's extra link targets. - public Dictionary> Children { get; } = new(StringComparer.Ordinal); + public SortedDictionary ConsolidatedGroups { get; } = new(state.ConsolidatedGroups.ToDictionary(), StringComparer.Ordinal); + public SortedDictionary FailedGroups { get; } = new(state.FailedGroups.ToDictionary(), StringComparer.Ordinal); + /// Slugs of the children of the articles split this run. + public HashSet SplitOff { get; } = new(StringComparer.Ordinal); /// Subjects whose de and en files both exist and are public at the run's start. public IReadOnlySet PublicSlugs { get; } = publicSlugs; /// Thin articles found without a nearby article this run: no fold decision pending. public HashSet Unfoldable { get; } = new(StringComparer.Ordinal); public int Assigned { get; set; } + public int Merged { get; set; } + public int Parented { get; set; } public int Split { get; set; } public int Folded { get; set; } - public EncyclopediaState State => new(Subjects, FailedBatches, Kept); + public EncyclopediaState State => new(Subjects, FailedBatches, Kept, ConsolidatedGroups, FailedGroups); } /// An unexpected failure is rethrown after what is staged is committed; caller cancellation commits nothing. @@ -75,7 +83,7 @@ public sealed class EncyclopediaReconciler( if (synced != head) { log.LogWarning("Catalog is at {Catalog}, the store at {Head}; skipping the encyclopedia run", synced, head); - return new ReconcileSummary(0, 0, 0, 0, 0, 0, 0, 0, false, "catalog-behind", null); + return new ReconcileSummary(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, "catalog-behind", null); } var notes = await catalog.ListDocumentsAsync(DocumentKind.Wiki, ct); @@ -83,7 +91,10 @@ public sealed class EncyclopediaReconciler( var planText = await store.ReadTextAsync(PlanPath, head, ct); var stateText = await store.ReadTextAsync(StatePath, head, ct); var existing = ParseArticles(articleDocs); - var run = new Run(notes, ReadPlan(planText), ReadState(stateText), clock.GetUtcNow(), PublicSlugs(existing)); + var storedPlan = ReadPlan(planText); + // what the state remembers belongs to the plan it was made with + var state = planText is not null && storedPlan is null ? EncyclopediaState.Empty : ReadState(stateText); + var run = new Run(notes, storedPlan ?? ArticlePlan.Empty(), state, clock.GetUtcNow(), PublicSlugs(existing)); var plan = run.Plan; var planned = plan.Articles.Select(a => a.Slug).ToList(); var changes = new List(); @@ -142,21 +153,22 @@ public sealed class EncyclopediaReconciler( changes.Add(Put(PlanPath, planJson, planText?.BlobSha)); var summary = await FinishAsync(changes, run.State, stateText, - new ReconcileSummary(plan.Articles.Count, run.Assigned, run.Split, run.Folded, written, deleted, rejected, remembered, - stopped, null, null), ct); + new ReconcileSummary(plan.Articles.Count, run.Assigned, run.Merged, run.Parented, run.Split, run.Folded, written, deleted, + rejected, remembered, stopped, null, null), ct); failure?.Throw(); return summary; } /// - /// Assign, split, fold, each within its cap. A text-model call that fails with skips its - /// batch or article; an unreachable embedding or index service ends the planning steps; what they changed stays. + /// Assign, consolidate, split, fold, each within its cap. A text-model call that fails with + /// skips its batch, group or article; an unreachable embedding or index service ends the planning steps; what they changed stays. /// private async Task PlanStepsAsync(Run run, CancellationToken ct) { try { await AssignAsync(run, ct); + await ConsolidateAsync(run, ct); await SplitAsync(run, ct); await FoldAsync(run, ct); } @@ -168,8 +180,10 @@ public sealed class EncyclopediaReconciler( /// /// Batches of similar new notes, seeded in slug order, each assigned in one call against the articles near its notes in the - /// plan as it stands, so later batches see the articles earlier ones created. A batch whose reply was unusable before, for - /// the same notes, is skipped without a call; a batch is tried once per run. + /// plan as it stands, so later batches see the articles earlier ones created. Near means: assigned to a note near a batch + /// note, or with an article vector near a batch note's vector; the two lists are interleaved, so both are shown when either + /// alone would fill the cap. A batch whose reply was unusable before, for the same notes, is skipped without a call; a batch + /// is tried once per run. /// private async Task AssignAsync(Run run, CancellationToken ct) { @@ -187,13 +201,15 @@ public sealed class EncyclopediaReconciler( var fingerprint = EncyclopediaState.BatchFingerprint(batch); if (run.FailedBatches.ContainsKey(fingerprint)) continue; - var nearby = new List(); + var byNotes = new List(); + var byVectors = new List(); foreach (var note in batch) { - foreach (var slug in await neighbours.NearbyArticlesAsync(note, plan, ct)) - if (nearby.Count < settings.CandidateArticles * 2 && !nearby.Contains(slug)) nearby.Add(slug); - if (nearby.Count >= settings.CandidateArticles * 2) break; + byNotes.AddRange(await neighbours.NearbyArticlesAsync(note, plan, ct)); + byVectors.AddRange(await neighbours.NearestArticlesAsync(note, plan, ct)); } + var nearby = Interleave(byNotes.Distinct(StringComparer.Ordinal).ToList(), byVectors.Distinct(StringComparer.Ordinal).ToList()) + .Distinct(StringComparer.Ordinal).Take(settings.CandidateArticles * 2).ToList(); calls++; AssignOutcome outcome; @@ -219,7 +235,97 @@ public sealed class EncyclopediaReconciler( log.LogInformation("Assign cap of {Cap} batches reached; the other new notes wait for the next run", settings.MaxAssignBatchesPerRun); } - /// Oversized articles, biggest first, at most calls. + private static IEnumerable Interleave(List a, List b) + { + for (var i = 0; i < Math.Max(a.Count, b.Count); i++) + { + if (i < a.Count) yield return a[i]; + if (i < b.Count) yield return b[i]; + } + } + + /// + /// Groups of nearby articles: seeds in slug order among the articles no group of this run covered yet, each with its + /// nearest articles by article vector, in all. A group whose + /// fingerprint is remembered is not asked again; at most calls. + /// After a usable reply the group's surviving members, as the reply left them, are remembered, so the result is not asked + /// about again; an unusable reply is remembered as failed; a failed call is not remembered. Once every seed is done, only + /// the fingerprints of this run's groups are kept. + /// + private async Task ConsolidateAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + if (settings.MaxConsolidationsPerRun <= 0 || settings.ConsolidateGroupSize < 2 || plan.Articles.Count < 2) return; + var covered = new HashSet(StringComparer.Ordinal); + var seen = new HashSet(StringComparer.Ordinal); + var calls = 0; + foreach (var slug in plan.Articles.Select(a => a.Slug).Order(StringComparer.Ordinal).ToList()) + { + if (covered.Contains(slug) || plan.Find(slug) is not { } seed) continue; + var nearest = await neighbours.NearestArticlesAsync(seed, plan, settings.ConsolidateGroupSize - 1, ct); + List group = [seed, .. nearest.Select(s => plan.Find(s)!)]; + var members = group.Select(a => a.Slug).ToList(); + var fingerprint = EncyclopediaState.GroupFingerprint(group); + seen.Add(fingerprint); + if (run.ConsolidatedGroups.ContainsKey(fingerprint) || run.FailedGroups.ContainsKey(fingerprint)) + { + covered.UnionWith(members); + continue; + } + if (calls >= settings.MaxConsolidationsPerRun) continue; + + calls++; + covered.UnionWith(members); + ConsolidateOutcome? outcome; + try + { + outcome = await consolidator.ConsolidateAsync(group, ct); + } + catch (HttpRequestException e) + { + log.LogWarning(e, "Consolidation of the group of {Seed} failed; asked again next run", slug); + continue; + } + if (outcome is null) + { + run.FailedGroups[fingerprint] = run.Now; + continue; + } + Apply(run, outcome); + // the members as the reply left them; the same as asked when nothing changed + var after = members.Select(plan.Find).OfType().ToList(); + if (after.Count < 2) continue; + var judged = EncyclopediaState.GroupFingerprint(after); + run.ConsolidatedGroups[judged] = run.Now; + seen.Add(judged); + } + if (calls >= settings.MaxConsolidationsPerRun && plan.Articles.Any(a => !covered.Contains(a.Slug))) + log.LogInformation("Consolidation cap of {Cap} groups reached; the other articles wait for the next run", settings.MaxConsolidationsPerRun); + foreach (var memory in new[] { run.ConsolidatedGroups, run.FailedGroups }) + foreach (var key in memory.Keys.Where(k => !seen.Contains(k)).ToList()) memory.Remove(key); + } + + /// Merges as folds (public redirect when the merged article was public at the run's start), then the parent links that close no cycle. + private void Apply(Run run, ConsolidateOutcome outcome) + { + var plan = run.Plan; + foreach (var m in outcome.Merges) + { + if (plan.Find(m.From) is null || plan.Find(m.Into) is null) continue; + log.LogInformation("Merging article {From} into {Into}", m.From, m.Into); + plan.Fold(m.From, m.Into, run.PublicSlugs.Contains(m.From)); + run.Merged++; + } + foreach (var p in outcome.Parents) + { + if (plan.Resolve(p.Child) is not { } child || plan.Resolve(p.Parent) is not { } parent || child == parent + || plan.Find(child)!.Parent == parent) continue; + if (plan.SetParent(child, parent)) run.Parented++; + else log.LogWarning("Dropping parent {Parent} of {Child}: it would close a cycle", parent, child); + } + } + + /// Oversized articles, biggest first, at most calls. The children get the overview as parent. private async Task SplitAsync(Run run, CancellationToken ct) { var plan = run.Plan; @@ -246,11 +352,11 @@ public sealed class EncyclopediaReconciler( // children first, so a note that moves wholly to a child never goes unassigned foreach (var child in outcome.Children) { - plan.AddArticle(child with { Sources = [] }); + plan.AddArticle(child with { Sources = [], Parent = article.Slug }); plan.ReplaceArticle(child); + run.SplitOff.Add(child.Slug); } plan.ReplaceArticle(outcome.Overview); - run.Children[article.Slug] = outcome.Children; run.Split++; } } @@ -264,7 +370,7 @@ public sealed class EncyclopediaReconciler( private async Task FoldAsync(Run run, CancellationToken ct) { var plan = run.Plan; - var children = run.Children.Values.SelectMany(c => c).Select(c => c.Slug).ToHashSet(StringComparer.Ordinal); + var children = run.SplitOff; var thin = plan.Articles.Where(a => FoldArticleUseCase.IsThin(a, run.NoteBySlug, settings)).Select(a => a.Slug).ToList(); var calls = 0; foreach (var slug in thin) @@ -338,8 +444,9 @@ public sealed class EncyclopediaReconciler( neighboursUp = false; } } - if (run.Children.TryGetValue(planned.Slug, out var children)) - targets = Articles(run.Plan, [.. targets.Select(t => t.Slug), .. children.Select(c => c.Slug)]); + var family = run.Plan.ChildrenOf(planned.Slug).Take(MaxChildLinks).Select(c => c.Slug).ToList(); + if (planned.Parent is { } parent) family.Insert(0, parent); + targets = Articles(run.Plan, [.. targets.Select(t => t.Slug), .. family]); var article = await writer.WriteAsync(planned, sources, targets, ct); foreach (var a in new[] { article.De, article.En }) @@ -473,7 +580,8 @@ public sealed class EncyclopediaReconciler( return summary; } - var message = $"articles: {summary.Assigned} assigned, {summary.Split} split, {summary.Folded} folded, " + var message = $"articles: {summary.Assigned} assigned, {summary.Merged} merged, {summary.Parented} parented, " + + $"{summary.Split} split, {summary.Folded} folded, " + $"{summary.Written} written, {summary.Deleted} deleted, {summary.Rejected} rejected"; try { @@ -511,12 +619,14 @@ public sealed class EncyclopediaReconciler( return articles; } - private ArticlePlan ReadPlan(StoredText? text) + /// The stored plan; empty when there is none; null when there is one in another format or malformed. + private ArticlePlan? ReadPlan(StoredText? text) { if (text is null) return ArticlePlan.Empty(); var plan = ArticlePlanJson.Read(text.Text); - if (plan is null) log.LogWarning("{Path} is malformed or in the old format; planning from scratch", PlanPath); - return plan ?? ArticlePlan.Empty(); + if (plan is null) + log.LogWarning("{Path} is malformed or in an older format; planning from scratch, without the attempt memory of that plan", PlanPath); + return plan; } private EncyclopediaState ReadState(StoredText? text) @@ -524,6 +634,7 @@ public sealed class EncyclopediaReconciler( if (text is null) return EncyclopediaState.Empty; var state = EncyclopediaStateJson.Read(text.Text); if (state.Subjects.Count == 0 && state.FailedBatches.Count == 0 && state.KeptArticles.Count == 0 + && state.ConsolidatedGroups.Count == 0 && state.FailedGroups.Count == 0 && text.Text.Trim() != EncyclopediaStateJson.Write(EncyclopediaState.Empty)) log.LogWarning("{Path} is malformed or empty; starting without attempt memory", StatePath); return state; diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs index ba0f33a..378a8c5 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs @@ -20,7 +20,11 @@ public sealed record EncyclopediaSettings( int MaxSourcesPerArticle = 120, int FoldMinSources = 3, int FoldMinChars = 4000, + // articles per consolidation group, the seed included + int ConsolidateGroupSize = 8, + int MaxConsolidationsPerRun = 60, // thinking tokens count against max_tokens int AssignMaxTokens = 32000, int SplitMaxTokens = 32000, - int FoldMaxTokens = 8000); + int FoldMaxTokens = 8000, + int ConsolidateMaxTokens = 8000); diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs index 48ab77a..c41cbab 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs @@ -22,19 +22,30 @@ public sealed record SubjectAttempt(string Fingerprint, AttemptOutcome Outcome, /// /// FailedBatches: fingerprint of an assignment batch whose reply was unusable → when. /// KeptArticles: article slug → at which the fold call did not fold it. +/// ConsolidatedGroups and FailedGroups: of a consolidation group whose reply +/// was usable, or unusable, → when. /// The failedPlan entry of earlier versions is ignored on read. /// public sealed record EncyclopediaState( IReadOnlyDictionary Subjects, IReadOnlyDictionary FailedBatches, - IReadOnlyDictionary KeptArticles) + IReadOnlyDictionary KeptArticles, + IReadOnlyDictionary ConsolidatedGroups, + IReadOnlyDictionary FailedGroups) { public static EncyclopediaState Empty => - new(new Dictionary(), new Dictionary(), new Dictionary()); + new(new Dictionary(), new Dictionary(), new Dictionary(), + new Dictionary(), new Dictionary()); public static string BatchFingerprint(IReadOnlyList notes) => Hash(Prompts.Version.ToString(), notes.Select(n => $"{n.Slug}@{n.ContentHash}").Order(StringComparer.Ordinal)); + /// The members and what the consolidation call judges of them: titles, category, scope and parent, not sources. + public static string GroupFingerprint(IReadOnlyList group) => + Hash(Prompts.Version.ToString(), group + .Select(a => string.Join('\u001f', a.Slug, a.TitleDe, a.TitleEn, a.Category.ToString(), a.Scope, a.Parent ?? "")) + .Order(StringComparer.Ordinal)); + public static string SubjectFingerprint(PlannedArticle planned, IReadOnlyList notes) { var hashes = notes.GroupBy(n => n.Slug).ToDictionary(g => g.Key, g => g.First().ContentHash); @@ -70,7 +81,9 @@ public static class EncyclopediaStateJson var subjects = s.Subjects.Where(kv => kv.Value is not null).ToDictionary(kv => kv.Key, kv => kv.Value); var failedBatches = (s.FailedBatches ?? new Dictionary()).ToDictionary(); var kept = (s.KeptArticles ?? new Dictionary()).Where(kv => kv.Value is not null).ToDictionary(); - return new EncyclopediaState(subjects, failedBatches, kept); + var consolidated = (s.ConsolidatedGroups ?? new Dictionary()).ToDictionary(); + var failedGroups = (s.FailedGroups ?? new Dictionary()).ToDictionary(); + return new EncyclopediaState(subjects, failedBatches, kept, consolidated, failedGroups); } catch (JsonException) { diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index 9e66fb2..1e71279 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Encyclopedia; /// public static class Prompts { - public const int Version = 7; + public const int Version = 8; public const string Assign = """ [assign] @@ -16,6 +16,7 @@ public static class Prompts For each note decide which articles it feeds: existing articles from the list, or new articles you propose. - Prefer an existing article from the list when the note fits its scope. - Granularity follows the material, like notability on Wikipedia. A subject gets its own new article only when the notes hold enough on it. A subject that was set once and forgotten is not an article of its own; its note feeds the article of its parent system as a section. + - A note about something in a specific room additionally feeds that room's article (category room), titled by floor and room, for example "EG Schlafzimmer". Create the room article when none is in the list. - Skip a note only when it is a meta note or about the wiki's own tooling. - A note may feed several articles. - Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses. Name the subject by what it is or does, for example "Netzwerkspeicher" rather than the server's hostname. @@ -54,6 +55,21 @@ public static class Prompts Reply with JSON only, no prose and no code fence: {"into":"article-slug"} to fold, or {"keep":true} to keep. """; + public const string Consolidate = """ + [consolidate] + You tidy a group of nearby articles of an encyclopedia about one household: the house and its homelab IT. Its readers are everyone who needs to know things about the house, from the owner to guests. + You receive the articles, one per line as `slug | title | category | scope | source count | current parent` (`-` for none). + + - Merge two articles when they are about the same subject, for example the same device under two names. Merge the one with fewer sources into the other. An article you merge away is never the target of another merge. + - Give an article a parent when the parent is the overview of a broader subject it belongs to, for example a heat pump under the heating system, or a device under the system it is part of. A room is the parent of nothing but its own parts; a device in a room keeps its system as parent. + - Change a current parent only when another article of the group is clearly the better overview. + - Articles about different subjects, and siblings, stay as they are. When nothing needs to change, reply with empty lists. + - Use only slugs from the list. + + Reply with JSON only, no prose and no code fence. Shape: + {"merges":[{"from":"article-slug","into":"article-slug"}],"parents":[{"child":"article-slug","parent":"article-slug"}]} + """; + /// The generic-examples rule, shared by and . public const string GenericExamples = "A generic example that explains a principle and names nothing concrete in this house, for example what an entity ID or an IP address looks like, is not a leak and stays public"; @@ -63,7 +79,7 @@ public static class Prompts You write one article of an encyclopedia about one household: the house and its homelab IT (Kubernetes, servers, network). You write in German. The readers are everyone who needs to know something about the house: the owner, the household, a friend fixing a light switch while the owner is away, a guest who wants to know how the lighting automation works. Many of them are not technical. Open plainly, so that a non-technical reader understands the start, and get more technical further down. - You receive the article's subject, the articles you may link to, the images you may use, and the source notes, each as .... Inside a note, &, <, > and " stand for &, <, > and "; a note ends only at its own . + You receive the article's subject, the articles you may link to (its main article and sub-articles are marked), the images you may use, and the source notes, each as .... Inside a note, &, <, > and " stand for &, <, > and "; a note ends only at its own . Shape, in this order: 1. A lead paragraph of a few sentences, without a heading, that a non-technical reader understands. @@ -73,6 +89,11 @@ public static class Prompts 5. Optionally a short History section last (`## Geschichte`), a few lines at most. History never dominates. Do not repeat the article title as a heading; the page shows it. + Article kinds: + - A room article (category room) describes what is in the room and what it does there: lights, switches, blinds, sensors, heating. It explains how things work only as far as the room needs; for the system behind them it links to the system's article. + - An article with sub-articles is their overview: it gives the big picture and, for each sub-article, a sentence or two and a link. Details stay in the sub-articles. + - The page links the main article and the sub-articles on its own; link them in the text where they help the reader. + Body conventions: - The infobox is a line `::: infobox`, then a two-column pipe table, then a line `:::`. - A private block is a line `::: private`, its content, then a line `:::`. diff --git a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs index 8380798..22021fa 100644 --- a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs @@ -254,9 +254,14 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe sb.Append("Category: ").Append(planned.Category.ToString().ToLowerInvariant()).Append('\n'); sb.Append("Scope: ").Append(Flatten(planned.Scope)).Append('\n'); - sb.Append("\nArticles you may link to (slug | German title):\n"); + sb.Append("\nArticles you may link to (slug | German title | relation):\n"); foreach (var a in linkTargets.Where(a => a.Slug != planned.Slug)) - sb.Append(a.Slug).Append(" | ").Append(Flatten(a.TitleDe)).Append('\n'); + { + sb.Append(a.Slug).Append(" | ").Append(Flatten(a.TitleDe)); + if (a.Slug == planned.Parent) sb.Append(" | main article"); + else if (a.Parent == planned.Slug) sb.Append(" | sub-article"); + sb.Append('\n'); + } var images = sources.SelectMany(Attachments).Distinct(StringComparer.Ordinal).ToList(); sb.Append("\nImages you may use:\n"); diff --git a/src/Wiki.Application/Reading/ArticleQueries.cs b/src/Wiki.Application/Reading/ArticleQueries.cs index 23e32eb..20a048b 100644 --- a/src/Wiki.Application/Reading/ArticleQueries.cs +++ b/src/Wiki.Application/Reading/ArticleQueries.cs @@ -6,11 +6,17 @@ using Wiki.Core.Ports; namespace Wiki.Application.Reading; +/// A link to an article: its slug and its title in the reader's language. +public sealed record ArticleLink(string Slug, string Title); + +/// The main article of an article and its sub-articles, by title; only those the reader may see. +public sealed record ArticleFamily(ArticleLink? Parent, IReadOnlyList Children); + /// /// Read side of the house encyclopedia, straight from the catalog. An article file that does not parse /// counts as missing. Anonymous visibility is per subject: both language files must exist and be public. -/// Redirects come from articles/_plan.json at the catalog's commit; anonymous readers get only those whose -/// retired article was public when it was folded. +/// Redirects and the article hierarchy come from articles/_plan.json at the catalog's commit; anonymous readers get +/// only the redirects whose retired article was public when it was folded. /// public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, ILogger log) { @@ -18,11 +24,13 @@ public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, I private const string PlanPath = "articles/_plan.json"; - private static readonly IReadOnlyDictionary NoRedirects = new Dictionary(); + private static readonly IReadOnlyDictionary None = new Dictionary(); - private sealed record Cached(string Commit, IReadOnlyDictionary All, IReadOnlyDictionary Public); + /// : article slug → its parent's slug, from the plan. + private sealed record Cached(string Commit, IReadOnlyDictionary All, IReadOnlyDictionary Public, + IReadOnlyDictionary Parents); - private volatile Cached? _redirects; + private volatile Cached? _plan; /// /// Retired slug → slug that took over; anonymous ( false): only redirects whose retired article was @@ -30,15 +38,38 @@ public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, I /// unreadable or the store fails. Cached per commit, a failure too (no store request per page view until the commit changes). /// public async Task> RedirectsAsync(bool signedIn, CancellationToken ct) + { + var cached = await PlanAsync(ct); + return cached is null ? None : signedIn ? cached.All : cached.Public; + } + + /// + /// The plan's parent and children of that exist in and that this reader may + /// see, exactly as for links (). None when the plan is missing, unreadable or the store fails. + /// + public async Task FamilyAsync(string lang, string slug, bool signedIn, CancellationToken ct) + { + var parents = (await PlanAsync(ct))?.Parents ?? None; + var titles = await VisibleTitlesAsync(lang, signedIn, ct); + var parent = parents.TryGetValue(slug, out var p) && titles.TryGetValue(p, out var title) ? new ArticleLink(p, title) : null; + var children = parents.Where(kv => kv.Value == slug && titles.ContainsKey(kv.Key)) + .Select(kv => new ArticleLink(kv.Key, titles[kv.Key])) + .OrderBy(c => c.Title, StringComparer.InvariantCultureIgnoreCase) + .ThenBy(c => c.Slug, StringComparer.Ordinal) + .ToList(); + return new ArticleFamily(parent, children); + } + + private async Task PlanAsync(CancellationToken ct) { var commit = (await catalog.GetStateAsync(ct)).Commit; - if (commit is null) return NoRedirects; - if (_redirects is not { } cached || cached.Commit != commit) + if (commit is null) return null; + if (_plan is not { } cached || cached.Commit != commit) { cached = await LoadAsync(commit, ct); - _redirects = cached; + _plan = cached; } - return signedIn ? cached.All : cached.Public; + return cached; } private async Task LoadAsync(string commit, CancellationToken ct) @@ -48,13 +79,14 @@ public sealed class ArticleQueries(IDocumentCatalog catalog, INoteStore store, I var text = await store.ReadTextAsync(PlanPath, commit, ct); if (text is not null && ArticlePlanJson.Read(text.Text) is { } plan) return new Cached(commit, plan.Redirects, - plan.Redirects.Where(r => plan.PublicRedirects.Contains(r.Key)).ToDictionary(StringComparer.Ordinal)); + plan.Redirects.Where(r => plan.PublicRedirects.Contains(r.Key)).ToDictionary(StringComparer.Ordinal), + plan.Articles.Where(a => a.Parent is not null).ToDictionary(a => a.Slug, a => a.Parent!, StringComparer.Ordinal)); } catch (Exception e) when (e is not OperationCanceledException) { - log.LogWarning(e, "Cannot read {Path} at {Commit}; serving without redirects", PlanPath, commit); + log.LogWarning(e, "Cannot read {Path} at {Commit}; serving without redirects and hierarchy", PlanPath, commit); } - return new Cached(commit, NoRedirects, NoRedirects); + return new Cached(commit, None, None, None); } /// diff --git a/src/Wiki.Host/Components/Pages/HouseArticle.razor b/src/Wiki.Host/Components/Pages/HouseArticle.razor index 3ef688a..c471ae5 100644 --- a/src/Wiki.Host/Components/Pages/HouseArticle.razor +++ b/src/Wiki.Host/Components/Pages/HouseArticle.razor @@ -29,6 +29,11 @@ else if (_article is not null && _rendered is not null) } + @if (_family?.Parent is { } parent) + { +
@(English ? "Main article" : "Hauptartikel"): @parent.Title
+ } + @if (_infobox is not null) { @((MarkupString)_infobox) @@ -48,6 +53,19 @@ else if (_article is not null && _rendered is not null) } @((MarkupString)_rendered.Html) + + @if (_family?.Children.Count > 0) + { + + } } @code { @@ -69,6 +87,7 @@ else if (_article is not null && _rendered is not null) private Article? _article; private RenderedArticle? _rendered; private string? _infobox; + private ArticleFamily? _family; private bool English => Lang == "en"; private string Other => English ? "de" : "en"; @@ -100,6 +119,7 @@ else if (_article is not null && _rendered is not null) _rendered = ArticleMarkdown.Render(article.Body, Lang, Slug, _signedIn, titles, redirects, Log); _infobox = WithTitleBar(_rendered.InfoboxHtml, article.Title); _otherExists = await Articles.GetAsync(Other, Slug, ct) is not null; + _family = await Articles.FamilyAsync(Lang, Slug, _signedIn, ct); } /// diff --git a/src/Wiki.Host/Components/Shared/StatusPanel.razor b/src/Wiki.Host/Components/Shared/StatusPanel.razor index a2bd405..4fd4d03 100644 --- a/src/Wiki.Host/Components/Shared/StatusPanel.razor +++ b/src/Wiki.Host/Components/Shared/StatusPanel.razor @@ -42,7 +42,7 @@ Last run@Status.LastEncyclopediaAttemptAt.Value.ToString("u") @if (Status.LastEncyclopediaRun is { } r) { - Result@r.Planned planned, @r.Assigned assigned, @r.Split split, @r.Folded folded, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") + Result@r.Planned planned, @r.Assigned assigned, @r.Merged merged, @r.Parented parented, @r.Split split, @r.Folded folded, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") } @if (Status.LastEncyclopediaError is not null) { diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs index 64c00b4..3bb8366 100644 --- a/src/Wiki.Host/Composition/WikiOptions.cs +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -101,8 +101,12 @@ public sealed class EncyclopediaOptions public int MaxSourcesPerArticle { get; set; } = 120; public int FoldMinSources { get; set; } = 3; public int FoldMinChars { get; set; } = 4000; + /// Articles per consolidation group, the seed included. + public int ConsolidateGroupSize { get; set; } = 8; + public int MaxConsolidationsPerRun { get; set; } = 60; // thinking tokens count against max_tokens public int AssignMaxTokens { get; set; } = 32000; public int SplitMaxTokens { get; set; } = 32000; public int FoldMaxTokens { get; set; } = 8000; + public int ConsolidateMaxTokens { get; set; } = 8000; } diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index b385f1b..2098d32 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -126,12 +126,16 @@ if (encyclopedia.Enabled) MaxSourcesPerArticle: encyclopedia.MaxSourcesPerArticle, FoldMinSources: encyclopedia.FoldMinSources, FoldMinChars: encyclopedia.FoldMinChars, + ConsolidateGroupSize: encyclopedia.ConsolidateGroupSize, + MaxConsolidationsPerRun: encyclopedia.MaxConsolidationsPerRun, AssignMaxTokens: encyclopedia.AssignMaxTokens, SplitMaxTokens: encyclopedia.SplitMaxTokens, - FoldMaxTokens: encyclopedia.FoldMaxTokens)); + FoldMaxTokens: encyclopedia.FoldMaxTokens, + ConsolidateMaxTokens: encyclopedia.ConsolidateMaxTokens)); // ArticleNeighbours takes the active IEmbeddingModel: the index holds its vectors. builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs index a7759e6..6b3cc5b 100644 --- a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs +++ b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs @@ -171,6 +171,88 @@ public class ArticleNeighboursTests Assert.Equal(24, rig.Index.LastK); } + /// Embeddings with a fixed vector per text; any other text gets the fake's hash vector. + private sealed class FixedEmbeddings(Dictionary vectors) : Wiki.Core.Ports.IEmbeddingModel + { + private readonly FakeEmbeddings _fallback = new(Model); + public List> Batches { get; } = []; + public string ModelId => Model; + public int Dimension => 4; + public string Name => "fixed"; + + public async Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct) + { + Batches.Add(texts); + var fallback = await _fallback.EmbedAsync(texts, ct); + return texts.Select((t, i) => vectors.TryGetValue(t, out var v) ? v : fallback[i]).ToList(); + } + + public Task CheckAsync(CancellationToken ct) => _fallback.CheckAsync(ct); + } + + private static PlannedArticle Titled(string slug, string scope = "scope") => new(slug, slug + " de", slug + " en", ArticleCategory.Topic, scope, ["n"]); + + [Fact] + public async Task NearestArticles_OfNote_ByCosineOfArticleVectors_Capped() + { + var note = Note("n"); + var near = Titled("near"); + var nearer = Titled("nearer"); + var far = Titled("far"); + var embeddings = new FixedEmbeddings(new() + { + [EmbeddingText.For(note, 1500)] = [1, 0, 0, 0], + [ArticleNeighbours.ArticleText(near)] = [2, 1, 0, 0], + [ArticleNeighbours.ArticleText(nearer)] = [5, 1, 0, 0], + [ArticleNeighbours.ArticleText(far)] = [0, 0, 3, 0], + }); + var plan = ArticlePlan.Empty(); + foreach (var a in new[] { far, near, nearer }) plan.AddArticle(a); + var neighbours = new ArticleNeighbours(embeddings, new FakeIndex(), Settings with { CandidateArticles = 2 }); + + var nearest = await neighbours.NearestArticlesAsync(note, plan, default); + + Assert.Equal(["nearer", "near"], nearest); + } + + [Fact] + public async Task NearestArticles_OfArticle_ExcludesItself() + { + var self = Titled("self"); + var other = Titled("other"); + var third = Titled("third"); + var embeddings = new FixedEmbeddings(new() + { + [ArticleNeighbours.ArticleText(self)] = [1, 0, 0, 0], + [ArticleNeighbours.ArticleText(other)] = [1, 1, 0, 0], + [ArticleNeighbours.ArticleText(third)] = [0, 1, 0, 0], + }); + var plan = ArticlePlan.Empty(); + foreach (var a in new[] { self, other, third }) plan.AddArticle(a); + var neighbours = new ArticleNeighbours(embeddings, new FakeIndex(), Settings); + + var nearest = await neighbours.NearestArticlesAsync(self, plan, 5, default); + + Assert.Equal(["other", "third"], nearest); + } + + [Fact] + public async Task ArticleVectors_EmbeddedInBatchesOnce_AgainWhenTheTextChanges() + { + var embeddings = new FixedEmbeddings([]); + var plan = ArticlePlan.Empty(); + foreach (var i in Enumerable.Range(0, 40)) plan.AddArticle(Titled($"a-{i:00}")); + var neighbours = new ArticleNeighbours(embeddings, new FakeIndex(), Settings); + + await neighbours.NearestArticlesAsync(plan.Find("a-00")!, plan, 3, default); + await neighbours.NearestArticlesAsync(plan.Find("a-01")!, plan, 3, default); + plan.ReplaceArticle(plan.Find("a-05")! with { Scope = "new scope" }); + await neighbours.NearestArticlesAsync(plan.Find("a-01")!, plan, 3, default); + + Assert.Equal([32, 8, 1], embeddings.Batches.Select(b => b.Count)); + Assert.Equal("a-05 de | a-05 en | new scope", Assert.Single(embeddings.Batches[^1])); + } + [Fact] public async Task Vector_CachedPerContentHash() { diff --git a/test/Wiki.Application.Tests/ArticlePlanTests.cs b/test/Wiki.Application.Tests/ArticlePlanTests.cs index 0f9a4c6..8240c27 100644 --- a/test/Wiki.Application.Tests/ArticlePlanTests.cs +++ b/test/Wiki.Application.Tests/ArticlePlanTests.cs @@ -75,7 +75,7 @@ public class ArticlePlanTests { // n3 is assigned to b without being among its sources var plan = ArticlePlanJson.Read(""" - {"articles":[{"slug":"b","category":"topic","sources":["n2"]}], + {"version":3,"articles":[{"slug":"b","category":"topic","sources":["n2"]}], "assignments":{"n2":["b"],"n3":["b"],"meta":[]}} """)!; @@ -112,7 +112,7 @@ public class ArticlePlanTests plan.RemoveNote("n1"); var read = ArticlePlanJson.Read(ArticlePlanJson.Write(plan))!; - var old = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{}}""")!; + var old = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{}}""")!; Assert.Equal(["gone"], read.Retired); Assert.True(read.SlugTaken("gone")); @@ -148,8 +148,8 @@ public class ArticlePlanTests plan.Fold("priv", "end", isPublic: false); var read = ArticlePlanJson.Read(ArticlePlanJson.Write(plan))!; - var old = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{"x":"y"}}""")!; - var stray = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{"x":"y"},"publicRedirects":["x","not-a-redirect"]}""")!; + var old = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{"x":"y"}}""")!; + var stray = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{"x":"y"},"publicRedirects":["x","not-a-redirect"]}""")!; Assert.Equal(["pub"], read.PublicRedirects); Assert.Empty(old.PublicRedirects); @@ -192,14 +192,14 @@ public class ArticlePlanTests Assert.Null(plan.Resolve("unknown")); var sixHops = ArticlePlanJson.Read(""" - {"articles":[{"slug":"end","category":"topic"}],"assignments":{}, + {"version":3,"articles":[{"slug":"end","category":"topic"}],"assignments":{}, "redirects":{"r1":"end","r2":"r1","r3":"r2","r4":"r3","r5":"r4","r6":"r5"}} """)!; Assert.Equal("end", sixHops.Resolve("r5")); Assert.Null(sixHops.Resolve("r6")); var cycle = ArticlePlanJson.Read(""" - {"articles":[],"assignments":{},"redirects":{"p":"q","q":"p"}} + {"version":3,"articles":[],"assignments":{},"redirects":{"p":"q","q":"p"}} """)!; Assert.Null(cycle.Resolve("p")); } @@ -247,7 +247,7 @@ public class ArticlePlanTests [Fact] public void Json_RedirectWithBadTargetSlug_Dropped() { - var read = ArticlePlanJson.Read("""{"articles":[],"assignments":{},"redirects":{"ok":"good-slug","bad":"../x","bad2":"A b"}}"""); + var read = ArticlePlanJson.Read("""{"version":3,"articles":[],"assignments":{},"redirects":{"ok":"good-slug","bad":"../x","bad2":"A b"}}"""); Assert.NotNull(read); Assert.Equal(["ok"], read.Redirects.Keys); @@ -257,6 +257,129 @@ public class ArticlePlanTests public void Json_OldFormat_IsNull() => Assert.Null(ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"topic","sources":["x"],"related":[]}],"coveredNotes":["x"]}""")); + [Theory] + [InlineData("""{"articles":[{"slug":"a","category":"topic","sources":["n1"]}],"assignments":{"n1":["a"]}}""")] + [InlineData("""{"version":2,"articles":[],"assignments":{}}""")] + [InlineData("""{"version":"3","articles":[],"assignments":{}}""")] + public void Json_EarlierFormatVersion_IsNull(string json) => Assert.Null(ArticlePlanJson.Read(json)); + + [Fact] + public void Json_Parent_RoundTrips() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("heating", "n1")); + plan.AddArticle(Article("heat-pump", "n2") with { Parent = "heating" }); + + var json = ArticlePlanJson.Write(plan); + var read = ArticlePlanJson.Read(json)!; + + Assert.StartsWith("{\n \"version\": 3,", json); + Assert.Null(read.Find("heating")!.Parent); + Assert.Equal("heating", read.Find("heat-pump")!.Parent); + Assert.Equal(json, ArticlePlanJson.Write(read)); + } + + [Fact] + public void Json_ParentUnknownOrCyclic_Dropped() + { + var plan = ArticlePlanJson.Read(""" + {"version":3,"articles":[ + {"slug":"a","category":"topic","parent":"ghost"}, + {"slug":"b","category":"topic","parent":"c"}, + {"slug":"c","category":"topic","parent":"b"}, + {"slug":"d","category":"topic","parent":"d"}, + {"slug":"e","category":"topic","parent":"b"}],"assignments":{}} + """)!; + + Assert.Null(plan.Find("a")!.Parent); + Assert.Null(plan.Find("d")!.Parent); + Assert.Equal("b", plan.Find("e")!.Parent); + // in file order: the link that would close the cycle is the one dropped + Assert.Equal("c", plan.Find("b")!.Parent); + Assert.Null(plan.Find("c")!.Parent); + } + + [Fact] + public void SetParent_RefusesCyclesAndSelf() + { + var plan = ArticlePlan.Empty(); + foreach (var s in new[] { "a", "b", "c" }) plan.AddArticle(Article(s)); + + Assert.True(plan.SetParent("b", "a")); + Assert.True(plan.SetParent("c", "b")); + Assert.False(plan.SetParent("a", "c")); + Assert.False(plan.SetParent("a", "a")); + Assert.True(plan.SetParent("c", "a")); + Assert.True(plan.SetParent("c", null)); + + Assert.Null(plan.Find("a")!.Parent); + Assert.Null(plan.Find("c")!.Parent); + Assert.Equal(["b"], plan.ChildrenOf("a").Select(c => c.Slug)); + Assert.Throws(() => plan.SetParent("ghost", "a")); + Assert.Throws(() => plan.SetParent("a", "ghost")); + } + + [Fact] + public void ReplaceArticle_KeepsParent() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a")); + plan.AddArticle(Article("b") with { Parent = "a" }); + + plan.ReplaceArticle(Article("b", "n1")); + + Assert.Equal("a", plan.Find("b")!.Parent); + } + + [Fact] + public void AddArticle_UnknownParent_Throws() + { + var plan = ArticlePlan.Empty(); + + Assert.Throws(() => plan.AddArticle(Article("b") with { Parent = "ghost" })); + } + + [Fact] + public void Fold_ChildrenMoveToTarget_TargetTakesFoldedParent() + { + var plan = ArticlePlan.Empty(); + foreach (var s in new[] { "top", "thin", "into", "kid" }) plan.AddArticle(Article(s, "n-" + s)); + plan.SetParent("thin", "top"); + plan.SetParent("into", "thin"); + plan.SetParent("kid", "thin"); + + plan.Fold("thin", "into", isPublic: false); + + Assert.Equal("top", plan.Find("into")!.Parent); + Assert.Equal("into", plan.Find("kid")!.Parent); + } + + [Fact] + public void Fold_ChildMoveWouldCycle_ChildLosesParent() + { + var plan = ArticlePlan.Empty(); + foreach (var s in new[] { "thin", "kid", "into" }) plan.AddArticle(Article(s, "n-" + s)); + plan.SetParent("kid", "thin"); + plan.SetParent("into", "kid"); + + plan.Fold("thin", "into", isPublic: false); + + Assert.Null(plan.Find("kid")!.Parent); + Assert.Equal("kid", plan.Find("into")!.Parent); + } + + [Fact] + public void RemoveNote_DeletedParent_ChildrenLoseParent() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a", "n1")); + plan.AddArticle(Article("b", "n2") with { Parent = "a" }); + + plan.RemoveNote("n1"); + + Assert.Null(plan.Find("b")!.Parent); + } + [Fact] public void Json_Write_MapKeysOrdinal_SameForSameContent() { @@ -285,7 +408,7 @@ public class ArticlePlanTests public void Json_BadSlugOrCategory_ArticleDropped() { var plan = ArticlePlanJson.Read(""" - {"articles":[{"slug":"Bad Slug","category":"topic"},{"slug":"a","category":"1"},{"slug":"b","category":"room"}],"assignments":{}} + {"version":3,"articles":[{"slug":"Bad Slug","category":"topic"},{"slug":"a","category":"1"},{"slug":"b","category":"room"}],"assignments":{}} """); Assert.Equal(["b"], plan!.Articles.Select(a => a.Slug)); } @@ -294,7 +417,7 @@ public class ArticlePlanTests public void Json_AssignmentToUnknownArticle_Dropped() { var plan = ArticlePlanJson.Read(""" - {"articles":[{"slug":"a","category":"topic","sources":["n1"]}], + {"version":3,"articles":[{"slug":"a","category":"topic","sources":["n1"]}], "assignments":{"n1":["a","ghost"],"n2":["ghost"],"n3":[]}} """)!; @@ -307,7 +430,7 @@ public class ArticlePlanTests { public bool Equals(PlannedArticle? x, PlannedArticle? y) => x!.Slug == y!.Slug && x.TitleDe == y.TitleDe && x.TitleEn == y.TitleEn && x.Category == y.Category - && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources); + && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources) && x.Parent == y.Parent; public int GetHashCode(PlannedArticle obj) => obj.Slug.GetHashCode(); } } diff --git a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs index ff153b0..5e18305 100644 --- a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs +++ b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs @@ -24,6 +24,13 @@ public class AssignNotesUseCaseTests private static Document[] Batch(params string[] slugs) => slugs.Select(s => TestNotes.Note(s)).ToArray(); + [Fact] + public void Prompt_RoomNotesAlsoFeedTheRoomArticle() + { + Assert.Contains("additionally feeds that room's article (category room)", Prompts.Assign); + Assert.Contains("\"EG Schlafzimmer\"", Prompts.Assign); + } + [Fact] public async Task Assign_ExistingAndNew() { diff --git a/test/Wiki.Application.Tests/ConsolidateArticlesUseCaseTests.cs b/test/Wiki.Application.Tests/ConsolidateArticlesUseCaseTests.cs new file mode 100644 index 0000000..d718c55 --- /dev/null +++ b/test/Wiki.Application.Tests/ConsolidateArticlesUseCaseTests.cs @@ -0,0 +1,130 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Encyclopedia; +using Wiki.Core.Model; + +namespace Wiki.Application.Tests; + +public class ConsolidateArticlesUseCaseTests +{ + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null) { ConsolidateGroupSize = 4 }; + + private static ConsolidateArticlesUseCase UseCase(FakeTextModel model) => + new(model, Settings, NullLogger.Instance); + + private static PlannedArticle Article(string slug, string? parent = null, int sources = 1) => + new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, + Enumerable.Range(0, sources).Select(i => $"{slug}-n{i}").ToList(), parent); + + private static readonly PlannedArticle[] Heating = + [Article("heating", sources: 2), Article("heat-pump", parent: "house"), Article("vitocal-heat-pump"), Article("radiators")]; + + [Fact] + public async Task Consolidate_MergesAndParents() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"vitocal-heat-pump","into":"heat-pump"}],"parents":[{"child":"heat-pump","parent":"heating"},{"child":"radiators","parent":"heating"}]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.NotNull(outcome); + Assert.Equal([new ArticleMerge("vitocal-heat-pump", "heat-pump")], outcome.Merges); + Assert.Equal([new ArticleParent("heat-pump", "heating"), new ArticleParent("radiators", "heating")], outcome.Parents); + var call = Assert.Single(model.Calls); + Assert.StartsWith("[consolidate]", call.System); + Assert.Equal(Settings.ConsolidateMaxTokens, call.MaxTokens); + Assert.Contains("heating | Titel heating | system | Scope heating | 2 | -\n", call.User); + Assert.Contains("heat-pump | Titel heat-pump | system | Scope heat-pump | 1 | house\n", call.User); + } + + [Fact] + public async Task Consolidate_ShowsAtMostTheGroupSize() + { + var model = new FakeTextModel().Reply("""{"merges":[],"parents":[]}"""); + + await UseCase(model).ConsolidateAsync([.. Heating, Article("garden")], CancellationToken.None); + + Assert.DoesNotContain("garden", model.Calls[0].User); + } + + [Fact] + public async Task Consolidate_EntriesOutsideTheGroupOrOnThemselves_Dropped() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"heating","into":"heating"},{"from":"garden","into":"heating"},{"from":"heating","into":"garden"},{"from":"radiators"}], + "parents":[{"child":"radiators","parent":"radiators"},{"child":"garden","parent":"heating"},{"child":"radiators","parent":"garden"}]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.Empty(outcome!.Merges); + Assert.Empty(outcome.Parents); + } + + [Fact] + public async Task Consolidate_MergedAwayArticleIsNeverATarget() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"vitocal-heat-pump","into":"heat-pump"},{"from":"heat-pump","into":"heating"}, + {"from":"radiators","into":"vitocal-heat-pump"},{"from":"vitocal-heat-pump","into":"heating"}],"parents":[]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.Equal([new ArticleMerge("vitocal-heat-pump", "heat-pump")], outcome!.Merges); + } + + [Fact] + public async Task Consolidate_ParentsFollowTheMergesAndAreUniquePerChild() + { + var model = new FakeTextModel().Reply(""" + {"merges":[{"from":"vitocal-heat-pump","into":"heat-pump"}], + "parents":[{"child":"vitocal-heat-pump","parent":"heating"},{"child":"radiators","parent":"vitocal-heat-pump"}, + {"child":"heat-pump","parent":"radiators"},{"child":"heat-pump","parent":"vitocal-heat-pump"}]} + """); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.Equal([new ArticleParent("heat-pump", "heating"), new ArticleParent("radiators", "heat-pump")], outcome!.Parents); + } + + [Fact] + public async Task Consolidate_NothingToDo_IsUsable() + { + var model = new FakeTextModel().Reply("""{"merges":[]}"""); + + var outcome = await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None); + + Assert.NotNull(outcome); + Assert.Empty(outcome.Merges); + Assert.Empty(outcome.Parents); + } + + [Theory] + [InlineData("not json", false)] + [InlineData("""{"keep":true}""", false)] + [InlineData("""{"merges":[],"parents":[]}""", true)] + public async Task Consolidate_UnusableReply_IsNull(string reply, bool truncated) + { + var model = new FakeTextModel().Reply(reply, truncated); + + Assert.Null(await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None)); + } + + [Fact] + public async Task Consolidate_BadRequest_IsNull() + { + var model = new FakeTextModel().Throw(new HttpRequestException("invalid", null, HttpStatusCode.BadRequest)); + + Assert.Null(await UseCase(model).ConsolidateAsync(Heating, CancellationToken.None)); + } + + [Fact] + public async Task Consolidate_OtherHttpError_Propagates() + { + var model = new FakeTextModel().Throw(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway)); + + await Assert.ThrowsAsync(() => UseCase(model).ConsolidateAsync(Heating, CancellationToken.None)); + } +} diff --git a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs index f2a4ab4..8009069 100644 --- a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs @@ -11,7 +11,7 @@ public class EncyclopediaCoordinatorTests public Task RunAsync(CancellationToken ct) => body(ct); } - private static readonly ReconcileSummary Done = new(3, 0, 0, 0, 2, 0, 0, 0, false, null, "c1"); + private static readonly ReconcileSummary Done = new(3, 0, 0, 0, 0, 0, 2, 0, 0, 0, false, null, "c1"); private static EncyclopediaCoordinator Coordinator(Func> body) => new(new ScriptedRun(body), new FakeTimeProvider(new DateTimeOffset(2026, 10, 24, 4, 0, 0, TimeSpan.Zero)), diff --git a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs index 8d3ae3f..98c6748 100644 --- a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs @@ -16,8 +16,8 @@ public class EncyclopediaReconcilerTests private const string StatePath = "articles/_state.json"; private static readonly DateTimeOffset Now = new(2026, 10, 24, 3, 0, 0, TimeSpan.Zero); - /// Fold is off (no article has fewer than zero sources) unless a test turns it on. - private static readonly EncyclopediaSettings Defaults = new(16000, 10, 300_000, [], null) { FoldMinSources = 0 }; + /// Fold (no article has fewer than zero sources) and consolidation are off unless a test turns them on. + private static readonly EncyclopediaSettings Defaults = new(16000, 10, 300_000, [], null) { FoldMinSources = 0, MaxConsolidationsPerRun = 0 }; private static readonly (string Slug, string[] Notes)[] HeatingAndKitchen = [("heating", ["boiler", "radiators"]), ("kitchen", ["kitchen"])]; @@ -58,6 +58,7 @@ public class EncyclopediaReconcilerTests var s = settings ?? Defaults; return new(_store, _catalog, new ArticleNeighbours(_embeddings, _index, s), new AssignNotesUseCase(_model, s, NullLogger.Instance), + new ConsolidateArticlesUseCase(_model, s, NullLogger.Instance), new SplitArticleUseCase(_model, s, NullLogger.Instance), new FoldArticleUseCase(_model, s, NullLogger.Instance), writer ?? _writer, s, _clock, NullLogger.Instance); @@ -100,12 +101,12 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); var commit = Assert.Single(_store.Commits); - Assert.Equal("articles: 3 assigned, 0 split, 0 folded, 2 written, 0 deleted, 0 rejected", commit.Message); + Assert.Equal("articles: 3 assigned, 0 merged, 0 parented, 0 split, 0 folded, 2 written, 0 deleted, 0 rejected", commit.Message); Assert.Equal( [PlanPath, StatePath, "articles/de/heating.md", "articles/de/kitchen.md", "articles/en/heating.md", "articles/en/kitchen.md"], commit.Changes.Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.All(commit.Changes, c => Assert.Equal(FileChangeOp.Create, c.Op)); - Assert.Equal(new ReconcileSummary(2, 3, 0, 0, 2, 0, 0, 0, false, null, "c2"), summary); + Assert.Equal(new ReconcileSummary(2, 3, 0, 0, 0, 0, 2, 0, 0, 0, false, null, "c2"), summary); Assert.Equal(AttemptOutcome.Written, State.Subjects["heating"].Outcome); Assert.Equal(Now, State.Subjects["heating"].At); Assert.Equal(["boiler", "radiators"], Plan.Find("heating")!.Sources); @@ -237,7 +238,7 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Run_SplitChildrenAreLinksOnlyInTheSplittingRun() + public async Task Run_SplitChildrenAreSubArticles_LinkedInLaterRunsToo() { await SeedAsync("boiler", "radiators", "pump", "valve"); _model.Reply(AssignReply(("heating", ["boiler", "radiators", "pump", "valve"]))); @@ -259,7 +260,10 @@ public class EncyclopediaReconcilerTests await Reconciler(settings).RunAsync(CancellationToken.None); Assert.Equal("heating", _writer.Calls[^1]); - Assert.Empty(_writer.Links["heating"]); + Assert.Equal(["heating-pump", "heating-radiators"], _writer.Links["heating"].Order(StringComparer.Ordinal)); + Assert.Equal(["heating"], _writer.Links["heating-pump"]); + Assert.Equal("heating", Plan.Find("heating-pump")!.Parent); + Assert.Null(Plan.Find("heating")!.Parent); } [Fact] @@ -507,7 +511,7 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); Assert.Equal(1, summary.Rejected); - Assert.Equal("articles: 0 assigned, 0 split, 0 folded, 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); + Assert.Equal("articles: 0 assigned, 0 merged, 0 parented, 0 split, 0 folded, 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); Assert.Equal([StatePath], LastCommit.Select(c => c.Path)); Assert.Equal(de, _store.Files["articles/de/heating.md"]); Assert.Equal(en, _store.Files["articles/en/heating.md"]); @@ -946,6 +950,243 @@ public class EncyclopediaReconcilerTests Assert.Equal(["heating"], _writer.Calls); } + private static readonly EncyclopediaSettings Consolidating = Defaults with { MaxConsolidationsPerRun = 60 }; + + private static string Consolidation(object[]? merges = null, object[]? parents = null) => + JsonSerializer.Serialize(new { merges = merges ?? [], parents = parents ?? [] }); + + private string[] Steps => _model.Calls.Select(c => c.System.Split('\n')[0]).ToArray(); + + [Fact] + public async Task Run_Consolidate_MergesAndSetsParents() + { + await SeedAsync("boiler", "pump", "radiators", "valve"); + _model.Reply(AssignReply(("heat-pump", ["boiler"]), ("vitocal", ["pump"]), ("heating", ["radiators", "valve"]))); + _model.Reply(Consolidation( + [new { from = "vitocal", into = "heat-pump" }], + [new { child = "heat-pump", parent = "heating" }, new { child = "vitocal", parent = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + Assert.Equal((1, 1, 2), (summary.Merged, summary.Parented, summary.Planned)); + Assert.StartsWith("articles: 4 assigned, 1 merged, 1 parented,", _store.Commits[^1].Message); + Assert.Equal("heat-pump", Plan.Redirects["vitocal"]); + Assert.Empty(Plan.PublicRedirects); + Assert.Equal(["boiler", "pump"], Plan.Find("heat-pump")!.Sources); + Assert.Equal(["heat-pump"], Plan.Assignments["pump"]); + Assert.Equal("heating", Plan.Find("heat-pump")!.Parent); + Assert.Contains("heating", _writer.Links["heat-pump"]); + Assert.Contains("heat-pump", _writer.Links["heating"]); + Assert.DoesNotContain("vitocal", _writer.Calls); + } + + [Fact] + public async Task Run_Consolidate_GroupIsTheSeedAndItsNearestArticles() + { + await SeedAsync("boiler", "pump", "radiators"); + _model.Reply(AssignReply(("heat-pump", ["boiler"]), ("vitocal", ["pump"]), ("heating", ["radiators"]))); + _model.Reply(Consolidation()); + + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + var user = _model.Calls[1].User; + Assert.StartsWith("Articles (slug | title | category | scope | source count | current parent):\nheat-pump | heat-pump de | topic | About heat-pump. | 1 | -\n", user); + Assert.Contains("vitocal | vitocal de |", user); + Assert.Contains("heating | heating de |", user); + } + + [Fact] + public async Task Run_Consolidate_RememberedGroupNotAskedAgain() + { + await SeedAsync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + _model.Reply(Consolidation()); + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + await Sync(); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + Assert.Single(State.ConsolidatedGroups); + Assert.Null(summary.Commit); + } + + [Fact] + public async Task Run_Consolidate_ChangedGroupRememberedAsChanged() + { + await SeedAsync("boiler", "pump", "radiators"); + _model.Reply(AssignReply(("heat-pump", ["boiler"]), ("vitocal", ["pump"]), ("heating", ["radiators"]))); + _model.Reply(Consolidation([new { from = "vitocal", into = "heat-pump" }], [new { child = "heat-pump", parent = "heating" }])); + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + await Sync(); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.Single(State.ConsolidatedGroups); + Assert.Null(summary.Commit); + } + + [Fact] + public async Task Run_Consolidate_UnusableReplyRememberedAsFailed() + { + await SeedAsync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + _model.Reply("I would rather not."); + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + await Sync(); + + await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + Assert.Single(State.FailedGroups); + Assert.Empty(State.ConsolidatedGroups); + } + + [Fact] + public async Task Run_Consolidate_HttpErrorSkipsOnlyThatGroup() + { + await SeedAsync("a1", "b1", "c1", "d1"); + _model.Reply(AssignReply(("a", ["a1"]), ("b", ["b1"]), ("c", ["c1"]), ("d", ["d1"]))); + _model.Throw(new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway)); + foreach (var _ in Enumerable.Range(0, 3)) _model.Reply(Consolidation()); + + var summary = await Reconciler(Consolidating with { ConsolidateGroupSize = 2 }).RunAsync(CancellationToken.None); + + Assert.True(Steps.Count(s => s == "[consolidate]") >= 2); + Assert.Empty(State.FailedGroups); + Assert.NotEmpty(State.ConsolidatedGroups); + Assert.Equal(4, summary.Written); + } + + [Fact] + public async Task Run_Consolidate_CapRespected() + { + await SeedAsync("a1", "b1", "c1", "d1"); + _model.Reply(AssignReply(("a", ["a1"]), ("b", ["b1"]), ("c", ["c1"]), ("d", ["d1"]))); + _model.Reply(Consolidation()); + + await Reconciler(Consolidating with { ConsolidateGroupSize = 2, MaxConsolidationsPerRun = 1 }).RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[consolidate]"], Steps); + } + + [Fact] + public async Task Run_Consolidate_MergeOfPublicArticle_LeavesPublicRedirectAndDeletesFiles() + { + await FirstRunAsync(); + _model.Reply(Consolidation([new { from = "kitchen", into = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal((1, 1), (summary.Merged, summary.Deleted)); + Assert.Equal("heating", Plan.Redirects["kitchen"]); + Assert.Equal(["kitchen"], Plan.PublicRedirects); + Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("kitchen", State.Subjects.Keys); + Assert.Equal("heating", _writer.Calls[^1]); + } + + [Fact] + public async Task Run_Consolidate_ParentLinkAloneRewritesNothing() + { + await FirstRunAsync(); + _model.Reply(Consolidation(parents: [new { child = "kitchen", parent = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(1, summary.Parented); + Assert.Equal(0, summary.Written); + Assert.Equal(2, _writer.Calls.Count); + Assert.Equal([PlanPath, StatePath], LastCommit.Select(c => c.Path).Order(StringComparer.Ordinal)); + Assert.Equal("heating", Plan.Find("kitchen")!.Parent); + } + + [Fact] + public async Task Run_Consolidate_ParentClosingACycle_Dropped() + { + await FirstRunAsync(); + var plan = Plan; + plan.SetParent("heating", "kitchen"); + _store.Files[PlanPath] = ArticlePlanJson.Write(plan); + await Sync(); + _model.Reply(Consolidation(parents: [new { child = "kitchen", parent = "heating" }])); + + var summary = await Reconciler(Consolidating).RunAsync(CancellationToken.None); + + Assert.Equal(0, summary.Parented); + Assert.Null(Plan.Find("kitchen")!.Parent); + Assert.Equal("kitchen", Plan.Find("heating")!.Parent); + } + + [Fact] + public async Task Run_AssignCandidates_IncludeArticlesNearByVector() + { + var settings = Defaults with { NeighbourNotes = 0 }; + await SeedAsync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + _store.Files["wiki/oven.md"] = Note("oven"); + await Sync(); + _model.Reply(ExistingReply(("oven", "kitchen"))); + + await Reconciler(settings).RunAsync(CancellationToken.None); + + // no note neighbours at all: both candidates come from the article vectors + Assert.Contains("\nheating | heating de |", _model.Calls[1].User); + Assert.Contains("\nkitchen | kitchen de |", _model.Calls[1].User); + } + + [Fact] + public async Task Run_WriteLinksParentAndAtMost20Children() + { + var children = Numbered(25); + await SeedAsync(["hub-note", .. children]); + var plan = ArticlePlan.Empty(); + plan.AddArticle(new PlannedArticle("hub", "Hub", "Hub", ArticleCategory.System, "The hub.", [])); + plan.Assign("hub-note", ["hub"]); + foreach (var c in children) + { + plan.AddArticle(new PlannedArticle("a-" + c, c, c, ArticleCategory.Device, "One.", [], Parent: "hub")); + plan.Assign(c, ["a-" + c]); + } + _store.Files[PlanPath] = ArticlePlanJson.Write(plan); + await Sync(); + + await Reconciler(Defaults with { CandidateArticles = 0, MaxArticlesPerRun = 30 }).RunAsync(CancellationToken.None); + + Assert.Equal(20, _writer.Links["hub"].Count); + Assert.Equal(["hub"], _writer.Links["a-note-25"]); + } + + [Fact] + public async Task Run_PlanInOlderFormat_BootstrapsAndDropsItsState() + { + await SeedAsync(); + _store.Files[PlanPath] = """ + {"articles":[{"slug":"heizung","titleDe":"Heizung","titleEn":"Heating","category":"system","scope":"x","sources":["boiler"]}], + "assignments":{"boiler":["heizung"],"radiators":[],"kitchen":[]},"redirects":{}} + """; + _store.Files[StatePath] = EncyclopediaStateJson.Write(EncyclopediaState.Empty with + { + Subjects = new Dictionary { ["heizung"] = new("f", AttemptOutcome.Rejected, Now) }, + KeptArticles = new Dictionary { ["heizung"] = "f" }, + }); + await Sync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(3, summary.Assigned); + Assert.Contains("boiler |", _model.Calls[0].User); + Assert.Null(Plan.Find("heizung")); + Assert.Equal(["heating", "kitchen"], State.Subjects.Keys); + Assert.Empty(State.KeptArticles); + Assert.StartsWith("{\n \"version\": 3,", _store.Files[PlanPath]); + } + /// Embeddings that fail like an unreachable service while , or with when set. private sealed class SwitchableEmbeddings(IEmbeddingModel inner) : IEmbeddingModel { diff --git a/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs b/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs index 78a553f..f58fa36 100644 --- a/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaSettingsTests.cs @@ -12,8 +12,18 @@ public class EncyclopediaSettingsTests Assert.Equal(32000, s.AssignMaxTokens); Assert.Equal(32000, s.SplitMaxTokens); Assert.Equal(8000, s.FoldMaxTokens); + Assert.Equal(8000, s.ConsolidateMaxTokens); } [Fact] - public void PromptsVersion_InvalidatesBatchesFailedAtTheOldCap() => Assert.Equal(7, Prompts.Version); + public void Defaults_ConsolidateABootstrapPlanInOneRun() + { + var s = new EncyclopediaSettings(16000, 10, 300_000, [], null); + + Assert.Equal(8, s.ConsolidateGroupSize); + Assert.Equal(60, s.MaxConsolidationsPerRun); + } + + [Fact] + public void PromptsVersion_InvalidatesMemoryOfThePreviousPlan() => Assert.Equal(8, Prompts.Version); } diff --git a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs index 11b4977..909d999 100644 --- a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs @@ -17,7 +17,9 @@ public class EncyclopediaStateTests ["sauna"] = new("def", AttemptOutcome.Rejected, At), }, new Dictionary(), - new Dictionary()); + new Dictionary(), + new Dictionary(), + new Dictionary()); var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); @@ -30,12 +32,42 @@ public class EncyclopediaStateTests var state = new EncyclopediaState( new Dictionary(), new Dictionary { ["fp1"] = At }, - new Dictionary { ["heating"] = "fp2" }); + new Dictionary { ["heating"] = "fp2" }, + new Dictionary { ["g1"] = At }, + new Dictionary { ["g2"] = At }); var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); Assert.Equal(state.FailedBatches, read.FailedBatches); Assert.Equal(state.KeptArticles, read.KeptArticles); + Assert.Equal(state.ConsolidatedGroups, read.ConsolidatedGroups); + Assert.Equal(state.FailedGroups, read.FailedGroups); + } + + [Fact] + public void Read_WithoutGroups_ReadsEmpty() + { + var read = EncyclopediaStateJson.Read("""{"subjects":{},"failedBatches":{},"keptArticles":{}}"""); + Assert.Empty(read.ConsolidatedGroups); + Assert.Empty(read.FailedGroups); + } + + [Fact] + public void GroupFingerprint_MembersAndWhatTheModelJudges_NotSources() + { + var a = new PlannedArticle("a", "A", "A", ArticleCategory.System, "About a.", ["n1"]); + var b = new PlannedArticle("b", "B", "B", ArticleCategory.Device, "About b.", ["n2"], Parent: "a"); + + var baseline = EncyclopediaState.GroupFingerprint([a, b]); + + Assert.Matches("^[0-9a-f]{64}$", baseline); + Assert.Equal(baseline, EncyclopediaState.GroupFingerprint([b, a])); + Assert.Equal(baseline, EncyclopediaState.GroupFingerprint([a, b with { Sources = ["n2", "n3"] }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { Parent = null }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { Scope = "Other." }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { TitleEn = "Bee" }])); + Assert.NotEqual(baseline, EncyclopediaState.GroupFingerprint([a, b with { Category = ArticleCategory.Room }])); } [Fact] @@ -45,6 +77,7 @@ public class EncyclopediaStateTests Assert.Empty(read.Subjects); Assert.Empty(read.FailedBatches); Assert.Empty(read.KeptArticles); + Assert.Empty(read.ConsolidatedGroups); Assert.DoesNotContain("failedPlan", EncyclopediaStateJson.Write(read)); } diff --git a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs index 6ea0ce3..0454141 100644 --- a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs @@ -62,6 +62,31 @@ public class WriteArticleUseCaseTests Assert.DoesNotContain("Related articles", user); } + [Fact] + public void Prompt_RoomArticlesDescribeTheRoomAndLinkTheSystems() + { + Assert.Contains("A room article (category room) describes what is in the room", Prompts.Write); + Assert.Contains("links to the system's article", Prompts.Write); + } + + [Fact] + public async Task Write_MainArticleAndSubArticlesMarked() + { + var planned = Planned with { Parent = "house" }; + var targets = new PlannedArticle[] + { + new("house", "Haus", "House", ArticleCategory.Topic, "", ["h"]), + new("lighting", "Licht", "Lighting", ArticleCategory.System, "", ["l"]), + new("dimmers", "Dimmer", "Dimmers", ArticleCategory.Device, "", ["d"], Parent: "wall-switches"), + }; + var model = Model(ThreeBlocks, Clean, """["Zero.","One.","Two."]""", Clean); + + await new WriteArticleUseCase(model, Settings, new FakeTimeProvider(Now)).WriteAsync(planned, [Source()], targets, CancellationToken.None); + + Assert.Contains("Articles you may link to (slug | German title | relation):\nhouse | Haus | main article\nlighting | Licht\ndimmers | Dimmer | sub-article\n", + model.Calls[0].User); + } + private static IReadOnlyList Public(string body) => ArticleBody.PublicBlocks(body).Select(b => b.Text).ToList(); [Fact] diff --git a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs index 094f00a..1a19d67 100644 --- a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs +++ b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs @@ -24,7 +24,7 @@ public sealed class EncyclopediaEndpointTests { CatalogCommits.Add((await catalog.GetStateAsync(ct)).Commit); if (Failure is not null) throw Failure; - return new ReconcileSummary(Planned, 0, 0, 0, 0, 0, 0, 0, StoppedByModel, null, null); + return new ReconcileSummary(Planned, 0, 0, 0, 0, 0, 0, 0, 0, 0, StoppedByModel, null, null); } } diff --git a/test/Wiki.Host.Tests/HouseTests.cs b/test/Wiki.Host.Tests/HouseTests.cs index 9ef63fa..062e753 100644 --- a/test/Wiki.Host.Tests/HouseTests.cs +++ b/test/Wiki.Host.Tests/HouseTests.cs @@ -44,9 +44,17 @@ public class HouseTests(WikiHostFactory factory) : IClassFixtureDas Haus", child); + Assert.DoesNotContain("Unterartikel", child); + Assert.Contains("

Unterartikel

", parent); + Assert.Contains("
  • Pfad
    • ", parent); + Assert.Contains("
  • Wandschalter
    • ", parent); + Assert.True(parent.IndexOf("/house/de/x\"", StringComparison.Ordinal) < parent.IndexOf("/house/de/schalter\"", StringComparison.Ordinal)); + Assert.DoesNotContain("network-internals", parent); + Assert.DoesNotContain("Netzwerk-Interna", parent); + Assert.DoesNotContain("heizung", parent); + Assert.DoesNotContain("geplant", parent); + Assert.DoesNotContain("Hauptartikel", parent); + } + + [Fact] + public async Task Navigation_Anonymous_PrivateMainArticleInvisible() + { + await SeedAsync(); + + var html = Text(await Html(await Browser().GetAsync("/house/de/y"))); + + Assert.DoesNotContain("Hauptartikel", html); + Assert.DoesNotContain("network-internals", html); + Assert.DoesNotContain("Netzwerk-Interna", html); + } + + [Fact] + public async Task Navigation_SignedIn_SeesPrivateRelatives() + { + await SeedAsync(); + var browser = await SignedInAsync(); + + var parent = Text(await Html(await browser.GetAsync("/house/de/haus"))); + var child = Text(await Html(await browser.GetAsync("/house/de/y"))); + + Assert.Contains("
  • Netzwerk-Interna
    • ", parent); + Assert.Contains("
  • Heizung
    • ", parent); + Assert.DoesNotContain("geplant", parent); + Assert.Contains("Hauptartikel: Netzwerk-Interna", child); + } + + [Fact] + public async Task Navigation_English_Labels() + { + await SeedAsync(); + + var child = Text(await Html(await Browser().GetAsync("/house/en/schalter"))); + var parent = Text(await Html(await Browser().GetAsync("/house/en/haus"))); + + Assert.Contains("Main article: The house", child); + Assert.Contains("

    Sub-articles

    ", parent); + Assert.Contains("
  • Wall switches
    • ", parent); + } + [Fact] public async Task FrontPage_Anonymous_ListsOnlyPublic() {