diff --git a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs index b81d638..26ab426 100644 --- a/src/Wiki.Application/Encyclopedia/ArticlePlan.cs +++ b/src/Wiki.Application/Encyclopedia/ArticlePlan.cs @@ -1,6 +1,5 @@ using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.RegularExpressions; using Wiki.Core.Model; namespace Wiki.Application.Encyclopedia; @@ -145,31 +144,26 @@ public sealed class ArticlePlan private void Replace(PlannedArticle a) => _articles[_articles.FindIndex(x => x.Slug == a.Slug)] = a; } -public static partial class ArticlePlanJson +public static class ArticlePlanJson { private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { WriteIndented = true }; - private static readonly JsonSerializerOptions CompactOptions = new(JsonSerializerDefaults.Web); - - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - + /// Map keys in ordinal order, so an unchanged plan serializes identically. public static string Write(ArticlePlan p) => - JsonSerializer.Serialize(new { Articles = Shape(p), p.Assignments, p.Redirects }, Options); - - /// Only the articles array, compact: the form the model sees and answers in. - public static string WriteArticlesCompact(ArticlePlan p) => - JsonSerializer.Serialize(Shape(p), CompactOptions); - - private static IEnumerable Shape(ArticlePlan p) => p.Articles.Select(a => new - { - a.Slug, - a.TitleDe, - a.TitleEn, - Category = a.Category.ToString().ToLowerInvariant(), - a.Scope, - a.Sources, - }); + JsonSerializer.Serialize(new + { + Articles = p.Articles.Select(a => new + { + a.Slug, + a.TitleDe, + a.TitleEn, + Category = a.Category.ToString().ToLowerInvariant(), + a.Scope, + a.Sources, + }), + Assignments = new SortedDictionary>(p.Assignments.ToDictionary(), StringComparer.Ordinal), + Redirects = new SortedDictionary(p.Redirects.ToDictionary(), StringComparer.Ordinal), + }, Options); /// /// The plan in , which may be wrapped in a code fence; null when it is not a plan, @@ -180,7 +174,7 @@ public static partial class ArticlePlanJson { try { - if (JsonNode.Parse(StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items + if (JsonNode.Parse(PlanParsing.StripFence(json)) is not JsonObject root || root["articles"] is not JsonArray items || root["assignments"] is not JsonObject assignments) return null; var plan = ArticlePlan.Empty(); foreach (var a in Articles(items)) @@ -188,13 +182,13 @@ public static partial class ArticlePlanJson if (root["redirects"] is JsonObject redirects) foreach (var (from, to) in redirects) { - var target = Text(to); - if (target is not null && SlugPattern().IsMatch(from) && plan.Find(from) is null) plan.SetRedirect(from, target); + var target = PlanParsing.Text(to); + if (target is not null && PlanParsing.SlugPattern().IsMatch(from) && plan.Find(from) is null) plan.SetRedirect(from, target); } foreach (var (note, value) in assignments) { if (value is not JsonArray slugs) continue; - var all = Strings(slugs); + var all = PlanParsing.Strings(slugs); var known = all.Where(s => plan.Find(s) is not null).Distinct(StringComparer.Ordinal).ToList(); if (all.Count == 0 || known.Count > 0) plan.SetAssignment(note, known); } @@ -206,48 +200,18 @@ public static partial class ArticlePlanJson } } - /// The articles array of a model reply in the one-shot format (no assignments); null when it is not JSON with one. Never throws. - internal static IReadOnlyList? ReadArticles(string json) - { - try - { - return JsonNode.Parse(StripFence(json)) is JsonObject root && root["articles"] is JsonArray items ? Articles(items) : null; - } - catch (Exception e) when (e is JsonException or InvalidOperationException) - { - return null; - } - } - private static List Articles(JsonArray items) { var articles = new List(); foreach (var item in items) { if (item is not JsonObject o) continue; - var slug = Text(o["slug"]); - var categoryText = Text(o["category"]); - if (slug is null || !SlugPattern().IsMatch(slug) || categoryText is null || !categoryText.All(char.IsAsciiLetter) - || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) continue; - articles.Add(new PlannedArticle(slug, Text(o["titleDe"]) ?? slug, Text(o["titleEn"]) ?? slug, category, - Text(o["scope"]) ?? "", Strings(o["sources"]))); + var slug = PlanParsing.Text(o["slug"]); + if (slug is null || !PlanParsing.SlugPattern().IsMatch(slug) + || !PlanParsing.TryCategory(PlanParsing.Text(o["category"]), out var category)) continue; + articles.Add(new PlannedArticle(slug, PlanParsing.Text(o["titleDe"]) ?? slug, PlanParsing.Text(o["titleEn"]) ?? slug, category, + PlanParsing.Text(o["scope"]) ?? "", PlanParsing.Strings(o["sources"]))); } return articles; } - - /// Trimmed, as frontmatter reads it back; null when blank. - private static string? Text(JsonNode? n) => - n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; - - private static List Strings(JsonNode? n) => - n is JsonArray a ? a.Select(Text).OfType().ToList() : []; - - internal static string StripFence(string text) - { - var t = text.Trim(); - if (!t.StartsWith("```")) return t; - var start = t.IndexOf('\n'); - var end = t.LastIndexOf("```", StringComparison.Ordinal); - return start >= 0 && end > start ? t[(start + 1)..end].Trim() : t; - } } diff --git a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs index 108a6de..d38beff 100644 --- a/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/AssignNotesUseCase.cs @@ -1,11 +1,10 @@ using System.Net; using System.Text; -using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Wiki.Core.Model; using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; namespace Wiki.Application.Encyclopedia; @@ -20,13 +19,10 @@ public sealed record AssignOutcome( bool Unusable); /// Assigns one batch of notes to nearby existing articles or to new ones, in one bounded model call. -public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +public sealed class AssignNotesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) { private const int FirstParagraphChars = 300; - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - private static AssignOutcome Unusable() => new([], new Dictionary>(), true); @@ -52,15 +48,7 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet return Unusable(); } - JsonObject? root; - try - { - root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; - } - catch (JsonException) - { - root = null; - } + var root = PlanParsing.Object(reply.Text); if (root is null) { log.LogWarning("Assign reply is not a JSON object"); @@ -137,9 +125,7 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet log.LogWarning("Dropping new article {Slug}: slug taken or duplicate", slug); return null; } - var categoryText = Text(o["category"]); - if (categoryText is null || !categoryText.All(char.IsAsciiLetter) - || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) + if (!TryCategory(Text(o["category"]), out var category)) { log.LogWarning("Dropping new article {Slug}: unknown category", slug); return null; @@ -158,12 +144,6 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet return new PlannedArticle(slug, titleDe, titleEn, category, scope, []); } - private static string? Text(JsonNode? n) => - n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; - - private static List Strings(JsonNode? n) => - n is JsonArray a ? a.Select(Text).OfType().ToList() : []; - private static string UserPrompt(IReadOnlyList batch, List candidates) { var sb = new StringBuilder(); @@ -187,7 +167,4 @@ public sealed partial class AssignNotesUseCase(ITextModel model, EncyclopediaSet var text = Flatten(string.Join(' ', lines)).Trim(); return text.Length > FirstParagraphChars ? text[..FirstParagraphChars] : text; } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); } diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs index 219b4af..72b6a60 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaReconciler.cs @@ -7,11 +7,13 @@ using Wiki.Core.Ports; namespace Wiki.Application.Encyclopedia; /// -/// What one run did. Counts are subjects, each in both languages. counts subjects -/// rejected earlier whose inputs are unchanged, so they were not retried. is null when nothing was committed. +/// What one run did. is the article count after the run; counts notes, +/// and articles; the write counts are subjects, each in both languages. +/// counts subjects rejected earlier whose inputs are unchanged, so they were not retried. +/// is null when nothing was committed. /// -public sealed record ReconcileSummary(int Planned, int Written, int Deleted, int Rejected, int Remembered, - bool StoppedByModel, string? Skipped, string? Commit); +public sealed record ReconcileSummary(int Planned, int Assigned, int Split, int Folded, int Written, int Deleted, int Rejected, + int Remembered, bool StoppedByModel, string? Skipped, string? Commit); public interface IEncyclopediaRun { @@ -19,14 +21,18 @@ public interface IEncyclopediaRun } /// -/// Brings articles/ up to date with the notes: replans when the note set changed, rewrites the articles whose -/// inputs changed, deletes the subjects a new plan dropped, and commits everything as one commit. +/// Brings articles/ up to date with the notes: updates the stored plan in bounded steps (drop removed notes, assign new +/// ones, split oversized and fold thin articles), rewrites the articles whose inputs changed, deletes the files of articles +/// the plan dropped, and commits everything as one commit. /// Works only on a catalog that reflects the store's head, so what it compares is what it commits onto. /// public sealed class EncyclopediaReconciler( INoteStore store, IDocumentCatalog catalog, - PlanArticlesUseCase planner, + ArticleNeighbours neighbours, + AssignNotesUseCase assigner, + SplitArticleUseCase splitter, + FoldArticleUseCase folder, IArticleWriter writer, EncyclopediaSettings settings, TimeProvider clock, @@ -36,7 +42,27 @@ public sealed class EncyclopediaReconciler( private const string StatePath = "articles/_state.json"; private static readonly string[] Languages = ["de", "en"]; - /// An unexpected failure while writing is rethrown after what is staged is committed; caller cancellation commits nothing. + /// The inputs and mutable outputs of one run. + private sealed class Run(IReadOnlyList notes, ArticlePlan plan, EncyclopediaState state, DateTimeOffset now) + { + public IReadOnlyList Notes { get; } = notes; + public Dictionary NoteBySlug { get; } = + notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + public ArticlePlan Plan { get; } = plan; + public DateTimeOffset Now { get; } = now; + public SortedDictionary Subjects { get; } = new(state.Subjects.ToDictionary(), StringComparer.Ordinal); + public SortedDictionary FailedBatches { get; } = new(state.FailedBatches.ToDictionary(), StringComparer.Ordinal); + public SortedDictionary Kept { get; } = new(state.KeptArticles.ToDictionary(), StringComparer.Ordinal); + /// Children of the articles split this run: their overview's extra link targets. + public Dictionary> Children { get; } = new(StringComparer.Ordinal); + public int Assigned { get; set; } + public int Split { get; set; } + public int Folded { get; set; } + + public EncyclopediaState State => new(Subjects, FailedBatches, Kept); + } + + /// An unexpected failure is rethrown after what is staged is committed; caller cancellation commits nothing. public async Task RunAsync(CancellationToken ct) { var head = (await store.GetHeadAsync(ct)).Commit; @@ -44,79 +70,236 @@ public sealed class EncyclopediaReconciler( if (synced != head) { log.LogWarning("Catalog is at {Catalog}, the store at {Head}; skipping the encyclopedia run", synced, head); - return new ReconcileSummary(0, 0, 0, 0, 0, false, "catalog-behind", null); + return new ReconcileSummary(0, 0, 0, 0, 0, 0, 0, 0, false, "catalog-behind", null); } var notes = await catalog.ListDocumentsAsync(DocumentKind.Wiki, ct); var articleDocs = await catalog.ListDocumentsAsync(DocumentKind.Article, ct); var planText = await store.ReadTextAsync(PlanPath, head, ct); var stateText = await store.ReadTextAsync(StatePath, head, ct); - var current = ReadPlan(planText); - var state = ReadState(stateText); - var subjects = new SortedDictionary(state.Subjects.ToDictionary(), StringComparer.Ordinal); - var failedPlan = state.FailedPlan; - var now = clock.GetUtcNow(); + var run = new Run(notes, ReadPlan(planText), ReadState(stateText), clock.GetUtcNow()); + var plan = run.Plan; + var planned = plan.Articles.Select(a => a.Slug).ToList(); var changes = new List(); - ArticlePlan? replanned = null; - if (PlanArticlesUseCase.NeedsReplan(current, failedPlan, notes)) - { - try - { - replanned = await planner.PlanAsync(current, notes, ct); - } - catch (ModelUnavailableException e) - { - log.LogWarning(e, "Text model unavailable while planning; keeping the current plan"); - return new ReconcileSummary(current?.Articles.Count ?? 0, 0, 0, 0, 0, true, null, null); - } - if (replanned is null) - { - failedPlan = new PlanAttempt(EncyclopediaState.NoteSetFingerprint(notes), now); - } - else - { - failedPlan = null; - changes.Add(Put(PlanPath, ArticlePlanJson.Write(replanned), planText?.BlobSha)); - } - } + foreach (var note in plan.Assignments.Keys.Where(n => !run.NoteBySlug.ContainsKey(n)).Order(StringComparer.Ordinal).ToList()) + foreach (var slug in plan.RemoveNote(note)) + log.LogInformation("Article {Slug} lost its last source {Note}; deleting it", slug, note); - var plan = replanned ?? current; - if (plan is null) + var stopped = false; + ExceptionDispatchInfo? failure = null; + try { - log.LogWarning("No usable article plan; no article is written or deleted"); - return await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText, - new ReconcileSummary(0, 0, 0, 0, 0, false, "no-plan", null), ct); + await PlanStepsAsync(run, ct); + } + catch (ModelUnavailableException e) + { + log.LogWarning(e, "Text model unavailable while planning; committing the plan so far, writing nothing"); + stopped = true; + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + log.LogError(e, "Planning failed unexpectedly; committing the plan so far, then failing the run"); + failure = ExceptionDispatchInfo.Capture(e); } var hashByPath = articleDocs.ToDictionary(d => d.Path, d => d.ContentHash, StringComparer.Ordinal); - var stale = Stale(plan, notes, ParseArticles(articleDocs)); - var remembered = InScope(stale, plan, settings).Count(p => IsRemembered(p, notes, state)); - var due = Due(stale, plan, notes, state, settings); - var noteBySlug = notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); - + var stale = Stale(plan, notes, ParseArticles(articleDocs), settings); + var remembered = InScope(stale, plan, settings).Count(p => IsRemembered(p, notes, run.State)); int written = 0, rejected = 0; - var stopped = false; - ExceptionDispatchInfo? failure = null; + if (!stopped && failure is null) + (written, rejected, stopped, failure) = await WriteAsync(run, Due(stale, plan, notes, run.State, settings), hashByPath, changes, ct); + + var deleted = 0; + foreach (var slug in planned.Where(s => plan.Find(s) is null)) + { + var any = false; + foreach (var lang in Languages) + { + var path = ArticlePath(lang, slug); + if (!hashByPath.TryGetValue(path, out var hash)) continue; + changes.Add(new FileChange(FileChangeOp.Delete, path, null, hash)); + any = true; + } + run.Subjects.Remove(slug); + run.Kept.Remove(slug); + if (any) deleted++; + } + + var planJson = ArticlePlanJson.Write(plan); + if (planJson != (planText?.Text ?? ArticlePlanJson.Write(ArticlePlan.Empty()))) + changes.Add(Put(PlanPath, planJson, planText?.BlobSha)); + + var summary = await FinishAsync(changes, run.State, stateText, + new ReconcileSummary(plan.Articles.Count, run.Assigned, run.Split, run.Folded, written, deleted, rejected, remembered, + stopped, null, null), ct); + failure?.Throw(); + return summary; + } + + /// + /// Assign, split, fold, each within its cap. An unreachable embedding or index service () + /// ends the planning steps; what they changed stays. + /// + private async Task PlanStepsAsync(Run run, CancellationToken ct) + { + try + { + await AssignAsync(run, ct); + await SplitAsync(run, ct); + await FoldAsync(run, ct); + } + catch (HttpRequestException e) + { + log.LogWarning(e, "Planning stopped: a service did not answer; writing with the plan so far"); + } + } + + /// + /// Batches of similar new notes, seeded in slug order, each assigned in one call against the articles near its notes in the + /// plan as it stands, so later batches see the articles earlier ones created. A batch whose reply was unusable before, for + /// the same notes, is skipped without a call; a batch is tried once per run. + /// + private async Task AssignAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + var fresh = run.NoteBySlug.Values.Where(n => !plan.IsAssigned(n.Slug)).OrderBy(n => n.Slug, StringComparer.Ordinal).ToList(); + var tried = new HashSet(StringComparer.Ordinal); + var calls = 0; + while (calls < settings.MaxAssignBatchesPerRun) + { + var open = fresh.Where(n => !tried.Contains(n.Slug) && !plan.IsAssigned(n.Slug)).ToList(); + if (open.Count == 0) return; + var batch = await neighbours.NearestAsync(open[0], open.Select(n => n.Slug).ToHashSet(StringComparer.Ordinal), + settings.AssignBatchSize, run.NoteBySlug, ct); + tried.UnionWith(batch.Select(n => n.Slug)); + var fingerprint = EncyclopediaState.BatchFingerprint(batch); + if (run.FailedBatches.ContainsKey(fingerprint)) continue; + + var nearby = new List(); + foreach (var note in batch) + { + foreach (var slug in await neighbours.NearbyArticlesAsync(note, plan, ct)) + if (nearby.Count < settings.CandidateArticles * 2 && !nearby.Contains(slug)) nearby.Add(slug); + if (nearby.Count >= settings.CandidateArticles * 2) break; + } + + calls++; + var outcome = await assigner.AssignAsync(batch, Articles(plan, nearby), plan, ct); + if (outcome.Unusable) + { + run.FailedBatches[fingerprint] = run.Now; + continue; + } + foreach (var a in outcome.New) plan.AddArticle(a); + foreach (var (note, articles) in outcome.Assignments) plan.Assign(note, articles); + run.Assigned += outcome.Assignments.Count; + } + if (fresh.Any(n => !tried.Contains(n.Slug) && !plan.IsAssigned(n.Slug))) + log.LogInformation("Assign cap of {Cap} batches reached; the other new notes wait for the next run", settings.MaxAssignBatchesPerRun); + } + + /// Oversized articles, biggest first, at most calls. + private async Task SplitAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + var oversized = plan.Articles + .Where(a => SplitArticleUseCase.NeedsSplit(a, run.NoteBySlug, settings)) + .OrderByDescending(a => a.Sources.Sum(s => run.NoteBySlug.TryGetValue(s, out var d) ? (long)d.Body.Length : 0)) + .ThenByDescending(a => a.Sources.Count) + .ThenBy(a => a.Slug, StringComparer.Ordinal) + .Take(settings.MaxSplitsPerRun) + .ToList(); + foreach (var article in oversized) + { + var outcome = await splitter.SplitAsync(article, run.NoteBySlug, plan, ct); + if (outcome is null) continue; + // children first, so a note that moves wholly to a child never goes unassigned + foreach (var child in outcome.Children) + { + plan.AddArticle(child with { Sources = [] }); + plan.ReplaceArticle(child); + } + plan.ReplaceArticle(outcome.Overview); + run.Children[article.Slug] = outcome.Children; + run.Split++; + } + } + + /// + /// Thin articles with a nearby article, not kept before at the same inputs, at most + /// calls. An answer other than a usable fold is remembered like a keep, so the article is not asked again until its inputs change. + /// + private async Task FoldAsync(Run run, CancellationToken ct) + { + var plan = run.Plan; + var thin = plan.Articles.Where(a => FoldArticleUseCase.IsThin(a, run.NoteBySlug, settings)).Select(a => a.Slug).ToList(); + var calls = 0; + foreach (var slug in thin) + { + if (calls >= settings.MaxFoldsPerRun) return; + if (plan.Find(slug) is not { } article || !FoldArticleUseCase.IsThin(article, run.NoteBySlug, settings)) continue; + var fingerprint = EncyclopediaState.SubjectFingerprint(article, run.Notes); + if (run.Kept.GetValueOrDefault(slug) == fingerprint) continue; + var nearby = Articles(plan, await neighbours.NearbyArticlesAsync(article, run.NoteBySlug, plan, ct)); + if (nearby.Count == 0) continue; + + calls++; + var into = await folder.FoldAsync(article, run.NoteBySlug, nearby, ct); + if (into is null) + { + run.Kept[slug] = fingerprint; + continue; + } + plan.Fold(slug, into); + run.Folded++; + } + } + + private async Task<(int Written, int Rejected, bool Stopped, ExceptionDispatchInfo? Failure)> WriteAsync( + Run run, IReadOnlyList due, Dictionary hashByPath, List changes, CancellationToken ct) + { + int written = 0, rejected = 0; + var neighboursUp = true; foreach (var planned in due) { - var fingerprint = EncyclopediaState.SubjectFingerprint(planned, notes); - var sources = planned.Sources.Where(noteBySlug.ContainsKey).Select(s => noteBySlug[s]).ToList(); + var fingerprint = EncyclopediaState.SubjectFingerprint(planned, run.Notes); + var sources = WriteSources(planned, run.NoteBySlug, settings); try { - var article = await writer.WriteAsync(planned, sources, plan.Articles, ct); + IReadOnlyList targets = []; + if (neighboursUp) + { + try + { + targets = Articles(run.Plan, await neighbours.NearbyArticlesAsync(planned, run.NoteBySlug, run.Plan, ct)); + } + catch (HttpRequestException e) + { + log.LogWarning(e, "Nearby articles unavailable; writing without link targets"); + neighboursUp = false; + } + } + if (run.Children.TryGetValue(planned.Slug, out var children)) + targets = Articles(run.Plan, [.. targets.Select(t => t.Slug), .. children.Select(c => c.Slug)]); + + var article = await writer.WriteAsync(planned, sources, targets, ct); foreach (var a in new[] { article.De, article.En }) { var path = ArticlePath(a.Lang, a.Slug); changes.Add(Put(path, ArticleFile.Build(a), hashByPath.GetValueOrDefault(path))); } - subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Written, now); + run.Subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Written, run.Now); written++; } catch (ArticleRejectedException e) { log.LogWarning("Article {Slug} rejected: {Reason}; kept as it was, retried when its inputs change", e.Slug, e.Reason); - subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Rejected, now); + run.Subjects[planned.Slug] = new SubjectAttempt(fingerprint, AttemptOutcome.Rejected, run.Now); rejected++; } catch (HttpRequestException e) @@ -127,8 +310,7 @@ public sealed class EncyclopediaReconciler( catch (ModelUnavailableException e) { log.LogWarning(e, "Text model unavailable; stopping after {Written} articles", written); - stopped = true; - break; + return (written, rejected, true, null); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { @@ -137,48 +319,29 @@ public sealed class EncyclopediaReconciler( catch (Exception e) { log.LogError(e, "Article {Slug} failed unexpectedly; committing what is done, then failing the run", planned.Slug); - failure = ExceptionDispatchInfo.Capture(e); - break; + return (written, rejected, false, ExceptionDispatchInfo.Capture(e)); } } - - // Deletes travel with the plan that drops the subjects: once that plan is committed, the difference is gone. - var deleted = 0; - if (replanned is not null && current is not null) - { - var kept = replanned.Articles.Select(a => a.Slug).ToHashSet(StringComparer.Ordinal); - foreach (var slug in current.Articles.Select(a => a.Slug).Where(s => !kept.Contains(s)).Distinct(StringComparer.Ordinal)) - { - var any = false; - foreach (var lang in Languages) - { - var path = ArticlePath(lang, slug); - if (!hashByPath.TryGetValue(path, out var hash)) continue; - changes.Add(new FileChange(FileChangeOp.Delete, path, null, hash)); - any = true; - } - subjects.Remove(slug); - if (any) deleted++; - } - } - - var summary = await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText, - new ReconcileSummary(plan.Articles.Count, written, deleted, rejected, remembered, stopped, null, null), ct); - failure?.Throw(); - return summary; + return (written, rejected, false, null); } /// /// Planned articles, in plan order, whose files no longer match the plan and the notes: missing in either language, - /// a different set of sources than the plan's sources that exist, a source hash other than the catalog's, or another title. + /// a different set of sources than the plan's sources the writer gets (), a source hash other + /// than the catalog's, or another title. /// - public static IReadOnlyList Stale(ArticlePlan plan, IReadOnlyList notes, IReadOnlyList
existing) + public static IReadOnlyList Stale(ArticlePlan plan, IReadOnlyList notes, IReadOnlyList
existing, + EncyclopediaSettings settings) { - var hashes = notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First().ContentHash, StringComparer.Ordinal); + var bySlug = notes.GroupBy(n => n.Slug, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); var articles = existing.GroupBy(a => (a.Lang, a.Slug)).ToDictionary(g => g.Key, g => g.First()); return plan.Articles - .Where(p => IsStale(p, hashes, articles.GetValueOrDefault(("de", p.Slug)), p.TitleDe) - || IsStale(p, hashes, articles.GetValueOrDefault(("en", p.Slug)), p.TitleEn)) + .Where(p => + { + var sources = WriteSources(p, bySlug, settings); + return IsStale(sources, articles.GetValueOrDefault(("de", p.Slug)), p.TitleDe) + || IsStale(sources, articles.GetValueOrDefault(("en", p.Slug)), p.TitleEn); + }) .ToList(); } @@ -194,6 +357,24 @@ public sealed class EncyclopediaReconciler( .Take(settings.MaxArticlesPerRun) .ToList(); + /// + /// The sources that exist, most relevant first, up to of body + /// (at least one): an article waiting for its split is written from its first sources. + /// + private static List WriteSources(PlannedArticle planned, IReadOnlyDictionary notes, EncyclopediaSettings settings) + { + var kept = new List(); + long total = 0; + foreach (var slug in planned.Sources) + { + if (!notes.TryGetValue(slug, out var note)) continue; + total += note.Body.Length; + if (kept.Count > 0 && total > settings.MaxSourceCharsPerArticle) break; + kept.Add(note); + } + return kept; + } + private static IEnumerable InScope(IReadOnlyList stale, ArticlePlan plan, EncyclopediaSettings settings) { if (settings.PilotSlugs.Count > 0) @@ -214,14 +395,18 @@ public sealed class EncyclopediaReconciler( && attempt.Outcome == AttemptOutcome.Rejected && attempt.Fingerprint == EncyclopediaState.SubjectFingerprint(planned, notes); - private static bool IsStale(PlannedArticle planned, Dictionary hashes, Article? article, string title) + private static bool IsStale(List sources, Article? article, string title) { if (article is null || article.Title != title) return true; - var sources = planned.Sources.Where(hashes.ContainsKey).ToHashSet(StringComparer.Ordinal); - return !article.Sources.Select(s => s.Slug).ToHashSet(StringComparer.Ordinal).SetEquals(sources) + var hashes = sources.ToDictionary(d => d.Slug, d => d.ContentHash, StringComparer.Ordinal); + return !article.Sources.Select(s => s.Slug).ToHashSet(StringComparer.Ordinal).SetEquals(hashes.Keys) || article.Sources.Any(s => hashes[s.Slug] != s.ContentHash); } + /// The live articles stand for, in order, each once. + private static List Articles(ArticlePlan plan, IEnumerable slugs) => + slugs.Select(plan.Resolve).OfType().Distinct(StringComparer.Ordinal).Select(s => plan.Find(s)!).ToList(); + private async Task FinishAsync(List changes, EncyclopediaState state, StoredText? stateText, ReconcileSummary summary, CancellationToken ct) { @@ -234,7 +419,8 @@ public sealed class EncyclopediaReconciler( return summary; } - var message = $"articles: {summary.Written} written, {summary.Deleted} deleted, {summary.Rejected} rejected"; + var message = $"articles: {summary.Assigned} assigned, {summary.Split} split, {summary.Folded} folded, " + + $"{summary.Written} written, {summary.Deleted} deleted, {summary.Rejected} rejected"; try { var commit = await store.CommitAsync(changes, message, ct); @@ -265,19 +451,20 @@ public sealed class EncyclopediaReconciler( return articles; } - private ArticlePlan? ReadPlan(StoredText? text) + private ArticlePlan ReadPlan(StoredText? text) { - if (text is null) return null; + if (text is null) return ArticlePlan.Empty(); var plan = ArticlePlanJson.Read(text.Text); - if (plan is null) log.LogWarning("{Path} is malformed; planning from scratch", PlanPath); - return plan; + if (plan is null) log.LogWarning("{Path} is malformed or in the old format; planning from scratch", PlanPath); + return plan ?? ArticlePlan.Empty(); } private EncyclopediaState ReadState(StoredText? text) { if (text is null) return EncyclopediaState.Empty; var state = EncyclopediaStateJson.Read(text.Text); - if (state.Subjects.Count == 0 && state.FailedPlan is null && text.Text.Trim() != EncyclopediaStateJson.Write(EncyclopediaState.Empty)) + if (state.Subjects.Count == 0 && state.FailedBatches.Count == 0 && state.KeptArticles.Count == 0 + && text.Text.Trim() != EncyclopediaStateJson.Write(EncyclopediaState.Empty)) log.LogWarning("{Path} is malformed or empty; starting without attempt memory", StatePath); return state; } diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs new file mode 100644 index 0000000..4fa2eab --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaSettings.cs @@ -0,0 +1,25 @@ +namespace Wiki.Application.Encyclopedia; + +/// +/// Limits of one encyclopedia run. and limit what is written, not what is planned. +/// The Max…PerRun fields cap each step's model calls; the rest bound what one call sees. +/// +public sealed record EncyclopediaSettings( + int ArticleMaxTokens, + int MaxArticlesPerRun, + int MaxSourceCharsPerArticle, + IReadOnlyList PilotSlugs, + int? MaxArticles, + int AssignBatchSize = 20, + int CandidateArticles = 10, + int NeighbourNotes = 30, + int NeighbourSources = 10, + int MaxAssignBatchesPerRun = 120, + int MaxSplitsPerRun = 5, + int MaxFoldsPerRun = 5, + int MaxSourcesPerArticle = 120, + int FoldMinSources = 3, + int FoldMinChars = 4000, + int AssignMaxTokens = 8000, + int SplitMaxTokens = 8000, + int FoldMaxTokens = 1000); diff --git a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs index 7c759cc..48ab77a 100644 --- a/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs +++ b/src/Wiki.Application/Encyclopedia/EncyclopediaState.cs @@ -15,32 +15,26 @@ public enum AttemptOutcome /// The last attempt at one subject: the inputs it was made from and how it ended. public sealed record SubjectAttempt(string Fingerprint, AttemptOutcome Outcome, DateTimeOffset At); -/// A planning attempt that produced no usable plan, for the note set it was made with. -public sealed record PlanAttempt(string NoteSetFingerprint, DateTimeOffset At); - /// /// What the nightly job remembers between runs (articles/_state.json), so it does not pay again /// for the same failure. An attempt is repeated only when its fingerprint changes. /// /// /// FailedBatches: fingerprint of an assignment batch whose reply was unusable → when. -/// KeptArticles: article slug → at which the fold call answered keep. +/// KeptArticles: article slug → at which the fold call did not fold it. +/// The failedPlan entry of earlier versions is ignored on read. /// public sealed record EncyclopediaState( IReadOnlyDictionary Subjects, - PlanAttempt? FailedPlan, IReadOnlyDictionary FailedBatches, IReadOnlyDictionary KeptArticles) { public static EncyclopediaState Empty => - new(new Dictionary(), null, new Dictionary(), new Dictionary()); + new(new Dictionary(), new Dictionary(), new Dictionary()); public static string BatchFingerprint(IReadOnlyList notes) => Hash(Prompts.Version.ToString(), notes.Select(n => $"{n.Slug}@{n.ContentHash}").Order(StringComparer.Ordinal)); - public static string NoteSetFingerprint(IReadOnlyList notes) => - Hash(Prompts.Version.ToString(), notes.Select(n => n.Slug).Order(StringComparer.Ordinal)); - public static string SubjectFingerprint(PlannedArticle planned, IReadOnlyList notes) { var hashes = notes.GroupBy(n => n.Slug).ToDictionary(g => g.Key, g => g.First().ContentHash); @@ -76,7 +70,7 @@ public static class EncyclopediaStateJson var subjects = s.Subjects.Where(kv => kv.Value is not null).ToDictionary(kv => kv.Key, kv => kv.Value); var failedBatches = (s.FailedBatches ?? new Dictionary()).ToDictionary(); var kept = (s.KeptArticles ?? new Dictionary()).Where(kv => kv.Value is not null).ToDictionary(); - return new EncyclopediaState(subjects, s.FailedPlan, failedBatches, kept); + return new EncyclopediaState(subjects, failedBatches, kept); } catch (JsonException) { diff --git a/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs index 3d15791..ecdc117 100644 --- a/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/FoldArticleUseCase.cs @@ -1,10 +1,9 @@ using System.Net; using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; using Microsoft.Extensions.Logging; using Wiki.Core.Model; using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; namespace Wiki.Application.Encyclopedia; @@ -40,44 +39,32 @@ public sealed class FoldArticleUseCase(ITextModel model, EncyclopediaSettings se return null; } - JsonObject? root; - try - { - root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; - } - catch (JsonException) - { - root = null; - } + var root = PlanParsing.Object(reply.Text); if (root is null) { log.LogWarning("Fold reply for {Slug} is not a JSON object", thin.Slug); return null; } - if (root["into"] is JsonValue v && v.TryGetValue(out var into) && !string.IsNullOrWhiteSpace(into)) + if (Text(root["into"]) is { } into) { - into = into.Trim(); if (shown.Any(n => n.Slug == into)) return into; log.LogWarning("Fold reply for {Slug} names {Target}, which is not a nearby article", thin.Slug, into); } return null; } - private static string UserPrompt(PlannedArticle thin, IReadOnlyDictionary notes, List nearby) + private string UserPrompt(PlannedArticle thin, IReadOnlyDictionary notes, List nearby) { var sb = new StringBuilder(); sb.Append("Thin article: ").Append(thin.Slug).Append(" | ").Append(Flatten(thin.TitleDe)).Append(" | ") .Append(Flatten(thin.TitleEn)).Append(" | ").Append(Flatten(thin.Scope)).Append('\n'); sb.Append("Its sources (title):\n"); - foreach (var s in thin.Sources) + foreach (var s in thin.Sources.Take(settings.FoldMinSources)) sb.Append(notes.TryGetValue(s, out var d) ? Flatten(d.Title) : s).Append('\n'); sb.Append("\nNearby articles (slug | title | scope):\n"); foreach (var n in nearby) sb.Append(n.Slug).Append(" | ").Append(Flatten(n.TitleDe)).Append(" | ").Append(Flatten(n.Scope)).Append('\n'); return sb.ToString(); } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); } diff --git a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs b/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs deleted file mode 100644 index 8d4b904..0000000 --- a/src/Wiki.Application/Encyclopedia/PlanArticlesUseCase.cs +++ /dev/null @@ -1,152 +0,0 @@ -using System.Net; -using System.Text; -using System.Text.RegularExpressions; -using Microsoft.Extensions.Logging; -using Wiki.Core.Model; -using Wiki.Core.Ports; - -namespace Wiki.Application.Encyclopedia; - -/// Limits of one encyclopedia run. and limit what is written, not what is planned. -public sealed record EncyclopediaSettings( - int PlanMaxTokens, - int ArticleMaxTokens, - int MaxArticlesPerRun, - int MaxSourceCharsPerArticle, - IReadOnlyList PilotSlugs, - int? MaxArticles) -{ - public int AssignBatchSize { get; init; } = 20; - public int CandidateArticles { get; init; } = 10; - public int NeighbourNotes { get; init; } = 30; - public int NeighbourSources { get; init; } = 10; - public int MaxAssignBatchesPerRun { get; init; } = 120; - public int MaxSplitsPerRun { get; init; } = 5; - public int MaxFoldsPerRun { get; init; } = 5; - public int MaxSourcesPerArticle { get; init; } = 120; - public int FoldMinSources { get; init; } = 3; - public int FoldMinChars { get; init; } = 4000; - public int AssignMaxTokens { get; init; } = 8000; - public int SplitMaxTokens { get; init; } = 8000; - public int FoldMaxTokens { get; init; } = 1000; -} - -/// -/// Asks the model which subjects get their own article and which notes feed each, and validates the answer. -/// A reply that cannot be used leaves the current plan in place. -/// -public sealed partial class PlanArticlesUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) -{ - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - - /// True when the notes changed since was planned, unless planning already failed for exactly this note set. - public static bool NeedsReplan(ArticlePlan? current, PlanAttempt? failedPlan, IReadOnlyList notes) - { - var changed = current is null - || !current.Assignments.Keys.ToHashSet(StringComparer.Ordinal).SetEquals(notes.Select(n => n.Slug)); - return changed && failedPlan?.NoteSetFingerprint != EncyclopediaState.NoteSetFingerprint(notes); - } - - /// The new plan, or null to keep . Model-unavailable and other errors propagate. - public async Task PlanAsync(ArticlePlan? current, IReadOnlyList notes, CancellationToken ct) - { - TextReply reply; - try - { - reply = await model.CompleteAsync( - Prompts.Plan.Replace(Prompts.MaxSourceCharsToken, settings.MaxSourceCharsPerArticle.ToString()), - UserPrompt(current, notes), settings.PlanMaxTokens, ct); - } - catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.BadRequest) - { - log.LogWarning(e, "Planning request was rejected; keeping the current plan"); - return null; - } - - if (reply.Truncated) - { - log.LogWarning("Plan reply hit the token cap ({Tokens}); keeping the current plan", settings.PlanMaxTokens); - return null; - } - - var parsed = ArticlePlanJson.ReadArticles(reply.Text); - if (parsed is null) - { - log.LogWarning("Plan reply is not valid JSON; keeping the current plan"); - return null; - } - - var articles = Validate(parsed, notes); - if (articles.Count == 0) - { - log.LogWarning("Plan reply has no valid article; keeping the current plan"); - return null; - } - var plan = ArticlePlan.Empty(); - foreach (var a in articles) plan.AddArticle(a); - foreach (var slug in notes.Select(n => n.Slug).Distinct(StringComparer.Ordinal)) - plan.Assign(slug, articles.Where(a => a.Sources.Contains(slug)).Select(a => a.Slug).ToList()); - return plan; - } - - private List Validate(IReadOnlyList proposed, IReadOnlyList notes) - { - var lengths = notes.GroupBy(n => n.Slug).ToDictionary(g => g.Key, g => g.First().Body.Length); - var seen = new HashSet(StringComparer.Ordinal); - var result = new List(); - foreach (var a in proposed) - { - if (!SlugPattern().IsMatch(a.Slug)) - { - log.LogWarning("Dropping planned article with invalid slug {Slug}", a.Slug); - continue; - } - if (!seen.Add(a.Slug)) - { - log.LogWarning("Dropping duplicate planned article {Slug}", a.Slug); - continue; - } - var sources = a.Sources.Where(lengths.ContainsKey).Distinct(StringComparer.Ordinal).ToList(); - if (sources.Count == 0) - { - log.LogWarning("Dropping planned article {Slug}: no known source", a.Slug); - continue; - } - sources = CapSources(a.Slug, sources, lengths); - result.Add(a with { Sources = sources }); - } - return result; - } - - private List CapSources(string slug, List sources, Dictionary lengths) - { - var kept = new List(); - long total = 0; - foreach (var s in sources) - { - total += lengths[s]; - if (kept.Count > 0 && total > settings.MaxSourceCharsPerArticle) break; - kept.Add(s); - } - if (kept.Count < sources.Count) - log.LogWarning("Planned article {Slug} exceeds {Cap} source chars; dropped {Dropped} lowest-ranked sources", - slug, settings.MaxSourceCharsPerArticle, sources.Count - kept.Count); - return kept; - } - - private static string UserPrompt(ArticlePlan? current, IReadOnlyList notes) - { - var sb = new StringBuilder(); - if (current is not null) - sb.Append("Current plan:\n").Append(ArticlePlanJson.WriteArticlesCompact(current)).Append("\n\n"); - sb.Append("Notes (slug | title | tags | body length in chars):\n"); - foreach (var n in notes.OrderBy(n => n.Slug, StringComparer.Ordinal)) - sb.Append(n.Slug).Append(" | ").Append(Flatten(n.Title)).Append(" | ").Append(Flatten(string.Join(", ", n.Tags))) - .Append(" | ").Append(n.Body.Length).Append('\n'); - return sb.ToString(); - } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); -} diff --git a/src/Wiki.Application/Encyclopedia/PlanParsing.cs b/src/Wiki.Application/Encyclopedia/PlanParsing.cs new file mode 100644 index 0000000..34a2e8d --- /dev/null +++ b/src/Wiki.Application/Encyclopedia/PlanParsing.cs @@ -0,0 +1,53 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Wiki.Core.Model; + +namespace Wiki.Application.Encyclopedia; + +/// Reading of plan files and planning replies, and the prompt-line form of their fields. +internal static partial class PlanParsing +{ + [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] + public static partial Regex SlugPattern(); + + /// Trimmed, as frontmatter reads it back; null when blank or not a string. + public static string? Text(JsonNode? n) => + n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; + + /// The non-blank strings of an array; empty when is not an array. + public static List Strings(JsonNode? n) => + n is JsonArray a ? a.Select(Text).OfType().ToList() : []; + + /// A category name in letters only, any case. + public static bool TryCategory(string? text, out ArticleCategory category) + { + category = default; + return text is not null && text.All(char.IsAsciiLetter) && Enum.TryParse(text, ignoreCase: true, out category); + } + + /// The JSON object in a reply, which may be wrapped in a code fence; null when it is not one. + public static JsonObject? Object(string reply) + { + try + { + return JsonNode.Parse(StripFence(reply)) as JsonObject; + } + catch (JsonException) + { + return null; + } + } + + public static string StripFence(string text) + { + var t = text.Trim(); + if (!t.StartsWith("```")) return t; + var start = t.IndexOf('\n'); + var end = t.LastIndexOf("```", StringComparison.Ordinal); + return start >= 0 && end > start ? t[(start + 1)..end].Trim() : t; + } + + /// Keeps a field on its line and inside its column. + public static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); +} diff --git a/src/Wiki.Application/Encyclopedia/Prompts.cs b/src/Wiki.Application/Encyclopedia/Prompts.cs index 1c3e871..e0c680a 100644 --- a/src/Wiki.Application/Encyclopedia/Prompts.cs +++ b/src/Wiki.Application/Encyclopedia/Prompts.cs @@ -54,29 +54,6 @@ public static class Prompts Reply with JSON only, no prose and no code fence: {"into":"article-slug"} to fold, or {"keep":true} to keep. """; - /// Placeholder in replaced by MaxSourceCharsPerArticle. - public const string MaxSourceCharsToken = "{maxSourceChars}"; - - public const string Plan = """ - [plan] - You plan the article set of an encyclopedia about one household: the house and its homelab IT. Its readers are everyone who needs to know things about the house, from the owner to guests. - You receive the wiki's notes, one per line as `slug | title | tags | body length in chars`, and the current plan as JSON if there is one. - - Decide which subjects get their own article and which notes feed each article. - - Granularity follows the material, like notability on Wikipedia. A subject gets its own article when the wiki holds enough on it: for example a heating valve controller with 17 automations and a complicated history. - - A subject that was set once and forgotten is not an article of its own. Its notes go into the article of its parent system as a section. - - Every note is used by at least one article, or is deliberately skipped. Skip only meta notes and notes about the wiki's own tooling. - - Keep the existing slugs for existing subjects. Do not rename or renumber them. - - Keep the existing titles for existing subjects unless the subject changed. - - Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses. Name the subject by what it is or does, for example "Netzwerkspeicher" rather than the server's hostname. - - List each article's sources most important first. When the sources of an article add up to more than {maxSourceChars} characters of body, split the subject into narrower articles. - - Slugs are lowercase ASCII words joined by single hyphens. Categories are room, system, device or topic. - - related lists the slugs of other articles in this plan that a reader of the article should continue with. - - Reply with JSON only, no prose and no code fence. Write it compact on a single line, with no indentation, no line breaks and no spaces between tokens, because the reply is size-limited. Shape: - {"articles":[{"slug":"...","titleDe":"...","titleEn":"...","category":"room|system|device|topic","sources":["note-slug"],"related":["article-slug"]}]} - """; - /// The generic-examples rule, shared by and . public const string GenericExamples = "A generic example that explains a principle and names nothing concrete in this house, for example what an entity ID or an IP address looks like, is not a leak and stays public"; diff --git a/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs index 3a1de92..373e91c 100644 --- a/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/SplitArticleUseCase.cs @@ -1,11 +1,10 @@ using System.Net; using System.Text; -using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Wiki.Core.Model; using Wiki.Core.Ports; +using static Wiki.Application.Encyclopedia.PlanParsing; namespace Wiki.Application.Encyclopedia; @@ -13,14 +12,11 @@ namespace Wiki.Application.Encyclopedia; public sealed record SplitOutcome(PlannedArticle Overview, IReadOnlyList Children); /// Splits one oversized article into an overview and narrower children, in one bounded model call. -public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) +public sealed class SplitArticleUseCase(ITextModel model, EncyclopediaSettings settings, ILogger log) { private const int MinChildren = 2; private const int MaxChildren = 6; - [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex SlugPattern(); - /// Body chars of the sources that exist in exceed the char cap, or the source count exceeds the count cap. public static bool NeedsSplit(PlannedArticle a, IReadOnlyDictionary notes, EncyclopediaSettings s) => a.Sources.Count > s.MaxSourcesPerArticle @@ -54,15 +50,7 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe return null; } - JsonObject? root; - try - { - root = JsonNode.Parse(ArticlePlanJson.StripFence(reply.Text)) as JsonObject; - } - catch (JsonException) - { - root = null; - } + var root = PlanParsing.Object(reply.Text); if (root is null) { log.LogWarning("Split reply for {Slug} is not a JSON object", article.Slug); @@ -110,8 +98,8 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe return null; } - var kept = overviewSources.Distinct(StringComparer.Ordinal) - .OrderBy(s => article.Sources.ToList().IndexOf(s)).ToList(); + var rank = article.Sources.Select((s, i) => (s, i)).DistinctBy(x => x.s).ToDictionary(x => x.s, x => x.i, StringComparer.Ordinal); + var kept = overviewSources.Distinct(StringComparer.Ordinal).OrderBy(s => rank[s]).ToList(); var rest = article.Sources.Where(s => !input.Contains(s)).ToList(); var overviewAll = kept.Concat(rest).ToList(); if (overviewAll.Count == 0) @@ -135,9 +123,7 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe log.LogWarning("Split of {Slug}: child slug {Child} is taken or duplicate", article.Slug, slug); return null; } - var categoryText = Text(o["category"]); - if (categoryText is null || !categoryText.All(char.IsAsciiLetter) - || !Enum.TryParse(categoryText, ignoreCase: true, out var category)) + if (!TryCategory(Text(o["category"]), out var category)) { log.LogWarning("Split of {Slug}: child {Child} has an unknown category", article.Slug, slug); return null; @@ -156,12 +142,6 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe return new PlannedArticle(slug, titleDe, titleEn, category, scope, sources); } - private static string? Text(JsonNode? n) => - n is JsonValue v && v.TryGetValue(out var s) && !string.IsNullOrWhiteSpace(s) ? s.Trim() : null; - - private static List Strings(JsonNode? n) => - n is JsonArray a ? a.Select(Text).OfType().ToList() : []; - private static string UserPrompt(PlannedArticle article, List shown, IReadOnlyDictionary notes) { var sb = new StringBuilder(); @@ -177,7 +157,4 @@ public sealed partial class SplitArticleUseCase(ITextModel model, EncyclopediaSe } return sb.ToString(); } - - /// Keeps a field on its line and inside its column. - private static string Flatten(string s) => s.Replace('\r', ' ').Replace('\n', ' ').Replace('|', ' '); } diff --git a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs index ec8dd9b..8380798 100644 --- a/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs +++ b/src/Wiki.Application/Encyclopedia/WriteArticleUseCase.cs @@ -200,7 +200,7 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe { try { - if (JsonNode.Parse(ArticlePlanJson.StripFence(text)) is not JsonObject root || root["privatize"] is not JsonArray items) return null; + if (JsonNode.Parse(PlanParsing.StripFence(text)) is not JsonObject root || root["privatize"] is not JsonArray items) return null; var result = new List(); foreach (var item in items) { @@ -219,7 +219,7 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe { try { - if (JsonNode.Parse(ArticlePlanJson.StripFence(text)) is not JsonArray items) return null; + if (JsonNode.Parse(PlanParsing.StripFence(text)) is not JsonArray items) return null; var result = new List(); foreach (var item in items) { diff --git a/src/Wiki.Host/Api/ApiV1.cs b/src/Wiki.Host/Api/ApiV1.cs index dea0a65..e7ee6d9 100644 --- a/src/Wiki.Host/Api/ApiV1.cs +++ b/src/Wiki.Host/Api/ApiV1.cs @@ -28,7 +28,7 @@ public static class ApiV1 }).RequireAuthorization(Auth.Policies.Write); // Syncs first so the run plans from the current notes. The run outlives a caller that hangs up; - // only shutdown stops it. A run the text model stopped is a failure for the caller. + // only shutdown stops it. A run the text model stopped, or one that ends without a planned article, is a failure for the caller. v1.MapPost("/encyclopedia/run", async (IServiceProvider services, SyncCoordinator sync, IHostApplicationLifetime lifetime) => { var encyclopedia = services.GetService(); @@ -38,7 +38,7 @@ public static class ApiV1 await sync.RunAsync(force: false, ct); var summary = await encyclopedia.RunAsync(ct); return summary is null ? Results.Json(new { error = encyclopedia.LastError }, statusCode: StatusCodes.Status502BadGateway) - : summary.StoppedByModel ? Results.Json(summary, statusCode: StatusCodes.Status502BadGateway) + : summary.StoppedByModel || summary.Planned == 0 ? Results.Json(summary, statusCode: StatusCodes.Status502BadGateway) : Results.Ok(summary); }).RequireAuthorization(Auth.Policies.Write); } diff --git a/src/Wiki.Host/Components/Shared/StatusPanel.razor b/src/Wiki.Host/Components/Shared/StatusPanel.razor index d3c07cd..a2bd405 100644 --- a/src/Wiki.Host/Components/Shared/StatusPanel.razor +++ b/src/Wiki.Host/Components/Shared/StatusPanel.razor @@ -42,7 +42,7 @@ Last run@Status.LastEncyclopediaAttemptAt.Value.ToString("u") @if (Status.LastEncyclopediaRun is { } r) { - Result@r.Planned planned, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") + Result@r.Planned planned, @r.Assigned assigned, @r.Split split, @r.Folded folded, @r.Written written, @r.Deleted deleted, @r.Rejected rejected@(r.Skipped is null ? "" : $" (skipped: {r.Skipped})")@(r.StoppedByModel ? " (stopped: model unavailable)" : "") } @if (Status.LastEncyclopediaError is not null) { diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs index 0d60fd9..6bbcab6 100644 --- a/src/Wiki.Host/Composition/WikiOptions.cs +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -84,11 +84,24 @@ public sealed class EncyclopediaOptions public string AnswerModel { get; set; } = "claude-sonnet"; public string Model { get; set; } = "claude-sonnet-5"; public int ArticleMaxTokens { get; set; } = 16000; - public int PlanMaxTokens { get; set; } = 32000; public int MaxArticlesPerRun { get; set; } = 10; public int MaxSourceCharsPerArticle { get; set; } = 300_000; /// When non-empty, only these subjects are written. public List PilotSlugs { get; set; } = []; /// Fallback write cap: the first N planned subjects. public int? MaxArticles { get; set; } = null; + public int AssignBatchSize { get; set; } = 20; + /// Nearby articles per note or article; an assign call sees at most twice this. + public int CandidateArticles { get; set; } = 10; + public int NeighbourNotes { get; set; } = 30; + public int NeighbourSources { get; set; } = 10; + public int MaxAssignBatchesPerRun { get; set; } = 120; + public int MaxSplitsPerRun { get; set; } = 5; + public int MaxFoldsPerRun { get; set; } = 5; + public int MaxSourcesPerArticle { get; set; } = 120; + public int FoldMinSources { get; set; } = 3; + public int FoldMinChars { get; set; } = 4000; + public int AssignMaxTokens { get; set; } = 8000; + public int SplitMaxTokens { get; set; } = 8000; + public int FoldMaxTokens { get; set; } = 1000; } diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index 1fe9d15..b385f1b 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -114,9 +114,26 @@ if (encyclopedia.Enabled) o.ApiKey = builder.Configuration[$"Wiki:AnswerModels:Models:{encyclopedia.AnswerModel}:ApiKey"] ?? string.Empty; }); builder.Services.AddSingleton(new Wiki.Application.Encyclopedia.EncyclopediaSettings( - encyclopedia.PlanMaxTokens, encyclopedia.ArticleMaxTokens, encyclopedia.MaxArticlesPerRun, - encyclopedia.MaxSourceCharsPerArticle, encyclopedia.PilotSlugs, encyclopedia.MaxArticles)); - builder.Services.AddSingleton(); + encyclopedia.ArticleMaxTokens, encyclopedia.MaxArticlesPerRun, encyclopedia.MaxSourceCharsPerArticle, + encyclopedia.PilotSlugs, encyclopedia.MaxArticles, + AssignBatchSize: encyclopedia.AssignBatchSize, + CandidateArticles: encyclopedia.CandidateArticles, + NeighbourNotes: encyclopedia.NeighbourNotes, + NeighbourSources: encyclopedia.NeighbourSources, + MaxAssignBatchesPerRun: encyclopedia.MaxAssignBatchesPerRun, + MaxSplitsPerRun: encyclopedia.MaxSplitsPerRun, + MaxFoldsPerRun: encyclopedia.MaxFoldsPerRun, + MaxSourcesPerArticle: encyclopedia.MaxSourcesPerArticle, + FoldMinSources: encyclopedia.FoldMinSources, + FoldMinChars: encyclopedia.FoldMinChars, + AssignMaxTokens: encyclopedia.AssignMaxTokens, + SplitMaxTokens: encyclopedia.SplitMaxTokens, + FoldMaxTokens: encyclopedia.FoldMaxTokens)); + // ArticleNeighbours takes the active IEmbeddingModel: the index holds its vectors. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs index ec56520..a4d72ae 100644 --- a/test/Wiki.Application.Tests/ArticleNeighboursTests.cs +++ b/test/Wiki.Application.Tests/ArticleNeighboursTests.cs @@ -8,9 +8,9 @@ public class ArticleNeighboursTests { private const string Model = "m"; - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null); - private static Document Note(string slug, string hash = "h") => PlanArticlesUseCaseTests.Note(slug, hash: hash); + private static Document Note(string slug, string hash = "h") => TestNotes.Note(slug, hash: hash); private static PlannedArticle Article(string slug, params string[] sources) => new(slug, slug, slug, ArticleCategory.Topic, "scope", sources); diff --git a/test/Wiki.Application.Tests/ArticlePlanTests.cs b/test/Wiki.Application.Tests/ArticlePlanTests.cs index f5fa602..a925856 100644 --- a/test/Wiki.Application.Tests/ArticlePlanTests.cs +++ b/test/Wiki.Application.Tests/ArticlePlanTests.cs @@ -152,7 +152,7 @@ public class ArticlePlanTests Assert.Contains("\"category\": \"system\"", json); Assert.NotNull(read); Assert.Equal(plan.Articles, read.Articles, new PlannedArticleComparer()); - Assert.Equal(["n1", "n2", "meta"], read.Assignments.Keys); + Assert.Equal(["meta", "n1", "n2"], read.Assignments.Keys); Assert.Equal(["heating"], read.Assignments["n2"]); Assert.Empty(read.Assignments["meta"]); Assert.Equal("heating", read.Redirects["old"]); @@ -162,6 +162,27 @@ public class ArticlePlanTests public void Json_OldFormat_IsNull() => Assert.Null(ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"topic","sources":["x"],"related":[]}],"coveredNotes":["x"]}""")); + [Fact] + public void Json_Write_MapKeysOrdinal_SameForSameContent() + { + var plan = ArticlePlan.Empty(); + plan.AddArticle(Article("a")); + plan.AddArticle(Article("c", "n9")); + plan.AddArticle(Article("d", "n0")); + plan.Assign("n2", ["a"]); + plan.Assign("n1", ["a"]); + plan.Fold("d", "c"); + plan.AddArticle(Article("b", "n8")); + plan.Fold("b", "c"); + + var json = ArticlePlanJson.Write(plan); + + var maps = json[json.IndexOf("\"assignments\"", StringComparison.Ordinal)..]; + Assert.True(maps.IndexOf("\"n1\"", StringComparison.Ordinal) < maps.IndexOf("\"n2\"", StringComparison.Ordinal)); + Assert.True(maps.IndexOf("\"b\": \"c\"", StringComparison.Ordinal) < maps.IndexOf("\"d\": \"c\"", StringComparison.Ordinal)); + Assert.Equal(json, ArticlePlanJson.Write(ArticlePlanJson.Read(json)!)); + } + [Fact] public void Json_Malformed_IsNull() => Assert.Null(ArticlePlanJson.Read("not json")); diff --git a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs index 5c1c7a2..76e6208 100644 --- a/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs +++ b/test/Wiki.Application.Tests/AssignNotesUseCaseTests.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Tests; public class AssignNotesUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null); private static AssignNotesUseCase UseCase(FakeTextModel model) => new(model, Settings, NullLogger.Instance); @@ -22,7 +22,7 @@ public class AssignNotesUseCaseTests } private static Document[] Batch(params string[] slugs) => - slugs.Select(s => PlanArticlesUseCaseTests.Note(s)).ToArray(); + slugs.Select(s => TestNotes.Note(s)).ToArray(); [Fact] public async Task Assign_ExistingAndNew() @@ -130,7 +130,7 @@ public class AssignNotesUseCaseTests [Fact] public async Task Assign_FirstParagraph_SkipsHeadingsAndBlankLines() { - var note = PlanArticlesUseCaseTests.Note("a") with { Body = "# T\n\n \nfirst line\ncontinued\n\nsecond para" }; + var note = TestNotes.Note("a") with { Body = "# T\n\n \nfirst line\ncontinued\n\nsecond para" }; var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); await UseCase(model).AssignAsync([note], [], ArticlePlan.Empty(), CancellationToken.None); @@ -142,7 +142,7 @@ public class AssignNotesUseCaseTests [Fact] public async Task Assign_FirstParagraph_CutTo300() { - var note = PlanArticlesUseCaseTests.Note("a") with { Body = new string('y', 500) }; + var note = TestNotes.Note("a") with { Body = new string('y', 500) }; var model = new FakeTextModel().Reply("""{"new":[],"notes":[]}"""); await UseCase(model).AssignAsync([note], [], ArticlePlan.Empty(), CancellationToken.None); diff --git a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs index 0a62188..f2a4ab4 100644 --- a/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaCoordinatorTests.cs @@ -11,7 +11,7 @@ public class EncyclopediaCoordinatorTests public Task RunAsync(CancellationToken ct) => body(ct); } - private static readonly ReconcileSummary Done = new(3, 2, 0, 0, 0, false, null, "c1"); + private static readonly ReconcileSummary Done = new(3, 0, 0, 0, 2, 0, 0, 0, false, null, "c1"); private static EncyclopediaCoordinator Coordinator(Func> body) => new(new ScriptedRun(body), new FakeTimeProvider(new DateTimeOffset(2026, 10, 24, 4, 0, 0, TimeSpan.Zero)), diff --git a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs index 58f9c4e..67affb5 100644 --- a/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaReconcilerTests.cs @@ -6,6 +6,7 @@ using Wiki.Application.Encyclopedia; using Wiki.Application.Sync; using Wiki.Core.Model; using Wiki.Core.Parsing; +using Wiki.Core.Ports; namespace Wiki.Application.Tests; @@ -14,39 +15,56 @@ public class EncyclopediaReconcilerTests private const string PlanPath = "articles/_plan.json"; private const string StatePath = "articles/_state.json"; private static readonly DateTimeOffset Now = new(2026, 10, 24, 3, 0, 0, TimeSpan.Zero); - private static readonly EncyclopediaSettings Defaults = new(8000, 16000, 10, 300_000, [], null); - private static readonly (string Slug, string[] Sources)[] HeatingAndKitchen = + /// Fold is off (no article has fewer than zero sources) unless a test turns it on. + private static readonly EncyclopediaSettings Defaults = new(16000, 10, 300_000, [], null) { FoldMinSources = 0 }; + + private static readonly (string Slug, string[] Notes)[] HeatingAndKitchen = [("heating", ["boiler", "radiators"]), ("kitchen", ["kitchen"])]; private readonly FakeStore _store = new(); private readonly FakeCatalog _catalog = new(); - private readonly FakeTextModel _planModel = new("plan"); + private readonly FakeIndex _index = new(); + private readonly SwitchableEmbeddings _embeddings = new(new FakeEmbeddings("m")); + private readonly FakeTextModel _model = new("plan"); private readonly FakeWriter _writer = new(); private readonly FakeTimeProvider _clock = new(Now); private static string Note(string title, string body = "Body.") => $"---\ntitle: {title}\ntype: wiki\n---\n\n{body}\n"; - private static string PlanReply(params (string Slug, string[] Sources)[] articles) => - "{\"articles\":[" + string.Join(",", articles.Select(a => - $"{{\"slug\":\"{a.Slug}\",\"titleDe\":\"{a.Slug} de\",\"titleEn\":\"{a.Slug} en\",\"category\":\"topic\"," + - $"\"sources\":[{string.Join(",", a.Sources.Select(s => $"\"{s}\""))}],\"related\":[]}}")) + "]}"; + /// An assign reply creating , each fed by its notes. + private static string AssignReply(params (string Slug, string[] Notes)[] articles) => + TitledAssignReply(articles.Select(a => (a.Slug, a.Slug + " de", a.Slug + " en", a.Notes)).ToArray()); - private static string TitledPlanReply(string titleDe, string titleEn) => + private static string TitledAssignReply(params (string Slug, string TitleDe, string TitleEn, string[] Notes)[] articles) => JsonSerializer.Serialize(new { - articles = new[] { new { slug = "heating", titleDe, titleEn, category = "system", sources = new[] { "boiler" }, related = Array.Empty() } }, + @new = articles.Select(a => new { slug = a.Slug, titleDe = a.TitleDe, titleEn = a.TitleEn, category = "topic", scope = "About " + a.Slug + "." }), + notes = articles.SelectMany(a => a.Notes.Select(n => (Note: n, a.Slug))) + .GroupBy(x => x.Note) + .Select(g => new { note = g.Key, articles = g.Select(x => x.Slug).ToArray() }), + }); + + /// An assign reply sending each note to existing articles. + private static string ExistingReply(params (string Note, string Article)[] notes) => + JsonSerializer.Serialize(new + { + @new = Array.Empty(), + notes = notes.Select(n => new { note = n.Note, articles = new[] { n.Article } }), }); private EncyclopediaReconciler Reconciler(EncyclopediaSettings? settings = null, IArticleWriter? writer = null) { var s = settings ?? Defaults; - return new(_store, _catalog, new PlanArticlesUseCase(_planModel, s, NullLogger.Instance), + return new(_store, _catalog, new ArticleNeighbours(_embeddings, _index, s), + new AssignNotesUseCase(_model, s, NullLogger.Instance), + new SplitArticleUseCase(_model, s, NullLogger.Instance), + new FoldArticleUseCase(_model, s, NullLogger.Instance), writer ?? _writer, s, _clock, NullLogger.Instance); } private Task Sync(bool force = true) => - new SyncUseCase(_store, _catalog, new FakeIndex(), [new FakeEmbeddings("m")], _clock, NullLogger.Instance) + new SyncUseCase(_store, _catalog, _index, [new FakeEmbeddings("m")], _clock, NullLogger.Instance) .RunAsync(force, CancellationToken.None); private async Task SeedAsync(params string[] notes) @@ -56,11 +74,13 @@ public class EncyclopediaReconcilerTests await Sync(); } - /// Seeds the default notes, runs once with and syncs the result. + private static string[] Numbered(int count) => Enumerable.Range(1, count).Select(i => $"note-{i:00}").ToArray(); + + /// Seeds the default notes, runs once assigning and syncs the result. private async Task FirstRunAsync(EncyclopediaSettings? settings = null) { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); await Reconciler(settings).RunAsync(CancellationToken.None); await Sync(); } @@ -69,23 +89,315 @@ public class EncyclopediaReconcilerTests private EncyclopediaState State => EncyclopediaStateJson.Read(_store.Files[StatePath]); + private ArticlePlan Plan => ArticlePlanJson.Read(_store.Files[PlanPath])!; + [Fact] - public async Task Reconcile_FirstRun_PlansWritesCommitsOnce() + public async Task Reconcile_FirstRun_AssignsWritesCommitsOnce() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); var summary = await Reconciler().RunAsync(CancellationToken.None); var commit = Assert.Single(_store.Commits); - Assert.Equal("articles: 2 written, 0 deleted, 0 rejected", commit.Message); + Assert.Equal("articles: 3 assigned, 0 split, 0 folded, 2 written, 0 deleted, 0 rejected", commit.Message); Assert.Equal( [PlanPath, StatePath, "articles/de/heating.md", "articles/de/kitchen.md", "articles/en/heating.md", "articles/en/kitchen.md"], commit.Changes.Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.All(commit.Changes, c => Assert.Equal(FileChangeOp.Create, c.Op)); - Assert.Equal(new ReconcileSummary(2, 2, 0, 0, 0, false, null, "c2"), summary); + Assert.Equal(new ReconcileSummary(2, 3, 0, 0, 2, 0, 0, 0, false, null, "c2"), summary); Assert.Equal(AttemptOutcome.Written, State.Subjects["heating"].Outcome); Assert.Equal(Now, State.Subjects["heating"].At); + Assert.Equal(["boiler", "radiators"], Plan.Find("heating")!.Sources); + Assert.StartsWith("[assign]", Assert.Single(_model.Calls).System); + } + + [Fact] + public async Task Run_Bootstrap_LaterBatchesSeeEarlierArticles() + { + var notes = Numbered(40); + await SeedAsync(notes); + _model.Reply(AssignReply(("alpha", notes))); + _model.Reply(AssignReply(("beta", notes))); + + var summary = await Reconciler(Defaults with { AssignBatchSize = 20 }).RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.DoesNotContain("alpha |", _model.Calls[0].User); + Assert.Contains("alpha |", _model.Calls[1].User); + Assert.Equal(40, summary.Assigned); + Assert.Equal(40, Plan.Assignments.Count); + Assert.Equal(20, Plan.Find("alpha")!.Sources.Count); + Assert.Equal(20, Plan.Find("beta")!.Sources.Count); + } + + [Fact] + public async Task Run_OnlyNewNotesAreAssigned() + { + await FirstRunAsync(); + _store.Files["wiki/oven.md"] = Note("oven"); + await Sync(); + _model.Reply(ExistingReply(("oven", "kitchen"))); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.Contains("oven |", _model.Calls[1].User); + Assert.DoesNotContain("boiler |", _model.Calls[1].User); + Assert.Equal(1, summary.Assigned); + Assert.Equal(["kitchen", "oven"], Plan.Find("kitchen")!.Sources); + Assert.Equal(["heating", "kitchen", "kitchen"], _writer.Calls); + } + + [Fact] + public async Task Run_RemovedNote_DropsSourceAndDeletesEmptyArticle() + { + await FirstRunAsync(); + _store.Files.Remove("wiki/kitchen.md"); + _store.Files.Remove("wiki/radiators.md"); + await Sync(); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Single(_model.Calls); + Assert.Equal(1, summary.Deleted); + Assert.Equal(1, summary.Written); + Assert.Equal(1, summary.Planned); + Assert.Equal(["articles/de/kitchen.md", "articles/en/kitchen.md"], + LastCommit.Where(c => c.Op == FileChangeOp.Delete).Select(c => c.Path).Order(StringComparer.Ordinal)); + Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("kitchen", State.Subjects.Keys); + Assert.Null(Plan.Find("kitchen")); + Assert.Equal(["boiler"], Plan.Find("heating")!.Sources); + Assert.Equal(["boiler"], Plan.Assignments.Keys); + Assert.Equal("heating", _writer.Calls[^1]); + } + + [Fact] + public async Task Run_UnusableBatch_RememberedAndNotRetried() + { + await SeedAsync(); + _model.Reply("I would rather not."); + + var first = await Reconciler().RunAsync(CancellationToken.None); + await Sync(); + var second = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(0, first.Planned); + Assert.Single(_model.Calls); + Assert.Single(State.FailedBatches); + Assert.Equal([StatePath], Assert.Single(_store.Commits).Changes.Select(c => c.Path)); + Assert.Null(second.Commit); + Assert.Empty(_writer.Calls); + } + + [Fact] + public async Task Run_UnusableBatch_RetriedWhenANoteChanges() + { + await SeedAsync(); + _model.Reply("I would rather not."); + await Reconciler().RunAsync(CancellationToken.None); + _store.Files["wiki/boiler.md"] = Note("boiler", "New boiler."); + await Sync(); + _model.Reply(AssignReply(HeatingAndKitchen)); + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Equal(2, _model.Calls.Count); + Assert.Equal(2, summary.Planned); + } + + [Fact] + public async Task Run_SplitThenWriteOverviewWithChildrenAsLinks() + { + await SeedAsync("boiler", "radiators", "pump", "valve", "kitchen"); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "pump", "valve"]), ("kitchen", ["kitchen"]))); + _model.Reply(JsonSerializer.Serialize(new + { + overview = new { scope = "Heating in general.", sources = new[] { "boiler" } }, + children = new[] + { + new { slug = "heating-radiators", titleDe = "Heizkörper", titleEn = "Radiators", category = "device", scope = "The radiators.", sources = new[] { "radiators" } }, + new { slug = "heating-pump", titleDe = "Pumpe", titleEn = "Pump", category = "device", scope = "The pump.", sources = new[] { "pump" } }, + }, + })); + + // no nearby articles: the overview's link targets are its children alone + var summary = await Reconciler(Defaults with { MaxSourcesPerArticle = 3, CandidateArticles = 0 }).RunAsync(CancellationToken.None); + + Assert.Equal(1, summary.Split); + Assert.StartsWith("[split]", _model.Calls[1].System); + Assert.Equal(["boiler", "valve"], Plan.Find("heating")!.Sources); + Assert.Equal("Heating in general.", Plan.Find("heating")!.Scope); + Assert.Equal(["heating-pump"], Plan.Assignments["pump"]); + Assert.Equal(4, summary.Planned); + Assert.Equal(["heating-pump", "heating-radiators"], _writer.Links["heating"].Order(StringComparer.Ordinal)); + Assert.Empty(_writer.Links["kitchen"]); + Assert.Equal(["heating", "heating-pump", "heating-radiators", "kitchen"], _writer.Calls.Order(StringComparer.Ordinal)); + } + + [Fact] + public async Task Run_SplitChildrenAreLinksOnlyInTheSplittingRun() + { + await SeedAsync("boiler", "radiators", "pump", "valve"); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "pump", "valve"]))); + _model.Reply(JsonSerializer.Serialize(new + { + overview = new { scope = "Heating in general.", sources = new[] { "boiler" } }, + children = new[] + { + new { slug = "heating-radiators", titleDe = "Heizkörper", titleEn = "Radiators", category = "device", scope = "The radiators.", sources = new[] { "radiators" } }, + new { slug = "heating-pump", titleDe = "Pumpe", titleEn = "Pump", category = "device", scope = "The pump.", sources = new[] { "pump" } }, + }, + })); + var settings = Defaults with { MaxSourcesPerArticle = 3, CandidateArticles = 0 }; + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + _store.Files["wiki/boiler.md"] = Note("boiler", "New boiler."); + await Sync(); + + await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal("heating", _writer.Calls[^1]); + Assert.Empty(_writer.Links["heating"]); + } + + [Fact] + public async Task Run_FoldDeletesFilesAndAddsRedirect() + { + await FirstRunAsync(); + _model.Reply("""{"into":"heating"}"""); + + var summary = await Reconciler(Defaults with { FoldMinSources = 2 }).RunAsync(CancellationToken.None); + + Assert.Equal(1, summary.Folded); + Assert.Equal(1, summary.Deleted); + Assert.StartsWith("[fold]", _model.Calls[^1].System); + Assert.Contains("kitchen |", _model.Calls[^1].User); + Assert.Equal("heating", Plan.Redirects["kitchen"]); + Assert.Equal(["boiler", "radiators", "kitchen"], Plan.Find("heating")!.Sources); + Assert.Equal(["heating"], Plan.Assignments["kitchen"]); + Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("articles/en/kitchen.md", _store.Files.Keys); + Assert.DoesNotContain("kitchen", State.Subjects.Keys); + Assert.Equal("heating", _writer.Calls[^1]); + } + + [Fact] + public async Task Run_KeptArticle_NotAskedAgain() + { + var settings = Defaults with { FoldMinSources = 2 }; + await FirstRunAsync(); + _model.Reply("""{"keep":true}"""); + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + var calls = _model.Calls.Count; + + var summary = await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal(calls, _model.Calls.Count); + Assert.Equal(0, summary.Folded); + Assert.Equal(["kitchen"], State.KeptArticles.Keys); + Assert.Null(summary.Commit); + } + + [Fact] + public async Task Run_EmbeddingsDown_StillWritesStale() + { + await FirstRunAsync(); + _store.Files["wiki/boiler.md"] = Note("boiler", "New boiler."); + _store.Files["wiki/garage.md"] = Note("garage"); + await Sync(); + _embeddings.Down = true; + + var summary = await Reconciler().RunAsync(CancellationToken.None); + + Assert.Single(_model.Calls); + Assert.False(summary.StoppedByModel); + Assert.Equal(0, summary.Assigned); + Assert.Equal(1, summary.Written); + Assert.Equal("heating", _writer.Calls[^1]); + Assert.Empty(_writer.Links["heating"]); + Assert.NotNull(summary.Commit); + Assert.False(Plan.IsAssigned("garage")); + } + + [Fact] + public async Task Run_StepCapsRespected() + { + var notes = Numbered(40); + await SeedAsync(notes); + _model.Reply(AssignReply(("alpha", notes))); + + var summary = await Reconciler(Defaults with { AssignBatchSize = 20, MaxAssignBatchesPerRun = 1 }).RunAsync(CancellationToken.None); + + Assert.Single(_model.Calls); + Assert.Equal(20, summary.Assigned); + Assert.Equal(20, Plan.Assignments.Count); + } + + [Fact] + public async Task Run_SplitAndFoldCapsRespected() + { + await SeedAsync("a1", "a2", "b1", "b2", "c1", "c2"); + _model.Reply(AssignReply(("a", ["a1", "a2"]), ("b", ["b1", "b2"]), ("c", ["c1", "c2"]))); + _model.Reply("{}"); + _model.Reply("""{"keep":true}"""); + + await Reconciler(Defaults with { MaxSourcesPerArticle = 1, MaxSplitsPerRun = 1, FoldMinSources = 3, MaxFoldsPerRun = 1 }) + .RunAsync(CancellationToken.None); + + Assert.Equal(["[assign]", "[split]", "[fold]"], _model.Calls.Select(c => c.System.Split('\n')[0])); + } + + [Fact] + public async Task Run_ModelUnavailable_CommitsStaged() + { + var notes = Numbered(40); + await SeedAsync(notes); + _model.Reply(AssignReply(("alpha", notes))); + _model.Throw(new ModelUnavailableException("credit exhausted")); + + var summary = await Reconciler(Defaults with { AssignBatchSize = 20 }).RunAsync(CancellationToken.None); + + Assert.True(summary.StoppedByModel); + Assert.Equal(20, summary.Assigned); + Assert.Empty(_writer.Calls); + Assert.Equal([PlanPath], Assert.Single(_store.Commits).Changes.Select(c => c.Path)); + Assert.Equal(20, Plan.Assignments.Count); + } + + [Fact] + public async Task Run_WriteLinkTargetsAreBounded() + { + var notes = Numbered(15); + await SeedAsync(notes); + _model.Reply(AssignReply(notes.Select(n => ("a-" + n, new[] { n })).ToArray())); + var settings = Defaults with { CandidateArticles = 3, MaxArticlesPerRun = 20 }; + + await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal(15, _writer.Calls.Count); + Assert.All(_writer.Links.Values, l => Assert.InRange(l.Count, 0, 3)); + Assert.Contains(_writer.Links.Values, l => l.Count == 3); + } + + [Fact] + public async Task Run_SourcesBeyondCharCap_LeftOutAndNotStale() + { + _store.Files["wiki/a.md"] = Note("a", new string('x', 60)); + _store.Files["wiki/b.md"] = Note("b", new string('y', 60)); + await Sync(); + _model.Reply(AssignReply(("heating", ["a", "b"]))); + var settings = Defaults with { MaxSourceCharsPerArticle = 100, MaxSourcesPerArticle = 120, MaxSplitsPerRun = 0 }; + await Reconciler(settings).RunAsync(CancellationToken.None); + await Sync(); + + var summary = await Reconciler(settings).RunAsync(CancellationToken.None); + + Assert.Equal(["a"], _writer.Sources["heating"]); + Assert.Null(summary.Commit); + Assert.Single(_writer.Calls); } [Fact] @@ -97,7 +409,7 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); Assert.Equal("catalog-behind", summary.Skipped); - Assert.Empty(_planModel.Calls); + Assert.Empty(_model.Calls); Assert.Empty(_writer.Calls); Assert.Empty(_store.Commits); } @@ -106,7 +418,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_AfterOwnCommit_IsNoop() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); await Reconciler().RunAsync(CancellationToken.None); var sync = await Sync(force: false); Assert.False(sync.Skipped); @@ -116,7 +428,7 @@ public class EncyclopediaReconcilerTests Assert.Null(summary.Skipped); Assert.Null(summary.Commit); Assert.Single(_store.Commits); - Assert.Single(_planModel.Calls); + Assert.Single(_model.Calls); Assert.Equal(2, _writer.Calls.Count); } @@ -131,7 +443,7 @@ public class EncyclopediaReconcilerTests Assert.Equal(1, summary.Written); Assert.Equal(["heating", "kitchen", "heating"], _writer.Calls); - Assert.Single(_planModel.Calls); + Assert.Single(_model.Calls); Assert.Equal(["articles/_state.json", "articles/de/heating.md", "articles/en/heating.md"], LastCommit.Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.All(LastCommit, c => Assert.Equal(FileChangeOp.Update, c.Op)); @@ -150,10 +462,10 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Reconcile_ModelUnavailable_CommitsWhatIsDone() + public async Task Reconcile_WriterModelUnavailable_CommitsWhatIsDone() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); _writer.Throws["kitchen"] = new ModelUnavailableException("credit exhausted"); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -166,13 +478,13 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Reconcile_PlanModelUnavailable_StopsKeepsPlan() + public async Task Reconcile_AssignModelUnavailable_StopsKeepsPlan() { await FirstRunAsync(); var plan = _store.Files[PlanPath]; _store.Files["wiki/garage.md"] = Note("garage"); await Sync(); - _planModel.Throw(new ModelUnavailableException("credit exhausted")); + _model.Throw(new ModelUnavailableException("credit exhausted")); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -180,7 +492,6 @@ public class EncyclopediaReconcilerTests Assert.Single(_store.Commits); Assert.Equal(plan, _store.Files[PlanPath]); Assert.Equal(2, _writer.Calls.Count); - Assert.Null(State.FailedPlan); } [Fact] @@ -196,7 +507,7 @@ public class EncyclopediaReconcilerTests var summary = await Reconciler().RunAsync(CancellationToken.None); Assert.Equal(1, summary.Rejected); - Assert.Equal("articles: 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); + Assert.Equal("articles: 0 assigned, 0 split, 0 folded, 0 written, 0 deleted, 1 rejected", _store.Commits[^1].Message); Assert.Equal([StatePath], LastCommit.Select(c => c.Path)); Assert.Equal(de, _store.Files["articles/de/heating.md"]); Assert.Equal(en, _store.Files["articles/en/heating.md"]); @@ -247,7 +558,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_Http400_RememberedAsRejected() { await SeedAsync(); - _planModel.Reply(PlanReply(("heating", ["boiler"]))); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "kitchen"]))); var writeModel = new FakeTextModel("write") .Throw(new HttpRequestException("invalid request", null, HttpStatusCode.BadRequest)); var writer = new WriteArticleUseCase(writeModel, Defaults, _clock); @@ -267,7 +578,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_TransientHttpError_NotRemembered() { await SeedAsync(); - _planModel.Reply(PlanReply(("heating", ["boiler"]))); + _model.Reply(AssignReply(("heating", ["boiler", "radiators", "kitchen"]))); _writer.Throws["heating"] = new HttpRequestException("bad gateway", null, HttpStatusCode.BadGateway); var first = await Reconciler().RunAsync(CancellationToken.None); @@ -284,7 +595,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_StalledStream_CommitsOthersNotRemembered() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); _writer.Throws["heating"] = new HttpRequestException(HttpRequestError.ResponseEnded, "stream stalled", new TimeoutException()); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -295,73 +606,19 @@ public class EncyclopediaReconcilerTests } [Fact] - public async Task Reconcile_PlanFailedSameNotes_DoesNotReplan() - { - await SeedAsync(); - _planModel.Reply("I would rather not."); - - var first = await Reconciler().RunAsync(CancellationToken.None); - await Sync(); - var second = await Reconciler().RunAsync(CancellationToken.None); - - Assert.Equal("no-plan", first.Skipped); - Assert.Equal("no-plan", second.Skipped); - Assert.Single(_planModel.Calls); - Assert.Single(_store.Commits); - Assert.NotNull(State.FailedPlan); - } - - [Fact] - public async Task Reconcile_NoPlan_WritesNothing() - { - var old = ArticleFile.Build(new Article("de", "old", "Alt", ArticleCategory.Topic, ArticleVisibility.Public, - [new ArticleSource("boiler", "h")], new DateOnly(2026, 1, 1), "m", "Alt.\n")); - _store.Files["articles/de/old.md"] = old; - await SeedAsync(); - _planModel.Reply("I would rather not."); - - var summary = await Reconciler().RunAsync(CancellationToken.None); - - Assert.Equal("no-plan", summary.Skipped); - Assert.Equal([StatePath], Assert.Single(_store.Commits).Changes.Select(c => c.Path)); - Assert.Equal(old, _store.Files["articles/de/old.md"]); - Assert.Empty(_writer.Calls); - } - - [Fact] - public async Task Reconcile_SubjectDropped_DeletesBothLanguages() - { - await FirstRunAsync(); - _store.Files["wiki/garage.md"] = Note("garage"); - await Sync(); - _planModel.Reply(PlanReply(("heating", ["boiler", "radiators"]), ("garage", ["garage"]))); - - var summary = await Reconciler().RunAsync(CancellationToken.None); - - Assert.Equal(1, summary.Deleted); - Assert.Equal(1, summary.Written); - Assert.Equal("articles: 1 written, 1 deleted, 0 rejected", _store.Commits[^1].Message); - Assert.Equal(["articles/de/kitchen.md", "articles/en/kitchen.md"], - LastCommit.Where(c => c.Op == FileChangeOp.Delete).Select(c => c.Path).Order(StringComparer.Ordinal)); - Assert.DoesNotContain("articles/de/kitchen.md", _store.Files.Keys); - Assert.DoesNotContain("kitchen", State.Subjects.Keys); - Assert.Equal(["heating", "kitchen", "garage"], _writer.Calls); - } - - [Fact] - public async Task Reconcile_ArticleNotInAnyPlan_NotDeleted() + public async Task Reconcile_ArticleNotInPlan_NotDeleted() { var legacy = ArticleFile.Build(new Article("de", "legacy", "Alt", ArticleCategory.Topic, ArticleVisibility.Public, [new ArticleSource("boiler", "h")], new DateOnly(2026, 1, 1), "m", "Alt.\n")); _store.Files["articles/de/legacy.md"] = legacy; await FirstRunAsync(); - _store.Files["wiki/garage.md"] = Note("garage"); + _store.Files.Remove("wiki/kitchen.md"); await Sync(); - _planModel.Reply(PlanReply([.. HeatingAndKitchen, ("garage", ["garage"])])); await Reconciler().RunAsync(CancellationToken.None); - Assert.All(_store.Commits.SelectMany(c => c.Changes), c => Assert.NotEqual(FileChangeOp.Delete, c.Op)); + Assert.Equal(["articles/de/kitchen.md", "articles/en/kitchen.md"], + LastCommit.Where(c => c.Op == FileChangeOp.Delete).Select(c => c.Path).Order(StringComparer.Ordinal)); Assert.Equal(legacy, _store.Files["articles/de/legacy.md"]); } @@ -369,7 +626,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_PilotCapShrinks_DeletesNothing() { await SeedAsync("boiler", "radiators", "kitchen", "garage"); - _planModel.Reply(PlanReply([.. HeatingAndKitchen, ("garage", ["garage"])])); + _model.Reply(AssignReply([.. HeatingAndKitchen, ("garage", ["garage"])])); await Reconciler().RunAsync(CancellationToken.None); await Sync(); Assert.Equal(6, _store.Files.Keys.Count(p => p.StartsWith("articles/d", StringComparison.Ordinal) @@ -386,7 +643,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_PilotSlugs_WritesOnlyListed() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); await Reconciler(Defaults with { PilotSlugs = ["kitchen"] }).RunAsync(CancellationToken.None); @@ -398,7 +655,7 @@ public class EncyclopediaReconcilerTests [Fact] public void Due_MaxArticles_KeepsOnlyFirstOfPlan() { - var notes = new[] { PlanArticlesUseCaseTests.Note("a"), PlanArticlesUseCaseTests.Note("b") }; + var notes = new[] { TestNotes.Note("a"), TestNotes.Note("b") }; var a = new PlannedArticle("a", "A", "A", ArticleCategory.Topic, "", ["a"]); var b = new PlannedArticle("b", "B", "B", ArticleCategory.Topic, "", ["b"]); var plan = ArticlePlan.Empty(); @@ -415,7 +672,7 @@ public class EncyclopediaReconcilerTests { await SeedAsync(); _store.Files["articles/de/heating.md"] = "written by someone else"; - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); var summary = await Reconciler().RunAsync(CancellationToken.None); @@ -430,7 +687,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_AmbiguousTitles_IsNoopOnSecondRun(string titleDe, string titleEn) { await SeedAsync(); - _planModel.Reply(TitledPlanReply(titleDe, titleEn)); + _model.Reply(TitledAssignReply(("heating", titleDe, titleEn, ["boiler", "radiators", "kitchen"]))); await Reconciler().RunAsync(CancellationToken.None); await Sync(); @@ -445,7 +702,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_WriterTimeout_CommitsStagedThenThrows() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); _writer.Throws["kitchen"] = new TaskCanceledException("HttpClient timeout"); await Assert.ThrowsAsync(() => Reconciler().RunAsync(CancellationToken.None)); @@ -459,7 +716,7 @@ public class EncyclopediaReconcilerTests public async Task Reconcile_CallerCancelled_NoCommit() { await SeedAsync(); - _planModel.Reply(PlanReply(HeatingAndKitchen)); + _model.Reply(AssignReply(HeatingAndKitchen)); using var cts = new CancellationTokenSource(); _writer.Before = slug => { if (slug == "kitchen") cts.Cancel(); }; _writer.Throws["kitchen"] = new OperationCanceledException(cts.Token); @@ -472,16 +729,16 @@ public class EncyclopediaReconcilerTests [Fact] public void Stale_TitleOnlyChange_IsStale() { - var notes = new[] { PlanArticlesUseCaseTests.Note("a", hash: "h1") }; + var notes = new[] { TestNotes.Note("a", hash: "h1") }; var planned = new PlannedArticle("a", "Neu", "New", ArticleCategory.Topic, "", ["a"]); Article Existing(string lang, string title) => new(lang, "a", title, ArticleCategory.Topic, ArticleVisibility.Public, [new ArticleSource("a", "h1")], new DateOnly(2026, 10, 1), "m", "x\n"); var plan = ArticlePlan.Empty(); plan.AddArticle(planned); - Assert.Empty(EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "New")])); - Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Alt"), Existing("en", "New")])); - Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "Old")])); + Assert.Empty(EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "New")], Defaults)); + Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Alt"), Existing("en", "New")], Defaults)); + Assert.Equal([planned], EncyclopediaReconciler.Stale(plan, notes, [Existing("de", "Neu"), Existing("en", "Old")], Defaults)); } [Fact] @@ -490,7 +747,7 @@ public class EncyclopediaReconcilerTests await FirstRunAsync(); _store.Files["wiki/attic.md"] = Note("attic"); await Sync(); - _planModel.Reply(PlanReply([.. HeatingAndKitchen, ("attic", ["attic"])])); + _model.Reply(AssignReply(("attic", ["attic"]))); await Reconciler().RunAsync(CancellationToken.None); await Sync(); var commits = _store.Commits.Count; @@ -502,10 +759,28 @@ public class EncyclopediaReconcilerTests Assert.Equal(["attic", "heating", "kitchen"], State.Subjects.Keys); } + /// Embeddings that fail like an unreachable service while . + private sealed class SwitchableEmbeddings(IEmbeddingModel inner) : IEmbeddingModel + { + public bool Down { get; set; } + public string ModelId => inner.ModelId; + public int Dimension => inner.Dimension; + public string Name => inner.Name; + + public Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct) => + Down ? Task.FromException>(new HttpRequestException("connection refused")) : inner.EmbedAsync(texts, ct); + + public Task CheckAsync(CancellationToken ct) => inner.CheckAsync(ct); + } + /// Writes "Text {slug}" in both languages from the given sources; throws the scripted exception for a slug. private sealed class FakeWriter : IArticleWriter { public List Calls { get; } = []; + /// Link target slugs of the last write of each article. + public Dictionary> Links { get; } = new(); + /// Source slugs of the last write of each article. + public Dictionary> Sources { get; } = new(); public Dictionary Throws { get; } = new(); public Action? Before { get; set; } @@ -513,6 +788,8 @@ public class EncyclopediaReconcilerTests IReadOnlyList linkTargets, CancellationToken ct) { Calls.Add(planned.Slug); + Links[planned.Slug] = linkTargets.Select(a => a.Slug).ToList(); + Sources[planned.Slug] = sources.Select(d => d.Slug).ToList(); Before?.Invoke(planned.Slug); if (Throws.TryGetValue(planned.Slug, out var e)) return Task.FromException(e); var de = new Article("de", planned.Slug, planned.TitleDe, planned.Category, ArticleVisibility.Public, diff --git a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs index 2a388db..11b4977 100644 --- a/test/Wiki.Application.Tests/EncyclopediaStateTests.cs +++ b/test/Wiki.Application.Tests/EncyclopediaStateTests.cs @@ -16,14 +16,12 @@ public class EncyclopediaStateTests ["heating"] = new("abc", AttemptOutcome.Written, At), ["sauna"] = new("def", AttemptOutcome.Rejected, At), }, - new PlanAttempt("zzz", At), new Dictionary(), new Dictionary()); var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state)); Assert.Equal(state.Subjects, read.Subjects); - Assert.Equal(state.FailedPlan, read.FailedPlan); } [Fact] @@ -31,7 +29,6 @@ public class EncyclopediaStateTests { var state = new EncyclopediaState( new Dictionary(), - null, new Dictionary { ["fp1"] = At }, new Dictionary { ["heating"] = "fp2" }); @@ -42,21 +39,22 @@ public class EncyclopediaStateTests } [Fact] - public void Read_OldJsonWithoutNewFields_ReadsEmptyBatchesAndKept() + public void Read_OldJsonWithFailedPlan_IgnoresIt() { var read = EncyclopediaStateJson.Read("""{"subjects":{},"failedPlan":{"noteSetFingerprint":"x","at":"2026-10-24T03:00:00+00:00"}}"""); - Assert.NotNull(read.FailedPlan); + Assert.Empty(read.Subjects); Assert.Empty(read.FailedBatches); Assert.Empty(read.KeptArticles); + Assert.DoesNotContain("failedPlan", EncyclopediaStateJson.Write(read)); } [Fact] public void BatchFingerprint_ChangesWhenANoteChanges() { - var ab = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "2")]); - var ba = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("b", hash: "2"), PlanArticlesUseCaseTests.Note("a", hash: "1")]); - var changed = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "3")]); - var fewer = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1")]); + var ab = EncyclopediaState.BatchFingerprint([TestNotes.Note("a", hash: "1"), TestNotes.Note("b", hash: "2")]); + var ba = EncyclopediaState.BatchFingerprint([TestNotes.Note("b", hash: "2"), TestNotes.Note("a", hash: "1")]); + var changed = EncyclopediaState.BatchFingerprint([TestNotes.Note("a", hash: "1"), TestNotes.Note("b", hash: "3")]); + var fewer = EncyclopediaState.BatchFingerprint([TestNotes.Note("a", hash: "1")]); Assert.Equal(ab, ba); Assert.NotEqual(ab, changed); Assert.NotEqual(ab, fewer); @@ -68,7 +66,6 @@ public class EncyclopediaStateTests { var read = EncyclopediaStateJson.Read("{{{ nope"); Assert.Empty(read.Subjects); - Assert.Null(read.FailedPlan); } [Fact] @@ -89,25 +86,15 @@ public class EncyclopediaStateTests public void SubjectFingerprint_ChangesWithSourceHashOrTitle() { var planned = new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, "", ["a", "b"]); - var notes = new[] { PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "2") }; + var notes = new[] { TestNotes.Note("a", hash: "1"), TestNotes.Note("b", hash: "2") }; var baseline = EncyclopediaState.SubjectFingerprint(planned, notes); Assert.Equal(baseline, EncyclopediaState.SubjectFingerprint(planned, notes)); Assert.Matches("^[0-9a-f]{64}$", baseline); - var changedHash = new[] { notes[0], PlanArticlesUseCaseTests.Note("b", hash: "3") }; + var changedHash = new[] { notes[0], TestNotes.Note("b", hash: "3") }; Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned, changedHash)); Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { TitleDe = "Heizungsanlage" }, notes)); Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { Scope = "Only boilers" }, notes)); } - - [Fact] - public void NoteSetFingerprint_IgnoresOrder_ChangesWithSlugs() - { - var ab = EncyclopediaState.NoteSetFingerprint([PlanArticlesUseCaseTests.Note("a"), PlanArticlesUseCaseTests.Note("b")]); - var ba = EncyclopediaState.NoteSetFingerprint([PlanArticlesUseCaseTests.Note("b"), PlanArticlesUseCaseTests.Note("a")]); - var abc = EncyclopediaState.NoteSetFingerprint([PlanArticlesUseCaseTests.Note("a"), PlanArticlesUseCaseTests.Note("b"), PlanArticlesUseCaseTests.Note("c")]); - Assert.Equal(ab, ba); - Assert.NotEqual(ab, abc); - } } diff --git a/test/Wiki.Application.Tests/Fakes.cs b/test/Wiki.Application.Tests/Fakes.cs index c30c12a..bfb93b3 100644 --- a/test/Wiki.Application.Tests/Fakes.cs +++ b/test/Wiki.Application.Tests/Fakes.cs @@ -6,6 +6,14 @@ using Wiki.Core.Ports; namespace Wiki.Application.Tests; +public static class TestNotes +{ + /// A wiki note with a body of chars, tagged heating. + public static Document Note(string slug, int bodyLength = 100, string hash = "h", string title = "T") => + new(DocumentKind.Wiki, slug, $"wiki/{slug}.md", hash, title, null, ["heating"], [], [], [], null, + new Dictionary(), new string('x', bodyLength), 0); +} + /// In-memory ports for use-case tests. Content hash = SHA-1 of the text, like a git blob. public sealed class FakeStore : INoteStore { diff --git a/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs b/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs index ab6cdcc..b7e76b3 100644 --- a/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/FoldArticleUseCaseTests.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Tests; public class FoldArticleUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null) { CandidateArticles = 2 }; + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null) { CandidateArticles = 2 }; private static FoldArticleUseCase UseCase(FakeTextModel model) => new(model, Settings, NullLogger.Instance); @@ -15,7 +15,7 @@ public class FoldArticleUseCaseTests new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, sources); private static Dictionary Notes(params string[] slugs) => - slugs.ToDictionary(s => s, s => PlanArticlesUseCaseTests.Note(s)); + slugs.ToDictionary(s => s, s => TestNotes.Note(s)); [Fact] public async Task Fold_Into() @@ -30,6 +30,23 @@ public class FoldArticleUseCaseTests Assert.StartsWith("[fold]", model.Calls[0].System); } + [Fact] + public async Task Fold_ListsAtMostFoldMinSources() + { + var model = new FakeTextModel().Reply("""{"keep":true}"""); + var thin = Article("valve", "a", "b", "c", "d"); + var notes = new Dictionary + { + ["a"] = TestNotes.Note("a", title: "Na"), ["b"] = TestNotes.Note("b", title: "Nb"), + ["c"] = TestNotes.Note("c", title: "Nc"), ["d"] = TestNotes.Note("d", title: "Nd"), + }; + + await UseCase(model).FoldAsync(thin, notes, [Article("heating", "h")], CancellationToken.None); + + Assert.Contains("Nc", model.Calls[0].User); + Assert.DoesNotContain("Nd", model.Calls[0].User); + } + [Fact] public async Task Fold_Keep_IsNull() { @@ -109,13 +126,13 @@ public class FoldArticleUseCaseTests [Fact] public void IsThin_Thresholds() { - var s = new EncyclopediaSettings(8000, 16000, 10, 300_000, [], null); + var s = new EncyclopediaSettings(16000, 10, 300_000, [], null); var notes = new Dictionary { - ["a"] = PlanArticlesUseCaseTests.Note("a", 1000), - ["b"] = PlanArticlesUseCaseTests.Note("b", 1000), - ["c"] = PlanArticlesUseCaseTests.Note("c", 1000), - ["big"] = PlanArticlesUseCaseTests.Note("big", 4000), + ["a"] = TestNotes.Note("a", 1000), + ["b"] = TestNotes.Note("b", 1000), + ["c"] = TestNotes.Note("c", 1000), + ["big"] = TestNotes.Note("big", 4000), }; Assert.True(FoldArticleUseCase.IsThin(Article("x", "a", "b"), notes, s)); diff --git a/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs b/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs deleted file mode 100644 index 4e172f1..0000000 --- a/test/Wiki.Application.Tests/PlanArticlesUseCaseTests.cs +++ /dev/null @@ -1,215 +0,0 @@ -using System.Net; -using Microsoft.Extensions.Logging.Abstractions; -using Wiki.Application.Encyclopedia; -using Wiki.Core.Model; - -namespace Wiki.Application.Tests; - -public class PlanArticlesUseCaseTests -{ - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); - - internal static Document Note(string slug, int bodyLength = 100, string hash = "h", string title = "T") => - new(DocumentKind.Wiki, slug, $"wiki/{slug}.md", hash, title, null, ["heating"], [], [], [], null, - new Dictionary(), new string('x', bodyLength), 0); - - private static PlanArticlesUseCase UseCase(FakeTextModel model, EncyclopediaSettings? settings = null) => - new(model, settings ?? Settings, NullLogger.Instance); - - private static ArticlePlan PlanFrom(PlannedArticle article, params string[] notes) - { - var plan = ArticlePlan.Empty(); - plan.AddArticle(article); - foreach (var n in notes) plan.Assign(n, article.Sources.Contains(n) ? [article.Slug] : []); - return plan; - } - - private static ArticlePlan PlanOf(params string[] covered) => - PlanFrom(new PlannedArticle("heating", "Heizung", "Heating", ArticleCategory.System, "", covered), covered); - - [Fact] - public void NeedsReplan_NullPlan_True() => - Assert.True(PlanArticlesUseCase.NeedsReplan(null, null, [Note("a")])); - - [Fact] - public void NeedsReplan_SameSlugs_False() => - Assert.False(PlanArticlesUseCase.NeedsReplan(PlanOf("b", "a"), null, [Note("a"), Note("b")])); - - [Fact] - public void NeedsReplan_NoteAdded_True() => - Assert.True(PlanArticlesUseCase.NeedsReplan(PlanOf("a"), null, [Note("a"), Note("b")])); - - [Fact] - public void NeedsReplan_FailedForSameNoteSet_False() - { - var notes = new[] { Note("a"), Note("b") }; - var failed = new PlanAttempt(EncyclopediaState.NoteSetFingerprint(notes), DateTimeOffset.UnixEpoch); - Assert.False(PlanArticlesUseCase.NeedsReplan(null, failed, notes)); - } - - [Fact] - public void NeedsReplan_FailedForOtherNoteSet_True() - { - var failed = new PlanAttempt(EncyclopediaState.NoteSetFingerprint([Note("a")]), DateTimeOffset.UnixEpoch); - Assert.True(PlanArticlesUseCase.NeedsReplan(null, failed, [Note("a"), Note("b")])); - } - - [Fact] - public async Task Plan_ParsesAndValidates() - { - var model = new FakeTextModel().Reply(""" - ```json - {"articles":[ - {"slug":"heating","titleDe":"Heizung","titleEn":"Heating","category":"system","sources":["a","ghost","b"]}, - {"slug":"Bad Slug","titleDe":"x","titleEn":"x","category":"topic","sources":["a"]}, - {"slug":"heating","titleDe":"dup","titleEn":"dup","category":"topic","sources":["a"]}, - {"slug":"empty","titleDe":"x","titleEn":"x","category":"topic","sources":["ghost"]} - ]} - ``` - """); - - var plan = await UseCase(model).PlanAsync(null, [Note("a"), Note("b")], CancellationToken.None); - - Assert.NotNull(plan); - var article = Assert.Single(plan.Articles); - Assert.Equal("heating", article.Slug); - Assert.Equal(ArticleCategory.System, article.Category); - Assert.Equal(["a", "b"], article.Sources); - Assert.Equal(["a", "b"], plan.Assignments.Keys.Order()); - } - - [Fact] - public async Task Plan_ModelGarbage_KeepsExistingPlan() - { - var model = new FakeTextModel().Reply("Sure! Here is"); - Assert.Null(await UseCase(model).PlanAsync(PlanOf("a"), [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_Truncated_KeepsExistingPlan() - { - var json = """{"articles":[{"slug":"heating","titleDe":"H","titleEn":"H","category":"system","sources":["a"]}]}"""; - var model = new FakeTextModel().Reply(json, truncated: true); - Assert.Null(await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_Http400_ReturnsNull() - { - var model = new FakeTextModel().Throw(new HttpRequestException("bad", null, HttpStatusCode.BadRequest)); - Assert.Null(await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_ModelUnavailable_Propagates() - { - var model = new FakeTextModel().Throw(new ModelUnavailableException("quota")); - await Assert.ThrowsAsync(() => UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_Http500_Propagates() - { - var model = new FakeTextModel().Throw(new HttpRequestException("boom", null, HttpStatusCode.InternalServerError)); - await Assert.ThrowsAsync(() => UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None)); - } - - [Fact] - public async Task Plan_SourcesOverCharCap_DropsLowestRanked() - { - var model = new FakeTextModel().Reply("""{"articles":[{"slug":"big","titleDe":"G","titleEn":"B","category":"topic","sources":["a","b","c"]}]}"""); - var notes = new[] { Note("a", 200_000), Note("b", 200_000), Note("c", 200_000) }; - - var plan = await UseCase(model).PlanAsync(null, notes, CancellationToken.None); - - Assert.Equal(["a"], Assert.Single(plan!.Articles).Sources); - } - - [Fact] - public async Task Plan_PromptContainsExistingSlugs() - { - var model = new FakeTextModel().Reply("nope"); - var current = PlanFrom(new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, "", ["a"]), ["a"]); - - await UseCase(model).PlanAsync(current, [Note("a", 42, title: "Valve notes")], CancellationToken.None); - - var call = Assert.Single(model.Calls); - Assert.Contains("heating-valves", call.User); - Assert.Contains("a | Valve notes | heating | 42", call.User); - Assert.StartsWith("[plan]", call.System); - Assert.Equal(Settings.PlanMaxTokens, call.MaxTokens); - } - - [Fact] - public async Task Plan_PromptForbidsConcreteTitles_AndKeepsTitles() - { - var model = new FakeTextModel().Reply("nope"); - - await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None); - - var system = Assert.Single(model.Calls).System; - Assert.Contains("Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses.", system); - Assert.Contains("Keep the existing titles for existing subjects unless the subject changed.", system); - } - - [Fact] - public async Task Plan_PromptAsksForCompactJson_AndOmitsCoveredNotes() - { - var model = new FakeTextModel().Reply("nope"); - var current = PlanFrom(new PlannedArticle("heating-valves", "H", "H", ArticleCategory.Device, "", ["a"]), ["a", "covered-only"]); - - await UseCase(model).PlanAsync(current, [Note("a")], CancellationToken.None); - - var call = Assert.Single(model.Calls); - Assert.Contains("compact", call.System); - Assert.DoesNotContain("coveredNotes", call.User); - Assert.DoesNotContain("covered-only", call.User); - var jsonLine = call.User.Split('\n').Single(l => l.StartsWith("[{")); - Assert.Equal("""[{"slug":"heating-valves","titleDe":"H","titleEn":"H","category":"device","scope":"","sources":["a"]}]""", jsonLine); - } - - [Fact] - public async Task Plan_TitleWithPipeOrNewline_Sanitized() - { - var model = new FakeTextModel().Reply("nope"); - var note = Note("a", title: "Left | Right\nSecond line"); - - await UseCase(model).PlanAsync(null, [note], CancellationToken.None); - - var line = Assert.Single(model.Calls[0].User.Split('\n'), l => l.StartsWith("a | ")); - Assert.Equal(4, line.Split(" | ").Length); - Assert.Contains("Left Right Second line", line); - } - - [Fact] - public void ArticlePlanJson_NumericCategory_ArticleSkipped() - { - var plan = ArticlePlanJson.Read("""{"articles":[{"slug":"a","category":"1","sources":["x"]},{"slug":"b","category":"room","sources":["x"]}],"assignments":{}}"""); - Assert.Equal(["b"], plan!.Articles.Select(a => a.Slug)); - } - - [Fact] - public void ArticlePlanJson_Malformed_ReadsNull() - { - Assert.Null(ArticlePlanJson.Read("not json")); - Assert.Null(ArticlePlanJson.Read("""{"foo":1}""")); - } - - [Fact] - public void ArticlePlanJson_RoundTrip() - { - var plan = PlanOf("a", "b"); - var read = ArticlePlanJson.Read(ArticlePlanJson.Write(plan)); - Assert.NotNull(read); - Assert.Equal(plan.Articles[0], read.Articles[0],new PlannedArticleComparer()); - Assert.Equal(plan.Assignments.Keys, read.Assignments.Keys); - } - - private sealed class PlannedArticleComparer : IEqualityComparer - { - public bool Equals(PlannedArticle? x, PlannedArticle? y) => - x!.Slug == y!.Slug && x.TitleDe == y.TitleDe && x.TitleEn == y.TitleEn && x.Category == y.Category - && x.Scope == y.Scope && x.Sources.SequenceEqual(y.Sources); - public int GetHashCode(PlannedArticle obj) => obj.Slug.GetHashCode(); - } -} diff --git a/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs b/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs index 094249c..ee1b387 100644 --- a/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/SplitArticleUseCaseTests.cs @@ -6,7 +6,7 @@ namespace Wiki.Application.Tests; public class SplitArticleUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null) { MaxSourcesPerArticle = 4 }; + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null) { MaxSourcesPerArticle = 4 }; private static SplitArticleUseCase UseCase(FakeTextModel model) => new(model, Settings, NullLogger.Instance); @@ -15,7 +15,7 @@ public class SplitArticleUseCaseTests new(slug, "Titel " + slug, "Title " + slug, ArticleCategory.System, "Scope " + slug, sources); private static Dictionary Notes(params string[] slugs) => - slugs.ToDictionary(s => s, s => PlanArticlesUseCaseTests.Note(s)); + slugs.ToDictionary(s => s, s => TestNotes.Note(s)); private static ArticlePlan PlanWith(params PlannedArticle[] articles) { @@ -193,7 +193,7 @@ public class SplitArticleUseCaseTests { var article = Article("heating", "a", "b", "c", "d"); var notes = Notes("a", "b", "c", "d"); - notes["a"] = PlanArticlesUseCaseTests.Note("a", title: "one | two\nthree"); + notes["a"] = TestNotes.Note("a", title: "one | two\nthree"); var model = new FakeTextModel().Reply(Valid); await UseCase(model).SplitAsync(article, notes, PlanWith(article), CancellationToken.None); @@ -204,13 +204,13 @@ public class SplitArticleUseCaseTests [Fact] public void NeedsSplit_ByCharsAndByCount() { - var s = new EncyclopediaSettings(8000, 16000, 10, 1000, [], null) { MaxSourcesPerArticle = 3 }; + var s = new EncyclopediaSettings(16000, 10, 1000, [], null) { MaxSourcesPerArticle = 3 }; var notes = new Dictionary { - ["a"] = PlanArticlesUseCaseTests.Note("a", 600), - ["b"] = PlanArticlesUseCaseTests.Note("b", 600), - ["c"] = PlanArticlesUseCaseTests.Note("c", 100), - ["d"] = PlanArticlesUseCaseTests.Note("d", 1), + ["a"] = TestNotes.Note("a", 600), + ["b"] = TestNotes.Note("b", 600), + ["c"] = TestNotes.Note("c", 100), + ["d"] = TestNotes.Note("d", 1), }; Assert.True(SplitArticleUseCase.NeedsSplit(Article("x", "a", "b"), notes, s)); diff --git a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs index 901ac26..6ea0ce3 100644 --- a/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs +++ b/test/Wiki.Application.Tests/WriteArticleUseCaseTests.cs @@ -8,7 +8,7 @@ namespace Wiki.Application.Tests; public class WriteArticleUseCaseTests { - private static readonly EncyclopediaSettings Settings = new(8000, 16000, 10, 300_000, [], null); + private static readonly EncyclopediaSettings Settings = new(16000, 10, 300_000, [], null); private static readonly DateTimeOffset Now = new(2026, 10, 24, 3, 0, 0, TimeSpan.Zero); private static readonly PlannedArticle Planned = diff --git a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs index af7c6c0..094f00a 100644 --- a/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs +++ b/test/Wiki.Host.Tests/EncyclopediaEndpointTests.cs @@ -18,12 +18,13 @@ public sealed class EncyclopediaEndpointTests public List CatalogCommits { get; } = []; public Exception? Failure { get; set; } public bool StoppedByModel { get; set; } + public int Planned { get; set; } = 3; public async Task RunAsync(CancellationToken ct) { CatalogCommits.Add((await catalog.GetStateAsync(ct)).Commit); if (Failure is not null) throw Failure; - return new ReconcileSummary(3, 0, 0, 0, 0, StoppedByModel, null, null); + return new ReconcileSummary(Planned, 0, 0, 0, 0, 0, 0, 0, StoppedByModel, null, null); } } @@ -135,4 +136,18 @@ public sealed class EncyclopediaEndpointTests Assert.Equal(HttpStatusCode.BadGateway, r.StatusCode); Assert.True((await r.Content.ReadFromJsonAsync()).GetProperty("stoppedByModel").GetBoolean()); } + + [Fact] + public async Task Run_EmptyPlan_Is502() + { + var (host, _, run) = Enabled(); + using var _h = host; + var client = host.CreateClient(); + run().Planned = 0; + + var r = await client.SendAsync(RunRequest()); + + Assert.Equal(HttpStatusCode.BadGateway, r.StatusCode); + Assert.Equal(0, (await r.Content.ReadFromJsonAsync()).GetProperty("planned").GetInt32()); + } }