fix(encyclopedia): leak-check slugs and titles, keep planned titles stable
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful

A scanner hit in the slug or titles, or the German leak check flagging them, makes the article private in both languages. The plan prompt forbids concrete identifiers in titles and slugs and keeps existing titles; prompt version 5.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:10:09 +01:00
co-authored by Claude Opus 5.5
parent 3f5c41a075
commit ca12438ebb
4 changed files with 73 additions and 16 deletions
+3 -1
View File
@@ -6,7 +6,7 @@ namespace Wiki.Application.Encyclopedia;
/// </summary>
public static class Prompts
{
public const int Version = 4;
public const int Version = 5;
/// <summary>Placeholder in <see cref="Plan"/> replaced by <c>MaxSourceCharsPerArticle</c>.</summary>
public const string MaxSourceCharsToken = "{maxSourceChars}";
@@ -21,6 +21,8 @@ public static class Prompts
- A subject that was set once and forgotten is not an article of its own. Its notes go into the article of its parent system as a section.
- Every note is used by at least one article, or is deliberately skipped. Skip only meta notes and notes about the wiki's own tooling.
- Keep the existing slugs for existing subjects. Do not rename or renumber them.
- Keep the existing titles for existing subjects unless the subject changed.
- Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses. Name the subject by what it is or does, for example "Netzwerkspeicher" rather than the server's hostname.
- List each article's sources most important first. When the sources of an article add up to more than {maxSourceChars} characters of body, split the subject into narrower articles.
- Slugs are lowercase ASCII words joined by single hyphens. Categories are room, system, device or topic.
- related lists the slugs of other articles in this plan that a reader of the article should continue with.
@@ -31,7 +31,8 @@ public interface IArticleWriter
/// <summary>
/// Writes an article in German, privatizes leaks (deterministic filter, then an LLM check of the public blocks),
/// translates it unit by unit into English, and filters and checks the English the same way, mirroring every
/// privatization into the German. Anything unexpected rejects the article.
/// privatization into the German. A leak in the slug or titles makes the whole article private.
/// Anything unexpected rejects the article.
/// </summary>
public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSettings settings, TimeProvider clock) : IArticleWriter
{
@@ -57,14 +58,20 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe
var (visibility, de) = ParseWritten(written.Text);
if (de.Length == 0) throw new ArticleRejectedException(slug, "the article body is empty");
de = ArticleBody.Privatize(de, await LeakIndexesAsync(slug, de, sourceTexts, ct));
// the slug and titles are shown to every reader of a public article, so a leak in them makes the article private;
// the slug comes first, so the line is one plain public block
var titleLeak = LeakScanner.Scan(Flatten($"{slug} {planned.TitleDe} {planned.TitleEn}"), sourceTexts).Count > 0;
var (germanLeaks, titleFlagged) = await LeakIndexesAsync(slug, de, sourceTexts,
titleLeak ? null : Flatten($"{slug} — {planned.TitleDe}"), ct);
if (titleLeak || titleFlagged) visibility = ArticleVisibility.Private;
de = ArticleBody.Privatize(de, germanLeaks);
var en = await TranslateAsync(slug, de, ct);
if (ArticleBody.PublicBlocks(de).Count != ArticleBody.PublicBlocks(en).Count)
throw new ArticleRejectedException(slug, "the translation changed the public blocks");
// indexes of the English public blocks are those of the German ones: Reassemble keeps every blank and structural line
var englishLeaks = await LeakIndexesAsync(slug, en, sourceTexts, ct);
var (englishLeaks, _) = await LeakIndexesAsync(slug, en, sourceTexts, null, ct);
de = ArticleBody.Privatize(de, englishLeaks);
en = ArticleBody.Privatize(en, englishLeaks);
if (!ArticleBody.Skeleton(de).SequenceEqual(ArticleBody.Skeleton(en), StringComparer.Ordinal))
@@ -104,27 +111,32 @@ public sealed partial class WriteArticleUseCase(ITextModel model, EncyclopediaSe
/// <summary>
/// Public block indexes of <paramref name="body"/> to privatize: the scanner's matches and every block with an
/// image (a text model cannot judge a photo), plus the blocks the model flags among the rest, which are all it sees.
/// The model sees those blocks numbered 0..n-1; an index outside that range rejects the article.
/// The model sees those blocks numbered 0..n-1 and, when given, <paramref name="title"/> as item n; <c>Title</c> is
/// true when it flags that item. An index outside the range shown rejects the article.
/// </summary>
private async Task<HashSet<int>> LeakIndexesAsync(string slug, string body, IReadOnlyList<string> sources, CancellationToken ct)
private async Task<(HashSet<int> Blocks, bool Title)> LeakIndexesAsync(string slug, string body, IReadOnlyList<string> sources,
string? title, CancellationToken ct)
{
var blocks = ArticleBody.PublicBlocks(body);
var flagged = LeakScanner.Scan(body, sources).Select(m => m.BlockIndex).ToHashSet();
flagged.UnionWith(blocks.Where(b => b.Text.Contains("![", StringComparison.Ordinal)).Select(b => b.Index));
var toCheck = blocks.Where(b => !flagged.Contains(b.Index)).ToList();
if (toCheck.Count == 0) return flagged;
var items = toCheck.Select((b, i) => (i, b.Text)).ToList();
if (title is not null) items.Add((toCheck.Count, title));
if (items.Count == 0) return (flagged, false);
var reply = await CompleteAsync(slug, "leak check", Prompts.LeakCheck,
Numbered(toCheck.Select((b, i) => (i, b.Text))), LeakCheckMaxTokens, ct);
var reply = await CompleteAsync(slug, "leak check", Prompts.LeakCheck, Numbered(items), LeakCheckMaxTokens, ct);
var chosen = ParseLeakCheck(reply.Text)
?? throw new ArticleRejectedException(slug, "the leak check reply is not {\"privatize\":[…]}");
var titleFlagged = false;
foreach (var i in chosen)
{
if (i < 0 || i >= toCheck.Count)
throw new ArticleRejectedException(slug, $"the leak check flagged block {i}, but only {toCheck.Count} were shown");
flagged.Add(toCheck[i].Index);
if (i < 0 || i >= items.Count)
throw new ArticleRejectedException(slug, $"the leak check flagged block {i}, but only {items.Count} were shown");
if (i == toCheck.Count) titleFlagged = true;
else flagged.Add(toCheck[i].Index);
}
return flagged;
return (flagged, titleFlagged);
}
/// <summary>Public and private units go in separate requests, so no private text can reach a public unit.</summary>
@@ -133,6 +133,18 @@ public class PlanArticlesUseCaseTests
Assert.Equal(Settings.PlanMaxTokens, call.MaxTokens);
}
[Fact]
public async Task Plan_PromptForbidsConcreteTitles_AndKeepsTitles()
{
var model = new FakeTextModel().Reply("nope");
await UseCase(model).PlanAsync(null, [Note("a")], CancellationToken.None);
var system = Assert.Single(model.Calls).System;
Assert.Contains("Titles and slugs name nothing concrete in this house: no hostnames, device IDs, addresses or IP addresses.", system);
Assert.Contains("Keep the existing titles for existing subjects unless the subject changed.", system);
}
[Fact]
public async Task Plan_PromptAsksForCompactJson_AndOmitsCoveredNotes()
{
@@ -38,8 +38,11 @@ public class WriteArticleUseCaseTests
}
private static Task<WrittenArticle> Write(FakeTextModel model, params Document[] sources) =>
Write(model, Planned, sources);
private static Task<WrittenArticle> Write(FakeTextModel model, PlannedArticle planned, params Document[] sources) =>
new WriteArticleUseCase(model, Settings, new FakeTimeProvider(Now))
.WriteAsync(Planned, sources.Length == 0 ? [Source()] : sources, All, CancellationToken.None);
.WriteAsync(planned, sources.Length == 0 ? [Source()] : sources, All, CancellationToken.None);
private static IReadOnlyList<string> Public(string body) => ArticleBody.PublicBlocks(body).Select(b => b.Text).ToList();
@@ -70,8 +73,36 @@ public class WriteArticleUseCaseTests
Assert.DoesNotContain("One.", ArticleBody.RedactPrivate(written.En.Body));
}
[Fact]
public async Task Write_TitleWithHouseHostname_ForcesPrivate()
{
var model = Model(ThreeBlocks, Clean, """["Zero.","One.","Two."]""", Clean);
var planned = Planned with { TitleDe = "Server nas-1.lan", TitleEn = "Server nas-1.lan" };
var written = await Write(model, planned, Source(body: "Der Server nas-1.lan steht im Keller."));
Assert.Equal(ArticleVisibility.Private, written.De.Visibility);
Assert.Equal(ArticleVisibility.Private, written.En.Visibility);
Assert.Equal(["Absatz null.", "Absatz eins.", "Absatz zwei."], Public(written.De.Body));
}
[Fact]
public async Task Write_LeakCheckFlagsTitle_ForcesPrivate()
{
var model = Model(ThreeBlocks, """{"privatize":[3]}""", """["Zero.","One.","Two."]""", Clean);
var written = await Write(model);
Assert.EndsWith("""{"i":3,"text":"wall-switches — Wandschalter"}]""", model.Calls[1].User);
Assert.DoesNotContain("Wandschalter", model.Calls[^1].User);
Assert.Equal(ArticleVisibility.Private, written.De.Visibility);
Assert.Equal(ArticleVisibility.Private, written.En.Visibility);
Assert.Equal(["Absatz null.", "Absatz eins.", "Absatz zwei."], Public(written.De.Body));
Assert.Equal(["Zero.", "One.", "Two."], Public(written.En.Body));
}
[Theory]
[InlineData("""{"privatize":[3]}""")]
[InlineData("""{"privatize":[4]}""")]
[InlineData("""{"privatize":[-1]}""")]
[InlineData("""{"privatize":[0,7]}""")]
public async Task LeakCheck_IndexOutOfRange_Rejected(string reply)
@@ -89,7 +120,7 @@ public class WriteArticleUseCaseTests
var written = await Write(model, Source(body: "Der Router hat die Adresse 10.2.40.1."));
Assert.Equal("""[{"i":0,"text":"Absatz null."},{"i":1,"text":"Absatz zwei."}]""", model.Calls[1].User);
Assert.Equal("""[{"i":0,"text":"Absatz null."},{"i":1,"text":"Absatz zwei."},{"i":2,"text":"wall-switches — Wandschalter"}]""", model.Calls[1].User);
Assert.Equal(["Absatz null."], Public(written.De.Body));
Assert.Equal(["Zero."], Public(written.En.Body));
Assert.Equal("""[{"i":0,"text":"Zero."}]""", model.Calls[^1].User);