feat(encyclopedia): state v2 with per-batch and keep memory

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-26 09:07:19 +01:00
co-authored by Claude Opus 5.5
parent 996ff182c0
commit 13d5495899
3 changed files with 61 additions and 7 deletions
@@ -85,7 +85,7 @@ public sealed class EncyclopediaReconciler(
if (plan is null)
{
log.LogWarning("No usable article plan; no article is written or deleted");
return await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan), stateText,
return await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText,
new ReconcileSummary(0, 0, 0, 0, 0, false, "no-plan", null), ct);
}
@@ -162,7 +162,7 @@ public sealed class EncyclopediaReconciler(
}
}
var summary = await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan), stateText,
var summary = await FinishAsync(changes, new EncyclopediaState(subjects, failedPlan, state.FailedBatches, state.KeptArticles), stateText,
new ReconcileSummary(plan.Articles.Count, written, deleted, rejected, remembered, stopped, null, null), ct);
failure?.Throw();
return summary;
@@ -22,9 +22,21 @@ public sealed record PlanAttempt(string NoteSetFingerprint, DateTimeOffset At);
/// What the nightly job remembers between runs (<c>articles/_state.json</c>), so it does not pay again
/// for the same failure. An attempt is repeated only when its fingerprint changes.
/// </summary>
public sealed record EncyclopediaState(IReadOnlyDictionary<string, SubjectAttempt> Subjects, PlanAttempt? FailedPlan)
/// <remarks>
/// <c>FailedBatches</c>: fingerprint of an assignment batch whose reply was unusable → when.
/// <c>KeptArticles</c>: article slug → <see cref="SubjectFingerprint"/> at which the fold call answered keep.
/// </remarks>
public sealed record EncyclopediaState(
IReadOnlyDictionary<string, SubjectAttempt> Subjects,
PlanAttempt? FailedPlan,
IReadOnlyDictionary<string, DateTimeOffset> FailedBatches,
IReadOnlyDictionary<string, string> KeptArticles)
{
public static EncyclopediaState Empty => new(new Dictionary<string, SubjectAttempt>(), null);
public static EncyclopediaState Empty =>
new(new Dictionary<string, SubjectAttempt>(), null, new Dictionary<string, DateTimeOffset>(), new Dictionary<string, string>());
public static string BatchFingerprint(IReadOnlyList<Document> notes) =>
Hash(Prompts.Version.ToString(), notes.Select(n => $"{n.Slug}@{n.ContentHash}").Order(StringComparer.Ordinal));
public static string NoteSetFingerprint(IReadOnlyList<Document> notes) =>
Hash(Prompts.Version.ToString(), notes.Select(n => n.Slug).Order(StringComparer.Ordinal));
@@ -35,7 +47,7 @@ public sealed record EncyclopediaState(IReadOnlyDictionary<string, SubjectAttemp
var sources = planned.Sources
.Select(s => $"{s}@{hashes.GetValueOrDefault(s, "")}")
.Order(StringComparer.Ordinal);
return Hash([Prompts.Version.ToString(), planned.TitleDe, planned.TitleEn, planned.Category.ToString(), .. sources]);
return Hash([Prompts.Version.ToString(), planned.TitleDe, planned.TitleEn, planned.Category.ToString(), planned.Scope, .. sources]);
}
private static string Hash(string first, IEnumerable<string> rest) => Hash([first, .. rest]);
@@ -62,7 +74,9 @@ public static class EncyclopediaStateJson
var s = JsonSerializer.Deserialize<EncyclopediaState>(json, Options);
if (s?.Subjects is null) return EncyclopediaState.Empty;
var subjects = s.Subjects.Where(kv => kv.Value is not null).ToDictionary(kv => kv.Key, kv => kv.Value);
return new EncyclopediaState(subjects, s.FailedPlan);
var failedBatches = (s.FailedBatches ?? new Dictionary<string, DateTimeOffset>()).ToDictionary();
var kept = (s.KeptArticles ?? new Dictionary<string, string>()).Where(kv => kv.Value is not null).ToDictionary();
return new EncyclopediaState(subjects, s.FailedPlan, failedBatches, kept);
}
catch (JsonException)
{
@@ -16,7 +16,9 @@ public class EncyclopediaStateTests
["heating"] = new("abc", AttemptOutcome.Written, At),
["sauna"] = new("def", AttemptOutcome.Rejected, At),
},
new PlanAttempt("zzz", At));
new PlanAttempt("zzz", At),
new Dictionary<string, DateTimeOffset>(),
new Dictionary<string, string>());
var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state));
@@ -24,6 +26,43 @@ public class EncyclopediaStateTests
Assert.Equal(state.FailedPlan, read.FailedPlan);
}
[Fact]
public void RoundTrip_FailedBatchesAndKept()
{
var state = new EncyclopediaState(
new Dictionary<string, SubjectAttempt>(),
null,
new Dictionary<string, DateTimeOffset> { ["fp1"] = At },
new Dictionary<string, string> { ["heating"] = "fp2" });
var read = EncyclopediaStateJson.Read(EncyclopediaStateJson.Write(state));
Assert.Equal(state.FailedBatches, read.FailedBatches);
Assert.Equal(state.KeptArticles, read.KeptArticles);
}
[Fact]
public void Read_OldJsonWithoutNewFields_ReadsEmptyBatchesAndKept()
{
var read = EncyclopediaStateJson.Read("""{"subjects":{},"failedPlan":{"noteSetFingerprint":"x","at":"2026-10-24T03:00:00+00:00"}}""");
Assert.NotNull(read.FailedPlan);
Assert.Empty(read.FailedBatches);
Assert.Empty(read.KeptArticles);
}
[Fact]
public void BatchFingerprint_ChangesWhenANoteChanges()
{
var ab = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "2")]);
var ba = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("b", hash: "2"), PlanArticlesUseCaseTests.Note("a", hash: "1")]);
var changed = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1"), PlanArticlesUseCaseTests.Note("b", hash: "3")]);
var fewer = EncyclopediaState.BatchFingerprint([PlanArticlesUseCaseTests.Note("a", hash: "1")]);
Assert.Equal(ab, ba);
Assert.NotEqual(ab, changed);
Assert.NotEqual(ab, fewer);
Assert.Matches("^[0-9a-f]{64}$", ab);
}
[Fact]
public void EncyclopediaStateJson_Malformed_ReadsEmpty()
{
@@ -59,6 +98,7 @@ public class EncyclopediaStateTests
var changedHash = new[] { notes[0], PlanArticlesUseCaseTests.Note("b", hash: "3") };
Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned, changedHash));
Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { TitleDe = "Heizungsanlage" }, notes));
Assert.NotEqual(baseline, EncyclopediaState.SubjectFingerprint(planned with { Scope = "Only boilers" }, notes));
}
[Fact]