ask: wiki_ask.py as a use case and POST /api/v1/ask; judges from configuration

Rank every query, unite the top notes into one context under a per-document and a total budget, expand context choices over paragraph or document ids plus NONE, add the covered gate, judge context choices alone and the rest batched, attach the answering paragraphs' text. A missing TypeSafe key is a call-time error shown in status, not a startup failure. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-24 13:57:17 +02:00
parent e10208ee2c
commit 34cf91ebaf
11 changed files with 623 additions and 5 deletions
@@ -0,0 +1,137 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Wiki.Application.Ask;
+using Wiki.Application.Retrieval;
+using Wiki.Application.Sync;
+using Wiki.Core.Model;
+using Wiki.Core.Parsing;
+using Wiki.Core.Ports;
+
+namespace Wiki.Application.Tests;
+
+/// <summary>A judge that records requests and answers by simple rules, so the use case's wiring is testable.</summary>
+public sealed class FakeJudge : IJudge
+{
+    public List<JudgeRequest> Requests { get; } = [];
+    public string ModelId => "fake-judge";
+    public int? MaxChoiceOptions { get; init; } = 255;
+    public string Name => "fake-judge";
+
+    public Task<JudgeResult> JudgeAsync(JudgeRequest request, CancellationToken ct)
+    {
+        Requests.Add(request);
+        var answers = new Dictionary<string, Judgement>();
+        foreach (var (id, q) in request.Questions)
+        {
+            answers[id] = q switch
+            {
+                TypedQuestion.Noul => new Judgement.Noul(id == AskUseCase.CoveredId ? 0.8 : 0.9),
+                TypedQuestion.Choice c => Spread(c),
+                TypedQuestion.Score => new Judgement.Score(0.5),
+                _ => throw new InvalidOperationException(),
+            };
+        }
+        return Task.FromResult(new JudgeResult(answers, new JudgeUsage(1, 100, request.Questions.Count, TimeSpan.FromMilliseconds(5))));
+    }
+
+    private static Judgement.Choice Spread(TypedQuestion.Choice c)
+    {
+        // First option gets 0.6, the rest share 0.4 equally (NONE included).
+        var keys = c.Options.Keys.ToList();
+        var probs = new Dictionary<string, double>();
+        for (var i = 0; i < keys.Count; i++)
+            probs[keys[i]] = keys.Count == 1 ? 1.0 : (i == 0 ? 0.6 : 0.4 / (keys.Count - 1));
+        return new Judgement.Choice(keys[0], probs, 0.6);
+    }
+
+    public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
+}
+
+public class AskUseCaseTests
+{
+    private readonly FakeStore _store = new();
+    private readonly FakeCatalog _catalog = new();
+    private readonly FakeIndex _index = new();
+    private readonly FakeEmbeddings _model = new("m1");
+    private readonly FakeJudge _judge = new();
+
+    private AskUseCase UseCase => new(new RankUseCase(_index, _catalog), _catalog, new ParagraphChunker());
+
+    private async Task SeedAsync()
+    {
+        _store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n\n## Details\n\nEven dig @1.1.1.1 is answered locally.\n";
+        _store.Files["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\ntags: [ceph]\n---\n\n# Ceph OSD\n\nOSD fsid mismatch after rebuild.\n";
+        _store.Files["wiki/long-note.md"] = "---\ntitle: Long\ntype: wiki\n---\n\n# Long\n\n" + string.Join("\n\n", Enumerable.Range(0, 40).Select(i => $"paragraph {i} " + new string('x', 400))) + "\n";
+        var sync = new SyncUseCase(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger<SyncUseCase>.Instance);
+        await sync.RunAsync(false, CancellationToken.None);
+    }
+
+    [Fact]
+    public async Task DefaultQuestionsPerQueryAndCoveredGate()
+    {
+        await SeedAsync();
+        var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!;
+        var request = new AskRequest([EmbeddingText.For(udm)], null, new AskOptions(K: 2));
+
+        var result = await UseCase.AskAsync(request, _model, _judge, CancellationToken.None);
+
+        Assert.Equal("udm-dns", result.Context[0].Slug);
+        Assert.Equal(0.8, result.Covered);
+        Assert.Equal(["answered_1", "where_1", AskUseCase.CoveredId], result.Answers.Keys.OrderBy(k => k == AskUseCase.CoveredId ? 2 : k.StartsWith("answered") ? 0 : 1));
+        var where = Assert.IsType<AskAnswer.Context>(result.Answers["where_1"]);
+        Assert.Equal("udm-dns#p1", where.Top[0].Id);
+        Assert.Contains("answers every query", where.Top[0].Text);
+        Assert.Equal("UDM intercepts DNS", where.Top[0].Title);
+        // the context choice went alone, the noul questions together
+        Assert.Equal(2, _judge.Requests.Count);
+        Assert.Contains(_judge.Requests, r => r.Questions.Count == 1 && r.Questions.ContainsKey("where_1"));
+        Assert.Equal(2, result.Usage.Requests);
+        Assert.Equal("fake-judge", result.JudgeModel);
+    }
+
+    [Fact]
+    public async Task ContextChoiceOptionsAreTheParagraphIdsPlusNone()
+    {
+        await SeedAsync();
+        var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!;
+        var request = new AskRequest([EmbeddingText.For(udm)], new Dictionary<string, AskQuestion>
+        {
+            ["src"] = new AskQuestion.ContextChoice("which note", ContextScope.Documents),
+            ["fact"] = new AskQuestion.Noul("is it intercepted", "yes", "no"),
+        }, new AskOptions(K: 2));
+
+        var result = await UseCase.AskAsync(request, _model, _judge, CancellationToken.None);
+
+        var sent = _judge.Requests.Single(r => r.Questions.ContainsKey("src"));
+        var choice = Assert.IsType<TypedQuestion.Choice>(sent.Questions["src"]);
+        Assert.Equal(result.Context.Select(d => d.Slug).Append(AskUseCase.None).OrderBy(x => x), choice.Options.Keys.OrderBy(x => x));
+        Assert.Equal(string.Empty, choice.Options[result.Context[0].Slug]);
+        var src = Assert.IsType<AskAnswer.Context>(result.Answers["src"]);
+        Assert.Equal("udm-dns", src.Top[0].Slug);
+        Assert.Null(src.Top[0].Text);
+        Assert.IsType<AskAnswer.Noul>(result.Answers["fact"]);
+        Assert.True(result.Answers.ContainsKey(AskUseCase.CoveredId));
+    }
+
+    [Fact]
+    public async Task BudgetsCapLongDocumentsAndDropTheLowest()
+    {
+        await SeedAsync();
+        var longNote = (await _catalog.GetAsync(DocumentKind.Wiki, "long-note", CancellationToken.None))!;
+        var (context, meta) = await UseCase.BuildContextAsync([EmbeddingText.For(longNote)], _model, new AskOptions(K: 3, DocChars: 3000, StateChars: 2950), CancellationToken.None);
+        Assert.Equal("long-note", context[0].Slug);
+        Assert.True(context[0].Truncated);
+        Assert.True(context[0].Paragraphs.Count < 40);
+        Assert.Equal(2, meta.DroppedForBudget.Count);
+        Assert.Single(context);
+    }
+
+    [Fact]
+    public async Task RejectsAContextChoiceOverTheJudgeCap()
+    {
+        await SeedAsync();
+        var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!;
+        var small = new FakeJudge { MaxChoiceOptions = 3 };
+        var request = new AskRequest([EmbeddingText.For(udm)], null, new AskOptions(K: 3));
+        await Assert.ThrowsAsync<ArgumentException>(() => UseCase.AskAsync(request, _model, small, CancellationToken.None));
+    }
+}
@@ -38,6 +38,11 @@ public sealed class WikiHostFactory : WebApplicationFactory<Program>
            services.AddKeyedSingleton<IEmbeddingModel>("bge-m3", fake);
            services.AddSingleton<IEmbeddingModel>(fake);
            services.AddSingleton<IReadOnlyList<IEmbeddingModel>>([fake]);
+            var judge = new FakeJudge();
+            services.AddKeyedSingleton<IJudge>("typesafe", judge);
+            services.RemoveAll<IJudge>();
+            services.AddSingleton<IJudge>(judge);
+            services.AddSingleton<IReadOnlyList<IJudge>>([judge]);
        });
    }
 }
@@ -159,6 +164,39 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture<WikiHostFact
        Assert.Equal(HttpStatusCode.OK, (await Client.GetAsync("/api/v1/reports/2026-09-21-wiki-audit")).StatusCode);
    }

+    [Fact]
+    public async Task AskReturnsJudgementsWithParagraphText()
+    {
+        factory.Store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n";
+        var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true");
+        req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
+        await Client.SendAsync(req);
+
+        var bad = await Client.PostAsJsonAsync("/api/v1/ask", new { queries = Array.Empty<string>() });
+        Assert.Equal(HttpStatusCode.BadRequest, bad.StatusCode);
+
+        var r = await Client.PostAsJsonAsync("/api/v1/ask", new
+        {
+            queries = new[] { "udm dns" },
+            k = 3,
+            questions = new Dictionary<string, object>
+            {
+                ["fact"] = new { type = "noul", instructions = "does it", criteria = new { @true = "yes", @false = "no" } },
+                ["where"] = new { type = "choice", instructions = "which paragraph", criteria = "paragraphs" },
+            },
+        });
+        Assert.Equal(HttpStatusCode.OK, r.StatusCode);
+        var body = await r.Content.ReadFromJsonAsync<AskResponse>();
+        Assert.NotNull(body);
+        Assert.Equal("fake-judge", body.Judge);
+        Assert.InRange(body.Covered ?? -1, 0, 1);
+        Assert.Equal(0.9, body.Answers["fact"].GetProperty("noul").GetDouble());
+        var top = body.Answers["where"].GetProperty("top");
+        Assert.True(top.GetArrayLength() >= 1);
+        Assert.Contains("answers every query", top[0].GetProperty("text").GetString());
+    }
+
+    private sealed record AskResponse(string Judge, double? Covered, Dictionary<string, System.Text.Json.JsonElement> Answers);
    private sealed record NoteRow(string Slug, string Title);
    private sealed record LinkRow(string? ToSlug, string? FromSlug, string Kind);
    private sealed record NoteBody(string Slug, string Body, List<LinkRow> Links, List<LinkRow> Backlinks);
@@ -44,6 +44,31 @@ public sealed class FakeStore : INoteStore
    public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
 }

+public sealed class FakeJudge : IJudge
+{
+    public string ModelId => "fake-judge";
+    public int? MaxChoiceOptions => 255;
+    public string Name => "fake-judge";
+
+    public Task<JudgeResult> JudgeAsync(JudgeRequest request, CancellationToken ct)
+    {
+        var answers = new Dictionary<string, Judgement>();
+        foreach (var (id, q) in request.Questions)
+        {
+            answers[id] = q switch
+            {
+                TypedQuestion.Noul => new Judgement.Noul(0.9),
+                TypedQuestion.Choice c => new Judgement.Choice(c.Options.Keys.First(), c.Options.Keys.ToDictionary(k => k, k => 1.0 / c.Options.Count), 0.5),
+                TypedQuestion.Score => new Judgement.Score(0.5),
+                _ => throw new InvalidOperationException(),
+            };
+        }
+        return Task.FromResult(new JudgeResult(answers, new JudgeUsage(1, 10, 1, TimeSpan.Zero)));
+    }
+
+    public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
+}
+
 public sealed class FakeEmbeddings(string modelId) : IEmbeddingModel
 {
    public string ModelId { get; } = modelId;