2026-09-24 14:57:42 +00:00
9 changed files with 453 additions and 1 deletions
@@ -27,4 +27,5 @@ dotnet run --project src/Wiki.Host
 Endpoints: `/health` (liveness), `/ready` (readiness), `/api/v1/status`, `POST /rank`,
 `POST /webhook`, `POST /api/v1/ask` (rank + judge), `POST /api/v1/questions` (ask + answer
 model, `save: true` writes `output/`), `POST /api/v1/facts` (raw source, API key),
-`GET /api/v1/{notes,tags,raw,reports}`.
+`GET /api/v1/{notes,tags,raw,reports}`, `POST /api/v1/lab/eval` (rank evaluation over labelled
+`output/` questions, API key).
@@ -0,0 +1,128 @@
+using System.Diagnostics;
+using Wiki.Application.Retrieval;
+using Wiki.Core.Model;
+using Wiki.Core.Ports;
+
+namespace Wiki.Application.Evaluation;
+
+public sealed record EvalOptions(int Sample = 0, int Seed = 1, int K = 50);
+
+public sealed record EvalQueryRow(string Slug, string Question, int GoldCount, int? FirstHitRank);
+
+public sealed record EvalResult(
+    string ModelId,
+    int QueriesEvaluated,
+    int QueriesSkipped,
+    IReadOnlyDictionary<int, double> Recall,
+    IReadOnlyDictionary<int, double> RecallHop,
+    IReadOnlyDictionary<int, double> HitRate,
+    double Mrr,
+    TimeSpan MeanElapsedPerQuery,
+    TimeSpan Wall,
+    IReadOnlyList<EvalQueryRow> Rows);
+
+/// <summary>
+/// bin/wiki_rank_eval.py as a use case: every output/ document with a `question:` and a
+/// `sources:` list is a labelled query, and gold is the cited slugs that still exist as wiki
+/// notes. Ranks each query with <see cref="RankUseCase"/> and reports recall@k, hit-rate@k and
+/// MRR for k in {1,5,10,20,50} (capped to <see cref="EvalOptions.K"/>), plus a one-hop variant
+/// of recall that also credits a gold note reachable from a top-k note by wikilink or `related:`.
+/// No judge involvement: this measures the embedding model's ranking alone.
+/// </summary>
+public sealed class EvaluateUseCase(IDocumentCatalog catalog, RankUseCase rank)
+{
+    private static readonly int[] AllKs = [1, 5, 10, 20, 50];
+
+    public async Task<EvalResult> EvaluateAsync(IEmbeddingModel model, EvalOptions options, CancellationToken ct)
+    {
+        var started = Stopwatch.GetTimestamp();
+        var (all, skipped) = await LoadQueriesAsync(ct);
+        var queries = Sample(all, options);
+
+        var k = Math.Clamp(options.K, 1, RankUseCase.MaxK);
+        var ks = AllKs.Where(x => x <= k).ToList();
+        if (ks.Count == 0) ks = [k];
+
+        var recallSums = ks.ToDictionary(x => x, _ => 0.0);
+        var recallHopSums = ks.ToDictionary(x => x, _ => 0.0);
+        var hitCounts = ks.ToDictionary(x => x, _ => 0);
+        var mrrSum = 0.0;
+        var elapsedTotal = TimeSpan.Zero;
+        var rows = new List<EvalQueryRow>(queries.Count);
+
+        foreach (var q in queries)
+        {
+            var queryStarted = Stopwatch.GetTimestamp();
+            var ranked = await rank.RankAsync(model, q.Question, k, ct);
+            elapsedTotal += Stopwatch.GetElapsedTime(queryStarted);
+
+            var rankedSlugs = ranked.Select(r => r.Slug).ToList();
+            var goldSet = q.Gold.ToHashSet();
+            int? firstHit = null;
+            for (var i = 0; i < rankedSlugs.Count; i++)
+            {
+                if (!goldSet.Contains(rankedSlugs[i])) continue;
+                firstHit = i + 1;
+                break;
+            }
+            mrrSum += firstHit is { } fh ? 1.0 / fh : 0.0;
+
+            foreach (var topK in ks)
+            {
+                var top = rankedSlugs.Take(topK).ToHashSet();
+                var recallAtK = q.Gold.Count(top.Contains) / (double)q.Gold.Count;
+                recallSums[topK] += recallAtK;
+                if (recallAtK > 0) hitCounts[topK]++;
+
+                var hop = new HashSet<string>(top);
+                foreach (var slug in top)
+                {
+                    foreach (var link in await catalog.OutgoingLinksAsync(slug, ct))
+                        hop.Add(link.ToSlug);
+                }
+                recallHopSums[topK] += q.Gold.Count(hop.Contains) / (double)q.Gold.Count;
+            }
+
+            rows.Add(new EvalQueryRow(q.Slug, q.Question, q.Gold.Count, firstHit));
+        }
+
+        var n = Math.Max(queries.Count, 1);
+        var recall = ks.ToDictionary(x => x, x => recallSums[x] / n);
+        var recallHop = ks.ToDictionary(x => x, x => recallHopSums[x] / n);
+        var hitRate = ks.ToDictionary(x => x, x => hitCounts[x] / (double)n);
+        var mrr = mrrSum / n;
+        var meanElapsed = queries.Count == 0 ? TimeSpan.Zero : elapsedTotal / queries.Count;
+
+        return new EvalResult(model.ModelId, queries.Count, skipped, recall, recallHop, hitRate, mrr, meanElapsed, Stopwatch.GetElapsedTime(started), rows);
+    }
+
+    private async Task<(List<LabelledQuery> Queries, int Skipped)> LoadQueriesAsync(CancellationToken ct)
+    {
+        var docs = await catalog.ListDocumentsAsync(DocumentKind.Output, ct);
+        var queries = new List<LabelledQuery>();
+        var skipped = 0;
+        foreach (var d in docs.OrderBy(d => d.Slug, StringComparer.Ordinal))
+        {
+            var question = d.Frontmatter.TryGetValue("question", out var v) ? v.Scalar : null;
+            var gold = new List<string>();
+            foreach (var s in d.Sources)
+            {
+                if (await catalog.GetAsync(DocumentKind.Wiki, s, ct) is not null) gold.Add(s);
+            }
+            if (string.IsNullOrWhiteSpace(question) || gold.Count == 0) { skipped++; continue; }
+            queries.Add(new LabelledQuery(d.Slug, question, gold));
+        }
+        return (queries, skipped);
+    }
+
+    /// <summary>A deterministic random sample: keys drawn from <paramref name="options"/>'s seed
+    /// over the queries in slug order, then the lowest-keyed <see cref="EvalOptions.Sample"/> kept.</summary>
+    private static List<LabelledQuery> Sample(List<LabelledQuery> queries, EvalOptions options)
+    {
+        if (options.Sample <= 0 || options.Sample >= queries.Count) return queries;
+        var rng = new Random(options.Seed);
+        return queries.OrderBy(_ => rng.Next()).Take(options.Sample).ToList();
+    }
+
+    private sealed record LabelledQuery(string Slug, string Question, IReadOnlyList<string> Gold);
+}
@@ -12,6 +12,7 @@ public static class ApiV1
        v1.MapReadEndpoints();
        v1.MapAskEndpoints();
        v1.MapFactEndpoints();
+        v1.MapLabEndpoints();
        v1.MapQuestionEndpoints();

        v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) =>
@@ -0,0 +1,26 @@
+using Wiki.Application.Evaluation;
+using Wiki.Core.Ports;
+
+namespace Wiki.Host.Api;
+
+/// <summary>Wire shape of an eval run request; every field is optional and falls back to a default.</summary>
+public sealed record EvalRequestDto(string? Model, int? Sample, int? Seed, int? K);
+
+public static class LabEndpoints
+{
+    public static void MapLabEndpoints(this RouteGroupBuilder v1)
+    {
+        v1.MapPost("/lab/eval", async (EvalRequestDto body, EvaluateUseCase evaluate, IEmbeddingModel activeModel, IServiceProvider services, CancellationToken ct) =>
+        {
+            var model = activeModel;
+            if (!string.IsNullOrEmpty(body.Model))
+            {
+                model = services.GetKeyedService<IEmbeddingModel>(body.Model);
+                if (model is null) return Results.BadRequest(new { error = $"unknown model '{body.Model}'" });
+            }
+            var options = new EvalOptions(Sample: body.Sample ?? 0, Seed: body.Seed ?? 1, K: body.K ?? 50);
+            var result = await evaluate.EvaluateAsync(model, options, ct);
+            return Results.Ok(result);
+        }).RequireAuthorization(Auth.Policies.Write);
+    }
+}
@@ -16,6 +16,7 @@
            <MudNavLink Href="answers" Icon="@Icons.Material.Outlined.QuestionAnswer">Answers</MudNavLink>
            <MudNavLink Href="browse" Icon="@Icons.Material.Outlined.Label">Browse</MudNavLink>
            <MudNavLink Href="maintenance" Icon="@Icons.Material.Outlined.Build">Maintenance</MudNavLink>
+            <MudNavLink Href="lab" Icon="@Icons.Material.Outlined.Science">Lab</MudNavLink>
            <MudNavLink Href="status" Icon="@Icons.Material.Outlined.MonitorHeart">Status</MudNavLink>
        </MudNavMenu>
    </MudDrawer>
@@ -0,0 +1,169 @@
+@page "/lab"
+@rendermode InteractiveServer
+@using Wiki.Application.Evaluation
+@inject IReadOnlyList<IEmbeddingModel> Models
+@inject IEmbeddingModel ActiveModel
+@inject EvaluateUseCase Evaluate
+
+<PageTitle>Lab</PageTitle>
+
+<MudText Typo="Typo.h4" GutterBottom="true">Lab</MudText>
+<MudText Typo="Typo.body2" Class="mb-4" Color="Color.Secondary">
+    Rank every labelled question from <code>output/</code> against a chosen embedding model and report recall, hit-rate and MRR. No judge involved.
+</MudText>
+
+<MudPaper Class="pa-4 mb-6">
+    <MudGrid>
+        <MudItem xs="12" sm="4">
+            <MudSelect T="string" @bind-Value="_selectedModel" Label="Model" Variant="Variant.Outlined">
+                @foreach (var m in Models)
+                {
+                    <MudSelectItem T="string" Value="@m.ModelId">@m.ModelId@(m.ModelId == ActiveModel.ModelId ? " (active)" : "")</MudSelectItem>
+                }
+            </MudSelect>
+        </MudItem>
+        <MudItem xs="6" sm="3">
+            <MudNumericField T="int" @bind-Value="_sample" Label="Sample size" Min="0" Variant="Variant.Outlined" HelperText="0 = all labelled queries" />
+        </MudItem>
+        <MudItem xs="6" sm="2">
+            <MudNumericField T="int" @bind-Value="_seed" Label="Seed" Min="0" Variant="Variant.Outlined" />
+        </MudItem>
+        <MudItem xs="12" sm="3">
+            <MudButton Variant="Variant.Filled" Color="Color.Primary" FullWidth="true" Disabled="@(_running || _selectedModel is null)" OnClick="RunAsync">Run</MudButton>
+        </MudItem>
+    </MudGrid>
+</MudPaper>
+
+@if (_running)
+{
+    <MudProgressLinear Indeterminate="true" Class="mb-4" />
+}
+
+@if (_error is not null)
+{
+    <MudAlert Severity="Severity.Error" Class="mb-4">Eval failed: @_error</MudAlert>
+}
+
+@if (_runs.Count == 0 && !_running)
+{
+    <MudAlert Severity="Severity.Info">No runs yet. Pick a model and hit Run.</MudAlert>
+}
+
+@foreach (var run in _runs)
+{
+    <MudPaper Class="pa-4 mb-6">
+        <MudText Typo="Typo.h6" GutterBottom="true">
+            @run.Model.ModelId — sample @(run.Sample == 0 ? "all" : run.Sample.ToString()), seed @run.Seed
+        </MudText>
+
+        @{ var ks = run.Result.Recall.Keys.OrderBy(k => k).ToList(); }
+        <MudSimpleTable Dense="true" Bordered="true" Class="mb-4">
+            <thead>
+                <tr>
+                    <th>Metric</th>
+                    @foreach (var k in ks)
+                    {
+                        <th>k=@k</th>
+                    }
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td>recall@k</td>
+                    @foreach (var k in ks)
+                    {
+                        <td>@run.Result.Recall[k].ToString("0.00")</td>
+                    }
+                </tr>
+                <tr>
+                    <td>recall_hop@k</td>
+                    @foreach (var k in ks)
+                    {
+                        <td>@run.Result.RecallHop[k].ToString("0.00")</td>
+                    }
+                </tr>
+                <tr>
+                    <td>hit_rate@k</td>
+                    @foreach (var k in ks)
+                    {
+                        <td>@run.Result.HitRate[k].ToString("0.00")</td>
+                    }
+                </tr>
+            </tbody>
+        </MudSimpleTable>
+
+        <MudSimpleTable Dense="true" Class="mb-4">
+            <tbody>
+                <tr><td>MRR</td><td>@run.Result.Mrr.ToString("0.000")</td></tr>
+                <tr><td>Queries evaluated / skipped</td><td>@run.Result.QueriesEvaluated / @run.Result.QueriesSkipped</td></tr>
+                <tr><td>Mean elapsed per query</td><td>@run.Result.MeanElapsedPerQuery.TotalMilliseconds.ToString("0") ms</td></tr>
+                <tr><td>Wall time</td><td>@run.Result.Wall.TotalSeconds.ToString("0.00") s</td></tr>
+            </tbody>
+        </MudSimpleTable>
+
+        <MudExpansionPanels>
+            <MudExpansionPanel Text="@($"Per-query results ({run.Result.Rows.Count})")">
+                <MudSimpleTable Dense="true" Striped="true">
+                    <thead>
+                        <tr>
+                            <th>Output</th>
+                            <th>Question</th>
+                            <th>Gold</th>
+                            <th>First hit rank</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        @foreach (var row in run.Result.Rows)
+                        {
+                            <tr>
+                                <td><MudLink Href="@($"/reports/{row.Slug}")">@row.Slug</MudLink></td>
+                                <td>@Truncate(row.Question, 80)</td>
+                                <td>@row.GoldCount</td>
+                                <td>@(row.FirstHitRank?.ToString() ?? "—")</td>
+                            </tr>
+                        }
+                    </tbody>
+                </MudSimpleTable>
+            </MudExpansionPanel>
+        </MudExpansionPanels>
+    </MudPaper>
+}
+
+@code {
+    private string? _selectedModel;
+    private int _sample = 40;
+    private int _seed = 1;
+    private bool _running;
+    private string? _error;
+    private readonly List<LabRun> _runs = [];
+
+    protected override void OnInitialized() => _selectedModel = ActiveModel.ModelId;
+
+    private async Task RunAsync()
+    {
+        if (_selectedModel is null) return;
+        var model = Models.FirstOrDefault(m => m.ModelId == _selectedModel);
+        if (model is null) return;
+
+        _error = null;
+        _running = true;
+        try
+        {
+            var options = new EvalOptions(Sample: _sample, Seed: _seed);
+            var result = await Evaluate.EvaluateAsync(model, options, CancellationToken.None);
+            _runs.Insert(0, new LabRun(model, _sample, _seed, result));
+        }
+        catch (Exception ex)
+        {
+            _error = ex.Message;
+        }
+        finally
+        {
+            _running = false;
+        }
+    }
+
+    private static string Truncate(string text, int max) => text.Length <= max ? text : text[..max] + "…";
+
+    private sealed record LabRun(IEmbeddingModel Model, int Sample, int Seed, EvalResult Result);
+}
@@ -114,6 +114,7 @@ builder.Services.AddSingleton<Wiki.Application.Reading.NoteQueries>();
 builder.Services.AddSingleton<IChunker, Wiki.Core.Parsing.ParagraphChunker>();
 builder.Services.AddSingleton<Wiki.Application.Ask.AskUseCase>();
 builder.Services.AddSingleton<Wiki.Application.Facts.DepositFactUseCase>();
+builder.Services.AddSingleton<Wiki.Application.Evaluation.EvaluateUseCase>();
 builder.Services.AddSingleton<Wiki.Application.Questions.QuestionUseCase>();
 builder.Services.AddSingleton<Wiki.Application.Questions.SaveAnswerUseCase>();
 builder.Services.AddSingleton<StatusQuery>();
@@ -0,0 +1,98 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Wiki.Application.Evaluation;
+using Wiki.Application.Retrieval;
+using Wiki.Application.Sync;
+using Wiki.Core.Model;
+using Wiki.Core.Parsing;
+
+namespace Wiki.Application.Tests;
+
+public class EvaluateUseCaseTests
+{
+    private readonly FakeStore _store = new();
+    private readonly FakeCatalog _catalog = new();
+    private readonly FakeIndex _index = new();
+    private readonly FakeEmbeddings _model = new("m1");
+
+    private SyncUseCase Sync => new(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger<SyncUseCase>.Instance);
+    private EvaluateUseCase UseCase => new(_catalog, new RankUseCase(_index, _catalog));
+
+    /// <summary>
+    /// Three wiki notes ("a", "b", "c"; "b" lists "c" under `related:`) and three output docs:
+    /// one asking about "a" (gold [a]), one asking about "b" (gold [b, c] — "c" only reachable via
+    /// the related hop), and one with no `sources:` at all (must be skipped). Each labelled
+    /// question is set to exactly `EmbeddingText.For` its target note, so with the deterministic
+    /// fake embeddings the target note ranks first with score 1.0 and every other note ranks lower.
+    /// </summary>
+    private async Task<(string QuestionA, string QuestionB)> SeedAsync()
+    {
+        _store.Files["wiki/a.md"] = "---\ntitle: A\ntype: wiki\ntags: [x]\nupdated: 2026-09-10\n---\n\n# A\n\nAlpha paragraph.\n";
+        _store.Files["wiki/b.md"] = "---\ntitle: B\ntype: wiki\ntags: [y]\nrelated: [c]\nupdated: 2026-09-11\n---\n\n# B\n\nBeta paragraph.\n";
+        _store.Files["wiki/c.md"] = "---\ntitle: C\ntype: wiki\ntags: [z]\nupdated: 2026-09-12\n---\n\n# C\n\nGamma paragraph.\n";
+        await Sync.RunAsync(false, CancellationToken.None);
+
+        var docA = (await _catalog.GetAsync(DocumentKind.Wiki, "a", CancellationToken.None))!;
+        var docB = (await _catalog.GetAsync(DocumentKind.Wiki, "b", CancellationToken.None))!;
+        var questionA = EmbeddingText.For(docA);
+        var questionB = EmbeddingText.For(docB);
+
+        _store.Files["output/2026-09-01-q-a.md"] = $"---\ntitle: Q about A\ntype: output\nquestion: \"{questionA}\"\nsources: [a]\ngenerated: 2026-09-01\n---\n\nAnswer about A.\n";
+        _store.Files["output/2026-09-02-q-b.md"] = $"---\ntitle: Q about B\ntype: output\nquestion: \"{questionB}\"\nsources: [b, c]\ngenerated: 2026-09-02\n---\n\nAnswer about B.\n";
+        _store.Files["output/2026-09-03-no-sources.md"] = "---\ntitle: No sources\ntype: output\nquestion: \"anything\"\ngenerated: 2026-09-03\n---\n\nNo sources here.\n";
+        await Sync.RunAsync(true, CancellationToken.None);
+
+        return (questionA, questionB);
+    }
+
+    [Fact]
+    public async Task SkipsOutputsWithoutSurvivingGold()
+    {
+        await SeedAsync();
+        var result = await UseCase.EvaluateAsync(_model, new EvalOptions(K: 1), CancellationToken.None);
+        Assert.Equal(2, result.QueriesEvaluated);
+        Assert.Equal(1, result.QueriesSkipped);
+    }
+
+    [Fact]
+    public async Task ComputesRecallHitRateMrrAndTheOneHopVariant()
+    {
+        var (questionA, questionB) = await SeedAsync();
+        var result = await UseCase.EvaluateAsync(_model, new EvalOptions(K: 1), CancellationToken.None);
+
+        // Only k=1 is <= EvalOptions.K, so exactly one k is reported.
+        Assert.Equal([1], result.Recall.Keys);
+
+        // query a: gold {a}, top-1 {a} -> recall@1 = 1.0
+        // query b: gold {b, c}, top-1 {b} -> recall@1 = 0.5 (c only reachable via the related hop)
+        Assert.Equal(0.75, result.Recall[1], 3);
+
+        // "c" is related: from "b", so the hop expansion of top-1 {b} is {b, c}, covering all of query b's gold too.
+        Assert.Equal(1.0, result.RecallHop[1], 3);
+
+        // Both queries have at least one hit in the top-1 (a fully, b partially).
+        Assert.Equal(1.0, result.HitRate[1], 3);
+
+        // Both queries' first gold hit is at rank 1.
+        Assert.Equal(1.0, result.Mrr, 3);
+
+        Assert.Equal(2, result.Rows.Count);
+        var rowA = Assert.Single(result.Rows, r => r.Slug == "2026-09-01-q-a");
+        Assert.Equal(questionA, rowA.Question);
+        Assert.Equal(1, rowA.GoldCount);
+        Assert.Equal(1, rowA.FirstHitRank);
+        var rowB = Assert.Single(result.Rows, r => r.Slug == "2026-09-02-q-b");
+        Assert.Equal(questionB, rowB.Question);
+        Assert.Equal(2, rowB.GoldCount);
+        Assert.Equal(1, rowB.FirstHitRank);
+    }
+
+    [Fact]
+    public async Task SampleIsDeterministicForAGivenSeed()
+    {
+        await SeedAsync();
+        var first = await UseCase.EvaluateAsync(_model, new EvalOptions(Sample: 1, Seed: 1, K: 1), CancellationToken.None);
+        var second = await UseCase.EvaluateAsync(_model, new EvalOptions(Sample: 1, Seed: 1, K: 1), CancellationToken.None);
+        Assert.Equal(1, first.QueriesEvaluated);
+        Assert.Equal(first.Rows[0].Slug, second.Rows[0].Slug);
+    }
+}
@@ -283,6 +283,33 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture<WikiHostFact
    private sealed record RawRow(string Slug, string Title);
    private sealed record ReportRow(string Slug, string Title, string? Type);

+    [Fact]
+    public async Task LabEvalRequiresAKeyAndReportsLabelledQueries()
+    {
+        factory.Store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n";
+        factory.Store.Files["output/2026-09-20-answer.md"] = "---\ntitle: An answer\ntype: output\nquestion: udm dns\nsources: [udm-dns]\ngenerated: 2026-09-20\n---\n\nYes.\n";
+        var sync = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true");
+        sync.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
+        Assert.Equal(HttpStatusCode.OK, (await Client.SendAsync(sync)).StatusCode);
+
+        var anonymous = await Client.PostAsJsonAsync("/api/v1/lab/eval", new { });
+        Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode);
+
+        var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/lab/eval") { Content = JsonContent.Create(new { }) };
+        req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
+        var response = await Client.SendAsync(req);
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        var result = await response.Content.ReadFromJsonAsync<EvalResultDto>();
+        Assert.NotNull(result);
+        // The fixture's store is shared with the other tests, so only the seeded query is asserted.
+        Assert.Equal(result.Rows.Count, result.QueriesEvaluated);
+        Assert.Contains(result.Rows, r => r.Slug == "2026-09-20-answer" && r.GoldCount == 1);
+        Assert.Equal("bge-m3", result.ModelId);
+    }
+
+    private sealed record EvalRow(string Slug, int GoldCount);
+    private sealed record EvalResultDto(string ModelId, int QueriesEvaluated, int QueriesSkipped, List<EvalRow> Rows);
+
    [Fact]
    public async Task StatusListsAdaptersAndIndexes()
    {