From 23c891db8f5977075cd5e2e13c6732be43d56235 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 24 Sep 2026 16:39:53 +0200 Subject: [PATCH 1/2] feat(lab): rank evaluation use case, API and Lab page Adds EvaluateUseCase, the C# port of bin/wiki_rank_eval.py: every output/ document with a question and a sources list labels a query, gold is the cited slugs that still exist as wiki notes, and RankUseCase ranks each one to report recall@k, hit-rate@k, MRR and a one-hop recall variant that also credits a gold note reachable from a top-k note by wikilink or related:. Exposes it as POST /api/v1/lab/eval (write-policy) and a new /lab page for running it against any registered embedding model and comparing runs side by side. Co-Authored-By: Claude Sonnet 5 --- .../Evaluation/EvaluateUseCase.cs | 128 +++++++++++++ src/Wiki.Host/Api/ApiV1.cs | 1 + src/Wiki.Host/Api/LabEndpoints.cs | 26 +++ .../Components/Layout/MainLayout.razor | 1 + src/Wiki.Host/Components/Pages/Lab.razor | 169 ++++++++++++++++++ src/Wiki.Host/Program.cs | 1 + .../EvaluateUseCaseTests.cs | 98 ++++++++++ test/Wiki.Host.Tests/EndpointTests.cs | 24 +++ 8 files changed, 448 insertions(+) create mode 100644 src/Wiki.Application/Evaluation/EvaluateUseCase.cs create mode 100644 src/Wiki.Host/Api/LabEndpoints.cs create mode 100644 src/Wiki.Host/Components/Pages/Lab.razor create mode 100644 test/Wiki.Application.Tests/EvaluateUseCaseTests.cs diff --git a/src/Wiki.Application/Evaluation/EvaluateUseCase.cs b/src/Wiki.Application/Evaluation/EvaluateUseCase.cs new file mode 100644 index 0000000..c95a5bc --- /dev/null +++ b/src/Wiki.Application/Evaluation/EvaluateUseCase.cs @@ -0,0 +1,128 @@ +using System.Diagnostics; +using Wiki.Application.Retrieval; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Evaluation; + +public sealed record EvalOptions(int Sample = 0, int Seed = 1, int K = 50); + +public sealed record EvalQueryRow(string Slug, string Question, int GoldCount, int? FirstHitRank); + +public sealed record EvalResult( + string ModelId, + int QueriesEvaluated, + int QueriesSkipped, + IReadOnlyDictionary Recall, + IReadOnlyDictionary RecallHop, + IReadOnlyDictionary HitRate, + double Mrr, + TimeSpan MeanElapsedPerQuery, + TimeSpan Wall, + IReadOnlyList Rows); + +/// +/// bin/wiki_rank_eval.py as a use case: every output/ document with a `question:` and a +/// `sources:` list is a labelled query, and gold is the cited slugs that still exist as wiki +/// notes. Ranks each query with and reports recall@k, hit-rate@k and +/// MRR for k in {1,5,10,20,50} (capped to ), plus a one-hop variant +/// of recall that also credits a gold note reachable from a top-k note by wikilink or `related:`. +/// No judge involvement: this measures the embedding model's ranking alone. +/// +public sealed class EvaluateUseCase(IDocumentCatalog catalog, RankUseCase rank) +{ + private static readonly int[] AllKs = [1, 5, 10, 20, 50]; + + public async Task EvaluateAsync(IEmbeddingModel model, EvalOptions options, CancellationToken ct) + { + var started = Stopwatch.GetTimestamp(); + var (all, skipped) = await LoadQueriesAsync(ct); + var queries = Sample(all, options); + + var k = Math.Clamp(options.K, 1, RankUseCase.MaxK); + var ks = AllKs.Where(x => x <= k).ToList(); + if (ks.Count == 0) ks = [k]; + + var recallSums = ks.ToDictionary(x => x, _ => 0.0); + var recallHopSums = ks.ToDictionary(x => x, _ => 0.0); + var hitCounts = ks.ToDictionary(x => x, _ => 0); + var mrrSum = 0.0; + var elapsedTotal = TimeSpan.Zero; + var rows = new List(queries.Count); + + foreach (var q in queries) + { + var queryStarted = Stopwatch.GetTimestamp(); + var ranked = await rank.RankAsync(model, q.Question, k, ct); + elapsedTotal += Stopwatch.GetElapsedTime(queryStarted); + + var rankedSlugs = ranked.Select(r => r.Slug).ToList(); + var goldSet = q.Gold.ToHashSet(); + int? firstHit = null; + for (var i = 0; i < rankedSlugs.Count; i++) + { + if (!goldSet.Contains(rankedSlugs[i])) continue; + firstHit = i + 1; + break; + } + mrrSum += firstHit is { } fh ? 1.0 / fh : 0.0; + + foreach (var topK in ks) + { + var top = rankedSlugs.Take(topK).ToHashSet(); + var recallAtK = q.Gold.Count(top.Contains) / (double)q.Gold.Count; + recallSums[topK] += recallAtK; + if (recallAtK > 0) hitCounts[topK]++; + + var hop = new HashSet(top); + foreach (var slug in top) + { + foreach (var link in await catalog.OutgoingLinksAsync(slug, ct)) + hop.Add(link.ToSlug); + } + recallHopSums[topK] += q.Gold.Count(hop.Contains) / (double)q.Gold.Count; + } + + rows.Add(new EvalQueryRow(q.Slug, q.Question, q.Gold.Count, firstHit)); + } + + var n = Math.Max(queries.Count, 1); + var recall = ks.ToDictionary(x => x, x => recallSums[x] / n); + var recallHop = ks.ToDictionary(x => x, x => recallHopSums[x] / n); + var hitRate = ks.ToDictionary(x => x, x => hitCounts[x] / (double)n); + var mrr = mrrSum / n; + var meanElapsed = queries.Count == 0 ? TimeSpan.Zero : elapsedTotal / queries.Count; + + return new EvalResult(model.ModelId, queries.Count, skipped, recall, recallHop, hitRate, mrr, meanElapsed, Stopwatch.GetElapsedTime(started), rows); + } + + private async Task<(List Queries, int Skipped)> LoadQueriesAsync(CancellationToken ct) + { + var docs = await catalog.ListDocumentsAsync(DocumentKind.Output, ct); + var queries = new List(); + var skipped = 0; + foreach (var d in docs.OrderBy(d => d.Slug, StringComparer.Ordinal)) + { + var question = d.Frontmatter.TryGetValue("question", out var v) ? v.Scalar : null; + var gold = new List(); + foreach (var s in d.Sources) + { + if (await catalog.GetAsync(DocumentKind.Wiki, s, ct) is not null) gold.Add(s); + } + if (string.IsNullOrWhiteSpace(question) || gold.Count == 0) { skipped++; continue; } + queries.Add(new LabelledQuery(d.Slug, question, gold)); + } + return (queries, skipped); + } + + /// A deterministic random sample: keys drawn from 's seed + /// over the queries in slug order, then the lowest-keyed kept. + private static List Sample(List queries, EvalOptions options) + { + if (options.Sample <= 0 || options.Sample >= queries.Count) return queries; + var rng = new Random(options.Seed); + return queries.OrderBy(_ => rng.Next()).Take(options.Sample).ToList(); + } + + private sealed record LabelledQuery(string Slug, string Question, IReadOnlyList Gold); +} diff --git a/src/Wiki.Host/Api/ApiV1.cs b/src/Wiki.Host/Api/ApiV1.cs index ec36fb0..c8a9751 100644 --- a/src/Wiki.Host/Api/ApiV1.cs +++ b/src/Wiki.Host/Api/ApiV1.cs @@ -12,6 +12,7 @@ public static class ApiV1 v1.MapReadEndpoints(); v1.MapAskEndpoints(); v1.MapFactEndpoints(); + v1.MapLabEndpoints(); v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) => Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous(); diff --git a/src/Wiki.Host/Api/LabEndpoints.cs b/src/Wiki.Host/Api/LabEndpoints.cs new file mode 100644 index 0000000..f778769 --- /dev/null +++ b/src/Wiki.Host/Api/LabEndpoints.cs @@ -0,0 +1,26 @@ +using Wiki.Application.Evaluation; +using Wiki.Core.Ports; + +namespace Wiki.Host.Api; + +/// Wire shape of an eval run request; every field is optional and falls back to a default. +public sealed record EvalRequestDto(string? Model, int? Sample, int? Seed, int? K); + +public static class LabEndpoints +{ + public static void MapLabEndpoints(this RouteGroupBuilder v1) + { + v1.MapPost("/lab/eval", async (EvalRequestDto body, EvaluateUseCase evaluate, IEmbeddingModel activeModel, IServiceProvider services, CancellationToken ct) => + { + var model = activeModel; + if (!string.IsNullOrEmpty(body.Model)) + { + model = services.GetKeyedService(body.Model); + if (model is null) return Results.BadRequest(new { error = $"unknown model '{body.Model}'" }); + } + var options = new EvalOptions(Sample: body.Sample ?? 0, Seed: body.Seed ?? 1, K: body.K ?? 50); + var result = await evaluate.EvaluateAsync(model, options, ct); + return Results.Ok(result); + }).RequireAuthorization(Auth.Policies.Write); + } +} diff --git a/src/Wiki.Host/Components/Layout/MainLayout.razor b/src/Wiki.Host/Components/Layout/MainLayout.razor index c22f8bd..8cd6c9b 100644 --- a/src/Wiki.Host/Components/Layout/MainLayout.razor +++ b/src/Wiki.Host/Components/Layout/MainLayout.razor @@ -15,6 +15,7 @@ Search Browse Maintenance + Lab Status diff --git a/src/Wiki.Host/Components/Pages/Lab.razor b/src/Wiki.Host/Components/Pages/Lab.razor new file mode 100644 index 0000000..f421e1c --- /dev/null +++ b/src/Wiki.Host/Components/Pages/Lab.razor @@ -0,0 +1,169 @@ +@page "/lab" +@rendermode InteractiveServer +@using Wiki.Application.Evaluation +@inject IReadOnlyList Models +@inject IEmbeddingModel ActiveModel +@inject EvaluateUseCase Evaluate + +Lab + +Lab + + Rank every labelled question from output/ against a chosen embedding model and report recall, hit-rate and MRR. No judge involved. + + + + + + + @foreach (var m in Models) + { + @m.ModelId@(m.ModelId == ActiveModel.ModelId ? " (active)" : "") + } + + + + + + + + + + Run + + + + +@if (_running) +{ + +} + +@if (_error is not null) +{ + Eval failed: @_error +} + +@if (_runs.Count == 0 && !_running) +{ + No runs yet. Pick a model and hit Run. +} + +@foreach (var run in _runs) +{ + + + @run.Model.ModelId — sample @(run.Sample == 0 ? "all" : run.Sample.ToString()), seed @run.Seed + + + @{ var ks = run.Result.Recall.Keys.OrderBy(k => k).ToList(); } + + + + Metric + @foreach (var k in ks) + { + k=@k + } + + + + + recall@k + @foreach (var k in ks) + { + @run.Result.Recall[k].ToString("0.00") + } + + + recall_hop@k + @foreach (var k in ks) + { + @run.Result.RecallHop[k].ToString("0.00") + } + + + hit_rate@k + @foreach (var k in ks) + { + @run.Result.HitRate[k].ToString("0.00") + } + + + + + + + MRR@run.Result.Mrr.ToString("0.000") + Queries evaluated / skipped@run.Result.QueriesEvaluated / @run.Result.QueriesSkipped + Mean elapsed per query@run.Result.MeanElapsedPerQuery.TotalMilliseconds.ToString("0") ms + Wall time@run.Result.Wall.TotalSeconds.ToString("0.00") s + + + + + + + + + Output + Question + Gold + First hit rank + + + + @foreach (var row in run.Result.Rows) + { + + @row.Slug + @Truncate(row.Question, 80) + @row.GoldCount + @(row.FirstHitRank?.ToString() ?? "—") + + } + + + + + +} + +@code { + private string? _selectedModel; + private int _sample = 40; + private int _seed = 1; + private bool _running; + private string? _error; + private readonly List _runs = []; + + protected override void OnInitialized() => _selectedModel = ActiveModel.ModelId; + + private async Task RunAsync() + { + if (_selectedModel is null) return; + var model = Models.FirstOrDefault(m => m.ModelId == _selectedModel); + if (model is null) return; + + _error = null; + _running = true; + try + { + var options = new EvalOptions(Sample: _sample, Seed: _seed); + var result = await Evaluate.EvaluateAsync(model, options, CancellationToken.None); + _runs.Insert(0, new LabRun(model, _sample, _seed, result)); + } + catch (Exception ex) + { + _error = ex.Message; + } + finally + { + _running = false; + } + } + + private static string Truncate(string text, int max) => text.Length <= max ? text : text[..max] + "…"; + + private sealed record LabRun(IEmbeddingModel Model, int Sample, int Seed, EvalResult Result); +} diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index d2425af..4c082ed 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -91,6 +91,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); diff --git a/test/Wiki.Application.Tests/EvaluateUseCaseTests.cs b/test/Wiki.Application.Tests/EvaluateUseCaseTests.cs new file mode 100644 index 0000000..10e4c02 --- /dev/null +++ b/test/Wiki.Application.Tests/EvaluateUseCaseTests.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Evaluation; +using Wiki.Application.Retrieval; +using Wiki.Application.Sync; +using Wiki.Core.Model; +using Wiki.Core.Parsing; + +namespace Wiki.Application.Tests; + +public class EvaluateUseCaseTests +{ + private readonly FakeStore _store = new(); + private readonly FakeCatalog _catalog = new(); + private readonly FakeIndex _index = new(); + private readonly FakeEmbeddings _model = new("m1"); + + private SyncUseCase Sync => new(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger.Instance); + private EvaluateUseCase UseCase => new(_catalog, new RankUseCase(_index, _catalog)); + + /// + /// Three wiki notes ("a", "b", "c"; "b" lists "c" under `related:`) and three output docs: + /// one asking about "a" (gold [a]), one asking about "b" (gold [b, c] — "c" only reachable via + /// the related hop), and one with no `sources:` at all (must be skipped). Each labelled + /// question is set to exactly `EmbeddingText.For` its target note, so with the deterministic + /// fake embeddings the target note ranks first with score 1.0 and every other note ranks lower. + /// + private async Task<(string QuestionA, string QuestionB)> SeedAsync() + { + _store.Files["wiki/a.md"] = "---\ntitle: A\ntype: wiki\ntags: [x]\nupdated: 2026-09-10\n---\n\n# A\n\nAlpha paragraph.\n"; + _store.Files["wiki/b.md"] = "---\ntitle: B\ntype: wiki\ntags: [y]\nrelated: [c]\nupdated: 2026-09-11\n---\n\n# B\n\nBeta paragraph.\n"; + _store.Files["wiki/c.md"] = "---\ntitle: C\ntype: wiki\ntags: [z]\nupdated: 2026-09-12\n---\n\n# C\n\nGamma paragraph.\n"; + await Sync.RunAsync(false, CancellationToken.None); + + var docA = (await _catalog.GetAsync(DocumentKind.Wiki, "a", CancellationToken.None))!; + var docB = (await _catalog.GetAsync(DocumentKind.Wiki, "b", CancellationToken.None))!; + var questionA = EmbeddingText.For(docA); + var questionB = EmbeddingText.For(docB); + + _store.Files["output/2026-09-01-q-a.md"] = $"---\ntitle: Q about A\ntype: output\nquestion: \"{questionA}\"\nsources: [a]\ngenerated: 2026-09-01\n---\n\nAnswer about A.\n"; + _store.Files["output/2026-09-02-q-b.md"] = $"---\ntitle: Q about B\ntype: output\nquestion: \"{questionB}\"\nsources: [b, c]\ngenerated: 2026-09-02\n---\n\nAnswer about B.\n"; + _store.Files["output/2026-09-03-no-sources.md"] = "---\ntitle: No sources\ntype: output\nquestion: \"anything\"\ngenerated: 2026-09-03\n---\n\nNo sources here.\n"; + await Sync.RunAsync(true, CancellationToken.None); + + return (questionA, questionB); + } + + [Fact] + public async Task SkipsOutputsWithoutSurvivingGold() + { + await SeedAsync(); + var result = await UseCase.EvaluateAsync(_model, new EvalOptions(K: 1), CancellationToken.None); + Assert.Equal(2, result.QueriesEvaluated); + Assert.Equal(1, result.QueriesSkipped); + } + + [Fact] + public async Task ComputesRecallHitRateMrrAndTheOneHopVariant() + { + var (questionA, questionB) = await SeedAsync(); + var result = await UseCase.EvaluateAsync(_model, new EvalOptions(K: 1), CancellationToken.None); + + // Only k=1 is <= EvalOptions.K, so exactly one k is reported. + Assert.Equal([1], result.Recall.Keys); + + // query a: gold {a}, top-1 {a} -> recall@1 = 1.0 + // query b: gold {b, c}, top-1 {b} -> recall@1 = 0.5 (c only reachable via the related hop) + Assert.Equal(0.75, result.Recall[1], 3); + + // "c" is related: from "b", so the hop expansion of top-1 {b} is {b, c}, covering all of query b's gold too. + Assert.Equal(1.0, result.RecallHop[1], 3); + + // Both queries have at least one hit in the top-1 (a fully, b partially). + Assert.Equal(1.0, result.HitRate[1], 3); + + // Both queries' first gold hit is at rank 1. + Assert.Equal(1.0, result.Mrr, 3); + + Assert.Equal(2, result.Rows.Count); + var rowA = Assert.Single(result.Rows, r => r.Slug == "2026-09-01-q-a"); + Assert.Equal(questionA, rowA.Question); + Assert.Equal(1, rowA.GoldCount); + Assert.Equal(1, rowA.FirstHitRank); + var rowB = Assert.Single(result.Rows, r => r.Slug == "2026-09-02-q-b"); + Assert.Equal(questionB, rowB.Question); + Assert.Equal(2, rowB.GoldCount); + Assert.Equal(1, rowB.FirstHitRank); + } + + [Fact] + public async Task SampleIsDeterministicForAGivenSeed() + { + await SeedAsync(); + var first = await UseCase.EvaluateAsync(_model, new EvalOptions(Sample: 1, Seed: 1, K: 1), CancellationToken.None); + var second = await UseCase.EvaluateAsync(_model, new EvalOptions(Sample: 1, Seed: 1, K: 1), CancellationToken.None); + Assert.Equal(1, first.QueriesEvaluated); + Assert.Equal(first.Rows[0].Slug, second.Rows[0].Slug); + } +} diff --git a/test/Wiki.Host.Tests/EndpointTests.cs b/test/Wiki.Host.Tests/EndpointTests.cs index 8a066a0..a28af91 100644 --- a/test/Wiki.Host.Tests/EndpointTests.cs +++ b/test/Wiki.Host.Tests/EndpointTests.cs @@ -245,6 +245,30 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture(); + Assert.NotNull(result); + Assert.Equal(1, result.QueriesEvaluated); + Assert.Equal("bge-m3", result.ModelId); + } + + private sealed record EvalResultDto(string ModelId, int QueriesEvaluated, int QueriesSkipped); + [Fact] public async Task StatusListsAdaptersAndIndexes() { -- 2.54.0 From e28a4032849287b3680b0b4dacb7a23e63532d73 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 24 Sep 2026 16:43:17 +0200 Subject: [PATCH 2/2] docs: list the lab eval endpoint Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 300624c..58daa8c 100644 --- a/README.md +++ b/README.md @@ -24,4 +24,4 @@ dotnet run --project src/Wiki.Host ``` Endpoints: `/health` (liveness), `/ready` (readiness), `/api/v1/status`, `POST /rank`, -`POST /webhook`. +`POST /webhook`, `POST /api/v1/lab/eval` (rank evaluation over labelled `output/` questions, API key). -- 2.54.0