diff --git a/src/Wiki.Adapters.TypeSafe/TypeSafeJudge.cs b/src/Wiki.Adapters.TypeSafe/TypeSafeJudge.cs index 0fc5518..017d58d 100644 --- a/src/Wiki.Adapters.TypeSafe/TypeSafeJudge.cs +++ b/src/Wiki.Adapters.TypeSafe/TypeSafeJudge.cs @@ -173,6 +173,9 @@ public sealed class TypeSafeJudge : IJudge var json = body.ToJsonString(JsonOptions); var delay = TimeSpan.FromSeconds(1); + // A missing key is a runtime condition the status page shows, not a reason the host cannot start. + if (string.IsNullOrWhiteSpace(_options.ApiKey)) + throw new InvalidOperationException("TypeSafe API key is not configured."); for (var attempt = 0; ; attempt++) { using var requestMessage = new HttpRequestMessage(HttpMethod.Post, _options.Url) diff --git a/src/Wiki.Adapters.TypeSafe/TypeSafeOptions.cs b/src/Wiki.Adapters.TypeSafe/TypeSafeOptions.cs index bcf34cf..ca90527 100644 --- a/src/Wiki.Adapters.TypeSafe/TypeSafeOptions.cs +++ b/src/Wiki.Adapters.TypeSafe/TypeSafeOptions.cs @@ -39,11 +39,6 @@ public sealed class TypeSafeOptions throw new ArgumentException("Model must be set.", nameof(Model)); } - if (string.IsNullOrWhiteSpace(ApiKey)) - { - throw new ArgumentException("ApiKey must be set.", nameof(ApiKey)); - } - if (MaxRetries < 0) { throw new ArgumentException("MaxRetries must not be negative.", nameof(MaxRetries)); diff --git a/src/Wiki.Application/Ask/AskUseCase.cs b/src/Wiki.Application/Ask/AskUseCase.cs new file mode 100644 index 0000000..7de3c36 --- /dev/null +++ b/src/Wiki.Application/Ask/AskUseCase.cs @@ -0,0 +1,253 @@ +using System.Diagnostics; +using Wiki.Application.Retrieval; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Ask; + +/// A caller's question, before expansion. Context choices are expanded over the context's paragraph or document ids. +public abstract record AskQuestion(string Instructions) +{ + public sealed record Noul(string Instructions, string TrueCriterion, string FalseCriterion) : AskQuestion(Instructions); + public sealed record Choice(string Instructions, IReadOnlyDictionary Options) : AskQuestion(Instructions); + public sealed record ContextChoice(string Instructions, ContextScope Scope) : AskQuestion(Instructions); + public sealed record Score(string Instructions, string Criteria) : AskQuestion(Instructions); +} + +public enum ContextScope +{ + Paragraphs, + Documents, +} + +public sealed record AskOptions(int K = 6, int DocChars = 12_000, int StateChars = 60_000, int MaxParallel = 8); + +public sealed record AskRequest(IReadOnlyList Queries, IReadOnlyDictionary? Questions, AskOptions Options); + +public sealed record ContextDocument(string Slug, string Title, IReadOnlyList Paragraphs, bool Truncated, float RankScore); + +public sealed record RetrievalMeta(IReadOnlyList DroppedForBudget, int Chars, int ParagraphCount, IReadOnlyDictionary RankScoreBySlug); + +/// A resolved context hit: a paragraph (with its text) or a document (with its title). +public sealed record ContextHit(string Id, double P, string Slug, string Title, string? Heading, string? Text); + +public abstract record AskAnswer +{ + public sealed record Noul(double Probability) : AskAnswer; + public sealed record Choice(string Selected, IReadOnlyDictionary Probabilities, double Confidence) : AskAnswer; + public sealed record Context(IReadOnlyList Top, double NoneP, double Confidence) : AskAnswer; + public sealed record Score(double Value) : AskAnswer; +} + +public sealed record AskResult( + IReadOnlyList Queries, + IReadOnlyList Context, + RetrievalMeta Retrieval, + IReadOnlyDictionary Answers, + double? Covered, + JudgeUsage Usage, + string JudgeModel, + TimeSpan Elapsed); + +/// +/// wiki_ask.py as a use case: rank every query, unite the top notes into one context (per-document +/// and total budget), expand context choices over the paragraph or document ids plus NONE, add the +/// covered gate, judge (every context choice in its own request, the rest batched), and attach the +/// answering paragraphs' text to the results. Judgements and quotations, never prose. +/// +public sealed class AskUseCase(RankUseCase rank, IDocumentCatalog catalog, IChunker chunker) +{ + public const string None = "NONE"; + public const string CoveredId = "_covered"; + + public async Task AskAsync(AskRequest request, IEmbeddingModel model, IJudge judge, CancellationToken ct) + { + if (request.Queries.Count == 0) throw new ArgumentException("at least one query is required", nameof(request)); + var started = Stopwatch.GetTimestamp(); + var (context, meta) = await BuildContextAsync(request.Queries, model, request.Options, ct); + + var questions = request.Questions ?? DefaultQuestions(request.Queries); + var (expanded, scopes) = Expand(questions, context, request.Queries, judge.MaxChoiceOptions); + + var (answers, usage) = await JudgeInParallelAsync(judge, context, expanded, scopes, request.Options.MaxParallel, ct); + var resolved = Resolve(answers, scopes, context); + double? covered = answers.TryGetValue(CoveredId, out var c) && c is Judgement.Noul n ? Math.Round(n.Probability, 3) : null; + return new AskResult(request.Queries, context, meta, resolved, covered, usage, judge.ModelId, Stopwatch.GetElapsedTime(started)); + } + + public async Task<(IReadOnlyList Context, RetrievalMeta Meta)> BuildContextAsync( + IReadOnlyList queries, IEmbeddingModel model, AskOptions options, CancellationToken ct) + { + var order = new Dictionary(); + foreach (var q in queries) + { + var rows = await rank.RankAsync(model, q, options.K, ct); + for (var pos = 0; pos < rows.Count; pos++) + { + var score = rows[pos].Score > 0 ? rows[pos].Score : 1e-6f / (pos + 1); + order[rows[pos].Slug] = Math.Max(order.GetValueOrDefault(rows[pos].Slug), score); + } + } + + var docs = new List(); + var dropped = new List(); + var used = 0; + foreach (var slug in order.Keys.OrderByDescending(s => order[s]).ThenBy(s => s)) + { + var document = await catalog.GetAsync(DocumentKind.Wiki, slug, ct); + if (document is null) continue; + var (paragraphs, truncated) = Cap(chunker.Split(document), options.DocChars); + var size = document.Title.Length + paragraphs.Sum(p => p.Text.Length + p.Heading.Length); + if (used + size > options.StateChars && docs.Count > 0) + { + dropped.Add(slug); + continue; + } + docs.Add(new ContextDocument(slug, document.Title, paragraphs, truncated, order[slug])); + used += size; + } + var meta = new RetrievalMeta(dropped, used, docs.Sum(d => d.Paragraphs.Count), + order.ToDictionary(kv => kv.Key, kv => (float)Math.Round(kv.Value, 3))); + return (docs, meta); + } + + private static (IReadOnlyList Paragraphs, bool Truncated) Cap(IReadOnlyList paragraphs, int docChars) + { + var kept = new List(); + var used = 0; + foreach (var p in paragraphs) + { + var size = p.Text.Length + p.Heading.Length; + if (kept.Count > 0 && used + size > docChars) break; + kept.Add(p); + used += size; + } + return (kept, kept.Count < paragraphs.Count); + } + + public static IReadOnlyDictionary DefaultQuestions(IReadOnlyList queries) + { + var qs = new Dictionary(); + for (var i = 0; i < queries.Count; i++) + { + var q = queries[i]; + qs[$"answered_{i + 1}"] = new AskQuestion.Noul( + $"Do the `documents` answer this question: {q}", + "At least one paragraph states what the question asks for.", + "The documents only touch the topic or do not address the question."); + qs[$"where_{i + 1}"] = new AskQuestion.ContextChoice( + $"Which paragraph (by `id`) most directly answers this question: {q} — pick {None} if none does.", + ContextScope.Paragraphs); + } + return qs; + } + + public static (IReadOnlyDictionary Questions, IReadOnlyDictionary Scopes) Expand( + IReadOnlyDictionary questions, IReadOnlyList context, IReadOnlyList queries, int? maxOptions) + { + var paragraphIds = context.SelectMany(d => d.Paragraphs.Select(p => p.Id)).ToList(); + var slugs = context.Select(d => d.Slug).ToList(); + var expanded = new Dictionary(); + var scopes = new Dictionary(); + foreach (var (id, q) in questions) + { + switch (q) + { + case AskQuestion.Noul n: + expanded[id] = new TypedQuestion.Noul(n.Instructions, n.TrueCriterion, n.FalseCriterion); + break; + case AskQuestion.Choice c: + expanded[id] = new TypedQuestion.Choice(c.Instructions, c.Options); + break; + case AskQuestion.Score s: + expanded[id] = new TypedQuestion.Score(s.Instructions, s.Criteria); + break; + case AskQuestion.ContextChoice cc: + var ids = cc.Scope == ContextScope.Paragraphs ? paragraphIds : slugs; + if (maxOptions is { } cap && ids.Count + 1 > cap) + throw new ArgumentException($"{id}: {ids.Count} {cc.Scope.ToString().ToLowerInvariant()} exceed the {cap - 1}-option Choice cap; lower k"); + var options = ids.ToDictionary(i => i, _ => string.Empty); + options[None] = $"No {(cc.Scope == ContextScope.Paragraphs ? "paragraph" : "document")} in the documents fits"; + expanded[id] = new TypedQuestion.Choice(cc.Instructions, options); + scopes[id] = cc.Scope; + break; + } + } + if (!expanded.ContainsKey(CoveredId)) + { + expanded[CoveredId] = new TypedQuestion.Noul( + "Do the `documents` together contain enough to answer these retrieval queries: " + string.Join(" | ", queries), + "The queries can be answered from the documents, possibly combining several.", + "The documents only touch the topic; a real answer would need material not here."); + } + return (expanded, scopes); + } + + private static async Task<(IReadOnlyDictionary Answers, JudgeUsage Usage)> JudgeInParallelAsync( + IJudge judge, IReadOnlyList context, IReadOnlyDictionary questions, + IReadOnlyDictionary scopes, int maxParallel, CancellationToken ct) + { + var state = new + { + documents = context.Select(d => new + { + slug = d.Slug, + title = d.Title, + paragraphs = d.Paragraphs.Select(p => new { id = p.Id, heading = p.Heading, text = p.Text }), + }), + }; + var groups = new List>(); + foreach (var (id, q) in questions.Where(kv => scopes.ContainsKey(kv.Key))) + groups.Add(new Dictionary { [id] = q }); + var rest = questions.Where(kv => !scopes.ContainsKey(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value); + if (rest.Count > 0) groups.Add(rest); + + using var gate = new SemaphoreSlim(Math.Max(1, maxParallel)); + var tasks = groups.Select(async g => + { + await gate.WaitAsync(ct); + try { return await judge.JudgeAsync(new JudgeRequest(state, g), ct); } + finally { gate.Release(); } + }).ToList(); + var results = await Task.WhenAll(tasks); + + var answers = new Dictionary(); + var usage = JudgeUsage.None; + foreach (var r in results) + { + foreach (var (id, a) in r.Answers) answers[id] = a; + usage = usage.Add(r.Usage); + } + return (answers, usage); + } + + private static IReadOnlyDictionary Resolve( + IReadOnlyDictionary answers, IReadOnlyDictionary scopes, IReadOnlyList context) + { + var byParagraph = context.SelectMany(d => d.Paragraphs.Select(p => (p, d))).ToDictionary(x => x.p.Id, x => x); + var titles = context.ToDictionary(d => d.Slug, d => d.Title); + var resolved = new Dictionary(); + foreach (var (id, a) in answers) + { + if (scopes.TryGetValue(id, out var scope) && a is Judgement.Choice c) + { + var top = c.Probabilities.Where(kv => kv.Key != None && kv.Value >= 0.03) + .OrderByDescending(kv => kv.Value).Take(3) + .Select(kv => scope == ContextScope.Paragraphs && byParagraph.TryGetValue(kv.Key, out var hit) + ? new ContextHit(kv.Key, Math.Round(kv.Value, 3), hit.d.Slug, hit.d.Title, hit.p.Heading, hit.p.Text) + : new ContextHit(kv.Key, Math.Round(kv.Value, 3), kv.Key, titles.GetValueOrDefault(kv.Key, kv.Key), null, null)) + .ToList(); + resolved[id] = new AskAnswer.Context(top, Math.Round(c.Probabilities.GetValueOrDefault(None), 3), c.Confidence); + continue; + } + resolved[id] = a switch + { + Judgement.Noul n => new AskAnswer.Noul(n.Probability), + Judgement.Choice ch => new AskAnswer.Choice(ch.Selected, ch.Probabilities, ch.Confidence), + Judgement.Score s => new AskAnswer.Score(s.Value), + _ => throw new InvalidOperationException($"unknown judgement for {id}"), + }; + } + return resolved; + } +} diff --git a/src/Wiki.Host/Api/ApiV1.cs b/src/Wiki.Host/Api/ApiV1.cs index f7e7dff..249d1f6 100644 --- a/src/Wiki.Host/Api/ApiV1.cs +++ b/src/Wiki.Host/Api/ApiV1.cs @@ -10,6 +10,7 @@ public static class ApiV1 { var v1 = app.MapGroup("/api/v1"); v1.MapReadEndpoints(); + v1.MapAskEndpoints(); v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) => Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous(); diff --git a/src/Wiki.Host/Api/AskEndpoints.cs b/src/Wiki.Host/Api/AskEndpoints.cs new file mode 100644 index 0000000..f27d757 --- /dev/null +++ b/src/Wiki.Host/Api/AskEndpoints.cs @@ -0,0 +1,122 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Wiki.Application.Ask; +using Wiki.Core.Ports; + +namespace Wiki.Host.Api; + +/// Wire shape of a caller's question, as bin/wiki_ask.py's --questions JSON: criteria is a +/// {true,false} object for noul, an options object or the strings "paragraphs"/"documents" for choice, a string for score. +public sealed record AskQuestionDto(string Type, string Instructions, JsonElement? Criteria); + +public sealed record AskRequestDto( + [property: JsonPropertyName("queries")] IReadOnlyList? Queries, + [property: JsonPropertyName("questions")] IReadOnlyDictionary? Questions, + [property: JsonPropertyName("k")] int? K, + [property: JsonPropertyName("model")] string? Model, + [property: JsonPropertyName("judge")] string? Judge); + +public static class AskEndpoints +{ + public static void MapAskEndpoints(this RouteGroupBuilder v1) + { + v1.MapPost("/ask", async (AskRequestDto body, AskUseCase ask, IEmbeddingModel activeModel, IJudge activeJudge, IServiceProvider services, CancellationToken ct) => + { + if (body.Queries is null || body.Queries.Count == 0 || body.Queries.All(string.IsNullOrWhiteSpace)) + return Results.BadRequest(new { error = "queries required" }); + + var model = activeModel; + if (!string.IsNullOrEmpty(body.Model)) + { + model = services.GetKeyedService(body.Model); + if (model is null) return Results.BadRequest(new { error = $"unknown model '{body.Model}'" }); + } + var judge = activeJudge; + if (!string.IsNullOrEmpty(body.Judge)) + { + judge = services.GetKeyedService(body.Judge); + if (judge is null) return Results.BadRequest(new { error = $"unknown judge '{body.Judge}'" }); + } + + IReadOnlyDictionary? questions; + try + { + questions = body.Questions is null ? null : body.Questions.ToDictionary(kv => kv.Key, kv => Parse(kv.Key, kv.Value)); + } + catch (ArgumentException e) + { + return Results.BadRequest(new { error = e.Message }); + } + + try + { + var result = await ask.AskAsync(new AskRequest(body.Queries.Where(q => !string.IsNullOrWhiteSpace(q)).ToList(), questions, new AskOptions(K: body.K ?? 6)), model, judge, ct); + return Results.Ok(new + { + result.Queries, + context = result.Context.Select(d => new { d.Slug, d.Title, paragraphs = d.Paragraphs.Count, d.Truncated, rankScore = d.RankScore }), + retrieval = new { result.Retrieval.DroppedForBudget, result.Retrieval.Chars, result.Retrieval.ParagraphCount }, + answers = result.Answers.ToDictionary(kv => kv.Key, kv => (object)(kv.Value switch + { + AskAnswer.Noul n => new { noul = n.Probability }, + AskAnswer.Score s => new { score = s.Value }, + AskAnswer.Choice c => new { choice = c.Selected, probabilities = c.Probabilities, confidence = c.Confidence }, + AskAnswer.Context x => new { top = x.Top, none_p = x.NoneP, confidence = x.Confidence }, + _ => new { }, + })), + result.Covered, + usage = result.Usage, + judge = result.JudgeModel, + model = model.ModelId, + elapsed_s = Math.Round(result.Elapsed.TotalSeconds, 3), + }); + } + catch (ArgumentException e) + { + return Results.BadRequest(new { error = e.Message }); + } + catch (HttpRequestException e) + { + return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway); + } + }).AllowAnonymous(); + } + + public static AskQuestion Parse(string id, AskQuestionDto dto) + { + var criteria = dto.Criteria; + switch (dto.Type?.ToLowerInvariant()) + { + case "noul": + { + if (criteria is not { ValueKind: JsonValueKind.Object } o || !o.TryGetProperty("true", out var t) || !o.TryGetProperty("false", out var f)) + throw new ArgumentException($"{id}: noul needs criteria {{true, false}}"); + return new AskQuestion.Noul(dto.Instructions, t.GetString() ?? string.Empty, f.GetString() ?? string.Empty); + } + case "choice": + { + if (criteria is { ValueKind: JsonValueKind.String } s) + { + return s.GetString() switch + { + "paragraphs" => new AskQuestion.ContextChoice(dto.Instructions, ContextScope.Paragraphs), + "documents" => new AskQuestion.ContextChoice(dto.Instructions, ContextScope.Documents), + var other => throw new ArgumentException($"{id}: unknown choice criteria '{other}'"), + }; + } + if (criteria is { ValueKind: JsonValueKind.Object } o) + { + var options = new Dictionary(); + foreach (var p in o.EnumerateObject()) options[p.Name] = p.Value.ValueKind == JsonValueKind.String ? p.Value.GetString() ?? string.Empty : string.Empty; + if (options.Count == 0) throw new ArgumentException($"{id}: choice needs options"); + return new AskQuestion.Choice(dto.Instructions, options); + } + throw new ArgumentException($"{id}: choice needs criteria (options object, \"paragraphs\" or \"documents\")"); + } + case "score": + return new AskQuestion.Score(dto.Instructions, criteria is { ValueKind: JsonValueKind.String } sc ? sc.GetString() ?? string.Empty : string.Empty); + default: + throw new ArgumentException($"{id}: unknown question type '{dto.Type}'"); + } + } +} diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs index 2b39673..92c95f5 100644 --- a/src/Wiki.Host/Composition/WikiOptions.cs +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -5,6 +5,7 @@ public sealed class WikiOptions { public NoteStoreOptions NoteStore { get; set; } = new(); public EmbeddingsOptions Embeddings { get; set; } = new(); + public JudgesOptions Judges { get; set; } = new(); public IndexOptions Index { get; set; } = new(); public SyncScheduleOptions Sync { get; set; } = new(); } @@ -36,6 +37,19 @@ public sealed class EmbeddingModelOptions public int Dimension { get; set; } = 1024; } +public sealed class JudgesOptions +{ + public string Active { get; set; } = "typesafe"; + public Dictionary Models { get; set; } = new(); +} + +public sealed class JudgeModelOptions +{ + public string Adapter { get; set; } = "TypeSafe"; + public string Url { get; set; } = "https://api.typesafe.ai/v1/systemone"; + public string Model { get; set; } = "jev-latest"; +} + public sealed class IndexOptions { public string Active { get; set; } = "file"; diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index 1ec622f..85f0796 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -3,6 +3,7 @@ using MudBlazor.Services; using Wiki.Adapters.FileIndex; using Wiki.Adapters.Gitea; using Wiki.Adapters.LlamaCpp; +using Wiki.Adapters.TypeSafe; using Wiki.Application.Retrieval; using Wiki.Application.Status; using Wiki.Application.Sync; @@ -47,6 +48,26 @@ foreach (var (name, model) in wiki.Embeddings.Models) builder.Services.AddSingleton>(sp => wiki.Embeddings.Models.Keys.Select(sp.GetRequiredKeyedService).ToList()); builder.Services.AddSingleton(sp => sp.GetRequiredKeyedService(wiki.Embeddings.Active)); +foreach (var (name, judge) in wiki.Judges.Models) +{ + switch (judge.Adapter) + { + case "TypeSafe": + builder.Services.AddTypeSafeJudge(name, o => + { + o.Url = judge.Url; + o.Model = judge.Model; + o.ModelId = name; + o.ApiKey = builder.Configuration[$"Wiki:Judges:Models:{name}:ApiKey"] ?? string.Empty; + }); + break; + default: + throw new InvalidOperationException($"unknown judge adapter '{judge.Adapter}' for '{name}'"); + } +} +builder.Services.AddSingleton>(sp => + wiki.Judges.Models.Keys.Select(sp.GetRequiredKeyedService).ToList()); +builder.Services.AddSingleton(sp => sp.GetRequiredKeyedService(wiki.Judges.Active)); // One adapter may serve several ports (Gitea is note store and fact sink); list it once. builder.Services.AddSingleton>(sp => { @@ -56,6 +77,7 @@ builder.Services.AddSingleton>(sp => sp.GetRequiredService(), sp.GetRequiredService(), .. sp.GetRequiredService>(), + .. sp.GetRequiredService>(), ]; return all.Distinct().ToList(); }); @@ -66,6 +88,8 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); diff --git a/src/Wiki.Host/appsettings.json b/src/Wiki.Host/appsettings.json index 0a93de3..0cd765a 100644 --- a/src/Wiki.Host/appsettings.json +++ b/src/Wiki.Host/appsettings.json @@ -18,6 +18,12 @@ "bge-m3": { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc.cluster.local:8080", "Model": "bge-m3", "Dimension": 1024 } } }, + "Judges": { + "Active": "typesafe", + "Models": { + "typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" } + } + }, "Index": { "Active": "file", "File": { "Directory": "/data" } }, "Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 } }, diff --git a/test/Wiki.Application.Tests/AskUseCaseTests.cs b/test/Wiki.Application.Tests/AskUseCaseTests.cs new file mode 100644 index 0000000..fbb7266 --- /dev/null +++ b/test/Wiki.Application.Tests/AskUseCaseTests.cs @@ -0,0 +1,137 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Ask; +using Wiki.Application.Retrieval; +using Wiki.Application.Sync; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Application.Tests; + +/// A judge that records requests and answers by simple rules, so the use case's wiring is testable. +public sealed class FakeJudge : IJudge +{ + public List Requests { get; } = []; + public string ModelId => "fake-judge"; + public int? MaxChoiceOptions { get; init; } = 255; + public string Name => "fake-judge"; + + public Task JudgeAsync(JudgeRequest request, CancellationToken ct) + { + Requests.Add(request); + var answers = new Dictionary(); + foreach (var (id, q) in request.Questions) + { + answers[id] = q switch + { + TypedQuestion.Noul => new Judgement.Noul(id == AskUseCase.CoveredId ? 0.8 : 0.9), + TypedQuestion.Choice c => Spread(c), + TypedQuestion.Score => new Judgement.Score(0.5), + _ => throw new InvalidOperationException(), + }; + } + return Task.FromResult(new JudgeResult(answers, new JudgeUsage(1, 100, request.Questions.Count, TimeSpan.FromMilliseconds(5)))); + } + + private static Judgement.Choice Spread(TypedQuestion.Choice c) + { + // First option gets 0.6, the rest share 0.4 equally (NONE included). + var keys = c.Options.Keys.ToList(); + var probs = new Dictionary(); + for (var i = 0; i < keys.Count; i++) + probs[keys[i]] = keys.Count == 1 ? 1.0 : (i == 0 ? 0.6 : 0.4 / (keys.Count - 1)); + return new Judgement.Choice(keys[0], probs, 0.6); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +public class AskUseCaseTests +{ + private readonly FakeStore _store = new(); + private readonly FakeCatalog _catalog = new(); + private readonly FakeIndex _index = new(); + private readonly FakeEmbeddings _model = new("m1"); + private readonly FakeJudge _judge = new(); + + private AskUseCase UseCase => new(new RankUseCase(_index, _catalog), _catalog, new ParagraphChunker()); + + private async Task SeedAsync() + { + _store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n\n## Details\n\nEven dig @1.1.1.1 is answered locally.\n"; + _store.Files["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\ntags: [ceph]\n---\n\n# Ceph OSD\n\nOSD fsid mismatch after rebuild.\n"; + _store.Files["wiki/long-note.md"] = "---\ntitle: Long\ntype: wiki\n---\n\n# Long\n\n" + string.Join("\n\n", Enumerable.Range(0, 40).Select(i => $"paragraph {i} " + new string('x', 400))) + "\n"; + var sync = new SyncUseCase(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger.Instance); + await sync.RunAsync(false, CancellationToken.None); + } + + [Fact] + public async Task DefaultQuestionsPerQueryAndCoveredGate() + { + await SeedAsync(); + var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + var request = new AskRequest([EmbeddingText.For(udm)], null, new AskOptions(K: 2)); + + var result = await UseCase.AskAsync(request, _model, _judge, CancellationToken.None); + + Assert.Equal("udm-dns", result.Context[0].Slug); + Assert.Equal(0.8, result.Covered); + Assert.Equal(["answered_1", "where_1", AskUseCase.CoveredId], result.Answers.Keys.OrderBy(k => k == AskUseCase.CoveredId ? 2 : k.StartsWith("answered") ? 0 : 1)); + var where = Assert.IsType(result.Answers["where_1"]); + Assert.Equal("udm-dns#p1", where.Top[0].Id); + Assert.Contains("answers every query", where.Top[0].Text); + Assert.Equal("UDM intercepts DNS", where.Top[0].Title); + // the context choice went alone, the noul questions together + Assert.Equal(2, _judge.Requests.Count); + Assert.Contains(_judge.Requests, r => r.Questions.Count == 1 && r.Questions.ContainsKey("where_1")); + Assert.Equal(2, result.Usage.Requests); + Assert.Equal("fake-judge", result.JudgeModel); + } + + [Fact] + public async Task ContextChoiceOptionsAreTheParagraphIdsPlusNone() + { + await SeedAsync(); + var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + var request = new AskRequest([EmbeddingText.For(udm)], new Dictionary + { + ["src"] = new AskQuestion.ContextChoice("which note", ContextScope.Documents), + ["fact"] = new AskQuestion.Noul("is it intercepted", "yes", "no"), + }, new AskOptions(K: 2)); + + var result = await UseCase.AskAsync(request, _model, _judge, CancellationToken.None); + + var sent = _judge.Requests.Single(r => r.Questions.ContainsKey("src")); + var choice = Assert.IsType(sent.Questions["src"]); + Assert.Equal(result.Context.Select(d => d.Slug).Append(AskUseCase.None).OrderBy(x => x), choice.Options.Keys.OrderBy(x => x)); + Assert.Equal(string.Empty, choice.Options[result.Context[0].Slug]); + var src = Assert.IsType(result.Answers["src"]); + Assert.Equal("udm-dns", src.Top[0].Slug); + Assert.Null(src.Top[0].Text); + Assert.IsType(result.Answers["fact"]); + Assert.True(result.Answers.ContainsKey(AskUseCase.CoveredId)); + } + + [Fact] + public async Task BudgetsCapLongDocumentsAndDropTheLowest() + { + await SeedAsync(); + var longNote = (await _catalog.GetAsync(DocumentKind.Wiki, "long-note", CancellationToken.None))!; + var (context, meta) = await UseCase.BuildContextAsync([EmbeddingText.For(longNote)], _model, new AskOptions(K: 3, DocChars: 3000, StateChars: 2950), CancellationToken.None); + Assert.Equal("long-note", context[0].Slug); + Assert.True(context[0].Truncated); + Assert.True(context[0].Paragraphs.Count < 40); + Assert.Equal(2, meta.DroppedForBudget.Count); + Assert.Single(context); + } + + [Fact] + public async Task RejectsAContextChoiceOverTheJudgeCap() + { + await SeedAsync(); + var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + var small = new FakeJudge { MaxChoiceOptions = 3 }; + var request = new AskRequest([EmbeddingText.For(udm)], null, new AskOptions(K: 3)); + await Assert.ThrowsAsync(() => UseCase.AskAsync(request, _model, small, CancellationToken.None)); + } +} diff --git a/test/Wiki.Host.Tests/EndpointTests.cs b/test/Wiki.Host.Tests/EndpointTests.cs index a148f8c..23cb173 100644 --- a/test/Wiki.Host.Tests/EndpointTests.cs +++ b/test/Wiki.Host.Tests/EndpointTests.cs @@ -38,6 +38,11 @@ public sealed class WikiHostFactory : WebApplicationFactory services.AddKeyedSingleton("bge-m3", fake); services.AddSingleton(fake); services.AddSingleton>([fake]); + var judge = new FakeJudge(); + services.AddKeyedSingleton("typesafe", judge); + services.RemoveAll(); + services.AddSingleton(judge); + services.AddSingleton>([judge]); }); } } @@ -159,6 +164,39 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture() }); + Assert.Equal(HttpStatusCode.BadRequest, bad.StatusCode); + + var r = await Client.PostAsJsonAsync("/api/v1/ask", new + { + queries = new[] { "udm dns" }, + k = 3, + questions = new Dictionary + { + ["fact"] = new { type = "noul", instructions = "does it", criteria = new { @true = "yes", @false = "no" } }, + ["where"] = new { type = "choice", instructions = "which paragraph", criteria = "paragraphs" }, + }, + }); + Assert.Equal(HttpStatusCode.OK, r.StatusCode); + var body = await r.Content.ReadFromJsonAsync(); + Assert.NotNull(body); + Assert.Equal("fake-judge", body.Judge); + Assert.InRange(body.Covered ?? -1, 0, 1); + Assert.Equal(0.9, body.Answers["fact"].GetProperty("noul").GetDouble()); + var top = body.Answers["where"].GetProperty("top"); + Assert.True(top.GetArrayLength() >= 1); + Assert.Contains("answers every query", top[0].GetProperty("text").GetString()); + } + + private sealed record AskResponse(string Judge, double? Covered, Dictionary Answers); private sealed record NoteRow(string Slug, string Title); private sealed record LinkRow(string? ToSlug, string? FromSlug, string Kind); private sealed record NoteBody(string Slug, string Body, List Links, List Backlinks); diff --git a/test/Wiki.Host.Tests/Fakes.cs b/test/Wiki.Host.Tests/Fakes.cs index cf29fcf..cfd8b7e 100644 --- a/test/Wiki.Host.Tests/Fakes.cs +++ b/test/Wiki.Host.Tests/Fakes.cs @@ -44,6 +44,31 @@ public sealed class FakeStore : INoteStore public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); } +public sealed class FakeJudge : IJudge +{ + public string ModelId => "fake-judge"; + public int? MaxChoiceOptions => 255; + public string Name => "fake-judge"; + + public Task JudgeAsync(JudgeRequest request, CancellationToken ct) + { + var answers = new Dictionary(); + foreach (var (id, q) in request.Questions) + { + answers[id] = q switch + { + TypedQuestion.Noul => new Judgement.Noul(0.9), + TypedQuestion.Choice c => new Judgement.Choice(c.Options.Keys.First(), c.Options.Keys.ToDictionary(k => k, k => 1.0 / c.Options.Count), 0.5), + TypedQuestion.Score => new Judgement.Score(0.5), + _ => throw new InvalidOperationException(), + }; + } + return Task.FromResult(new JudgeResult(answers, new JudgeUsage(1, 10, 1, TimeSpan.Zero))); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + public sealed class FakeEmbeddings(string modelId) : IEmbeddingModel { public string ModelId { get; } = modelId;