diff --git a/src/Wiki.Adapters.TypeSafe/ServiceCollectionExtensions.cs b/src/Wiki.Adapters.TypeSafe/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..e159c0b --- /dev/null +++ b/src/Wiki.Adapters.TypeSafe/ServiceCollectionExtensions.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.DependencyInjection; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.TypeSafe; + +public static class ServiceCollectionExtensions +{ + /// + /// Registers an backed by TypeSafe's System One API, keyed by . + /// Several judges may be registered under different names, each with its own . + /// + public static IServiceCollection AddTypeSafeJudge( + this IServiceCollection services, + string name, + Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(configure); + + var options = new TypeSafeOptions(); + configure(options); + options.Validate(); + + var httpClientName = $"typesafe-judge:{name}"; + + services.AddHttpClient(httpClientName, client => + { + client.Timeout = options.Timeout; + }); + + services.AddKeyedSingleton(name, (sp, _) => + { + var httpClientFactory = sp.GetRequiredService(); + var client = httpClientFactory.CreateClient(httpClientName); + return new TypeSafeJudge(client, options); + }); + + return services; + } +} diff --git a/src/Wiki.Adapters.TypeSafe/TypeSafeJudge.cs b/src/Wiki.Adapters.TypeSafe/TypeSafeJudge.cs new file mode 100644 index 0000000..017d58d --- /dev/null +++ b/src/Wiki.Adapters.TypeSafe/TypeSafeJudge.cs @@ -0,0 +1,231 @@ +using System.Diagnostics; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.TypeSafe; + +/// +/// Judges typed questions via TypeSafe's System One API. One HTTP request per +/// call, retried on 429/529 with the server's Retry-After (or exponential backoff), never retried +/// for anything else. +/// +public sealed class TypeSafeJudge : IJudge +{ + private const int ChoiceCap = 255; + + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient _http; + private readonly TypeSafeOptions _options; + private readonly Func _delay; + + public TypeSafeJudge(HttpClient http, TypeSafeOptions options, Func? delay = null) + { + _http = http; + _options = options; + _delay = delay ?? Task.Delay; + } + + public string ModelId => _options.ModelId; + + public int? MaxChoiceOptions => ChoiceCap; + + public string Name => $"typesafe:{ModelId}"; + + public async Task JudgeAsync(JudgeRequest request, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(request); + + foreach (var question in request.Questions.Values) + { + if (question is TypedQuestion.Choice choice && choice.Options.Count > ChoiceCap) + { + throw new ArgumentException( + $"Choice has {choice.Options.Count} options; TypeSafe's cap is {ChoiceCap}.", nameof(request)); + } + } + + if (request.Questions.Count == 0) + { + return new JudgeResult(new Dictionary(), JudgeUsage.None); + } + + var body = BuildRequestBody(request); + var stopwatch = Stopwatch.StartNew(); + using var response = await PostWithRetryAsync(body, ct).ConfigureAwait(false); + var responseBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + stopwatch.Stop(); + + using var document = JsonDocument.Parse(responseBody); + var root = document.RootElement; + + var answers = new Dictionary(); + foreach (var property in root.GetProperty("answers").EnumerateObject()) + { + answers[property.Name] = ParseAnswer(property.Value); + } + + var usageElement = root.GetProperty("usage"); + var usage = new JudgeUsage( + 1, + usageElement.GetProperty("input_tokens").GetInt64(), + usageElement.GetProperty("output_tokens").GetInt64(), + stopwatch.Elapsed); + + return new JudgeResult(answers, usage); + } + + public async Task CheckAsync(CancellationToken ct) + { + var stopwatch = Stopwatch.StartNew(); + try + { + var request = new JudgeRequest(new { ping = true }, new Dictionary + { + ["ping"] = new TypedQuestion.Noul("Is ping true?", "it is true", "it is not true"), + }); + + var result = await JudgeAsync(request, ct).ConfigureAwait(false); + var detail = $"{result.Usage.InputTokens} input / {result.Usage.OutputTokens} output tokens"; + return new AdapterHealth(Name, true, detail, stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException) + { + return new AdapterHealth(Name, false, ex.Message, stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + } + + private JsonObject BuildRequestBody(JudgeRequest request) + { + var questions = new JsonObject(); + foreach (var (id, question) in request.Questions) + { + questions[id] = SerializeQuestion(question); + } + + return new JsonObject + { + ["model"] = _options.Model, + ["state"] = JsonSerializer.SerializeToNode(request.State, JsonOptions), + ["questions"] = questions, + }; + } + + private static JsonNode SerializeQuestion(TypedQuestion question) => question switch + { + TypedQuestion.Noul noul => new JsonObject + { + ["type"] = "noul", + ["instructions"] = noul.Instructions, + ["criteria"] = new JsonObject + { + ["true"] = noul.TrueCriterion, + ["false"] = noul.FalseCriterion, + }, + }, + TypedQuestion.Choice choice => new JsonObject + { + ["type"] = "choice", + ["instructions"] = choice.Instructions, + ["criteria"] = new JsonObject(choice.Options.Select(o => KeyValuePair.Create(o.Key, (JsonNode?)JsonValue.Create(o.Value)))), + }, + TypedQuestion.Score score => new JsonObject + { + ["type"] = "score", + ["instructions"] = score.Instructions, + ["criteria"] = score.Criteria, + }, + _ => throw new ArgumentOutOfRangeException(nameof(question), question, "Unknown TypedQuestion."), + }; + + private static Judgement ParseAnswer(JsonElement value) + { + if (value.TryGetProperty("noul", out var noul)) + { + return new Judgement.Noul(noul.GetDouble()); + } + + if (value.TryGetProperty("choice", out var choice)) + { + var probabilities = new Dictionary(); + foreach (var probability in value.GetProperty("probabilities").EnumerateObject()) + { + probabilities[probability.Name] = probability.Value.GetDouble(); + } + + return new Judgement.Choice(choice.GetString()!, probabilities, value.GetProperty("confidence").GetDouble()); + } + + if (value.TryGetProperty("score", out var score)) + { + return new Judgement.Score(score.GetDouble()); + } + + throw new InvalidOperationException($"typesafe: answer had none of noul/choice/score: {value.GetRawText()}"); + } + + private async Task PostWithRetryAsync(JsonObject body, CancellationToken ct) + { + var json = body.ToJsonString(JsonOptions); + var delay = TimeSpan.FromSeconds(1); + + // A missing key is a runtime condition the status page shows, not a reason the host cannot start. + if (string.IsNullOrWhiteSpace(_options.ApiKey)) + throw new InvalidOperationException("TypeSafe API key is not configured."); + for (var attempt = 0; ; attempt++) + { + using var requestMessage = new HttpRequestMessage(HttpMethod.Post, _options.Url) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiKey); + + var response = await _http.SendAsync(requestMessage, ct).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + return response; + } + + var statusCode = (int)response.StatusCode; + if ((statusCode == 429 || statusCode == 529) && attempt < _options.MaxRetries) + { + var wait = RetryAfter(response) ?? delay; + response.Dispose(); + await _delay(wait, ct).ConfigureAwait(false); + delay = TimeSpan.FromSeconds(Math.Min(delay.TotalSeconds * 2, 30)); + continue; + } + + var errorBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + response.Dispose(); + var truncated = errorBody.Length > 500 ? errorBody[..500] : errorBody; + throw new HttpRequestException($"typesafe: HTTP {statusCode}: {truncated}"); + } + } + + private static TimeSpan? RetryAfter(HttpResponseMessage response) + { + var header = response.Headers.RetryAfter; + if (header is null) + { + return null; + } + + if (header.Delta is { } delta) + { + return delta; + } + + if (header.Date is { } date) + { + var diff = date - DateTimeOffset.UtcNow; + return diff > TimeSpan.Zero ? diff : TimeSpan.Zero; + } + + return null; + } +} diff --git a/src/Wiki.Adapters.TypeSafe/TypeSafeOptions.cs b/src/Wiki.Adapters.TypeSafe/TypeSafeOptions.cs new file mode 100644 index 0000000..ca90527 --- /dev/null +++ b/src/Wiki.Adapters.TypeSafe/TypeSafeOptions.cs @@ -0,0 +1,47 @@ +namespace Wiki.Adapters.TypeSafe; + +/// Configuration for one TypeSafe System One judge. +public sealed class TypeSafeOptions +{ + private string? _modelId; + + /// The System One endpoint. + public string Url { get; set; } = "https://api.typesafe.ai/v1/systemone"; + + /// Model name sent in the request body. + public string Model { get; set; } = "jev-latest"; + + /// Identifier reported as . Defaults to . + public string ModelId + { + get => string.IsNullOrWhiteSpace(_modelId) ? Model : _modelId; + set => _modelId = value; + } + + /// Bearer token sent as Authorization. + public string ApiKey { get; set; } = ""; + + /// Per-request timeout. + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(60); + + /// How many times a 429/529 response is retried before giving up. + public int MaxRetries { get; set; } = 5; + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(Url)) + { + throw new ArgumentException("Url must be set.", nameof(Url)); + } + + if (string.IsNullOrWhiteSpace(Model)) + { + throw new ArgumentException("Model must be set.", nameof(Model)); + } + + if (MaxRetries < 0) + { + throw new ArgumentException("MaxRetries must not be negative.", nameof(MaxRetries)); + } + } +} diff --git a/src/Wiki.Adapters.TypeSafe/Wiki.Adapters.TypeSafe.csproj b/src/Wiki.Adapters.TypeSafe/Wiki.Adapters.TypeSafe.csproj index f5a3032..9440b48 100644 --- a/src/Wiki.Adapters.TypeSafe/Wiki.Adapters.TypeSafe.csproj +++ b/src/Wiki.Adapters.TypeSafe/Wiki.Adapters.TypeSafe.csproj @@ -4,6 +4,11 @@ + + + + + net10.0 enable diff --git a/src/Wiki.Application/Ask/AskUseCase.cs b/src/Wiki.Application/Ask/AskUseCase.cs new file mode 100644 index 0000000..7de3c36 --- /dev/null +++ b/src/Wiki.Application/Ask/AskUseCase.cs @@ -0,0 +1,253 @@ +using System.Diagnostics; +using Wiki.Application.Retrieval; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Ask; + +/// A caller's question, before expansion. Context choices are expanded over the context's paragraph or document ids. +public abstract record AskQuestion(string Instructions) +{ + public sealed record Noul(string Instructions, string TrueCriterion, string FalseCriterion) : AskQuestion(Instructions); + public sealed record Choice(string Instructions, IReadOnlyDictionary Options) : AskQuestion(Instructions); + public sealed record ContextChoice(string Instructions, ContextScope Scope) : AskQuestion(Instructions); + public sealed record Score(string Instructions, string Criteria) : AskQuestion(Instructions); +} + +public enum ContextScope +{ + Paragraphs, + Documents, +} + +public sealed record AskOptions(int K = 6, int DocChars = 12_000, int StateChars = 60_000, int MaxParallel = 8); + +public sealed record AskRequest(IReadOnlyList Queries, IReadOnlyDictionary? Questions, AskOptions Options); + +public sealed record ContextDocument(string Slug, string Title, IReadOnlyList Paragraphs, bool Truncated, float RankScore); + +public sealed record RetrievalMeta(IReadOnlyList DroppedForBudget, int Chars, int ParagraphCount, IReadOnlyDictionary RankScoreBySlug); + +/// A resolved context hit: a paragraph (with its text) or a document (with its title). +public sealed record ContextHit(string Id, double P, string Slug, string Title, string? Heading, string? Text); + +public abstract record AskAnswer +{ + public sealed record Noul(double Probability) : AskAnswer; + public sealed record Choice(string Selected, IReadOnlyDictionary Probabilities, double Confidence) : AskAnswer; + public sealed record Context(IReadOnlyList Top, double NoneP, double Confidence) : AskAnswer; + public sealed record Score(double Value) : AskAnswer; +} + +public sealed record AskResult( + IReadOnlyList Queries, + IReadOnlyList Context, + RetrievalMeta Retrieval, + IReadOnlyDictionary Answers, + double? Covered, + JudgeUsage Usage, + string JudgeModel, + TimeSpan Elapsed); + +/// +/// wiki_ask.py as a use case: rank every query, unite the top notes into one context (per-document +/// and total budget), expand context choices over the paragraph or document ids plus NONE, add the +/// covered gate, judge (every context choice in its own request, the rest batched), and attach the +/// answering paragraphs' text to the results. Judgements and quotations, never prose. +/// +public sealed class AskUseCase(RankUseCase rank, IDocumentCatalog catalog, IChunker chunker) +{ + public const string None = "NONE"; + public const string CoveredId = "_covered"; + + public async Task AskAsync(AskRequest request, IEmbeddingModel model, IJudge judge, CancellationToken ct) + { + if (request.Queries.Count == 0) throw new ArgumentException("at least one query is required", nameof(request)); + var started = Stopwatch.GetTimestamp(); + var (context, meta) = await BuildContextAsync(request.Queries, model, request.Options, ct); + + var questions = request.Questions ?? DefaultQuestions(request.Queries); + var (expanded, scopes) = Expand(questions, context, request.Queries, judge.MaxChoiceOptions); + + var (answers, usage) = await JudgeInParallelAsync(judge, context, expanded, scopes, request.Options.MaxParallel, ct); + var resolved = Resolve(answers, scopes, context); + double? covered = answers.TryGetValue(CoveredId, out var c) && c is Judgement.Noul n ? Math.Round(n.Probability, 3) : null; + return new AskResult(request.Queries, context, meta, resolved, covered, usage, judge.ModelId, Stopwatch.GetElapsedTime(started)); + } + + public async Task<(IReadOnlyList Context, RetrievalMeta Meta)> BuildContextAsync( + IReadOnlyList queries, IEmbeddingModel model, AskOptions options, CancellationToken ct) + { + var order = new Dictionary(); + foreach (var q in queries) + { + var rows = await rank.RankAsync(model, q, options.K, ct); + for (var pos = 0; pos < rows.Count; pos++) + { + var score = rows[pos].Score > 0 ? rows[pos].Score : 1e-6f / (pos + 1); + order[rows[pos].Slug] = Math.Max(order.GetValueOrDefault(rows[pos].Slug), score); + } + } + + var docs = new List(); + var dropped = new List(); + var used = 0; + foreach (var slug in order.Keys.OrderByDescending(s => order[s]).ThenBy(s => s)) + { + var document = await catalog.GetAsync(DocumentKind.Wiki, slug, ct); + if (document is null) continue; + var (paragraphs, truncated) = Cap(chunker.Split(document), options.DocChars); + var size = document.Title.Length + paragraphs.Sum(p => p.Text.Length + p.Heading.Length); + if (used + size > options.StateChars && docs.Count > 0) + { + dropped.Add(slug); + continue; + } + docs.Add(new ContextDocument(slug, document.Title, paragraphs, truncated, order[slug])); + used += size; + } + var meta = new RetrievalMeta(dropped, used, docs.Sum(d => d.Paragraphs.Count), + order.ToDictionary(kv => kv.Key, kv => (float)Math.Round(kv.Value, 3))); + return (docs, meta); + } + + private static (IReadOnlyList Paragraphs, bool Truncated) Cap(IReadOnlyList paragraphs, int docChars) + { + var kept = new List(); + var used = 0; + foreach (var p in paragraphs) + { + var size = p.Text.Length + p.Heading.Length; + if (kept.Count > 0 && used + size > docChars) break; + kept.Add(p); + used += size; + } + return (kept, kept.Count < paragraphs.Count); + } + + public static IReadOnlyDictionary DefaultQuestions(IReadOnlyList queries) + { + var qs = new Dictionary(); + for (var i = 0; i < queries.Count; i++) + { + var q = queries[i]; + qs[$"answered_{i + 1}"] = new AskQuestion.Noul( + $"Do the `documents` answer this question: {q}", + "At least one paragraph states what the question asks for.", + "The documents only touch the topic or do not address the question."); + qs[$"where_{i + 1}"] = new AskQuestion.ContextChoice( + $"Which paragraph (by `id`) most directly answers this question: {q} — pick {None} if none does.", + ContextScope.Paragraphs); + } + return qs; + } + + public static (IReadOnlyDictionary Questions, IReadOnlyDictionary Scopes) Expand( + IReadOnlyDictionary questions, IReadOnlyList context, IReadOnlyList queries, int? maxOptions) + { + var paragraphIds = context.SelectMany(d => d.Paragraphs.Select(p => p.Id)).ToList(); + var slugs = context.Select(d => d.Slug).ToList(); + var expanded = new Dictionary(); + var scopes = new Dictionary(); + foreach (var (id, q) in questions) + { + switch (q) + { + case AskQuestion.Noul n: + expanded[id] = new TypedQuestion.Noul(n.Instructions, n.TrueCriterion, n.FalseCriterion); + break; + case AskQuestion.Choice c: + expanded[id] = new TypedQuestion.Choice(c.Instructions, c.Options); + break; + case AskQuestion.Score s: + expanded[id] = new TypedQuestion.Score(s.Instructions, s.Criteria); + break; + case AskQuestion.ContextChoice cc: + var ids = cc.Scope == ContextScope.Paragraphs ? paragraphIds : slugs; + if (maxOptions is { } cap && ids.Count + 1 > cap) + throw new ArgumentException($"{id}: {ids.Count} {cc.Scope.ToString().ToLowerInvariant()} exceed the {cap - 1}-option Choice cap; lower k"); + var options = ids.ToDictionary(i => i, _ => string.Empty); + options[None] = $"No {(cc.Scope == ContextScope.Paragraphs ? "paragraph" : "document")} in the documents fits"; + expanded[id] = new TypedQuestion.Choice(cc.Instructions, options); + scopes[id] = cc.Scope; + break; + } + } + if (!expanded.ContainsKey(CoveredId)) + { + expanded[CoveredId] = new TypedQuestion.Noul( + "Do the `documents` together contain enough to answer these retrieval queries: " + string.Join(" | ", queries), + "The queries can be answered from the documents, possibly combining several.", + "The documents only touch the topic; a real answer would need material not here."); + } + return (expanded, scopes); + } + + private static async Task<(IReadOnlyDictionary Answers, JudgeUsage Usage)> JudgeInParallelAsync( + IJudge judge, IReadOnlyList context, IReadOnlyDictionary questions, + IReadOnlyDictionary scopes, int maxParallel, CancellationToken ct) + { + var state = new + { + documents = context.Select(d => new + { + slug = d.Slug, + title = d.Title, + paragraphs = d.Paragraphs.Select(p => new { id = p.Id, heading = p.Heading, text = p.Text }), + }), + }; + var groups = new List>(); + foreach (var (id, q) in questions.Where(kv => scopes.ContainsKey(kv.Key))) + groups.Add(new Dictionary { [id] = q }); + var rest = questions.Where(kv => !scopes.ContainsKey(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value); + if (rest.Count > 0) groups.Add(rest); + + using var gate = new SemaphoreSlim(Math.Max(1, maxParallel)); + var tasks = groups.Select(async g => + { + await gate.WaitAsync(ct); + try { return await judge.JudgeAsync(new JudgeRequest(state, g), ct); } + finally { gate.Release(); } + }).ToList(); + var results = await Task.WhenAll(tasks); + + var answers = new Dictionary(); + var usage = JudgeUsage.None; + foreach (var r in results) + { + foreach (var (id, a) in r.Answers) answers[id] = a; + usage = usage.Add(r.Usage); + } + return (answers, usage); + } + + private static IReadOnlyDictionary Resolve( + IReadOnlyDictionary answers, IReadOnlyDictionary scopes, IReadOnlyList context) + { + var byParagraph = context.SelectMany(d => d.Paragraphs.Select(p => (p, d))).ToDictionary(x => x.p.Id, x => x); + var titles = context.ToDictionary(d => d.Slug, d => d.Title); + var resolved = new Dictionary(); + foreach (var (id, a) in answers) + { + if (scopes.TryGetValue(id, out var scope) && a is Judgement.Choice c) + { + var top = c.Probabilities.Where(kv => kv.Key != None && kv.Value >= 0.03) + .OrderByDescending(kv => kv.Value).Take(3) + .Select(kv => scope == ContextScope.Paragraphs && byParagraph.TryGetValue(kv.Key, out var hit) + ? new ContextHit(kv.Key, Math.Round(kv.Value, 3), hit.d.Slug, hit.d.Title, hit.p.Heading, hit.p.Text) + : new ContextHit(kv.Key, Math.Round(kv.Value, 3), kv.Key, titles.GetValueOrDefault(kv.Key, kv.Key), null, null)) + .ToList(); + resolved[id] = new AskAnswer.Context(top, Math.Round(c.Probabilities.GetValueOrDefault(None), 3), c.Confidence); + continue; + } + resolved[id] = a switch + { + Judgement.Noul n => new AskAnswer.Noul(n.Probability), + Judgement.Choice ch => new AskAnswer.Choice(ch.Selected, ch.Probabilities, ch.Confidence), + Judgement.Score s => new AskAnswer.Score(s.Value), + _ => throw new InvalidOperationException($"unknown judgement for {id}"), + }; + } + return resolved; + } +} diff --git a/src/Wiki.Host/Api/ApiV1.cs b/src/Wiki.Host/Api/ApiV1.cs index f7e7dff..249d1f6 100644 --- a/src/Wiki.Host/Api/ApiV1.cs +++ b/src/Wiki.Host/Api/ApiV1.cs @@ -10,6 +10,7 @@ public static class ApiV1 { var v1 = app.MapGroup("/api/v1"); v1.MapReadEndpoints(); + v1.MapAskEndpoints(); v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) => Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous(); diff --git a/src/Wiki.Host/Api/AskEndpoints.cs b/src/Wiki.Host/Api/AskEndpoints.cs new file mode 100644 index 0000000..f27d757 --- /dev/null +++ b/src/Wiki.Host/Api/AskEndpoints.cs @@ -0,0 +1,122 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Wiki.Application.Ask; +using Wiki.Core.Ports; + +namespace Wiki.Host.Api; + +/// Wire shape of a caller's question, as bin/wiki_ask.py's --questions JSON: criteria is a +/// {true,false} object for noul, an options object or the strings "paragraphs"/"documents" for choice, a string for score. +public sealed record AskQuestionDto(string Type, string Instructions, JsonElement? Criteria); + +public sealed record AskRequestDto( + [property: JsonPropertyName("queries")] IReadOnlyList? Queries, + [property: JsonPropertyName("questions")] IReadOnlyDictionary? Questions, + [property: JsonPropertyName("k")] int? K, + [property: JsonPropertyName("model")] string? Model, + [property: JsonPropertyName("judge")] string? Judge); + +public static class AskEndpoints +{ + public static void MapAskEndpoints(this RouteGroupBuilder v1) + { + v1.MapPost("/ask", async (AskRequestDto body, AskUseCase ask, IEmbeddingModel activeModel, IJudge activeJudge, IServiceProvider services, CancellationToken ct) => + { + if (body.Queries is null || body.Queries.Count == 0 || body.Queries.All(string.IsNullOrWhiteSpace)) + return Results.BadRequest(new { error = "queries required" }); + + var model = activeModel; + if (!string.IsNullOrEmpty(body.Model)) + { + model = services.GetKeyedService(body.Model); + if (model is null) return Results.BadRequest(new { error = $"unknown model '{body.Model}'" }); + } + var judge = activeJudge; + if (!string.IsNullOrEmpty(body.Judge)) + { + judge = services.GetKeyedService(body.Judge); + if (judge is null) return Results.BadRequest(new { error = $"unknown judge '{body.Judge}'" }); + } + + IReadOnlyDictionary? questions; + try + { + questions = body.Questions is null ? null : body.Questions.ToDictionary(kv => kv.Key, kv => Parse(kv.Key, kv.Value)); + } + catch (ArgumentException e) + { + return Results.BadRequest(new { error = e.Message }); + } + + try + { + var result = await ask.AskAsync(new AskRequest(body.Queries.Where(q => !string.IsNullOrWhiteSpace(q)).ToList(), questions, new AskOptions(K: body.K ?? 6)), model, judge, ct); + return Results.Ok(new + { + result.Queries, + context = result.Context.Select(d => new { d.Slug, d.Title, paragraphs = d.Paragraphs.Count, d.Truncated, rankScore = d.RankScore }), + retrieval = new { result.Retrieval.DroppedForBudget, result.Retrieval.Chars, result.Retrieval.ParagraphCount }, + answers = result.Answers.ToDictionary(kv => kv.Key, kv => (object)(kv.Value switch + { + AskAnswer.Noul n => new { noul = n.Probability }, + AskAnswer.Score s => new { score = s.Value }, + AskAnswer.Choice c => new { choice = c.Selected, probabilities = c.Probabilities, confidence = c.Confidence }, + AskAnswer.Context x => new { top = x.Top, none_p = x.NoneP, confidence = x.Confidence }, + _ => new { }, + })), + result.Covered, + usage = result.Usage, + judge = result.JudgeModel, + model = model.ModelId, + elapsed_s = Math.Round(result.Elapsed.TotalSeconds, 3), + }); + } + catch (ArgumentException e) + { + return Results.BadRequest(new { error = e.Message }); + } + catch (HttpRequestException e) + { + return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway); + } + }).AllowAnonymous(); + } + + public static AskQuestion Parse(string id, AskQuestionDto dto) + { + var criteria = dto.Criteria; + switch (dto.Type?.ToLowerInvariant()) + { + case "noul": + { + if (criteria is not { ValueKind: JsonValueKind.Object } o || !o.TryGetProperty("true", out var t) || !o.TryGetProperty("false", out var f)) + throw new ArgumentException($"{id}: noul needs criteria {{true, false}}"); + return new AskQuestion.Noul(dto.Instructions, t.GetString() ?? string.Empty, f.GetString() ?? string.Empty); + } + case "choice": + { + if (criteria is { ValueKind: JsonValueKind.String } s) + { + return s.GetString() switch + { + "paragraphs" => new AskQuestion.ContextChoice(dto.Instructions, ContextScope.Paragraphs), + "documents" => new AskQuestion.ContextChoice(dto.Instructions, ContextScope.Documents), + var other => throw new ArgumentException($"{id}: unknown choice criteria '{other}'"), + }; + } + if (criteria is { ValueKind: JsonValueKind.Object } o) + { + var options = new Dictionary(); + foreach (var p in o.EnumerateObject()) options[p.Name] = p.Value.ValueKind == JsonValueKind.String ? p.Value.GetString() ?? string.Empty : string.Empty; + if (options.Count == 0) throw new ArgumentException($"{id}: choice needs options"); + return new AskQuestion.Choice(dto.Instructions, options); + } + throw new ArgumentException($"{id}: choice needs criteria (options object, \"paragraphs\" or \"documents\")"); + } + case "score": + return new AskQuestion.Score(dto.Instructions, criteria is { ValueKind: JsonValueKind.String } sc ? sc.GetString() ?? string.Empty : string.Empty); + default: + throw new ArgumentException($"{id}: unknown question type '{dto.Type}'"); + } + } +} diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs index 2b39673..92c95f5 100644 --- a/src/Wiki.Host/Composition/WikiOptions.cs +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -5,6 +5,7 @@ public sealed class WikiOptions { public NoteStoreOptions NoteStore { get; set; } = new(); public EmbeddingsOptions Embeddings { get; set; } = new(); + public JudgesOptions Judges { get; set; } = new(); public IndexOptions Index { get; set; } = new(); public SyncScheduleOptions Sync { get; set; } = new(); } @@ -36,6 +37,19 @@ public sealed class EmbeddingModelOptions public int Dimension { get; set; } = 1024; } +public sealed class JudgesOptions +{ + public string Active { get; set; } = "typesafe"; + public Dictionary Models { get; set; } = new(); +} + +public sealed class JudgeModelOptions +{ + public string Adapter { get; set; } = "TypeSafe"; + public string Url { get; set; } = "https://api.typesafe.ai/v1/systemone"; + public string Model { get; set; } = "jev-latest"; +} + public sealed class IndexOptions { public string Active { get; set; } = "file"; diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index 1ec622f..85f0796 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -3,6 +3,7 @@ using MudBlazor.Services; using Wiki.Adapters.FileIndex; using Wiki.Adapters.Gitea; using Wiki.Adapters.LlamaCpp; +using Wiki.Adapters.TypeSafe; using Wiki.Application.Retrieval; using Wiki.Application.Status; using Wiki.Application.Sync; @@ -47,6 +48,26 @@ foreach (var (name, model) in wiki.Embeddings.Models) builder.Services.AddSingleton>(sp => wiki.Embeddings.Models.Keys.Select(sp.GetRequiredKeyedService).ToList()); builder.Services.AddSingleton(sp => sp.GetRequiredKeyedService(wiki.Embeddings.Active)); +foreach (var (name, judge) in wiki.Judges.Models) +{ + switch (judge.Adapter) + { + case "TypeSafe": + builder.Services.AddTypeSafeJudge(name, o => + { + o.Url = judge.Url; + o.Model = judge.Model; + o.ModelId = name; + o.ApiKey = builder.Configuration[$"Wiki:Judges:Models:{name}:ApiKey"] ?? string.Empty; + }); + break; + default: + throw new InvalidOperationException($"unknown judge adapter '{judge.Adapter}' for '{name}'"); + } +} +builder.Services.AddSingleton>(sp => + wiki.Judges.Models.Keys.Select(sp.GetRequiredKeyedService).ToList()); +builder.Services.AddSingleton(sp => sp.GetRequiredKeyedService(wiki.Judges.Active)); // One adapter may serve several ports (Gitea is note store and fact sink); list it once. builder.Services.AddSingleton>(sp => { @@ -56,6 +77,7 @@ builder.Services.AddSingleton>(sp => sp.GetRequiredService(), sp.GetRequiredService(), .. sp.GetRequiredService>(), + .. sp.GetRequiredService>(), ]; return all.Distinct().ToList(); }); @@ -66,6 +88,8 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); diff --git a/src/Wiki.Host/appsettings.json b/src/Wiki.Host/appsettings.json index 0a93de3..0cd765a 100644 --- a/src/Wiki.Host/appsettings.json +++ b/src/Wiki.Host/appsettings.json @@ -18,6 +18,12 @@ "bge-m3": { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc.cluster.local:8080", "Model": "bge-m3", "Dimension": 1024 } } }, + "Judges": { + "Active": "typesafe", + "Models": { + "typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" } + } + }, "Index": { "Active": "file", "File": { "Directory": "/data" } }, "Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 } }, diff --git a/test/Wiki.Application.Tests/AskUseCaseTests.cs b/test/Wiki.Application.Tests/AskUseCaseTests.cs new file mode 100644 index 0000000..fbb7266 --- /dev/null +++ b/test/Wiki.Application.Tests/AskUseCaseTests.cs @@ -0,0 +1,137 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Ask; +using Wiki.Application.Retrieval; +using Wiki.Application.Sync; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Application.Tests; + +/// A judge that records requests and answers by simple rules, so the use case's wiring is testable. +public sealed class FakeJudge : IJudge +{ + public List Requests { get; } = []; + public string ModelId => "fake-judge"; + public int? MaxChoiceOptions { get; init; } = 255; + public string Name => "fake-judge"; + + public Task JudgeAsync(JudgeRequest request, CancellationToken ct) + { + Requests.Add(request); + var answers = new Dictionary(); + foreach (var (id, q) in request.Questions) + { + answers[id] = q switch + { + TypedQuestion.Noul => new Judgement.Noul(id == AskUseCase.CoveredId ? 0.8 : 0.9), + TypedQuestion.Choice c => Spread(c), + TypedQuestion.Score => new Judgement.Score(0.5), + _ => throw new InvalidOperationException(), + }; + } + return Task.FromResult(new JudgeResult(answers, new JudgeUsage(1, 100, request.Questions.Count, TimeSpan.FromMilliseconds(5)))); + } + + private static Judgement.Choice Spread(TypedQuestion.Choice c) + { + // First option gets 0.6, the rest share 0.4 equally (NONE included). + var keys = c.Options.Keys.ToList(); + var probs = new Dictionary(); + for (var i = 0; i < keys.Count; i++) + probs[keys[i]] = keys.Count == 1 ? 1.0 : (i == 0 ? 0.6 : 0.4 / (keys.Count - 1)); + return new Judgement.Choice(keys[0], probs, 0.6); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +public class AskUseCaseTests +{ + private readonly FakeStore _store = new(); + private readonly FakeCatalog _catalog = new(); + private readonly FakeIndex _index = new(); + private readonly FakeEmbeddings _model = new("m1"); + private readonly FakeJudge _judge = new(); + + private AskUseCase UseCase => new(new RankUseCase(_index, _catalog), _catalog, new ParagraphChunker()); + + private async Task SeedAsync() + { + _store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n\n## Details\n\nEven dig @1.1.1.1 is answered locally.\n"; + _store.Files["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\ntags: [ceph]\n---\n\n# Ceph OSD\n\nOSD fsid mismatch after rebuild.\n"; + _store.Files["wiki/long-note.md"] = "---\ntitle: Long\ntype: wiki\n---\n\n# Long\n\n" + string.Join("\n\n", Enumerable.Range(0, 40).Select(i => $"paragraph {i} " + new string('x', 400))) + "\n"; + var sync = new SyncUseCase(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger.Instance); + await sync.RunAsync(false, CancellationToken.None); + } + + [Fact] + public async Task DefaultQuestionsPerQueryAndCoveredGate() + { + await SeedAsync(); + var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + var request = new AskRequest([EmbeddingText.For(udm)], null, new AskOptions(K: 2)); + + var result = await UseCase.AskAsync(request, _model, _judge, CancellationToken.None); + + Assert.Equal("udm-dns", result.Context[0].Slug); + Assert.Equal(0.8, result.Covered); + Assert.Equal(["answered_1", "where_1", AskUseCase.CoveredId], result.Answers.Keys.OrderBy(k => k == AskUseCase.CoveredId ? 2 : k.StartsWith("answered") ? 0 : 1)); + var where = Assert.IsType(result.Answers["where_1"]); + Assert.Equal("udm-dns#p1", where.Top[0].Id); + Assert.Contains("answers every query", where.Top[0].Text); + Assert.Equal("UDM intercepts DNS", where.Top[0].Title); + // the context choice went alone, the noul questions together + Assert.Equal(2, _judge.Requests.Count); + Assert.Contains(_judge.Requests, r => r.Questions.Count == 1 && r.Questions.ContainsKey("where_1")); + Assert.Equal(2, result.Usage.Requests); + Assert.Equal("fake-judge", result.JudgeModel); + } + + [Fact] + public async Task ContextChoiceOptionsAreTheParagraphIdsPlusNone() + { + await SeedAsync(); + var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + var request = new AskRequest([EmbeddingText.For(udm)], new Dictionary + { + ["src"] = new AskQuestion.ContextChoice("which note", ContextScope.Documents), + ["fact"] = new AskQuestion.Noul("is it intercepted", "yes", "no"), + }, new AskOptions(K: 2)); + + var result = await UseCase.AskAsync(request, _model, _judge, CancellationToken.None); + + var sent = _judge.Requests.Single(r => r.Questions.ContainsKey("src")); + var choice = Assert.IsType(sent.Questions["src"]); + Assert.Equal(result.Context.Select(d => d.Slug).Append(AskUseCase.None).OrderBy(x => x), choice.Options.Keys.OrderBy(x => x)); + Assert.Equal(string.Empty, choice.Options[result.Context[0].Slug]); + var src = Assert.IsType(result.Answers["src"]); + Assert.Equal("udm-dns", src.Top[0].Slug); + Assert.Null(src.Top[0].Text); + Assert.IsType(result.Answers["fact"]); + Assert.True(result.Answers.ContainsKey(AskUseCase.CoveredId)); + } + + [Fact] + public async Task BudgetsCapLongDocumentsAndDropTheLowest() + { + await SeedAsync(); + var longNote = (await _catalog.GetAsync(DocumentKind.Wiki, "long-note", CancellationToken.None))!; + var (context, meta) = await UseCase.BuildContextAsync([EmbeddingText.For(longNote)], _model, new AskOptions(K: 3, DocChars: 3000, StateChars: 2950), CancellationToken.None); + Assert.Equal("long-note", context[0].Slug); + Assert.True(context[0].Truncated); + Assert.True(context[0].Paragraphs.Count < 40); + Assert.Equal(2, meta.DroppedForBudget.Count); + Assert.Single(context); + } + + [Fact] + public async Task RejectsAContextChoiceOverTheJudgeCap() + { + await SeedAsync(); + var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + var small = new FakeJudge { MaxChoiceOptions = 3 }; + var request = new AskRequest([EmbeddingText.For(udm)], null, new AskOptions(K: 3)); + await Assert.ThrowsAsync(() => UseCase.AskAsync(request, _model, small, CancellationToken.None)); + } +} diff --git a/test/Wiki.Contracts/JudgeContract.cs b/test/Wiki.Contracts/JudgeContract.cs new file mode 100644 index 0000000..d1a7a25 --- /dev/null +++ b/test/Wiki.Contracts/JudgeContract.cs @@ -0,0 +1,70 @@ +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Contracts; + +/// +/// Every adapter derives from this. The adapter under test may talk to a fake +/// server; the fake must honour the request shape and answer every question it is asked. +/// +public abstract class JudgeContract +{ + protected abstract Task CreateJudgeAsync(); + + private static readonly object State = new { documents = new[] { new { id = "a", text = "The UDM answers every DNS query itself." } } }; + + [Fact] + public async Task AnswersAllThreePrimitivesInOneRequest() + { + var judge = await CreateJudgeAsync(); + var request = new JudgeRequest(State, new Dictionary + { + ["fact"] = new TypedQuestion.Noul("Do the documents say the UDM answers DNS itself?", "they do", "they do not"), + ["where"] = new TypedQuestion.Choice("Which document says it?", new Dictionary { ["a"] = "document a", ["NONE"] = "none of them" }), + ["how_sure"] = new TypedQuestion.Score("How directly is it stated?", "0 = not at all, 1 = verbatim"), + }); + + var result = await judge.JudgeAsync(request, CancellationToken.None); + + Assert.Equal(new[] { "fact", "how_sure", "where" }, result.Answers.Keys.OrderBy(k => k)); + var fact = Assert.IsType(result.Answers["fact"]); + Assert.InRange(fact.Probability, 0.0, 1.0); + var where = Assert.IsType(result.Answers["where"]); + Assert.Contains(where.Selected, new[] { "a", "NONE" }); + Assert.Equal(new[] { "NONE", "a" }, where.Probabilities.Keys.OrderBy(k => k)); + Assert.InRange(where.Probabilities.Values.Sum(), 0.99, 1.01); + Assert.InRange(where.Confidence, 0.0, 1.0); + var score = Assert.IsType(result.Answers["how_sure"]); + Assert.InRange(score.Value, 0.0, 1.0); + Assert.Equal(1, result.Usage.Requests); + Assert.True(result.Usage.InputTokens > 0); + } + + [Fact] + public async Task EmptyQuestionSetNeedsNoRequest() + { + var judge = await CreateJudgeAsync(); + var result = await judge.JudgeAsync(new JudgeRequest(State, new Dictionary()), CancellationToken.None); + Assert.Empty(result.Answers); + Assert.Equal(0, result.Usage.Requests); + } + + [Fact] + public async Task ChoiceAboveTheCapIsRejectedBeforeSending() + { + var judge = await CreateJudgeAsync(); + if (judge.MaxChoiceOptions is not { } cap) return; + var options = Enumerable.Range(0, cap + 1).ToDictionary(i => $"o{i}", i => $"option {i}"); + var request = new JudgeRequest(State, new Dictionary { ["too_many"] = new TypedQuestion.Choice("pick", options) }); + await Assert.ThrowsAsync(() => judge.JudgeAsync(request, CancellationToken.None)); + } + + [Fact] + public async Task ModelIdAndHealth() + { + var judge = await CreateJudgeAsync(); + Assert.False(string.IsNullOrWhiteSpace(judge.ModelId)); + var health = await judge.CheckAsync(CancellationToken.None); + Assert.True(health.Ok, health.Detail); + } +} diff --git a/test/Wiki.Contracts/TypeSafe/FakeTypeSafeHandler.cs b/test/Wiki.Contracts/TypeSafe/FakeTypeSafeHandler.cs new file mode 100644 index 0000000..5465309 --- /dev/null +++ b/test/Wiki.Contracts/TypeSafe/FakeTypeSafeHandler.cs @@ -0,0 +1,138 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Wiki.Contracts.TypeSafe; + +/// +/// Stands in for the TypeSafe System One endpoint. Validates the request shape and answers every +/// question deterministically: noul is 0.9 when the instructions mention "UDM", else 0.1; choice +/// spreads probability over the given options and selects the first one; score is always 0.5. +/// Can be scripted to return a run of 429s (with Retry-After: 0) before succeeding, or a 400 with +/// a JSON error body, to exercise the adapter's retry and error handling. +/// +internal sealed class FakeTypeSafeHandler : HttpMessageHandler +{ + public int TooManyRequestsBeforeSuccess { get; set; } + + public bool RespondBadRequest { get; set; } + + public string BadRequestBody { get; set; } = "{\"error\":\"bad request\"}"; + + public string? LastRequestJson { get; private set; } + + public AuthenticationHeaderValue? LastAuthorization { get; private set; } + + private int _tooManyRequestsServed; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + LastAuthorization = request.Headers.Authorization; + var body = await request.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + LastRequestJson = body; + + if (RespondBadRequest) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent(BadRequestBody, Encoding.UTF8, "application/json"), + }; + } + + if (_tooManyRequestsServed < TooManyRequestsBeforeSuccess) + { + _tooManyRequestsServed++; + var tooMany = new HttpResponseMessage((HttpStatusCode)429) + { + Content = new StringContent("{\"error\":\"rate limited\"}", Encoding.UTF8, "application/json"), + }; + tooMany.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.Zero); + return tooMany; + } + + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + + if (!root.TryGetProperty("model", out _) || !root.TryGetProperty("state", out _) || !root.TryGetProperty("questions", out var questions)) + { + throw new InvalidOperationException($"Malformed TypeSafe request: {body}"); + } + + var answers = new JsonObject(); + var questionCount = 0; + foreach (var question in questions.EnumerateObject()) + { + questionCount++; + var type = question.Value.GetProperty("type").GetString(); + if (string.IsNullOrEmpty(question.Value.GetProperty("instructions").GetString())) + { + throw new InvalidOperationException("Question missing instructions."); + } + + if (!question.Value.TryGetProperty("criteria", out _)) + { + throw new InvalidOperationException("Question missing criteria."); + } + + answers[question.Name] = type switch + { + "noul" => AnswerNoul(question.Value), + "choice" => AnswerChoice(question.Value), + "score" => AnswerScore(), + _ => throw new InvalidOperationException($"Unknown question type: {type}"), + }; + } + + var responseBody = new JsonObject + { + ["answers"] = answers, + ["usage"] = new JsonObject + { + ["input_tokens"] = body.Length / 4, + ["output_tokens"] = questionCount, + }, + }; + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(responseBody.ToJsonString(), Encoding.UTF8, "application/json"), + }; + } + + private static JsonObject AnswerNoul(JsonElement question) + { + var instructions = question.GetProperty("instructions").GetString() ?? ""; + var probability = instructions.Contains("UDM", StringComparison.Ordinal) ? 0.9 : 0.1; + return new JsonObject { ["noul"] = probability }; + } + + private static JsonObject AnswerChoice(JsonElement question) + { + var options = question.GetProperty("criteria").EnumerateObject().Select(p => p.Name).ToArray(); + var probabilities = new JsonObject(); + var share = 1.0 / options.Length; + double firstProbability = share; + for (var i = 0; i < options.Length; i++) + { + // The last option absorbs rounding so the probabilities sum to exactly 1. + var value = i == options.Length - 1 ? 1.0 - share * (options.Length - 1) : share; + if (i == 0) + { + firstProbability = value; + } + + probabilities[options[i]] = value; + } + + return new JsonObject + { + ["choice"] = options[0], + ["probabilities"] = probabilities, + ["confidence"] = firstProbability, + }; + } + + private static JsonObject AnswerScore() => new() { ["score"] = 0.5 }; +} diff --git a/test/Wiki.Contracts/TypeSafe/TypeSafeJudgeContractTests.cs b/test/Wiki.Contracts/TypeSafe/TypeSafeJudgeContractTests.cs new file mode 100644 index 0000000..e8118b5 --- /dev/null +++ b/test/Wiki.Contracts/TypeSafe/TypeSafeJudgeContractTests.cs @@ -0,0 +1,23 @@ +using Wiki.Adapters.TypeSafe; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.TypeSafe; + +/// Runs the shared against the TypeSafe adapter over a fake server. +public sealed class TypeSafeJudgeContractTests : JudgeContract +{ + protected override Task CreateJudgeAsync() + { + var handler = new FakeTypeSafeHandler(); + var client = new HttpClient(handler); + var options = new TypeSafeOptions + { + Url = "https://typesafe.test/v1/systemone", + Model = "jev-latest", + ApiKey = "test-key", + }; + + IJudge judge = new TypeSafeJudge(client, options, delay: (_, _) => Task.CompletedTask); + return Task.FromResult(judge); + } +} diff --git a/test/Wiki.Contracts/TypeSafe/TypeSafeJudgeTests.cs b/test/Wiki.Contracts/TypeSafe/TypeSafeJudgeTests.cs new file mode 100644 index 0000000..45dceef --- /dev/null +++ b/test/Wiki.Contracts/TypeSafe/TypeSafeJudgeTests.cs @@ -0,0 +1,126 @@ +using System.Text.Json; +using Wiki.Adapters.TypeSafe; +using Wiki.Core.Model; + +namespace Wiki.Contracts.TypeSafe; + +public sealed class TypeSafeJudgeTests +{ + private static TypeSafeJudge CreateJudge(FakeTypeSafeHandler handler, TypeSafeOptions? options = null) + { + var client = new HttpClient(handler); + options ??= new TypeSafeOptions + { + Url = "https://typesafe.test/v1/systemone", + Model = "jev-latest", + ApiKey = "test-key", + }; + + return new TypeSafeJudge(client, options, delay: (_, _) => Task.CompletedTask); + } + + [Fact] + public async Task RequestBodyMatchesTheDocumentedShape() + { + var handler = new FakeTypeSafeHandler(); + var judge = CreateJudge(handler); + + var state = new { question = "does the UDM answer DNS itself?" }; + var request = new JudgeRequest(state, new Dictionary + { + ["fact"] = new TypedQuestion.Noul("Do the docs say the UDM answers DNS itself?", "they do", "they do not"), + ["where"] = new TypedQuestion.Choice("Which doc says it?", new Dictionary { ["a"] = "document a", ["NONE"] = "none" }), + ["how_sure"] = new TypedQuestion.Score("How directly is it stated?", "0 = not at all, 1 = verbatim"), + }); + + await judge.JudgeAsync(request, CancellationToken.None); + + Assert.NotNull(handler.LastRequestJson); + using var document = JsonDocument.Parse(handler.LastRequestJson!); + var root = document.RootElement; + + Assert.Equal("jev-latest", root.GetProperty("model").GetString()); + Assert.Equal("does the UDM answer DNS itself?", root.GetProperty("state").GetProperty("question").GetString()); + + var fact = root.GetProperty("questions").GetProperty("fact"); + Assert.Equal("noul", fact.GetProperty("type").GetString()); + Assert.Equal("Do the docs say the UDM answers DNS itself?", fact.GetProperty("instructions").GetString()); + Assert.Equal("they do", fact.GetProperty("criteria").GetProperty("true").GetString()); + Assert.Equal("they do not", fact.GetProperty("criteria").GetProperty("false").GetString()); + + var where = root.GetProperty("questions").GetProperty("where"); + Assert.Equal("choice", where.GetProperty("type").GetString()); + Assert.Equal("document a", where.GetProperty("criteria").GetProperty("a").GetString()); + Assert.Equal("none", where.GetProperty("criteria").GetProperty("NONE").GetString()); + + var howSure = root.GetProperty("questions").GetProperty("how_sure"); + Assert.Equal("score", howSure.GetProperty("type").GetString()); + Assert.Equal("0 = not at all, 1 = verbatim", howSure.GetProperty("criteria").GetString()); + } + + [Fact] + public async Task SendsABearerAuthorizationHeader() + { + var handler = new FakeTypeSafeHandler(); + var judge = CreateJudge(handler, new TypeSafeOptions + { + Url = "https://typesafe.test/v1/systemone", + Model = "jev-latest", + ApiKey = "super-secret", + }); + + await judge.JudgeAsync(new JudgeRequest(new { }, new Dictionary + { + ["q"] = new TypedQuestion.Score("rate it", "0 = low, 1 = high"), + }), CancellationToken.None); + + Assert.NotNull(handler.LastAuthorization); + Assert.Equal("Bearer", handler.LastAuthorization!.Scheme); + Assert.Equal("super-secret", handler.LastAuthorization.Parameter); + } + + [Fact] + public async Task RetriesOn429ThenSucceedsAndCountsOneRequest() + { + var handler = new FakeTypeSafeHandler { TooManyRequestsBeforeSuccess = 2 }; + var judge = CreateJudge(handler); + + var result = await judge.JudgeAsync(new JudgeRequest(new { }, new Dictionary + { + ["q"] = new TypedQuestion.Score("rate it", "0 = low, 1 = high"), + }), CancellationToken.None); + + Assert.Equal(1, result.Usage.Requests); + Assert.IsType(result.Answers["q"]); + } + + [Fact] + public async Task BadRequestThrowsWithTheResponseBody() + { + var handler = new FakeTypeSafeHandler { RespondBadRequest = true, BadRequestBody = "{\"error\":\"nope\"}" }; + var judge = CreateJudge(handler); + + var ex = await Assert.ThrowsAsync(() => judge.JudgeAsync(new JudgeRequest(new { }, new Dictionary + { + ["q"] = new TypedQuestion.Score("rate it", "0 = low, 1 = high"), + }), CancellationToken.None)); + + Assert.Contains("nope", ex.Message); + } + + [Fact] + public async Task ChoiceOverTheCapIsRejectedWithoutSendingARequest() + { + var handler = new FakeTypeSafeHandler(); + var judge = CreateJudge(handler); + + var options = Enumerable.Range(0, 256).ToDictionary(i => $"o{i}", i => $"option {i}"); + var request = new JudgeRequest(new { }, new Dictionary + { + ["too_many"] = new TypedQuestion.Choice("pick", options), + }); + + await Assert.ThrowsAsync(() => judge.JudgeAsync(request, CancellationToken.None)); + Assert.Null(handler.LastRequestJson); + } +} diff --git a/test/Wiki.Host.Tests/EndpointTests.cs b/test/Wiki.Host.Tests/EndpointTests.cs index a148f8c..23cb173 100644 --- a/test/Wiki.Host.Tests/EndpointTests.cs +++ b/test/Wiki.Host.Tests/EndpointTests.cs @@ -38,6 +38,11 @@ public sealed class WikiHostFactory : WebApplicationFactory services.AddKeyedSingleton("bge-m3", fake); services.AddSingleton(fake); services.AddSingleton>([fake]); + var judge = new FakeJudge(); + services.AddKeyedSingleton("typesafe", judge); + services.RemoveAll(); + services.AddSingleton(judge); + services.AddSingleton>([judge]); }); } } @@ -159,6 +164,39 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture() }); + Assert.Equal(HttpStatusCode.BadRequest, bad.StatusCode); + + var r = await Client.PostAsJsonAsync("/api/v1/ask", new + { + queries = new[] { "udm dns" }, + k = 3, + questions = new Dictionary + { + ["fact"] = new { type = "noul", instructions = "does it", criteria = new { @true = "yes", @false = "no" } }, + ["where"] = new { type = "choice", instructions = "which paragraph", criteria = "paragraphs" }, + }, + }); + Assert.Equal(HttpStatusCode.OK, r.StatusCode); + var body = await r.Content.ReadFromJsonAsync(); + Assert.NotNull(body); + Assert.Equal("fake-judge", body.Judge); + Assert.InRange(body.Covered ?? -1, 0, 1); + Assert.Equal(0.9, body.Answers["fact"].GetProperty("noul").GetDouble()); + var top = body.Answers["where"].GetProperty("top"); + Assert.True(top.GetArrayLength() >= 1); + Assert.Contains("answers every query", top[0].GetProperty("text").GetString()); + } + + private sealed record AskResponse(string Judge, double? Covered, Dictionary Answers); private sealed record NoteRow(string Slug, string Title); private sealed record LinkRow(string? ToSlug, string? FromSlug, string Kind); private sealed record NoteBody(string Slug, string Body, List Links, List Backlinks); diff --git a/test/Wiki.Host.Tests/Fakes.cs b/test/Wiki.Host.Tests/Fakes.cs index cf29fcf..cfd8b7e 100644 --- a/test/Wiki.Host.Tests/Fakes.cs +++ b/test/Wiki.Host.Tests/Fakes.cs @@ -44,6 +44,31 @@ public sealed class FakeStore : INoteStore public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); } +public sealed class FakeJudge : IJudge +{ + public string ModelId => "fake-judge"; + public int? MaxChoiceOptions => 255; + public string Name => "fake-judge"; + + public Task JudgeAsync(JudgeRequest request, CancellationToken ct) + { + var answers = new Dictionary(); + foreach (var (id, q) in request.Questions) + { + answers[id] = q switch + { + TypedQuestion.Noul => new Judgement.Noul(0.9), + TypedQuestion.Choice c => new Judgement.Choice(c.Options.Keys.First(), c.Options.Keys.ToDictionary(k => k, k => 1.0 / c.Options.Count), 0.5), + TypedQuestion.Score => new Judgement.Score(0.5), + _ => throw new InvalidOperationException(), + }; + } + return Task.FromResult(new JudgeResult(answers, new JudgeUsage(1, 10, 1, TimeSpan.Zero))); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + public sealed class FakeEmbeddings(string modelId) : IEmbeddingModel { public string ModelId { get; } = modelId;