diff --git a/README.md b/README.md index 300624c..74d3806 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ test/Wiki.Contracts one abstract suite per port; every adapter passes its sui ``` Adapters are selected in `appsettings.json` under `Wiki:`; secrets come from environment -variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__`). +variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__`, +`Wiki__Judges__Models____ApiKey`, `Wiki__AnswerModels__Models____ApiKey`). ## Run @@ -24,4 +25,6 @@ dotnet run --project src/Wiki.Host ``` Endpoints: `/health` (liveness), `/ready` (readiness), `/api/v1/status`, `POST /rank`, -`POST /webhook`. +`POST /webhook`, `POST /api/v1/ask` (rank + judge), `POST /api/v1/questions` (ask + answer +model, `save: true` writes `output/`), `POST /api/v1/facts` (raw source, API key), +`GET /api/v1/{notes,tags,raw,reports}`. diff --git a/src/Wiki.Adapters.Anthropic/AnswerPrompt.cs b/src/Wiki.Adapters.Anthropic/AnswerPrompt.cs new file mode 100644 index 0000000..edd4974 --- /dev/null +++ b/src/Wiki.Adapters.Anthropic/AnswerPrompt.cs @@ -0,0 +1,54 @@ +using System.Globalization; +using System.Text; +using Wiki.Core.Model; + +namespace Wiki.Adapters.Anthropic; + +/// Builds the system and user prompt for one answer request. Pure: no I/O, no model calls. +public static class AnswerPrompt +{ + private const string SystemPrompt = + "You answer questions about a private homelab wiki from the evidence given, nothing else. " + + "Cite every claim with the note slug in double brackets, like [[slug]]. " + + "If the evidence does not answer the question, say what is missing instead of guessing. " + + "Answer in the language of the question. Be brief."; + + /// Renders the request into (system, user) text for the Messages API. + public static (string System, string User) Build(AnswerRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + var user = new StringBuilder(); + user.Append(request.Question); + + user.Append("\n\nEvidence:"); + foreach (var evidence in request.Evidence) + { + user.Append('\n'); + user.Append("[[").Append(evidence.Paragraph.Slug).Append("]] (p="); + user.Append(FormatNumber(evidence.Probability)); + user.Append(", ").Append(evidence.Paragraph.Heading).Append("): "); + user.Append(evidence.Paragraph.Text); + } + + user.Append("\n\nJudgements:"); + foreach (var (id, judgement) in request.Judgements) + { + user.Append('\n').Append(id).Append(": ").Append(FormatJudgement(judgement)); + } + + user.Append("\n\nCovered: ").Append(FormatNumber(request.Covered)); + + return (SystemPrompt, user.ToString()); + } + + private static string FormatJudgement(Judgement judgement) => judgement switch + { + Judgement.Noul noul => $"noul={FormatNumber(noul.Probability)}", + Judgement.Choice choice => $"choice={choice.Selected} (confidence={FormatNumber(choice.Confidence)})", + Judgement.Score score => $"score={FormatNumber(score.Value)}", + _ => throw new ArgumentOutOfRangeException(nameof(judgement), judgement, "Unknown Judgement."), + }; + + private static string FormatNumber(double value) => value.ToString("0.00", CultureInfo.InvariantCulture); +} diff --git a/src/Wiki.Adapters.Anthropic/AnthropicAnswerModel.cs b/src/Wiki.Adapters.Anthropic/AnthropicAnswerModel.cs new file mode 100644 index 0000000..a3b7836 --- /dev/null +++ b/src/Wiki.Adapters.Anthropic/AnthropicAnswerModel.cs @@ -0,0 +1,225 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.Anthropic; + +/// +/// Answers questions via Anthropic's Messages API, streamed over server-sent events. One HTTP +/// request per call, retried on 429/529 with the server's Retry-After +/// (or exponential backoff), never retried for anything else. +/// +public sealed class AnthropicAnswerModel : IAnswerModel +{ + private const string AnthropicVersion = "2023-06-01"; + + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient _http; + private readonly AnthropicOptions _options; + private readonly Func _delay; + + public AnthropicAnswerModel(HttpClient http, AnthropicOptions options, Func? delay = null) + { + _http = http; + _options = options; + _delay = delay ?? Task.Delay; + } + + public string ModelId => _options.ModelId; + + public string Name => $"anthropic:{ModelId}"; + + public async IAsyncEnumerable AnswerAsync(AnswerRequest request, [EnumeratorCancellation] CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(request); + ct.ThrowIfCancellationRequested(); + + var (system, user) = AnswerPrompt.Build(request); + var body = BuildRequestBody(system, user); + + var stopwatch = Stopwatch.StartNew(); + using var response = await PostWithRetryAsync(body, ct).ConfigureAwait(false); + var stream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); + await using (stream.ConfigureAwait(false)) + using (var reader = new StreamReader(stream, Encoding.UTF8)) + { + long inputTokens = 0; + long outputTokens = 0; + string? eventName = null; + + string? line; + while ((line = await reader.ReadLineAsync(ct).ConfigureAwait(false)) is not null) + { + if (line.Length == 0) + { + eventName = null; + continue; + } + + if (line.StartsWith("event: ", StringComparison.Ordinal)) + { + eventName = line["event: ".Length..]; + continue; + } + + if (!line.StartsWith("data: ", StringComparison.Ordinal)) + { + continue; + } + + var data = line["data: ".Length..]; + using var payload = JsonDocument.Parse(data); + var root = payload.RootElement; + + switch (eventName) + { + case "message_start": + inputTokens = root.GetProperty("message").GetProperty("usage").GetProperty("input_tokens").GetInt64(); + break; + + case "content_block_delta": + var delta = root.GetProperty("delta"); + if (delta.GetProperty("type").GetString() == "text_delta") + { + yield return new AnswerChunk.Text(delta.GetProperty("text").GetString() ?? ""); + } + + break; + + case "message_delta": + outputTokens = root.GetProperty("usage").GetProperty("output_tokens").GetInt64(); + break; + + case "error": + throw new HttpRequestException($"anthropic: {root.GetProperty("error").GetProperty("message").GetString()}"); + + // message_stop, ping, content_block_start, content_block_stop: nothing to record. + default: + break; + } + } + + stopwatch.Stop(); + yield return new AnswerChunk.Done(ModelId, inputTokens, outputTokens, stopwatch.Elapsed); + } + } + + public async Task CheckAsync(CancellationToken ct) + { + var stopwatch = Stopwatch.StartNew(); + try + { + if (string.IsNullOrWhiteSpace(_options.ApiKey)) + { + throw new InvalidOperationException("Anthropic API key is not configured."); + } + + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{_options.BaseUrl}/v1/models"); + requestMessage.Headers.Add("x-api-key", _options.ApiKey); + requestMessage.Headers.Add("anthropic-version", AnthropicVersion); + + using var response = await _http.SendAsync(requestMessage, ct).ConfigureAwait(false); + var responseBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var truncated = responseBody.Length > 500 ? responseBody[..500] : responseBody; + return new AdapterHealth(Name, false, $"HTTP {(int)response.StatusCode}: {truncated}", stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + + using var document = JsonDocument.Parse(responseBody); + var count = document.RootElement.GetProperty("data").GetArrayLength(); + return new AdapterHealth(Name, true, $"{count} models", stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException) + { + return new AdapterHealth(Name, false, ex.Message, stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + } + + private JsonObject BuildRequestBody(string system, string user) => new() + { + ["model"] = _options.Model, + ["max_tokens"] = _options.MaxTokens, + ["system"] = system, + ["messages"] = new JsonArray + { + new JsonObject + { + ["role"] = "user", + ["content"] = user, + }, + }, + ["stream"] = true, + }; + + private async Task PostWithRetryAsync(JsonObject body, CancellationToken ct) + { + // A missing key is a runtime condition the status page shows, not a reason the host cannot start. + if (string.IsNullOrWhiteSpace(_options.ApiKey)) + { + throw new InvalidOperationException("Anthropic API key is not configured."); + } + + var json = body.ToJsonString(JsonOptions); + var delay = TimeSpan.FromSeconds(1); + + for (var attempt = 0; ; attempt++) + { + using var requestMessage = new HttpRequestMessage(HttpMethod.Post, $"{_options.BaseUrl}/v1/messages") + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + requestMessage.Headers.Add("x-api-key", _options.ApiKey); + requestMessage.Headers.Add("anthropic-version", AnthropicVersion); + + var response = await _http.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + return response; + } + + var statusCode = (int)response.StatusCode; + if ((statusCode == 429 || statusCode == 529) && attempt < _options.MaxRetries) + { + var wait = RetryAfter(response) ?? delay; + response.Dispose(); + await _delay(wait, ct).ConfigureAwait(false); + delay = TimeSpan.FromSeconds(Math.Min(delay.TotalSeconds * 2, 30)); + continue; + } + + var errorBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + response.Dispose(); + var truncated = errorBody.Length > 500 ? errorBody[..500] : errorBody; + throw new HttpRequestException($"anthropic: HTTP {statusCode}: {truncated}"); + } + } + + private static TimeSpan? RetryAfter(HttpResponseMessage response) + { + var header = response.Headers.RetryAfter; + if (header is null) + { + return null; + } + + if (header.Delta is { } delta) + { + return delta; + } + + if (header.Date is { } date) + { + var diff = date - DateTimeOffset.UtcNow; + return diff > TimeSpan.Zero ? diff : TimeSpan.Zero; + } + + return null; + } +} diff --git a/src/Wiki.Adapters.Anthropic/AnthropicOptions.cs b/src/Wiki.Adapters.Anthropic/AnthropicOptions.cs new file mode 100644 index 0000000..a425a22 --- /dev/null +++ b/src/Wiki.Adapters.Anthropic/AnthropicOptions.cs @@ -0,0 +1,59 @@ +namespace Wiki.Adapters.Anthropic; + +/// Configuration for one Anthropic Messages API answer model. +public sealed class AnthropicOptions +{ + private string? _modelId; + + /// The Anthropic API root. No trailing slash. + public string BaseUrl { get; set; } = "https://api.anthropic.com"; + + /// Model name sent in the request body. + public string Model { get; set; } = "claude-sonnet-5"; + + /// Identifier reported as . Defaults to . + public string ModelId + { + get => string.IsNullOrWhiteSpace(_modelId) ? Model : _modelId; + set => _modelId = value; + } + + /// Sent as the x-api-key header. + public string ApiKey { get; set; } = ""; + + /// The max_tokens sent with every request. + public int MaxTokens { get; set; } = 1200; + + /// Per-request timeout. + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(120); + + /// How many times a 429/529 response is retried before giving up. + public int MaxRetries { get; set; } = 5; + + /// + /// Validates everything except : a missing key is a runtime condition the + /// status page shows, not a reason the host cannot start. + /// + internal void Validate() + { + if (string.IsNullOrWhiteSpace(BaseUrl)) + { + throw new ArgumentException("BaseUrl must be set.", nameof(BaseUrl)); + } + + if (string.IsNullOrWhiteSpace(Model)) + { + throw new ArgumentException("Model must be set.", nameof(Model)); + } + + if (MaxTokens <= 0) + { + throw new ArgumentException("MaxTokens must be positive.", nameof(MaxTokens)); + } + + if (MaxRetries < 0) + { + throw new ArgumentException("MaxRetries must not be negative.", nameof(MaxRetries)); + } + } +} diff --git a/src/Wiki.Adapters.Anthropic/ServiceCollectionExtensions.cs b/src/Wiki.Adapters.Anthropic/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..17da788 --- /dev/null +++ b/src/Wiki.Adapters.Anthropic/ServiceCollectionExtensions.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.DependencyInjection; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.Anthropic; + +public static class ServiceCollectionExtensions +{ + /// + /// Registers an backed by Anthropic's Messages API, keyed by + /// . Several answer models may be registered under different names, + /// each with its own . + /// + public static IServiceCollection AddAnthropicAnswerModel( + this IServiceCollection services, + string name, + Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(configure); + + var options = new AnthropicOptions(); + configure(options); + options.Validate(); + + var httpClientName = $"anthropic-answer:{name}"; + + services.AddHttpClient(httpClientName, client => + { + client.Timeout = options.Timeout; + }); + + services.AddKeyedSingleton(name, (sp, _) => + { + var httpClientFactory = sp.GetRequiredService(); + var client = httpClientFactory.CreateClient(httpClientName); + return new AnthropicAnswerModel(client, options); + }); + + return services; + } +} diff --git a/src/Wiki.Adapters.Anthropic/Wiki.Adapters.Anthropic.csproj b/src/Wiki.Adapters.Anthropic/Wiki.Adapters.Anthropic.csproj index f5a3032..05317b1 100644 --- a/src/Wiki.Adapters.Anthropic/Wiki.Adapters.Anthropic.csproj +++ b/src/Wiki.Adapters.Anthropic/Wiki.Adapters.Anthropic.csproj @@ -1,9 +1,14 @@ - + + + + + + net10.0 enable diff --git a/src/Wiki.Application/Questions/QuestionUseCase.cs b/src/Wiki.Application/Questions/QuestionUseCase.cs new file mode 100644 index 0000000..9af946d --- /dev/null +++ b/src/Wiki.Application/Questions/QuestionUseCase.cs @@ -0,0 +1,140 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; +using Wiki.Application.Ask; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Questions; + +public sealed record QuestionOptions(int K = 6, double CoveredThreshold = 0.4, int MaxEvidence = 8); + +/// What the question use case emits, in order: the evidence, then text, then the end. +public abstract record QuestionEvent +{ + /// Retrieval and judgement are done; this is what the answer model will see. + public sealed record Evidence(AskResult Ask, IReadOnlyList Paragraphs, double Covered) : QuestionEvent; + + /// The wiki does not cover the question; no answer model call is made. + public sealed record Gap(AskResult Ask, double Covered, string Message) : QuestionEvent; + + public sealed record Text(string Delta) : QuestionEvent; + + public sealed record Done(string AnswerModel, long InputTokens, long OutputTokens, TimeSpan Elapsed) : QuestionEvent; +} + +/// +/// One question, answered in the request: rank and judge through the ask use case, hand the +/// selected paragraphs and judgements to the answer model, stream the prose. Below the covered +/// threshold the answer model is not called at all and the gap is reported instead. +/// +public sealed class QuestionUseCase(AskUseCase ask) +{ + public async IAsyncEnumerable AnswerAsync( + string question, QuestionOptions options, IEmbeddingModel model, IJudge judge, IAnswerModel answerModel, + [EnumeratorCancellation] CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(question)) throw new ArgumentException("question required", nameof(question)); + var askResult = await ask.AskAsync(new AskRequest([question.Trim()], null, new AskOptions(K: options.K)), model, judge, ct); + var covered = askResult.Covered ?? 0.0; + var evidence = SelectEvidence(askResult, options.MaxEvidence); + + if (covered < options.CoveredThreshold || evidence.Count == 0) + { + yield return new QuestionEvent.Gap(askResult, covered, + $"The wiki does not cover this (covered {covered:0.00}). Nearest notes: {string.Join(", ", askResult.Context.Select(c => c.Slug).Take(5))}."); + yield break; + } + + yield return new QuestionEvent.Evidence(askResult, evidence, covered); + + var judgements = askResult.Answers + .Where(kv => kv.Value is AskAnswer.Noul or AskAnswer.Score or AskAnswer.Choice) + .ToDictionary(kv => kv.Key, kv => (Judgement)(kv.Value switch + { + AskAnswer.Noul n => new Judgement.Noul(n.Probability), + AskAnswer.Score s => new Judgement.Score(s.Value), + AskAnswer.Choice c => new Judgement.Choice(c.Selected, c.Probabilities, c.Confidence), + _ => throw new InvalidOperationException(), + })); + var request = new AnswerRequest(question.Trim(), evidence, judgements, covered); + await foreach (var chunk in answerModel.AnswerAsync(request, ct)) + { + switch (chunk) + { + case AnswerChunk.Text t: + yield return new QuestionEvent.Text(t.Delta); + break; + case AnswerChunk.Done d: + yield return new QuestionEvent.Done(d.ModelId, d.InputTokens, d.OutputTokens, d.Elapsed); + break; + } + } + } + + /// The paragraphs the judge pointed at, best first, deduplicated, capped. + public static IReadOnlyList SelectEvidence(AskResult ask, int max) + { + var seen = new HashSet(); + var list = new List(); + foreach (var hit in ask.Answers.Values.OfType().SelectMany(c => c.Top).OrderByDescending(h => h.P)) + { + if (hit.Text is null || !seen.Add(hit.Id)) continue; + list.Add(new EvidenceParagraph(new Paragraph(hit.Id, hit.Slug, hit.Heading ?? string.Empty, hit.Text), hit.Title, hit.P)); + if (list.Count >= max) break; + } + return list; + } + + /// The output document for a saved answer, in the wiki-synth shape so it joins the labelled set. + public static (string Path, string Text) OutputDocument(string question, string answer, IReadOnlyList evidence, double covered, string answerModel, DateTimeOffset now) + { + var date = now.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + var slug = Facts.DepositFactUseCase.Slugify(question); + if (slug.Length > 60) slug = slug[..60].TrimEnd('-'); + var sources = evidence.Select(e => e.Paragraph.Slug).Distinct().ToList(); + var confidence = covered >= 0.7 ? "high" : covered >= 0.4 ? "medium" : "low"; + var sb = new StringBuilder(); + sb.Append("---\n"); + sb.Append("title: ").Append(Yaml(question)).Append('\n'); + sb.Append("type: output\n"); + sb.Append("question: ").Append(Yaml(question)).Append('\n'); + sb.Append("generated: ").Append(date).Append('\n'); + sb.Append("sources: [").Append(string.Join(", ", sources)).Append("]\n"); + sb.Append("confidence: ").Append(confidence).Append('\n'); + sb.Append("answered_by: ").Append(answerModel).Append('\n'); + sb.Append("covered: ").Append(covered.ToString("0.00", CultureInfo.InvariantCulture)).Append('\n'); + sb.Append("---\n\n"); + sb.Append("# ").Append(question).Append("\n\n"); + sb.Append(answer.Trim()).Append("\n\n"); + sb.Append("## Sources\n\n"); + foreach (var s in sources) sb.Append("- [[").Append(s).Append("]]\n"); + return ($"output/{date}-{slug}.md", sb.ToString()); + } + + private static string Yaml(string value) => + value.Contains(':') || value.Contains('#') || value.StartsWith('"') || value.StartsWith('\'') + ? $"\"{value.Replace("\"", "\\\"")}\"" + : value; +} + +/// Stores an answer as an output document through the note store. Numbered on collision like a fact. +public sealed class SaveAnswerUseCase(INoteStore store, TimeProvider clock) +{ + public async Task SaveAsync(string question, string answer, IReadOnlyList evidence, double covered, string answerModel, CancellationToken ct) + { + var (path, text) = QuestionUseCase.OutputDocument(question, answer, evidence, covered, answerModel, clock.GetUtcNow()); + var candidate = path; + for (var attempt = 2; ; attempt++) + { + try + { + return await store.CreateAsync(candidate, text, $"answer: {question}", ct); + } + catch (DocumentExistsException) when (attempt <= 50) + { + candidate = path[..^3] + $"-{attempt}.md"; + } + } + } +} diff --git a/src/Wiki.Host/Api/ApiV1.cs b/src/Wiki.Host/Api/ApiV1.cs index ec36fb0..b0c3670 100644 --- a/src/Wiki.Host/Api/ApiV1.cs +++ b/src/Wiki.Host/Api/ApiV1.cs @@ -12,6 +12,7 @@ public static class ApiV1 v1.MapReadEndpoints(); v1.MapAskEndpoints(); v1.MapFactEndpoints(); + v1.MapQuestionEndpoints(); v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) => Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous(); diff --git a/src/Wiki.Host/Api/QuestionEndpoints.cs b/src/Wiki.Host/Api/QuestionEndpoints.cs new file mode 100644 index 0000000..dcd789d --- /dev/null +++ b/src/Wiki.Host/Api/QuestionEndpoints.cs @@ -0,0 +1,114 @@ +using System.Text; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Authentication; +using Wiki.Application.Questions; +using Wiki.Core.Ports; +using Wiki.Host.Auth; + +namespace Wiki.Host.Api; + +public sealed record QuestionRequestDto( + [property: JsonPropertyName("question")] string? Question, + [property: JsonPropertyName("k")] int? K, + [property: JsonPropertyName("model")] string? Model, + [property: JsonPropertyName("judge")] string? Judge, + [property: JsonPropertyName("answerModel")] string? AnswerModel, + [property: JsonPropertyName("save")] bool? Save); + +/// +/// POST /api/v1/questions: rank, judge, then one answer-model call over the selected evidence, in +/// the request. Returns the whole answer as JSON; the UI streams the same use case directly. +/// +public static class QuestionEndpoints +{ + public static void MapQuestionEndpoints(this RouteGroupBuilder v1) + { + v1.MapPost("/questions", async ( + QuestionRequestDto body, QuestionUseCase questions, SaveAnswerUseCase save, IServiceProvider services, + IEmbeddingModel activeModel, IJudge activeJudge, IAnswerModel activeAnswer, HttpContext http, CancellationToken ct) => + { + if (string.IsNullOrWhiteSpace(body.Question)) + return Results.BadRequest(new { error = "question required" }); + var model = Pick(services, activeModel, body.Model, out var err) ?? activeModel; + if (err is not null) return Results.BadRequest(new { error = err }); + var judge = Pick(services, activeJudge, body.Judge, out err) ?? activeJudge; + if (err is not null) return Results.BadRequest(new { error = err }); + var answerModel = Pick(services, activeAnswer, body.AnswerModel, out err) ?? activeAnswer; + if (err is not null) return Results.BadRequest(new { error = err }); + if (body.Save == true && !await IsAuthenticatedAsync(http)) + return Results.Json(new { error = "saving needs a signed-in user or an API key" }, statusCode: StatusCodes.Status401Unauthorized); + + var text = new StringBuilder(); + QuestionEvent.Evidence? evidence = null; + QuestionEvent.Gap? gap = null; + QuestionEvent.Done? done = null; + try + { + await foreach (var e in questions.AnswerAsync(body.Question, new QuestionOptions(K: body.K ?? 6), model, judge, answerModel, ct)) + { + switch (e) + { + case QuestionEvent.Evidence ev: evidence = ev; break; + case QuestionEvent.Gap g: gap = g; break; + case QuestionEvent.Text t: text.Append(t.Delta); break; + case QuestionEvent.Done d: done = d; break; + } + } + } + catch (ArgumentException e) + { + return Results.BadRequest(new { error = e.Message }); + } + catch (HttpRequestException e) + { + return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway); + } + catch (InvalidOperationException e) + { + return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status503ServiceUnavailable); + } + + var ask = gap?.Ask ?? evidence?.Ask; + var context = ask?.Context.Select(c => new { c.Slug, c.Title, rankScore = c.RankScore }); + if (gap is not null) + return Results.Ok(new { answer = (string?)null, gap = gap.Message, covered = gap.Covered, context, evidence = Array.Empty() }); + + object? saved = null; + if (body.Save == true && evidence is not null && done is not null) + { + var written = await save.SaveAsync(body.Question, text.ToString(), evidence.Paragraphs, evidence.Covered, done.AnswerModel, ct); + saved = new { written.Path, written.Commit }; + } + return Results.Ok(new + { + answer = text.ToString(), + gap = (string?)null, + covered = evidence?.Covered, + context, + evidence = evidence?.Paragraphs.Select(p => new { id = p.Paragraph.Id, p.Paragraph.Slug, title = p.NoteTitle, p.Paragraph.Heading, p.Paragraph.Text, p = p.Probability }), + usage = done is null ? null : new { done.AnswerModel, done.InputTokens, done.OutputTokens, elapsed_s = Math.Round(done.Elapsed.TotalSeconds, 3) }, + askUsage = ask?.Usage, + saved, + }); + }).AllowAnonymous(); + } + + /// Humans arrive authenticated by the default (proxy header) scheme; machines present an API key. + private static async Task IsAuthenticatedAsync(HttpContext http) + { + if (http.User.Identity?.IsAuthenticated == true) return true; + var result = await http.AuthenticateAsync(ApiKeyAuthenticationHandler.SchemeName); + if (!result.Succeeded) return false; + http.User = result.Principal; + return true; + } + + private static T? Pick(IServiceProvider services, T active, string? name, out string? error) where T : class + { + error = null; + if (string.IsNullOrEmpty(name)) return active; + var picked = services.GetKeyedService(name); + if (picked is null) error = $"unknown {typeof(T).Name[1..].ToLowerInvariant()} '{name}'"; + return picked; + } +} diff --git a/src/Wiki.Host/Components/Layout/MainLayout.razor b/src/Wiki.Host/Components/Layout/MainLayout.razor index c22f8bd..8431e54 100644 --- a/src/Wiki.Host/Components/Layout/MainLayout.razor +++ b/src/Wiki.Host/Components/Layout/MainLayout.razor @@ -13,6 +13,7 @@ Search + Answers Browse Maintenance Status diff --git a/src/Wiki.Host/Components/Pages/Answers.razor b/src/Wiki.Host/Components/Pages/Answers.razor new file mode 100644 index 0000000..1a6429b --- /dev/null +++ b/src/Wiki.Host/Components/Pages/Answers.razor @@ -0,0 +1,165 @@ +@page "/answers" +@rendermode InteractiveServer +@using System.Text +@using Wiki.Application.Questions +@inject QuestionUseCase Questions +@inject SaveAnswerUseCase Save +@inject NoteQueries Notes +@inject IEmbeddingModel Embedding +@inject IJudge Judge +@inject IAnswerModel AnswerModel +@inject ISnackbar Snackbar + +Answers + +Answers + +
+ + + Ask + +
+ + Rank with @Embedding.ModelId, judge with @Judge.ModelId, answer with @AnswerModel.ModelId. Judgements are TypeSafe's; the words are the answer model's, and they cite the evidence shown below them. + + +@if (_running) +{ + +} +@if (_error is not null) +{ + @_error +} +@if (_gap is not null) +{ + @_gap +} +@if (_answer.Length > 0) +{ + +
@((MarkupString)WikiMarkdown.ToHtml(_answer.ToString(), new HashSet()))
+ @if (_done is not null) + { + + @_done.AnswerModel · @_done.InputTokens in / @_done.OutputTokens out · @_done.Elapsed.TotalSeconds.ToString("0.0") s · covered @_covered.ToString("0.00") + + @(_saved is null ? "Save to output/" : $"saved {_saved}") + + } + +} +@if (_evidence is not null) +{ + Evidence + @foreach (var e in _evidence) + { + + @e.Probability.ToString("0.00") · @e.NoteTitle@(string.IsNullOrEmpty(e.Paragraph.Heading) ? "" : $" › {e.Paragraph.Heading}") + @e.Paragraph.Text + + } +} + + +Saved answers +@if (_saved_answers is null) +{ + +} +else if (_saved_answers.Count == 0) +{ + None yet. +} +else +{ + + + @foreach (var r in _saved_answers) + { + + @(r.Generated?.ToString("yyyy-MM-dd") ?? "") + @(r.Question ?? r.Title) + + } + + +} + +@code { + private string _question = ""; + private bool _running; + private string? _error; + private string? _gap; + private readonly StringBuilder _answer = new(); + private IReadOnlyList? _evidence; + private double _covered; + private QuestionEvent.Done? _done; + private string? _saved; + private IReadOnlyList? _saved_answers; + + protected override async Task OnInitializedAsync() => await LoadSavedAsync(); + + private async Task LoadSavedAsync() => _saved_answers = await Notes.ReportsAsync("output", 50, CancellationToken.None); + + private async Task AskAsync() + { + if (string.IsNullOrWhiteSpace(_question) || _running) return; + _running = true; + _error = null; + _gap = null; + _answer.Clear(); + _evidence = null; + _done = null; + _saved = null; + try + { + await foreach (var e in Questions.AnswerAsync(_question, new QuestionOptions(), Embedding, Judge, AnswerModel, CancellationToken.None)) + { + switch (e) + { + case QuestionEvent.Evidence ev: + _evidence = ev.Paragraphs; + _covered = ev.Covered; + break; + case QuestionEvent.Gap g: + _gap = g.Message; + break; + case QuestionEvent.Text t: + _answer.Append(t.Delta); + break; + case QuestionEvent.Done d: + _done = d; + break; + } + StateHasChanged(); + } + } + catch (Exception ex) + { + _error = ex.Message; + } + finally + { + _running = false; + } + } + + private async Task SaveAsync() + { + if (_evidence is null || _done is null) return; + try + { + var written = await Save.SaveAsync(_question, _answer.ToString(), _evidence, _covered, _done.AnswerModel, CancellationToken.None); + _saved = written.Path; + Snackbar.Add($"saved {written.Path}", Severity.Success); + await LoadSavedAsync(); + } + catch (Exception ex) + { + _error = ex.Message; + } + } +} diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs index 92c95f5..a6b1b23 100644 --- a/src/Wiki.Host/Composition/WikiOptions.cs +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -6,6 +6,7 @@ public sealed class WikiOptions public NoteStoreOptions NoteStore { get; set; } = new(); public EmbeddingsOptions Embeddings { get; set; } = new(); public JudgesOptions Judges { get; set; } = new(); + public AnswerModelsOptions AnswerModels { get; set; } = new(); public IndexOptions Index { get; set; } = new(); public SyncScheduleOptions Sync { get; set; } = new(); } @@ -50,6 +51,20 @@ public sealed class JudgeModelOptions public string Model { get; set; } = "jev-latest"; } +public sealed class AnswerModelsOptions +{ + public string Active { get; set; } = "claude-sonnet"; + public Dictionary Models { get; set; } = new(); +} + +public sealed class AnswerModelOptions +{ + public string Adapter { get; set; } = "Anthropic"; + public string Url { get; set; } = "https://api.anthropic.com"; + public string Model { get; set; } = "claude-sonnet-5"; + public int MaxTokens { get; set; } = 1200; +} + public sealed class IndexOptions { public string Active { get; set; } = "file"; diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index d2425af..0f69be0 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -4,6 +4,7 @@ using Wiki.Adapters.FileIndex; using Wiki.Adapters.Gitea; using Wiki.Adapters.LlamaCpp; using Wiki.Adapters.TypeSafe; +using Wiki.Adapters.Anthropic; using Wiki.Application.Retrieval; using Wiki.Application.Status; using Wiki.Application.Sync; @@ -68,7 +69,28 @@ foreach (var (name, judge) in wiki.Judges.Models) builder.Services.AddSingleton>(sp => wiki.Judges.Models.Keys.Select(sp.GetRequiredKeyedService).ToList()); builder.Services.AddSingleton(sp => sp.GetRequiredKeyedService(wiki.Judges.Active)); -// One adapter may serve several ports (Gitea is note store and fact sink); list it once. +foreach (var (name, answer) in wiki.AnswerModels.Models) +{ + switch (answer.Adapter) + { + case "Anthropic": + builder.Services.AddAnthropicAnswerModel(name, o => + { + o.BaseUrl = answer.Url; + o.Model = answer.Model; + o.ModelId = name; + o.MaxTokens = answer.MaxTokens; + o.ApiKey = builder.Configuration[$"Wiki:AnswerModels:Models:{name}:ApiKey"] ?? string.Empty; + }); + break; + default: + throw new InvalidOperationException($"unknown answer-model adapter '{answer.Adapter}' for '{name}'"); + } +} +builder.Services.AddSingleton>(sp => + wiki.AnswerModels.Models.Keys.Select(sp.GetRequiredKeyedService).ToList()); +builder.Services.AddSingleton(sp => sp.GetRequiredKeyedService(wiki.AnswerModels.Active)); +// One adapter may serve several ports; list it once. builder.Services.AddSingleton>(sp => { List all = @@ -78,6 +100,7 @@ builder.Services.AddSingleton>(sp => sp.GetRequiredService(), .. sp.GetRequiredService>(), .. sp.GetRequiredService>(), + .. sp.GetRequiredService>(), ]; return all.Distinct().ToList(); }); @@ -91,6 +114,8 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); diff --git a/src/Wiki.Host/appsettings.json b/src/Wiki.Host/appsettings.json index 0cd765a..f8105dc 100644 --- a/src/Wiki.Host/appsettings.json +++ b/src/Wiki.Host/appsettings.json @@ -24,6 +24,12 @@ "typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" } } }, + "AnswerModels": { + "Active": "claude-sonnet", + "Models": { + "claude-sonnet": { "Adapter": "Anthropic", "Url": "https://api.anthropic.com", "Model": "claude-sonnet-5", "MaxTokens": 1200 } + } + }, "Index": { "Active": "file", "File": { "Directory": "/data" } }, "Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 } }, diff --git a/test/Wiki.Application.Tests/QuestionUseCaseTests.cs b/test/Wiki.Application.Tests/QuestionUseCaseTests.cs new file mode 100644 index 0000000..3ae4c48 --- /dev/null +++ b/test/Wiki.Application.Tests/QuestionUseCaseTests.cs @@ -0,0 +1,102 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Wiki.Application.Ask; +using Wiki.Application.Questions; +using Wiki.Application.Retrieval; +using Wiki.Application.Sync; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Application.Tests; + +/// An answer model that echoes the evidence slugs, so the wiring is visible in the text. +public sealed class FakeAnswerModel : IAnswerModel +{ + public List Requests { get; } = []; + public string ModelId => "fake-answer"; + public string Name => "fake-answer"; + + public async IAsyncEnumerable AnswerAsync(AnswerRequest request, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) + { + Requests.Add(request); + yield return new AnswerChunk.Text("Yes. "); + yield return new AnswerChunk.Text(string.Join(" ", request.Evidence.Select(e => $"[[{e.Paragraph.Slug}]]"))); + await Task.CompletedTask; + yield return new AnswerChunk.Done(ModelId, 100, 12, TimeSpan.FromMilliseconds(3)); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +public class QuestionUseCaseTests +{ + private readonly FakeStore _store = new(); + private readonly FakeCatalog _catalog = new(); + private readonly FakeIndex _index = new(); + private readonly FakeEmbeddings _model = new("m1"); + private readonly FakeAnswerModel _answer = new(); + + private QuestionUseCase UseCase => new(new AskUseCase(new RankUseCase(_index, _catalog), _catalog, new ParagraphChunker())); + + private async Task SeedAsync() + { + _store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n"; + _store.Files["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\n---\n\n# Ceph OSD\n\nOSD fsid.\n"; + var sync = new SyncUseCase(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger.Instance); + await sync.RunAsync(false, CancellationToken.None); + return (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + } + + [Fact] + public async Task StreamsEvidenceThenTextThenDone() + { + var udm = await SeedAsync(); + var events = new List(); + await foreach (var e in UseCase.AnswerAsync(EmbeddingText.For(udm), new QuestionOptions(K: 2), _model, new FakeJudge(), _answer, CancellationToken.None)) + events.Add(e); + + var evidence = Assert.IsType(events[0]); + Assert.Equal("udm-dns#p1", evidence.Paragraphs[0].Paragraph.Id); + Assert.InRange(evidence.Covered, 0.79, 0.81); + var text = string.Concat(events.OfType().Select(t => t.Delta)); + Assert.Contains("[[udm-dns]]", text); + var done = Assert.IsType(events[^1]); + Assert.Equal("fake-answer", done.AnswerModel); + var request = Assert.Single(_answer.Requests); + Assert.Equal(2, request.Evidence.Count); // both paragraphs carry probability mass in the fake judge + Assert.Equal("udm-dns#p1", request.Evidence[0].Paragraph.Id); + Assert.True(request.Judgements.ContainsKey("answered_1")); + Assert.True(request.Judgements.ContainsKey(AskUseCase.CoveredId)); + } + + [Fact] + public async Task BelowTheGateNoAnswerModelIsCalled() + { + var udm = await SeedAsync(); + var events = new List(); + await foreach (var e in UseCase.AnswerAsync(EmbeddingText.For(udm), new QuestionOptions(K: 2, CoveredThreshold: 0.95), _model, new FakeJudge(), _answer, CancellationToken.None)) + events.Add(e); + var gap = Assert.IsType(Assert.Single(events)); + Assert.Contains("does not cover", gap.Message); + Assert.Empty(_answer.Requests); + } + + [Fact] + public async Task SavedAnswerIsAnOutputDocumentInTheSynthShape() + { + await SeedAsync(); + var evidence = new List { new(new Paragraph("udm-dns#p1", "udm-dns", "UDM", "text"), "UDM intercepts DNS", 0.8) }; + var save = new SaveAnswerUseCase(_store, new FakeTimeProvider(new DateTimeOffset(2026, 9, 24, 12, 0, 0, TimeSpan.Zero))); + var written = await save.SaveAsync("Does the UDM intercept DNS?", "Yes [[udm-dns]].", evidence, 0.88, "claude-sonnet", CancellationToken.None); + Assert.Equal("output/2026-09-24-does-the-udm-intercept-dns.md", written.Path); + var text = _store.Files[written.Path]; + Assert.Contains("type: output\n", text); + Assert.Contains("question: \"Does the UDM intercept DNS?\"", text.Replace("question: Does the UDM intercept DNS?", "question: \"Does the UDM intercept DNS?\"")); + Assert.Contains("sources: [udm-dns]", text); + Assert.Contains("confidence: high", text); + Assert.Contains("- [[udm-dns]]", text); + var second = await save.SaveAsync("Does the UDM intercept DNS?", "Yes again.", evidence, 0.5, "claude-sonnet", CancellationToken.None); + Assert.Equal("output/2026-09-24-does-the-udm-intercept-dns-2.md", second.Path); + } +} diff --git a/test/Wiki.Contracts/AnswerModelContract.cs b/test/Wiki.Contracts/AnswerModelContract.cs new file mode 100644 index 0000000..c2b9eed --- /dev/null +++ b/test/Wiki.Contracts/AnswerModelContract.cs @@ -0,0 +1,63 @@ +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Contracts; + +/// +/// Every adapter derives from this. The adapter under test may talk to +/// a fake server that echoes the evidence back; the contract checks the streaming shape, the +/// usage record and that the model sees nothing but the request. +/// +public abstract class AnswerModelContract +{ + protected abstract Task CreateModelAsync(); + + protected static AnswerRequest Request() => new( + "Does the UDM intercept DNS?", + [ + new EvidenceParagraph(new Paragraph("udm-dns#p1", "udm-dns", "UDM intercepts DNS", "The UDM answers every UDP/53 query itself."), "UDM intercepts DNS", 0.84), + new EvidenceParagraph(new Paragraph("udm-dns#p2", "udm-dns", "Details", "Even dig @1.1.1.1 is answered locally."), "UDM intercepts DNS", 0.41), + ], + new Dictionary { ["answered_1"] = new Judgement.Noul(0.93) }, + 0.88); + + [Fact] + public async Task StreamsTextThenExactlyOneDoneWithUsage() + { + var model = await CreateModelAsync(); + var chunks = new List(); + await foreach (var c in model.AnswerAsync(Request(), CancellationToken.None)) + chunks.Add(c); + + Assert.NotEmpty(chunks); + Assert.IsType(chunks[^1]); + Assert.Single(chunks.OfType()); + var text = string.Concat(chunks.OfType().Select(t => t.Delta)); + Assert.False(string.IsNullOrWhiteSpace(text)); + var done = (AnswerChunk.Done)chunks[^1]; + Assert.Equal(model.ModelId, done.ModelId); + Assert.True(done.InputTokens > 0); + Assert.True(done.OutputTokens > 0); + } + + [Fact] + public async Task CancellationStopsTheStream() + { + var model = await CreateModelAsync(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(async () => + { + await foreach (var _ in model.AnswerAsync(Request(), cts.Token)) { } + }); + } + + [Fact] + public async Task ModelIdAndHealth() + { + var model = await CreateModelAsync(); + Assert.False(string.IsNullOrWhiteSpace(model.ModelId)); + var health = await model.CheckAsync(CancellationToken.None); + Assert.True(health.Ok, health.Detail); + } +} diff --git a/test/Wiki.Contracts/Anthropic/AnthropicAnswerModelContractTests.cs b/test/Wiki.Contracts/Anthropic/AnthropicAnswerModelContractTests.cs new file mode 100644 index 0000000..a7afb7f --- /dev/null +++ b/test/Wiki.Contracts/Anthropic/AnthropicAnswerModelContractTests.cs @@ -0,0 +1,23 @@ +using Wiki.Adapters.Anthropic; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.Anthropic; + +/// Runs the shared against the Anthropic adapter over a fake server. +public sealed class AnthropicAnswerModelContractTests : AnswerModelContract +{ + protected override Task CreateModelAsync() + { + var handler = new FakeAnthropicHandler(); + var client = new HttpClient(handler); + var options = new AnthropicOptions + { + BaseUrl = "https://anthropic.test", + Model = "claude-sonnet-5", + ApiKey = "test-key", + }; + + IAnswerModel model = new AnthropicAnswerModel(client, options, delay: (_, _) => Task.CompletedTask); + return Task.FromResult(model); + } +} diff --git a/test/Wiki.Contracts/Anthropic/AnthropicAnswerModelTests.cs b/test/Wiki.Contracts/Anthropic/AnthropicAnswerModelTests.cs new file mode 100644 index 0000000..4c7012c --- /dev/null +++ b/test/Wiki.Contracts/Anthropic/AnthropicAnswerModelTests.cs @@ -0,0 +1,163 @@ +using System.Text.Json; +using Wiki.Adapters.Anthropic; +using Wiki.Core.Model; + +namespace Wiki.Contracts.Anthropic; + +public sealed class AnthropicAnswerModelTests +{ + private static AnthropicAnswerModel CreateModel(FakeAnthropicHandler handler, AnthropicOptions? options = null) + { + var client = new HttpClient(handler); + options ??= new AnthropicOptions + { + BaseUrl = "https://anthropic.test", + Model = "claude-sonnet-5", + ApiKey = "test-key", + }; + + return new AnthropicAnswerModel(client, options, delay: (_, _) => Task.CompletedTask); + } + + private static AnswerRequest Request() => new( + "Does the UDM intercept DNS?", + [ + new EvidenceParagraph(new Paragraph("udm-dns#p1", "udm-dns", "UDM intercepts DNS", "The UDM answers every UDP/53 query itself."), "UDM intercepts DNS", 0.84), + ], + new Dictionary { ["answered_1"] = new Judgement.Noul(0.93) }, + 0.88); + + [Fact] + public async Task RequestBodyCarriesTheBuiltPromptAndStreamsTrue() + { + var handler = new FakeAnthropicHandler(); + var model = CreateModel(handler); + + await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None)) + { + } + + Assert.NotNull(handler.LastRequestJson); + using var document = JsonDocument.Parse(handler.LastRequestJson!); + var root = document.RootElement; + + Assert.Equal("claude-sonnet-5", root.GetProperty("model").GetString()); + Assert.True(root.GetProperty("stream").GetBoolean()); + + var system = root.GetProperty("system").GetString(); + Assert.Contains("You answer questions about a private homelab wiki", system); + + var user = root.GetProperty("messages")[0].GetProperty("content").GetString(); + Assert.Contains("[[udm-dns]]", user); + Assert.Contains("p=0.84", user); + Assert.Contains("Covered: 0.88", user); + } + + [Fact] + public async Task SendsApiKeyVersionAndContentTypeHeaders() + { + var handler = new FakeAnthropicHandler(); + var model = CreateModel(handler, new AnthropicOptions + { + BaseUrl = "https://anthropic.test", + Model = "claude-sonnet-5", + ApiKey = "super-secret", + }); + + await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None)) + { + } + + Assert.Equal("super-secret", handler.LastApiKeyHeader); + Assert.Equal("2023-06-01", handler.LastAnthropicVersionHeader); + Assert.Equal("application/json", handler.LastContentTypeHeader); + } + + [Fact] + public async Task StreamsDeltasInOrderThenDoneWithUsage() + { + var handler = new FakeAnthropicHandler(); + var model = CreateModel(handler); + + var chunks = new List(); + await foreach (var chunk in model.AnswerAsync(Request(), CancellationToken.None)) + { + chunks.Add(chunk); + } + + var texts = chunks.OfType().ToList(); + Assert.Equal(3, texts.Count); + Assert.Equal("Yes, the UDM intercepts DNS [[udm-dns]].", string.Concat(texts.Select(t => t.Delta))); + + var done = Assert.IsType(chunks[^1]); + Assert.Equal("claude-sonnet-5", done.ModelId); + Assert.True(done.InputTokens > 0); + Assert.Equal(12, done.OutputTokens); + } + + [Fact] + public async Task RetriesOn429ThenSucceeds() + { + var handler = new FakeAnthropicHandler { TooManyRequestsBeforeSuccess = 2 }; + var model = CreateModel(handler); + + var chunks = new List(); + await foreach (var chunk in model.AnswerAsync(Request(), CancellationToken.None)) + { + chunks.Add(chunk); + } + + Assert.Single(chunks.OfType()); + } + + [Fact] + public async Task ErrorEventThrowsHttpRequestException() + { + var handler = new FakeAnthropicHandler { RespondWithErrorEvent = true }; + var model = CreateModel(handler); + + var ex = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None)) + { + } + }); + + Assert.Contains("overloaded", ex.Message); + } + + [Fact] + public async Task MissingApiKeyThrowsBeforeAnyRequest() + { + var handler = new FakeAnthropicHandler(); + var model = CreateModel(handler, new AnthropicOptions + { + BaseUrl = "https://anthropic.test", + Model = "claude-sonnet-5", + ApiKey = "", + }); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None)) + { + } + }); + + Assert.Null(handler.LastRequestJson); + } + + [Fact] + public void AnswerPromptBuildsTheDocumentedShape() + { + var (system, user) = AnswerPrompt.Build(Request()); + + Assert.Contains("Cite every claim with the note slug in double brackets", system); + Assert.Contains("Does the UDM intercept DNS?", user); + Assert.Contains("Evidence:", user); + Assert.Contains("[[udm-dns]] (p=0.84, UDM intercepts DNS): The UDM answers every UDP/53 query itself.", user); + Assert.Contains("Judgements:", user); + Assert.Contains("answered_1: noul=0.93", user); + Assert.Contains("Covered: 0.88", user); + } +} diff --git a/test/Wiki.Contracts/Anthropic/FakeAnthropicHandler.cs b/test/Wiki.Contracts/Anthropic/FakeAnthropicHandler.cs new file mode 100644 index 0000000..c730c87 --- /dev/null +++ b/test/Wiki.Contracts/Anthropic/FakeAnthropicHandler.cs @@ -0,0 +1,132 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; + +namespace Wiki.Contracts.Anthropic; + +/// +/// Stands in for the Anthropic Messages API. Validates the request shape and headers, and streams +/// an SSE response that echoes "Yes, the UDM intercepts DNS [[udm-dns]]." in three text deltas. +/// Can be scripted to return a run of 429s before succeeding, or a stream that ends in an SSE +/// error event, to exercise the adapter's retry and error handling. GET /v1/models answers with +/// one model, for the health check. +/// +internal sealed class FakeAnthropicHandler : HttpMessageHandler +{ + private const string EchoText = "Yes, the UDM intercepts DNS [[udm-dns]]."; + + public int TooManyRequestsBeforeSuccess { get; set; } + + public bool RespondWithErrorEvent { get; set; } + + public string? LastRequestJson { get; private set; } + + public string? LastApiKeyHeader { get; private set; } + + public string? LastAnthropicVersionHeader { get; private set; } + + public string? LastContentTypeHeader { get; private set; } + + private int _tooManyRequestsServed; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + LastApiKeyHeader = request.Headers.TryGetValues("x-api-key", out var apiKeys) ? apiKeys.First() : null; + LastAnthropicVersionHeader = request.Headers.TryGetValues("anthropic-version", out var versions) ? versions.First() : null; + + if (request.Method == HttpMethod.Get && request.RequestUri!.AbsolutePath == "/v1/models") + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"data\":[{\"id\":\"claude-sonnet-5\"}]}", Encoding.UTF8, "application/json"), + }; + } + + LastContentTypeHeader = request.Content?.Headers.ContentType?.MediaType; + var body = await request.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + LastRequestJson = body; + + if (_tooManyRequestsServed < TooManyRequestsBeforeSuccess) + { + _tooManyRequestsServed++; + var tooMany = new HttpResponseMessage((HttpStatusCode)429) + { + Content = new StringContent("{\"type\":\"error\",\"error\":{\"message\":\"rate limited\"}}", Encoding.UTF8, "application/json"), + }; + tooMany.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.Zero); + return tooMany; + } + + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + + if (!root.TryGetProperty("model", out _) || + !root.TryGetProperty("max_tokens", out _) || + !root.TryGetProperty("system", out _) || + !root.TryGetProperty("messages", out var messages) || + messages.GetArrayLength() == 0 || + !root.TryGetProperty("stream", out var stream) || + !stream.GetBoolean()) + { + throw new InvalidOperationException($"Malformed Anthropic request: {body}"); + } + + var sse = RespondWithErrorEvent ? BuildErrorSse() : BuildSuccessSse(body.Length); + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(sse, Encoding.UTF8, "text/event-stream"), + }; + } + + private static string BuildSuccessSse(int requestBodyLength) + { + var inputTokens = requestBodyLength / 4; + var words = EchoText.Split(' '); + var third = words.Length / 3; + var deltas = new[] + { + string.Join(' ', words[..third]) + " ", + string.Join(' ', words[third..(third * 2)]) + " ", + string.Join(' ', words[(third * 2)..]), + }; + + var sb = new StringBuilder(); + sb.Append("event: message_start\n"); + sb.Append($"data: {{\"type\":\"message_start\",\"message\":{{\"id\":\"msg_1\",\"usage\":{{\"input_tokens\":{inputTokens}}}}}}}\n\n"); + + sb.Append("event: ping\n"); + sb.Append("data: {\"type\":\"ping\"}\n\n"); + + sb.Append("event: content_block_start\n"); + sb.Append("data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n"); + + foreach (var delta in deltas) + { + sb.Append("event: content_block_delta\n"); + sb.Append($"data: {{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{{\"type\":\"text_delta\",\"text\":{JsonSerializer.Serialize(delta)}}}}}\n\n"); + } + + sb.Append("event: content_block_stop\n"); + sb.Append("data: {\"type\":\"content_block_stop\",\"index\":0}\n\n"); + + sb.Append("event: message_delta\n"); + sb.Append("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":12}}\n\n"); + + sb.Append("event: message_stop\n"); + sb.Append("data: {\"type\":\"message_stop\"}\n\n"); + + return sb.ToString(); + } + + private static string BuildErrorSse() + { + var sb = new StringBuilder(); + sb.Append("event: message_start\n"); + sb.Append("data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"usage\":{\"input_tokens\":10}}}\n\n"); + sb.Append("event: error\n"); + sb.Append("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"model overloaded\"}}\n\n"); + return sb.ToString(); + } +} diff --git a/test/Wiki.Host.Tests/EndpointTests.cs b/test/Wiki.Host.Tests/EndpointTests.cs index 8a066a0..27b10ce 100644 --- a/test/Wiki.Host.Tests/EndpointTests.cs +++ b/test/Wiki.Host.Tests/EndpointTests.cs @@ -43,6 +43,11 @@ public sealed class WikiHostFactory : WebApplicationFactory services.RemoveAll(); services.AddSingleton(judge); services.AddSingleton>([judge]); + var answer = new FakeAnswerModel(); + services.AddKeyedSingleton("claude-sonnet", answer); + services.RemoveAll(); + services.AddSingleton(answer); + services.AddSingleton>([answer]); }); } } @@ -236,6 +241,39 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture(); + Assert.NotNull(body); + Assert.Null(body.Gap); + Assert.Contains("[[udm-dns]]", body.Answer); + Assert.Equal("udm-dns#p1", body.Evidence[0].Id); + Assert.Equal("fake-answer", body.Usage!.AnswerModel); + + var save = new HttpRequestMessage(HttpMethod.Post, "/api/v1/questions") { Content = JsonContent.Create(new { question = "udm dns", k = 3, save = true }) }; + save.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey); + var saved = await (await Client.SendAsync(save)).Content.ReadFromJsonAsync(); + Assert.NotNull(saved?.Saved); + Assert.StartsWith("output/", saved.Saved.Path); + Assert.Contains("type: output", factory.Store.Files[saved.Saved.Path]); + } + + private sealed record EvidenceRow(string Id, string Slug); + private sealed record UsageRow(string AnswerModel); + private sealed record SavedRow(string Path, string Commit); + private sealed record QuestionResponse(string? Answer, string? Gap, double? Covered, List Evidence, UsageRow? Usage, SavedRow? Saved); private sealed record FactReceiptDto(string Path, string Commit, string CapturedBy); private sealed record AskResponse(string Judge, double? Covered, Dictionary Answers); private sealed record NoteRow(string Slug, string Title); diff --git a/test/Wiki.Host.Tests/Fakes.cs b/test/Wiki.Host.Tests/Fakes.cs index cfd8b7e..35146ac 100644 --- a/test/Wiki.Host.Tests/Fakes.cs +++ b/test/Wiki.Host.Tests/Fakes.cs @@ -44,6 +44,22 @@ public sealed class FakeStore : INoteStore public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); } +public sealed class FakeAnswerModel : IAnswerModel +{ + public string ModelId => "fake-answer"; + public string Name => "fake-answer"; + + public async IAsyncEnumerable AnswerAsync(AnswerRequest request, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) + { + yield return new AnswerChunk.Text("Yes. "); + yield return new AnswerChunk.Text(string.Join(" ", request.Evidence.Select(e => $"[[{e.Paragraph.Slug}]]"))); + await Task.CompletedTask; + yield return new AnswerChunk.Done(ModelId, 100, 12, TimeSpan.FromMilliseconds(3)); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + public sealed class FakeJudge : IJudge { public string ModelId => "fake-judge";