Phase 5: questions endpoint, Anthropic answer model, Answers page #7

Merged
k.renner merged 2 commits from feat/answer-model into main 2026-09-24 14:57:29 +00:00
21 changed files with 1396 additions and 4 deletions
+5 -2
View File
@@ -14,7 +14,8 @@ test/Wiki.Contracts one abstract suite per port; every adapter passes its sui
```
Adapters are selected in `appsettings.json` under `Wiki:`; secrets come from environment
variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__<name>`).
variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__<name>`,
`Wiki__Judges__Models__<name>__ApiKey`, `Wiki__AnswerModels__Models__<name>__ApiKey`).
## Run
@@ -24,4 +25,6 @@ dotnet run --project src/Wiki.Host
```
Endpoints: `/health` (liveness), `/ready` (readiness), `/api/v1/status`, `POST /rank`,
`POST /webhook`.
`POST /webhook`, `POST /api/v1/ask` (rank + judge), `POST /api/v1/questions` (ask + answer
model, `save: true` writes `output/`), `POST /api/v1/facts` (raw source, API key),
`GET /api/v1/{notes,tags,raw,reports}`.
@@ -0,0 +1,54 @@
using System.Globalization;
using System.Text;
using Wiki.Core.Model;
namespace Wiki.Adapters.Anthropic;
/// <summary>Builds the system and user prompt for one answer request. Pure: no I/O, no model calls.</summary>
public static class AnswerPrompt
{
private const string SystemPrompt =
"You answer questions about a private homelab wiki from the evidence given, nothing else. " +
"Cite every claim with the note slug in double brackets, like [[slug]]. " +
"If the evidence does not answer the question, say what is missing instead of guessing. " +
"Answer in the language of the question. Be brief.";
/// <summary>Renders the request into (system, user) text for the Messages API.</summary>
public static (string System, string User) Build(AnswerRequest request)
{
ArgumentNullException.ThrowIfNull(request);
var user = new StringBuilder();
user.Append(request.Question);
user.Append("\n\nEvidence:");
foreach (var evidence in request.Evidence)
{
user.Append('\n');
user.Append("[[").Append(evidence.Paragraph.Slug).Append("]] (p=");
user.Append(FormatNumber(evidence.Probability));
user.Append(", ").Append(evidence.Paragraph.Heading).Append("): ");
user.Append(evidence.Paragraph.Text);
}
user.Append("\n\nJudgements:");
foreach (var (id, judgement) in request.Judgements)
{
user.Append('\n').Append(id).Append(": ").Append(FormatJudgement(judgement));
}
user.Append("\n\nCovered: ").Append(FormatNumber(request.Covered));
return (SystemPrompt, user.ToString());
}
private static string FormatJudgement(Judgement judgement) => judgement switch
{
Judgement.Noul noul => $"noul={FormatNumber(noul.Probability)}",
Judgement.Choice choice => $"choice={choice.Selected} (confidence={FormatNumber(choice.Confidence)})",
Judgement.Score score => $"score={FormatNumber(score.Value)}",
_ => throw new ArgumentOutOfRangeException(nameof(judgement), judgement, "Unknown Judgement."),
};
private static string FormatNumber(double value) => value.ToString("0.00", CultureInfo.InvariantCulture);
}
@@ -0,0 +1,225 @@
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using Wiki.Core.Model;
using Wiki.Core.Ports;
namespace Wiki.Adapters.Anthropic;
/// <summary>
/// Answers questions via Anthropic's Messages API, streamed over server-sent events. One HTTP
/// request per <see cref="AnswerAsync"/> call, retried on 429/529 with the server's Retry-After
/// (or exponential backoff), never retried for anything else.
/// </summary>
public sealed class AnthropicAnswerModel : IAnswerModel
{
private const string AnthropicVersion = "2023-06-01";
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly HttpClient _http;
private readonly AnthropicOptions _options;
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
public AnthropicAnswerModel(HttpClient http, AnthropicOptions options, Func<TimeSpan, CancellationToken, Task>? delay = null)
{
_http = http;
_options = options;
_delay = delay ?? Task.Delay;
}
public string ModelId => _options.ModelId;
public string Name => $"anthropic:{ModelId}";
public async IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, [EnumeratorCancellation] CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(request);
ct.ThrowIfCancellationRequested();
var (system, user) = AnswerPrompt.Build(request);
var body = BuildRequestBody(system, user);
var stopwatch = Stopwatch.StartNew();
using var response = await PostWithRetryAsync(body, ct).ConfigureAwait(false);
var stream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
await using (stream.ConfigureAwait(false))
using (var reader = new StreamReader(stream, Encoding.UTF8))
{
long inputTokens = 0;
long outputTokens = 0;
string? eventName = null;
string? line;
while ((line = await reader.ReadLineAsync(ct).ConfigureAwait(false)) is not null)
{
if (line.Length == 0)
{
eventName = null;
continue;
}
if (line.StartsWith("event: ", StringComparison.Ordinal))
{
eventName = line["event: ".Length..];
continue;
}
if (!line.StartsWith("data: ", StringComparison.Ordinal))
{
continue;
}
var data = line["data: ".Length..];
using var payload = JsonDocument.Parse(data);
var root = payload.RootElement;
switch (eventName)
{
case "message_start":
inputTokens = root.GetProperty("message").GetProperty("usage").GetProperty("input_tokens").GetInt64();
break;
case "content_block_delta":
var delta = root.GetProperty("delta");
if (delta.GetProperty("type").GetString() == "text_delta")
{
yield return new AnswerChunk.Text(delta.GetProperty("text").GetString() ?? "");
}
break;
case "message_delta":
outputTokens = root.GetProperty("usage").GetProperty("output_tokens").GetInt64();
break;
case "error":
throw new HttpRequestException($"anthropic: {root.GetProperty("error").GetProperty("message").GetString()}");
// message_stop, ping, content_block_start, content_block_stop: nothing to record.
default:
break;
}
}
stopwatch.Stop();
yield return new AnswerChunk.Done(ModelId, inputTokens, outputTokens, stopwatch.Elapsed);
}
}
public async Task<AdapterHealth> CheckAsync(CancellationToken ct)
{
var stopwatch = Stopwatch.StartNew();
try
{
if (string.IsNullOrWhiteSpace(_options.ApiKey))
{
throw new InvalidOperationException("Anthropic API key is not configured.");
}
using var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{_options.BaseUrl}/v1/models");
requestMessage.Headers.Add("x-api-key", _options.ApiKey);
requestMessage.Headers.Add("anthropic-version", AnthropicVersion);
using var response = await _http.SendAsync(requestMessage, ct).ConfigureAwait(false);
var responseBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var truncated = responseBody.Length > 500 ? responseBody[..500] : responseBody;
return new AdapterHealth(Name, false, $"HTTP {(int)response.StatusCode}: {truncated}", stopwatch.Elapsed, DateTimeOffset.UtcNow);
}
using var document = JsonDocument.Parse(responseBody);
var count = document.RootElement.GetProperty("data").GetArrayLength();
return new AdapterHealth(Name, true, $"{count} models", stopwatch.Elapsed, DateTimeOffset.UtcNow);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException)
{
return new AdapterHealth(Name, false, ex.Message, stopwatch.Elapsed, DateTimeOffset.UtcNow);
}
}
private JsonObject BuildRequestBody(string system, string user) => new()
{
["model"] = _options.Model,
["max_tokens"] = _options.MaxTokens,
["system"] = system,
["messages"] = new JsonArray
{
new JsonObject
{
["role"] = "user",
["content"] = user,
},
},
["stream"] = true,
};
private async Task<HttpResponseMessage> PostWithRetryAsync(JsonObject body, CancellationToken ct)
{
// A missing key is a runtime condition the status page shows, not a reason the host cannot start.
if (string.IsNullOrWhiteSpace(_options.ApiKey))
{
throw new InvalidOperationException("Anthropic API key is not configured.");
}
var json = body.ToJsonString(JsonOptions);
var delay = TimeSpan.FromSeconds(1);
for (var attempt = 0; ; attempt++)
{
using var requestMessage = new HttpRequestMessage(HttpMethod.Post, $"{_options.BaseUrl}/v1/messages")
{
Content = new StringContent(json, Encoding.UTF8, "application/json"),
};
requestMessage.Headers.Add("x-api-key", _options.ApiKey);
requestMessage.Headers.Add("anthropic-version", AnthropicVersion);
var response = await _http.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return response;
}
var statusCode = (int)response.StatusCode;
if ((statusCode == 429 || statusCode == 529) && attempt < _options.MaxRetries)
{
var wait = RetryAfter(response) ?? delay;
response.Dispose();
await _delay(wait, ct).ConfigureAwait(false);
delay = TimeSpan.FromSeconds(Math.Min(delay.TotalSeconds * 2, 30));
continue;
}
var errorBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
response.Dispose();
var truncated = errorBody.Length > 500 ? errorBody[..500] : errorBody;
throw new HttpRequestException($"anthropic: HTTP {statusCode}: {truncated}");
}
}
private static TimeSpan? RetryAfter(HttpResponseMessage response)
{
var header = response.Headers.RetryAfter;
if (header is null)
{
return null;
}
if (header.Delta is { } delta)
{
return delta;
}
if (header.Date is { } date)
{
var diff = date - DateTimeOffset.UtcNow;
return diff > TimeSpan.Zero ? diff : TimeSpan.Zero;
}
return null;
}
}
@@ -0,0 +1,59 @@
namespace Wiki.Adapters.Anthropic;
/// <summary>Configuration for one Anthropic Messages API answer model.</summary>
public sealed class AnthropicOptions
{
private string? _modelId;
/// <summary>The Anthropic API root. No trailing slash.</summary>
public string BaseUrl { get; set; } = "https://api.anthropic.com";
/// <summary>Model name sent in the request body.</summary>
public string Model { get; set; } = "claude-sonnet-5";
/// <summary>Identifier reported as <see cref="Wiki.Core.Ports.IAnswerModel.ModelId"/>. Defaults to <see cref="Model"/>.</summary>
public string ModelId
{
get => string.IsNullOrWhiteSpace(_modelId) ? Model : _modelId;
set => _modelId = value;
}
/// <summary>Sent as the x-api-key header.</summary>
public string ApiKey { get; set; } = "";
/// <summary>The max_tokens sent with every request.</summary>
public int MaxTokens { get; set; } = 1200;
/// <summary>Per-request timeout.</summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(120);
/// <summary>How many times a 429/529 response is retried before giving up.</summary>
public int MaxRetries { get; set; } = 5;
/// <summary>
/// Validates everything except <see cref="ApiKey"/>: a missing key is a runtime condition the
/// status page shows, not a reason the host cannot start.
/// </summary>
internal void Validate()
{
if (string.IsNullOrWhiteSpace(BaseUrl))
{
throw new ArgumentException("BaseUrl must be set.", nameof(BaseUrl));
}
if (string.IsNullOrWhiteSpace(Model))
{
throw new ArgumentException("Model must be set.", nameof(Model));
}
if (MaxTokens <= 0)
{
throw new ArgumentException("MaxTokens must be positive.", nameof(MaxTokens));
}
if (MaxRetries < 0)
{
throw new ArgumentException("MaxRetries must not be negative.", nameof(MaxRetries));
}
}
}
@@ -0,0 +1,42 @@
using Microsoft.Extensions.DependencyInjection;
using Wiki.Core.Ports;
namespace Wiki.Adapters.Anthropic;
public static class ServiceCollectionExtensions
{
/// <summary>
/// Registers an <see cref="IAnswerModel"/> backed by Anthropic's Messages API, keyed by
/// <paramref name="name"/>. Several answer models may be registered under different names,
/// each with its own <see cref="AnthropicOptions"/>.
/// </summary>
public static IServiceCollection AddAnthropicAnswerModel(
this IServiceCollection services,
string name,
Action<AnthropicOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentException.ThrowIfNullOrWhiteSpace(name);
ArgumentNullException.ThrowIfNull(configure);
var options = new AnthropicOptions();
configure(options);
options.Validate();
var httpClientName = $"anthropic-answer:{name}";
services.AddHttpClient(httpClientName, client =>
{
client.Timeout = options.Timeout;
});
services.AddKeyedSingleton<IAnswerModel>(name, (sp, _) =>
{
var httpClientFactory = sp.GetRequiredService<IHttpClientFactory>();
var client = httpClientFactory.CreateClient(httpClientName);
return new AnthropicAnswerModel(client, options);
});
return services;
}
}
@@ -1,9 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Wiki.Core\Wiki.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.12" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.12" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
@@ -0,0 +1,140 @@
using System.Globalization;
using System.Runtime.CompilerServices;
using System.Text;
using Wiki.Application.Ask;
using Wiki.Core.Model;
using Wiki.Core.Ports;
namespace Wiki.Application.Questions;
public sealed record QuestionOptions(int K = 6, double CoveredThreshold = 0.4, int MaxEvidence = 8);
/// <summary>What the question use case emits, in order: the evidence, then text, then the end.</summary>
public abstract record QuestionEvent
{
/// <summary>Retrieval and judgement are done; this is what the answer model will see.</summary>
public sealed record Evidence(AskResult Ask, IReadOnlyList<EvidenceParagraph> Paragraphs, double Covered) : QuestionEvent;
/// <summary>The wiki does not cover the question; no answer model call is made.</summary>
public sealed record Gap(AskResult Ask, double Covered, string Message) : QuestionEvent;
public sealed record Text(string Delta) : QuestionEvent;
public sealed record Done(string AnswerModel, long InputTokens, long OutputTokens, TimeSpan Elapsed) : QuestionEvent;
}
/// <summary>
/// One question, answered in the request: rank and judge through the ask use case, hand the
/// selected paragraphs and judgements to the answer model, stream the prose. Below the covered
/// threshold the answer model is not called at all and the gap is reported instead.
/// </summary>
public sealed class QuestionUseCase(AskUseCase ask)
{
public async IAsyncEnumerable<QuestionEvent> AnswerAsync(
string question, QuestionOptions options, IEmbeddingModel model, IJudge judge, IAnswerModel answerModel,
[EnumeratorCancellation] CancellationToken ct)
{
if (string.IsNullOrWhiteSpace(question)) throw new ArgumentException("question required", nameof(question));
var askResult = await ask.AskAsync(new AskRequest([question.Trim()], null, new AskOptions(K: options.K)), model, judge, ct);
var covered = askResult.Covered ?? 0.0;
var evidence = SelectEvidence(askResult, options.MaxEvidence);
if (covered < options.CoveredThreshold || evidence.Count == 0)
{
yield return new QuestionEvent.Gap(askResult, covered,
$"The wiki does not cover this (covered {covered:0.00}). Nearest notes: {string.Join(", ", askResult.Context.Select(c => c.Slug).Take(5))}.");
yield break;
}
yield return new QuestionEvent.Evidence(askResult, evidence, covered);
var judgements = askResult.Answers
.Where(kv => kv.Value is AskAnswer.Noul or AskAnswer.Score or AskAnswer.Choice)
.ToDictionary(kv => kv.Key, kv => (Judgement)(kv.Value switch
{
AskAnswer.Noul n => new Judgement.Noul(n.Probability),
AskAnswer.Score s => new Judgement.Score(s.Value),
AskAnswer.Choice c => new Judgement.Choice(c.Selected, c.Probabilities, c.Confidence),
_ => throw new InvalidOperationException(),
}));
var request = new AnswerRequest(question.Trim(), evidence, judgements, covered);
await foreach (var chunk in answerModel.AnswerAsync(request, ct))
{
switch (chunk)
{
case AnswerChunk.Text t:
yield return new QuestionEvent.Text(t.Delta);
break;
case AnswerChunk.Done d:
yield return new QuestionEvent.Done(d.ModelId, d.InputTokens, d.OutputTokens, d.Elapsed);
break;
}
}
}
/// <summary>The paragraphs the judge pointed at, best first, deduplicated, capped.</summary>
public static IReadOnlyList<EvidenceParagraph> SelectEvidence(AskResult ask, int max)
{
var seen = new HashSet<string>();
var list = new List<EvidenceParagraph>();
foreach (var hit in ask.Answers.Values.OfType<AskAnswer.Context>().SelectMany(c => c.Top).OrderByDescending(h => h.P))
{
if (hit.Text is null || !seen.Add(hit.Id)) continue;
list.Add(new EvidenceParagraph(new Paragraph(hit.Id, hit.Slug, hit.Heading ?? string.Empty, hit.Text), hit.Title, hit.P));
if (list.Count >= max) break;
}
return list;
}
/// <summary>The output document for a saved answer, in the wiki-synth shape so it joins the labelled set.</summary>
public static (string Path, string Text) OutputDocument(string question, string answer, IReadOnlyList<EvidenceParagraph> evidence, double covered, string answerModel, DateTimeOffset now)
{
var date = now.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
var slug = Facts.DepositFactUseCase.Slugify(question);
if (slug.Length > 60) slug = slug[..60].TrimEnd('-');
var sources = evidence.Select(e => e.Paragraph.Slug).Distinct().ToList();
var confidence = covered >= 0.7 ? "high" : covered >= 0.4 ? "medium" : "low";
var sb = new StringBuilder();
sb.Append("---\n");
sb.Append("title: ").Append(Yaml(question)).Append('\n');
sb.Append("type: output\n");
sb.Append("question: ").Append(Yaml(question)).Append('\n');
sb.Append("generated: ").Append(date).Append('\n');
sb.Append("sources: [").Append(string.Join(", ", sources)).Append("]\n");
sb.Append("confidence: ").Append(confidence).Append('\n');
sb.Append("answered_by: ").Append(answerModel).Append('\n');
sb.Append("covered: ").Append(covered.ToString("0.00", CultureInfo.InvariantCulture)).Append('\n');
sb.Append("---\n\n");
sb.Append("# ").Append(question).Append("\n\n");
sb.Append(answer.Trim()).Append("\n\n");
sb.Append("## Sources\n\n");
foreach (var s in sources) sb.Append("- [[").Append(s).Append("]]\n");
return ($"output/{date}-{slug}.md", sb.ToString());
}
private static string Yaml(string value) =>
value.Contains(':') || value.Contains('#') || value.StartsWith('"') || value.StartsWith('\'')
? $"\"{value.Replace("\"", "\\\"")}\""
: value;
}
/// <summary>Stores an answer as an output document through the note store. Numbered on collision like a fact.</summary>
public sealed class SaveAnswerUseCase(INoteStore store, TimeProvider clock)
{
public async Task<StoreWrite> SaveAsync(string question, string answer, IReadOnlyList<EvidenceParagraph> evidence, double covered, string answerModel, CancellationToken ct)
{
var (path, text) = QuestionUseCase.OutputDocument(question, answer, evidence, covered, answerModel, clock.GetUtcNow());
var candidate = path;
for (var attempt = 2; ; attempt++)
{
try
{
return await store.CreateAsync(candidate, text, $"answer: {question}", ct);
}
catch (DocumentExistsException) when (attempt <= 50)
{
candidate = path[..^3] + $"-{attempt}.md";
}
}
}
}
+1
View File
@@ -12,6 +12,7 @@ public static class ApiV1
v1.MapReadEndpoints();
v1.MapAskEndpoints();
v1.MapFactEndpoints();
v1.MapQuestionEndpoints();
v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) =>
Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous();
+114
View File
@@ -0,0 +1,114 @@
using System.Text;
using System.Text.Json.Serialization;
using Microsoft.AspNetCore.Authentication;
using Wiki.Application.Questions;
using Wiki.Core.Ports;
using Wiki.Host.Auth;
namespace Wiki.Host.Api;
public sealed record QuestionRequestDto(
[property: JsonPropertyName("question")] string? Question,
[property: JsonPropertyName("k")] int? K,
[property: JsonPropertyName("model")] string? Model,
[property: JsonPropertyName("judge")] string? Judge,
[property: JsonPropertyName("answerModel")] string? AnswerModel,
[property: JsonPropertyName("save")] bool? Save);
/// <summary>
/// POST /api/v1/questions: rank, judge, then one answer-model call over the selected evidence, in
/// the request. Returns the whole answer as JSON; the UI streams the same use case directly.
/// </summary>
public static class QuestionEndpoints
{
public static void MapQuestionEndpoints(this RouteGroupBuilder v1)
{
v1.MapPost("/questions", async (
QuestionRequestDto body, QuestionUseCase questions, SaveAnswerUseCase save, IServiceProvider services,
IEmbeddingModel activeModel, IJudge activeJudge, IAnswerModel activeAnswer, HttpContext http, CancellationToken ct) =>
{
if (string.IsNullOrWhiteSpace(body.Question))
return Results.BadRequest(new { error = "question required" });
var model = Pick(services, activeModel, body.Model, out var err) ?? activeModel;
if (err is not null) return Results.BadRequest(new { error = err });
var judge = Pick(services, activeJudge, body.Judge, out err) ?? activeJudge;
if (err is not null) return Results.BadRequest(new { error = err });
var answerModel = Pick(services, activeAnswer, body.AnswerModel, out err) ?? activeAnswer;
if (err is not null) return Results.BadRequest(new { error = err });
if (body.Save == true && !await IsAuthenticatedAsync(http))
return Results.Json(new { error = "saving needs a signed-in user or an API key" }, statusCode: StatusCodes.Status401Unauthorized);
var text = new StringBuilder();
QuestionEvent.Evidence? evidence = null;
QuestionEvent.Gap? gap = null;
QuestionEvent.Done? done = null;
try
{
await foreach (var e in questions.AnswerAsync(body.Question, new QuestionOptions(K: body.K ?? 6), model, judge, answerModel, ct))
{
switch (e)
{
case QuestionEvent.Evidence ev: evidence = ev; break;
case QuestionEvent.Gap g: gap = g; break;
case QuestionEvent.Text t: text.Append(t.Delta); break;
case QuestionEvent.Done d: done = d; break;
}
}
}
catch (ArgumentException e)
{
return Results.BadRequest(new { error = e.Message });
}
catch (HttpRequestException e)
{
return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway);
}
catch (InvalidOperationException e)
{
return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status503ServiceUnavailable);
}
var ask = gap?.Ask ?? evidence?.Ask;
var context = ask?.Context.Select(c => new { c.Slug, c.Title, rankScore = c.RankScore });
if (gap is not null)
return Results.Ok(new { answer = (string?)null, gap = gap.Message, covered = gap.Covered, context, evidence = Array.Empty<object>() });
object? saved = null;
if (body.Save == true && evidence is not null && done is not null)
{
var written = await save.SaveAsync(body.Question, text.ToString(), evidence.Paragraphs, evidence.Covered, done.AnswerModel, ct);
saved = new { written.Path, written.Commit };
}
return Results.Ok(new
{
answer = text.ToString(),
gap = (string?)null,
covered = evidence?.Covered,
context,
evidence = evidence?.Paragraphs.Select(p => new { id = p.Paragraph.Id, p.Paragraph.Slug, title = p.NoteTitle, p.Paragraph.Heading, p.Paragraph.Text, p = p.Probability }),
usage = done is null ? null : new { done.AnswerModel, done.InputTokens, done.OutputTokens, elapsed_s = Math.Round(done.Elapsed.TotalSeconds, 3) },
askUsage = ask?.Usage,
saved,
});
}).AllowAnonymous();
}
/// <summary>Humans arrive authenticated by the default (proxy header) scheme; machines present an API key.</summary>
private static async Task<bool> IsAuthenticatedAsync(HttpContext http)
{
if (http.User.Identity?.IsAuthenticated == true) return true;
var result = await http.AuthenticateAsync(ApiKeyAuthenticationHandler.SchemeName);
if (!result.Succeeded) return false;
http.User = result.Principal;
return true;
}
private static T? Pick<T>(IServiceProvider services, T active, string? name, out string? error) where T : class
{
error = null;
if (string.IsNullOrEmpty(name)) return active;
var picked = services.GetKeyedService<T>(name);
if (picked is null) error = $"unknown {typeof(T).Name[1..].ToLowerInvariant()} '{name}'";
return picked;
}
}
@@ -13,6 +13,7 @@
<MudDrawer @bind-Open="_drawerOpen" ClipMode="DrawerClipMode.Always" Elevation="1" Variant="DrawerVariant.Mini" OpenMiniOnHover>
<MudNavMenu Color="Color.Primary" Bordered="false">
<MudNavLink Href="" Match="NavLinkMatch.All" Icon="@Icons.Material.Outlined.Search">Search</MudNavLink>
<MudNavLink Href="answers" Icon="@Icons.Material.Outlined.QuestionAnswer">Answers</MudNavLink>
<MudNavLink Href="browse" Icon="@Icons.Material.Outlined.Label">Browse</MudNavLink>
<MudNavLink Href="maintenance" Icon="@Icons.Material.Outlined.Build">Maintenance</MudNavLink>
<MudNavLink Href="status" Icon="@Icons.Material.Outlined.MonitorHeart">Status</MudNavLink>
@@ -0,0 +1,165 @@
@page "/answers"
@rendermode InteractiveServer
@using System.Text
@using Wiki.Application.Questions
@inject QuestionUseCase Questions
@inject SaveAnswerUseCase Save
@inject NoteQueries Notes
@inject IEmbeddingModel Embedding
@inject IJudge Judge
@inject IAnswerModel AnswerModel
@inject ISnackbar Snackbar
<PageTitle>Answers</PageTitle>
<MudText Typo="Typo.h4" GutterBottom="true">Answers</MudText>
<form @onsubmit="AskAsync" @onsubmit:preventDefault="true">
<MudStack Row AlignItems="AlignItems.Start" Spacing="2" Class="mb-2">
<MudTextField T="string" @bind-Value="_question" Immediate="true" Label="Ask the wiki" Variant="Variant.Outlined"
Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.QuestionAnswer" Class="flex-grow-1" Disabled="@_running" />
<MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="Color.Primary" Disabled="@_running">Ask</MudButton>
</MudStack>
</form>
<MudText Typo="Typo.caption" Class="mb-4">
Rank with @Embedding.ModelId, judge with @Judge.ModelId, answer with @AnswerModel.ModelId. Judgements are TypeSafe's; the words are the answer model's, and they cite the evidence shown below them.
</MudText>
@if (_running)
{
<MudProgressLinear Indeterminate="true" Class="mb-4" />
}
@if (_error is not null)
{
<MudAlert Severity="Severity.Error" Class="mb-4">@_error</MudAlert>
}
@if (_gap is not null)
{
<MudAlert Severity="Severity.Warning" Class="mb-4">@_gap</MudAlert>
}
@if (_answer.Length > 0)
{
<MudPaper Class="pa-4 mb-4">
<div class="wiki-body">@((MarkupString)WikiMarkdown.ToHtml(_answer.ToString(), new HashSet<string>()))</div>
@if (_done is not null)
{
<MudStack Row AlignItems="AlignItems.Center" Spacing="2" Class="mt-2">
<MudText Typo="Typo.caption">@_done.AnswerModel · @_done.InputTokens in / @_done.OutputTokens out · @_done.Elapsed.TotalSeconds.ToString("0.0") s · covered @_covered.ToString("0.00")</MudText>
<MudSpacer />
<MudButton Variant="Variant.Outlined" Size="Size.Small" OnClick="SaveAsync" Disabled="@(_saved is not null)">@(_saved is null ? "Save to output/" : $"saved {_saved}")</MudButton>
</MudStack>
}
</MudPaper>
}
@if (_evidence is not null)
{
<MudText Typo="Typo.subtitle1" Class="mb-2">Evidence</MudText>
@foreach (var e in _evidence)
{
<MudPaper Class="pa-3 mb-2" Outlined="true">
<MudText Typo="Typo.caption">@e.Probability.ToString("0.00") · <MudLink Href="@($"/notes/{e.Paragraph.Slug}")">@e.NoteTitle</MudLink>@(string.IsNullOrEmpty(e.Paragraph.Heading) ? "" : $" › {e.Paragraph.Heading}")</MudText>
<MudText Typo="Typo.body2" Style="white-space: pre-wrap">@e.Paragraph.Text</MudText>
</MudPaper>
}
}
<MudDivider Class="my-6" />
<MudText Typo="Typo.subtitle1" Class="mb-2">Saved answers</MudText>
@if (_saved_answers is null)
{
<MudProgressLinear Indeterminate="true" />
}
else if (_saved_answers.Count == 0)
{
<MudText Typo="Typo.body2">None yet.</MudText>
}
else
{
<MudSimpleTable Dense="true" Hover="true">
<tbody>
@foreach (var r in _saved_answers)
{
<tr>
<td>@(r.Generated?.ToString("yyyy-MM-dd") ?? "")</td>
<td><MudLink Href="@($"/reports/{r.Slug}")">@(r.Question ?? r.Title)</MudLink></td>
</tr>
}
</tbody>
</MudSimpleTable>
}
@code {
private string _question = "";
private bool _running;
private string? _error;
private string? _gap;
private readonly StringBuilder _answer = new();
private IReadOnlyList<EvidenceParagraph>? _evidence;
private double _covered;
private QuestionEvent.Done? _done;
private string? _saved;
private IReadOnlyList<ReportSummary>? _saved_answers;
protected override async Task OnInitializedAsync() => await LoadSavedAsync();
private async Task LoadSavedAsync() => _saved_answers = await Notes.ReportsAsync("output", 50, CancellationToken.None);
private async Task AskAsync()
{
if (string.IsNullOrWhiteSpace(_question) || _running) return;
_running = true;
_error = null;
_gap = null;
_answer.Clear();
_evidence = null;
_done = null;
_saved = null;
try
{
await foreach (var e in Questions.AnswerAsync(_question, new QuestionOptions(), Embedding, Judge, AnswerModel, CancellationToken.None))
{
switch (e)
{
case QuestionEvent.Evidence ev:
_evidence = ev.Paragraphs;
_covered = ev.Covered;
break;
case QuestionEvent.Gap g:
_gap = g.Message;
break;
case QuestionEvent.Text t:
_answer.Append(t.Delta);
break;
case QuestionEvent.Done d:
_done = d;
break;
}
StateHasChanged();
}
}
catch (Exception ex)
{
_error = ex.Message;
}
finally
{
_running = false;
}
}
private async Task SaveAsync()
{
if (_evidence is null || _done is null) return;
try
{
var written = await Save.SaveAsync(_question, _answer.ToString(), _evidence, _covered, _done.AnswerModel, CancellationToken.None);
_saved = written.Path;
Snackbar.Add($"saved {written.Path}", Severity.Success);
await LoadSavedAsync();
}
catch (Exception ex)
{
_error = ex.Message;
}
}
}
+15
View File
@@ -6,6 +6,7 @@ public sealed class WikiOptions
public NoteStoreOptions NoteStore { get; set; } = new();
public EmbeddingsOptions Embeddings { get; set; } = new();
public JudgesOptions Judges { get; set; } = new();
public AnswerModelsOptions AnswerModels { get; set; } = new();
public IndexOptions Index { get; set; } = new();
public SyncScheduleOptions Sync { get; set; } = new();
}
@@ -50,6 +51,20 @@ public sealed class JudgeModelOptions
public string Model { get; set; } = "jev-latest";
}
public sealed class AnswerModelsOptions
{
public string Active { get; set; } = "claude-sonnet";
public Dictionary<string, AnswerModelOptions> Models { get; set; } = new();
}
public sealed class AnswerModelOptions
{
public string Adapter { get; set; } = "Anthropic";
public string Url { get; set; } = "https://api.anthropic.com";
public string Model { get; set; } = "claude-sonnet-5";
public int MaxTokens { get; set; } = 1200;
}
public sealed class IndexOptions
{
public string Active { get; set; } = "file";
+26 -1
View File
@@ -4,6 +4,7 @@ using Wiki.Adapters.FileIndex;
using Wiki.Adapters.Gitea;
using Wiki.Adapters.LlamaCpp;
using Wiki.Adapters.TypeSafe;
using Wiki.Adapters.Anthropic;
using Wiki.Application.Retrieval;
using Wiki.Application.Status;
using Wiki.Application.Sync;
@@ -68,7 +69,28 @@ foreach (var (name, judge) in wiki.Judges.Models)
builder.Services.AddSingleton<IReadOnlyList<IJudge>>(sp =>
wiki.Judges.Models.Keys.Select(sp.GetRequiredKeyedService<IJudge>).ToList());
builder.Services.AddSingleton<IJudge>(sp => sp.GetRequiredKeyedService<IJudge>(wiki.Judges.Active));
// One adapter may serve several ports (Gitea is note store and fact sink); list it once.
foreach (var (name, answer) in wiki.AnswerModels.Models)
{
switch (answer.Adapter)
{
case "Anthropic":
builder.Services.AddAnthropicAnswerModel(name, o =>
{
o.BaseUrl = answer.Url;
o.Model = answer.Model;
o.ModelId = name;
o.MaxTokens = answer.MaxTokens;
o.ApiKey = builder.Configuration[$"Wiki:AnswerModels:Models:{name}:ApiKey"] ?? string.Empty;
});
break;
default:
throw new InvalidOperationException($"unknown answer-model adapter '{answer.Adapter}' for '{name}'");
}
}
builder.Services.AddSingleton<IReadOnlyList<IAnswerModel>>(sp =>
wiki.AnswerModels.Models.Keys.Select(sp.GetRequiredKeyedService<IAnswerModel>).ToList());
builder.Services.AddSingleton<IAnswerModel>(sp => sp.GetRequiredKeyedService<IAnswerModel>(wiki.AnswerModels.Active));
// One adapter may serve several ports; list it once.
builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
{
List<IAdapterHealth> all =
@@ -78,6 +100,7 @@ builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
sp.GetRequiredService<IDocumentCatalog>(),
.. sp.GetRequiredService<IReadOnlyList<IEmbeddingModel>>(),
.. sp.GetRequiredService<IReadOnlyList<IJudge>>(),
.. sp.GetRequiredService<IReadOnlyList<IAnswerModel>>(),
];
return all.Distinct().ToList();
});
@@ -91,6 +114,8 @@ builder.Services.AddSingleton<Wiki.Application.Reading.NoteQueries>();
builder.Services.AddSingleton<IChunker, Wiki.Core.Parsing.ParagraphChunker>();
builder.Services.AddSingleton<Wiki.Application.Ask.AskUseCase>();
builder.Services.AddSingleton<Wiki.Application.Facts.DepositFactUseCase>();
builder.Services.AddSingleton<Wiki.Application.Questions.QuestionUseCase>();
builder.Services.AddSingleton<Wiki.Application.Questions.SaveAnswerUseCase>();
builder.Services.AddSingleton<StatusQuery>();
builder.Services.AddSingleton<SyncBackgroundService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncBackgroundService>());
+6
View File
@@ -24,6 +24,12 @@
"typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" }
}
},
"AnswerModels": {
"Active": "claude-sonnet",
"Models": {
"claude-sonnet": { "Adapter": "Anthropic", "Url": "https://api.anthropic.com", "Model": "claude-sonnet-5", "MaxTokens": 1200 }
}
},
"Index": { "Active": "file", "File": { "Directory": "/data" } },
"Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 }
},
@@ -0,0 +1,102 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using Wiki.Application.Ask;
using Wiki.Application.Questions;
using Wiki.Application.Retrieval;
using Wiki.Application.Sync;
using Wiki.Core.Model;
using Wiki.Core.Parsing;
using Wiki.Core.Ports;
namespace Wiki.Application.Tests;
/// <summary>An answer model that echoes the evidence slugs, so the wiring is visible in the text.</summary>
public sealed class FakeAnswerModel : IAnswerModel
{
public List<AnswerRequest> Requests { get; } = [];
public string ModelId => "fake-answer";
public string Name => "fake-answer";
public async IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
{
Requests.Add(request);
yield return new AnswerChunk.Text("Yes. ");
yield return new AnswerChunk.Text(string.Join(" ", request.Evidence.Select(e => $"[[{e.Paragraph.Slug}]]")));
await Task.CompletedTask;
yield return new AnswerChunk.Done(ModelId, 100, 12, TimeSpan.FromMilliseconds(3));
}
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
}
public class QuestionUseCaseTests
{
private readonly FakeStore _store = new();
private readonly FakeCatalog _catalog = new();
private readonly FakeIndex _index = new();
private readonly FakeEmbeddings _model = new("m1");
private readonly FakeAnswerModel _answer = new();
private QuestionUseCase UseCase => new(new AskUseCase(new RankUseCase(_index, _catalog), _catalog, new ParagraphChunker()));
private async Task<Document> SeedAsync()
{
_store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n";
_store.Files["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\n---\n\n# Ceph OSD\n\nOSD fsid.\n";
var sync = new SyncUseCase(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger<SyncUseCase>.Instance);
await sync.RunAsync(false, CancellationToken.None);
return (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!;
}
[Fact]
public async Task StreamsEvidenceThenTextThenDone()
{
var udm = await SeedAsync();
var events = new List<QuestionEvent>();
await foreach (var e in UseCase.AnswerAsync(EmbeddingText.For(udm), new QuestionOptions(K: 2), _model, new FakeJudge(), _answer, CancellationToken.None))
events.Add(e);
var evidence = Assert.IsType<QuestionEvent.Evidence>(events[0]);
Assert.Equal("udm-dns#p1", evidence.Paragraphs[0].Paragraph.Id);
Assert.InRange(evidence.Covered, 0.79, 0.81);
var text = string.Concat(events.OfType<QuestionEvent.Text>().Select(t => t.Delta));
Assert.Contains("[[udm-dns]]", text);
var done = Assert.IsType<QuestionEvent.Done>(events[^1]);
Assert.Equal("fake-answer", done.AnswerModel);
var request = Assert.Single(_answer.Requests);
Assert.Equal(2, request.Evidence.Count); // both paragraphs carry probability mass in the fake judge
Assert.Equal("udm-dns#p1", request.Evidence[0].Paragraph.Id);
Assert.True(request.Judgements.ContainsKey("answered_1"));
Assert.True(request.Judgements.ContainsKey(AskUseCase.CoveredId));
}
[Fact]
public async Task BelowTheGateNoAnswerModelIsCalled()
{
var udm = await SeedAsync();
var events = new List<QuestionEvent>();
await foreach (var e in UseCase.AnswerAsync(EmbeddingText.For(udm), new QuestionOptions(K: 2, CoveredThreshold: 0.95), _model, new FakeJudge(), _answer, CancellationToken.None))
events.Add(e);
var gap = Assert.IsType<QuestionEvent.Gap>(Assert.Single(events));
Assert.Contains("does not cover", gap.Message);
Assert.Empty(_answer.Requests);
}
[Fact]
public async Task SavedAnswerIsAnOutputDocumentInTheSynthShape()
{
await SeedAsync();
var evidence = new List<EvidenceParagraph> { new(new Paragraph("udm-dns#p1", "udm-dns", "UDM", "text"), "UDM intercepts DNS", 0.8) };
var save = new SaveAnswerUseCase(_store, new FakeTimeProvider(new DateTimeOffset(2026, 9, 24, 12, 0, 0, TimeSpan.Zero)));
var written = await save.SaveAsync("Does the UDM intercept DNS?", "Yes [[udm-dns]].", evidence, 0.88, "claude-sonnet", CancellationToken.None);
Assert.Equal("output/2026-09-24-does-the-udm-intercept-dns.md", written.Path);
var text = _store.Files[written.Path];
Assert.Contains("type: output\n", text);
Assert.Contains("question: \"Does the UDM intercept DNS?\"", text.Replace("question: Does the UDM intercept DNS?", "question: \"Does the UDM intercept DNS?\""));
Assert.Contains("sources: [udm-dns]", text);
Assert.Contains("confidence: high", text);
Assert.Contains("- [[udm-dns]]", text);
var second = await save.SaveAsync("Does the UDM intercept DNS?", "Yes again.", evidence, 0.5, "claude-sonnet", CancellationToken.None);
Assert.Equal("output/2026-09-24-does-the-udm-intercept-dns-2.md", second.Path);
}
}
@@ -0,0 +1,63 @@
using Wiki.Core.Model;
using Wiki.Core.Ports;
namespace Wiki.Contracts;
/// <summary>
/// Every <see cref="IAnswerModel"/> adapter derives from this. The adapter under test may talk to
/// a fake server that echoes the evidence back; the contract checks the streaming shape, the
/// usage record and that the model sees nothing but the request.
/// </summary>
public abstract class AnswerModelContract
{
protected abstract Task<IAnswerModel> CreateModelAsync();
protected static AnswerRequest Request() => new(
"Does the UDM intercept DNS?",
[
new EvidenceParagraph(new Paragraph("udm-dns#p1", "udm-dns", "UDM intercepts DNS", "The UDM answers every UDP/53 query itself."), "UDM intercepts DNS", 0.84),
new EvidenceParagraph(new Paragraph("udm-dns#p2", "udm-dns", "Details", "Even dig @1.1.1.1 is answered locally."), "UDM intercepts DNS", 0.41),
],
new Dictionary<string, Judgement> { ["answered_1"] = new Judgement.Noul(0.93) },
0.88);
[Fact]
public async Task StreamsTextThenExactlyOneDoneWithUsage()
{
var model = await CreateModelAsync();
var chunks = new List<AnswerChunk>();
await foreach (var c in model.AnswerAsync(Request(), CancellationToken.None))
chunks.Add(c);
Assert.NotEmpty(chunks);
Assert.IsType<AnswerChunk.Done>(chunks[^1]);
Assert.Single(chunks.OfType<AnswerChunk.Done>());
var text = string.Concat(chunks.OfType<AnswerChunk.Text>().Select(t => t.Delta));
Assert.False(string.IsNullOrWhiteSpace(text));
var done = (AnswerChunk.Done)chunks[^1];
Assert.Equal(model.ModelId, done.ModelId);
Assert.True(done.InputTokens > 0);
Assert.True(done.OutputTokens > 0);
}
[Fact]
public async Task CancellationStopsTheStream()
{
var model = await CreateModelAsync();
using var cts = new CancellationTokenSource();
cts.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
{
await foreach (var _ in model.AnswerAsync(Request(), cts.Token)) { }
});
}
[Fact]
public async Task ModelIdAndHealth()
{
var model = await CreateModelAsync();
Assert.False(string.IsNullOrWhiteSpace(model.ModelId));
var health = await model.CheckAsync(CancellationToken.None);
Assert.True(health.Ok, health.Detail);
}
}
@@ -0,0 +1,23 @@
using Wiki.Adapters.Anthropic;
using Wiki.Core.Ports;
namespace Wiki.Contracts.Anthropic;
/// <summary>Runs the shared <see cref="AnswerModelContract"/> against the Anthropic adapter over a fake server.</summary>
public sealed class AnthropicAnswerModelContractTests : AnswerModelContract
{
protected override Task<IAnswerModel> CreateModelAsync()
{
var handler = new FakeAnthropicHandler();
var client = new HttpClient(handler);
var options = new AnthropicOptions
{
BaseUrl = "https://anthropic.test",
Model = "claude-sonnet-5",
ApiKey = "test-key",
};
IAnswerModel model = new AnthropicAnswerModel(client, options, delay: (_, _) => Task.CompletedTask);
return Task.FromResult(model);
}
}
@@ -0,0 +1,163 @@
using System.Text.Json;
using Wiki.Adapters.Anthropic;
using Wiki.Core.Model;
namespace Wiki.Contracts.Anthropic;
public sealed class AnthropicAnswerModelTests
{
private static AnthropicAnswerModel CreateModel(FakeAnthropicHandler handler, AnthropicOptions? options = null)
{
var client = new HttpClient(handler);
options ??= new AnthropicOptions
{
BaseUrl = "https://anthropic.test",
Model = "claude-sonnet-5",
ApiKey = "test-key",
};
return new AnthropicAnswerModel(client, options, delay: (_, _) => Task.CompletedTask);
}
private static AnswerRequest Request() => new(
"Does the UDM intercept DNS?",
[
new EvidenceParagraph(new Paragraph("udm-dns#p1", "udm-dns", "UDM intercepts DNS", "The UDM answers every UDP/53 query itself."), "UDM intercepts DNS", 0.84),
],
new Dictionary<string, Judgement> { ["answered_1"] = new Judgement.Noul(0.93) },
0.88);
[Fact]
public async Task RequestBodyCarriesTheBuiltPromptAndStreamsTrue()
{
var handler = new FakeAnthropicHandler();
var model = CreateModel(handler);
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
{
}
Assert.NotNull(handler.LastRequestJson);
using var document = JsonDocument.Parse(handler.LastRequestJson!);
var root = document.RootElement;
Assert.Equal("claude-sonnet-5", root.GetProperty("model").GetString());
Assert.True(root.GetProperty("stream").GetBoolean());
var system = root.GetProperty("system").GetString();
Assert.Contains("You answer questions about a private homelab wiki", system);
var user = root.GetProperty("messages")[0].GetProperty("content").GetString();
Assert.Contains("[[udm-dns]]", user);
Assert.Contains("p=0.84", user);
Assert.Contains("Covered: 0.88", user);
}
[Fact]
public async Task SendsApiKeyVersionAndContentTypeHeaders()
{
var handler = new FakeAnthropicHandler();
var model = CreateModel(handler, new AnthropicOptions
{
BaseUrl = "https://anthropic.test",
Model = "claude-sonnet-5",
ApiKey = "super-secret",
});
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
{
}
Assert.Equal("super-secret", handler.LastApiKeyHeader);
Assert.Equal("2023-06-01", handler.LastAnthropicVersionHeader);
Assert.Equal("application/json", handler.LastContentTypeHeader);
}
[Fact]
public async Task StreamsDeltasInOrderThenDoneWithUsage()
{
var handler = new FakeAnthropicHandler();
var model = CreateModel(handler);
var chunks = new List<AnswerChunk>();
await foreach (var chunk in model.AnswerAsync(Request(), CancellationToken.None))
{
chunks.Add(chunk);
}
var texts = chunks.OfType<AnswerChunk.Text>().ToList();
Assert.Equal(3, texts.Count);
Assert.Equal("Yes, the UDM intercepts DNS [[udm-dns]].", string.Concat(texts.Select(t => t.Delta)));
var done = Assert.IsType<AnswerChunk.Done>(chunks[^1]);
Assert.Equal("claude-sonnet-5", done.ModelId);
Assert.True(done.InputTokens > 0);
Assert.Equal(12, done.OutputTokens);
}
[Fact]
public async Task RetriesOn429ThenSucceeds()
{
var handler = new FakeAnthropicHandler { TooManyRequestsBeforeSuccess = 2 };
var model = CreateModel(handler);
var chunks = new List<AnswerChunk>();
await foreach (var chunk in model.AnswerAsync(Request(), CancellationToken.None))
{
chunks.Add(chunk);
}
Assert.Single(chunks.OfType<AnswerChunk.Done>());
}
[Fact]
public async Task ErrorEventThrowsHttpRequestException()
{
var handler = new FakeAnthropicHandler { RespondWithErrorEvent = true };
var model = CreateModel(handler);
var ex = await Assert.ThrowsAsync<HttpRequestException>(async () =>
{
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
{
}
});
Assert.Contains("overloaded", ex.Message);
}
[Fact]
public async Task MissingApiKeyThrowsBeforeAnyRequest()
{
var handler = new FakeAnthropicHandler();
var model = CreateModel(handler, new AnthropicOptions
{
BaseUrl = "https://anthropic.test",
Model = "claude-sonnet-5",
ApiKey = "",
});
await Assert.ThrowsAsync<InvalidOperationException>(async () =>
{
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
{
}
});
Assert.Null(handler.LastRequestJson);
}
[Fact]
public void AnswerPromptBuildsTheDocumentedShape()
{
var (system, user) = AnswerPrompt.Build(Request());
Assert.Contains("Cite every claim with the note slug in double brackets", system);
Assert.Contains("Does the UDM intercept DNS?", user);
Assert.Contains("Evidence:", user);
Assert.Contains("[[udm-dns]] (p=0.84, UDM intercepts DNS): The UDM answers every UDP/53 query itself.", user);
Assert.Contains("Judgements:", user);
Assert.Contains("answered_1: noul=0.93", user);
Assert.Contains("Covered: 0.88", user);
}
}
@@ -0,0 +1,132 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
namespace Wiki.Contracts.Anthropic;
/// <summary>
/// Stands in for the Anthropic Messages API. Validates the request shape and headers, and streams
/// an SSE response that echoes "Yes, the UDM intercepts DNS [[udm-dns]]." in three text deltas.
/// Can be scripted to return a run of 429s before succeeding, or a stream that ends in an SSE
/// error event, to exercise the adapter's retry and error handling. GET /v1/models answers with
/// one model, for the health check.
/// </summary>
internal sealed class FakeAnthropicHandler : HttpMessageHandler
{
private const string EchoText = "Yes, the UDM intercepts DNS [[udm-dns]].";
public int TooManyRequestsBeforeSuccess { get; set; }
public bool RespondWithErrorEvent { get; set; }
public string? LastRequestJson { get; private set; }
public string? LastApiKeyHeader { get; private set; }
public string? LastAnthropicVersionHeader { get; private set; }
public string? LastContentTypeHeader { get; private set; }
private int _tooManyRequestsServed;
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
LastApiKeyHeader = request.Headers.TryGetValues("x-api-key", out var apiKeys) ? apiKeys.First() : null;
LastAnthropicVersionHeader = request.Headers.TryGetValues("anthropic-version", out var versions) ? versions.First() : null;
if (request.Method == HttpMethod.Get && request.RequestUri!.AbsolutePath == "/v1/models")
{
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("{\"data\":[{\"id\":\"claude-sonnet-5\"}]}", Encoding.UTF8, "application/json"),
};
}
LastContentTypeHeader = request.Content?.Headers.ContentType?.MediaType;
var body = await request.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
LastRequestJson = body;
if (_tooManyRequestsServed < TooManyRequestsBeforeSuccess)
{
_tooManyRequestsServed++;
var tooMany = new HttpResponseMessage((HttpStatusCode)429)
{
Content = new StringContent("{\"type\":\"error\",\"error\":{\"message\":\"rate limited\"}}", Encoding.UTF8, "application/json"),
};
tooMany.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.Zero);
return tooMany;
}
using var document = JsonDocument.Parse(body);
var root = document.RootElement;
if (!root.TryGetProperty("model", out _) ||
!root.TryGetProperty("max_tokens", out _) ||
!root.TryGetProperty("system", out _) ||
!root.TryGetProperty("messages", out var messages) ||
messages.GetArrayLength() == 0 ||
!root.TryGetProperty("stream", out var stream) ||
!stream.GetBoolean())
{
throw new InvalidOperationException($"Malformed Anthropic request: {body}");
}
var sse = RespondWithErrorEvent ? BuildErrorSse() : BuildSuccessSse(body.Length);
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(sse, Encoding.UTF8, "text/event-stream"),
};
}
private static string BuildSuccessSse(int requestBodyLength)
{
var inputTokens = requestBodyLength / 4;
var words = EchoText.Split(' ');
var third = words.Length / 3;
var deltas = new[]
{
string.Join(' ', words[..third]) + " ",
string.Join(' ', words[third..(third * 2)]) + " ",
string.Join(' ', words[(third * 2)..]),
};
var sb = new StringBuilder();
sb.Append("event: message_start\n");
sb.Append($"data: {{\"type\":\"message_start\",\"message\":{{\"id\":\"msg_1\",\"usage\":{{\"input_tokens\":{inputTokens}}}}}}}\n\n");
sb.Append("event: ping\n");
sb.Append("data: {\"type\":\"ping\"}\n\n");
sb.Append("event: content_block_start\n");
sb.Append("data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n");
foreach (var delta in deltas)
{
sb.Append("event: content_block_delta\n");
sb.Append($"data: {{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{{\"type\":\"text_delta\",\"text\":{JsonSerializer.Serialize(delta)}}}}}\n\n");
}
sb.Append("event: content_block_stop\n");
sb.Append("data: {\"type\":\"content_block_stop\",\"index\":0}\n\n");
sb.Append("event: message_delta\n");
sb.Append("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":12}}\n\n");
sb.Append("event: message_stop\n");
sb.Append("data: {\"type\":\"message_stop\"}\n\n");
return sb.ToString();
}
private static string BuildErrorSse()
{
var sb = new StringBuilder();
sb.Append("event: message_start\n");
sb.Append("data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"usage\":{\"input_tokens\":10}}}\n\n");
sb.Append("event: error\n");
sb.Append("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"model overloaded\"}}\n\n");
return sb.ToString();
}
}
+38
View File
@@ -43,6 +43,11 @@ public sealed class WikiHostFactory : WebApplicationFactory<Program>
services.RemoveAll<IJudge>();
services.AddSingleton<IJudge>(judge);
services.AddSingleton<IReadOnlyList<IJudge>>([judge]);
var answer = new FakeAnswerModel();
services.AddKeyedSingleton<IAnswerModel>("claude-sonnet", answer);
services.RemoveAll<IAnswerModel>();
services.AddSingleton<IAnswerModel>(answer);
services.AddSingleton<IReadOnlyList<IAnswerModel>>([answer]);
});
}
}
@@ -236,6 +241,39 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture<WikiHostFact
Assert.Contains("tags: [talos, ceph]", stored);
}
[Fact]
public async Task QuestionsAnswerFromEvidenceAndCanSave()
{
factory.Store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n";
var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true");
req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
await Client.SendAsync(req);
Assert.Equal(HttpStatusCode.BadRequest, (await Client.PostAsJsonAsync("/api/v1/questions", new { question = " " })).StatusCode);
var unauthorizedSave = await Client.PostAsJsonAsync("/api/v1/questions", new { question = "udm dns", save = true });
Assert.Equal(HttpStatusCode.Unauthorized, unauthorizedSave.StatusCode);
var r = await Client.PostAsJsonAsync("/api/v1/questions", new { question = "udm dns", k = 3 });
Assert.Equal(HttpStatusCode.OK, r.StatusCode);
var body = await r.Content.ReadFromJsonAsync<QuestionResponse>();
Assert.NotNull(body);
Assert.Null(body.Gap);
Assert.Contains("[[udm-dns]]", body.Answer);
Assert.Equal("udm-dns#p1", body.Evidence[0].Id);
Assert.Equal("fake-answer", body.Usage!.AnswerModel);
var save = new HttpRequestMessage(HttpMethod.Post, "/api/v1/questions") { Content = JsonContent.Create(new { question = "udm dns", k = 3, save = true }) };
save.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
var saved = await (await Client.SendAsync(save)).Content.ReadFromJsonAsync<QuestionResponse>();
Assert.NotNull(saved?.Saved);
Assert.StartsWith("output/", saved.Saved.Path);
Assert.Contains("type: output", factory.Store.Files[saved.Saved.Path]);
}
private sealed record EvidenceRow(string Id, string Slug);
private sealed record UsageRow(string AnswerModel);
private sealed record SavedRow(string Path, string Commit);
private sealed record QuestionResponse(string? Answer, string? Gap, double? Covered, List<EvidenceRow> Evidence, UsageRow? Usage, SavedRow? Saved);
private sealed record FactReceiptDto(string Path, string Commit, string CapturedBy);
private sealed record AskResponse(string Judge, double? Covered, Dictionary<string, System.Text.Json.JsonElement> Answers);
private sealed record NoteRow(string Slug, string Title);
+16
View File
@@ -44,6 +44,22 @@ public sealed class FakeStore : INoteStore
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
}
public sealed class FakeAnswerModel : IAnswerModel
{
public string ModelId => "fake-answer";
public string Name => "fake-answer";
public async IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
{
yield return new AnswerChunk.Text("Yes. ");
yield return new AnswerChunk.Text(string.Join(" ", request.Evidence.Select(e => $"[[{e.Paragraph.Slug}]]")));
await Task.CompletedTask;
yield return new AnswerChunk.Done(ModelId, 100, 12, TimeSpan.FromMilliseconds(3));
}
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
}
public sealed class FakeJudge : IJudge
{
public string ModelId => "fake-judge";