Phase 5: questions endpoint, Anthropic answer model, Answers page #7
@@ -14,7 +14,8 @@ test/Wiki.Contracts one abstract suite per port; every adapter passes its sui
|
||||
```
|
||||
|
||||
Adapters are selected in `appsettings.json` under `Wiki:`; secrets come from environment
|
||||
variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__<name>`).
|
||||
variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__<name>`,
|
||||
`Wiki__Judges__Models__<name>__ApiKey`, `Wiki__AnswerModels__Models__<name>__ApiKey`).
|
||||
|
||||
## Run
|
||||
|
||||
@@ -24,4 +25,6 @@ dotnet run --project src/Wiki.Host
|
||||
```
|
||||
|
||||
Endpoints: `/health` (liveness), `/ready` (readiness), `/api/v1/status`, `POST /rank`,
|
||||
`POST /webhook`.
|
||||
`POST /webhook`, `POST /api/v1/ask` (rank + judge), `POST /api/v1/questions` (ask + answer
|
||||
model, `save: true` writes `output/`), `POST /api/v1/facts` (raw source, API key),
|
||||
`GET /api/v1/{notes,tags,raw,reports}`.
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Adapters.Anthropic;
|
||||
|
||||
/// <summary>Builds the system and user prompt for one answer request. Pure: no I/O, no model calls.</summary>
|
||||
public static class AnswerPrompt
|
||||
{
|
||||
private const string SystemPrompt =
|
||||
"You answer questions about a private homelab wiki from the evidence given, nothing else. " +
|
||||
"Cite every claim with the note slug in double brackets, like [[slug]]. " +
|
||||
"If the evidence does not answer the question, say what is missing instead of guessing. " +
|
||||
"Answer in the language of the question. Be brief.";
|
||||
|
||||
/// <summary>Renders the request into (system, user) text for the Messages API.</summary>
|
||||
public static (string System, string User) Build(AnswerRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var user = new StringBuilder();
|
||||
user.Append(request.Question);
|
||||
|
||||
user.Append("\n\nEvidence:");
|
||||
foreach (var evidence in request.Evidence)
|
||||
{
|
||||
user.Append('\n');
|
||||
user.Append("[[").Append(evidence.Paragraph.Slug).Append("]] (p=");
|
||||
user.Append(FormatNumber(evidence.Probability));
|
||||
user.Append(", ").Append(evidence.Paragraph.Heading).Append("): ");
|
||||
user.Append(evidence.Paragraph.Text);
|
||||
}
|
||||
|
||||
user.Append("\n\nJudgements:");
|
||||
foreach (var (id, judgement) in request.Judgements)
|
||||
{
|
||||
user.Append('\n').Append(id).Append(": ").Append(FormatJudgement(judgement));
|
||||
}
|
||||
|
||||
user.Append("\n\nCovered: ").Append(FormatNumber(request.Covered));
|
||||
|
||||
return (SystemPrompt, user.ToString());
|
||||
}
|
||||
|
||||
private static string FormatJudgement(Judgement judgement) => judgement switch
|
||||
{
|
||||
Judgement.Noul noul => $"noul={FormatNumber(noul.Probability)}",
|
||||
Judgement.Choice choice => $"choice={choice.Selected} (confidence={FormatNumber(choice.Confidence)})",
|
||||
Judgement.Score score => $"score={FormatNumber(score.Value)}",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(judgement), judgement, "Unknown Judgement."),
|
||||
};
|
||||
|
||||
private static string FormatNumber(double value) => value.ToString("0.00", CultureInfo.InvariantCulture);
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.Anthropic;
|
||||
|
||||
/// <summary>
|
||||
/// Answers questions via Anthropic's Messages API, streamed over server-sent events. One HTTP
|
||||
/// request per <see cref="AnswerAsync"/> call, retried on 429/529 with the server's Retry-After
|
||||
/// (or exponential backoff), never retried for anything else.
|
||||
/// </summary>
|
||||
public sealed class AnthropicAnswerModel : IAnswerModel
|
||||
{
|
||||
private const string AnthropicVersion = "2023-06-01";
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly HttpClient _http;
|
||||
private readonly AnthropicOptions _options;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
|
||||
|
||||
public AnthropicAnswerModel(HttpClient http, AnthropicOptions options, Func<TimeSpan, CancellationToken, Task>? delay = null)
|
||||
{
|
||||
_http = http;
|
||||
_options = options;
|
||||
_delay = delay ?? Task.Delay;
|
||||
}
|
||||
|
||||
public string ModelId => _options.ModelId;
|
||||
|
||||
public string Name => $"anthropic:{ModelId}";
|
||||
|
||||
public async IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, [EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var (system, user) = AnswerPrompt.Build(request);
|
||||
var body = BuildRequestBody(system, user);
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
using var response = await PostWithRetryAsync(body, ct).ConfigureAwait(false);
|
||||
var stream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
|
||||
await using (stream.ConfigureAwait(false))
|
||||
using (var reader = new StreamReader(stream, Encoding.UTF8))
|
||||
{
|
||||
long inputTokens = 0;
|
||||
long outputTokens = 0;
|
||||
string? eventName = null;
|
||||
|
||||
string? line;
|
||||
while ((line = await reader.ReadLineAsync(ct).ConfigureAwait(false)) is not null)
|
||||
{
|
||||
if (line.Length == 0)
|
||||
{
|
||||
eventName = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("event: ", StringComparison.Ordinal))
|
||||
{
|
||||
eventName = line["event: ".Length..];
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!line.StartsWith("data: ", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var data = line["data: ".Length..];
|
||||
using var payload = JsonDocument.Parse(data);
|
||||
var root = payload.RootElement;
|
||||
|
||||
switch (eventName)
|
||||
{
|
||||
case "message_start":
|
||||
inputTokens = root.GetProperty("message").GetProperty("usage").GetProperty("input_tokens").GetInt64();
|
||||
break;
|
||||
|
||||
case "content_block_delta":
|
||||
var delta = root.GetProperty("delta");
|
||||
if (delta.GetProperty("type").GetString() == "text_delta")
|
||||
{
|
||||
yield return new AnswerChunk.Text(delta.GetProperty("text").GetString() ?? "");
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case "message_delta":
|
||||
outputTokens = root.GetProperty("usage").GetProperty("output_tokens").GetInt64();
|
||||
break;
|
||||
|
||||
case "error":
|
||||
throw new HttpRequestException($"anthropic: {root.GetProperty("error").GetProperty("message").GetString()}");
|
||||
|
||||
// message_stop, ping, content_block_start, content_block_stop: nothing to record.
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
yield return new AnswerChunk.Done(ModelId, inputTokens, outputTokens, stopwatch.Elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<AdapterHealth> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_options.ApiKey))
|
||||
{
|
||||
throw new InvalidOperationException("Anthropic API key is not configured.");
|
||||
}
|
||||
|
||||
using var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{_options.BaseUrl}/v1/models");
|
||||
requestMessage.Headers.Add("x-api-key", _options.ApiKey);
|
||||
requestMessage.Headers.Add("anthropic-version", AnthropicVersion);
|
||||
|
||||
using var response = await _http.SendAsync(requestMessage, ct).ConfigureAwait(false);
|
||||
var responseBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var truncated = responseBody.Length > 500 ? responseBody[..500] : responseBody;
|
||||
return new AdapterHealth(Name, false, $"HTTP {(int)response.StatusCode}: {truncated}", stopwatch.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(responseBody);
|
||||
var count = document.RootElement.GetProperty("data").GetArrayLength();
|
||||
return new AdapterHealth(Name, true, $"{count} models", stopwatch.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException)
|
||||
{
|
||||
return new AdapterHealth(Name, false, ex.Message, stopwatch.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
}
|
||||
|
||||
private JsonObject BuildRequestBody(string system, string user) => new()
|
||||
{
|
||||
["model"] = _options.Model,
|
||||
["max_tokens"] = _options.MaxTokens,
|
||||
["system"] = system,
|
||||
["messages"] = new JsonArray
|
||||
{
|
||||
new JsonObject
|
||||
{
|
||||
["role"] = "user",
|
||||
["content"] = user,
|
||||
},
|
||||
},
|
||||
["stream"] = true,
|
||||
};
|
||||
|
||||
private async Task<HttpResponseMessage> PostWithRetryAsync(JsonObject body, CancellationToken ct)
|
||||
{
|
||||
// A missing key is a runtime condition the status page shows, not a reason the host cannot start.
|
||||
if (string.IsNullOrWhiteSpace(_options.ApiKey))
|
||||
{
|
||||
throw new InvalidOperationException("Anthropic API key is not configured.");
|
||||
}
|
||||
|
||||
var json = body.ToJsonString(JsonOptions);
|
||||
var delay = TimeSpan.FromSeconds(1);
|
||||
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
using var requestMessage = new HttpRequestMessage(HttpMethod.Post, $"{_options.BaseUrl}/v1/messages")
|
||||
{
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
requestMessage.Headers.Add("x-api-key", _options.ApiKey);
|
||||
requestMessage.Headers.Add("anthropic-version", AnthropicVersion);
|
||||
|
||||
var response = await _http.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
return response;
|
||||
}
|
||||
|
||||
var statusCode = (int)response.StatusCode;
|
||||
if ((statusCode == 429 || statusCode == 529) && attempt < _options.MaxRetries)
|
||||
{
|
||||
var wait = RetryAfter(response) ?? delay;
|
||||
response.Dispose();
|
||||
await _delay(wait, ct).ConfigureAwait(false);
|
||||
delay = TimeSpan.FromSeconds(Math.Min(delay.TotalSeconds * 2, 30));
|
||||
continue;
|
||||
}
|
||||
|
||||
var errorBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
|
||||
response.Dispose();
|
||||
var truncated = errorBody.Length > 500 ? errorBody[..500] : errorBody;
|
||||
throw new HttpRequestException($"anthropic: HTTP {statusCode}: {truncated}");
|
||||
}
|
||||
}
|
||||
|
||||
private static TimeSpan? RetryAfter(HttpResponseMessage response)
|
||||
{
|
||||
var header = response.Headers.RetryAfter;
|
||||
if (header is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (header.Delta is { } delta)
|
||||
{
|
||||
return delta;
|
||||
}
|
||||
|
||||
if (header.Date is { } date)
|
||||
{
|
||||
var diff = date - DateTimeOffset.UtcNow;
|
||||
return diff > TimeSpan.Zero ? diff : TimeSpan.Zero;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
namespace Wiki.Adapters.Anthropic;
|
||||
|
||||
/// <summary>Configuration for one Anthropic Messages API answer model.</summary>
|
||||
public sealed class AnthropicOptions
|
||||
{
|
||||
private string? _modelId;
|
||||
|
||||
/// <summary>The Anthropic API root. No trailing slash.</summary>
|
||||
public string BaseUrl { get; set; } = "https://api.anthropic.com";
|
||||
|
||||
/// <summary>Model name sent in the request body.</summary>
|
||||
public string Model { get; set; } = "claude-sonnet-5";
|
||||
|
||||
/// <summary>Identifier reported as <see cref="Wiki.Core.Ports.IAnswerModel.ModelId"/>. Defaults to <see cref="Model"/>.</summary>
|
||||
public string ModelId
|
||||
{
|
||||
get => string.IsNullOrWhiteSpace(_modelId) ? Model : _modelId;
|
||||
set => _modelId = value;
|
||||
}
|
||||
|
||||
/// <summary>Sent as the x-api-key header.</summary>
|
||||
public string ApiKey { get; set; } = "";
|
||||
|
||||
/// <summary>The max_tokens sent with every request.</summary>
|
||||
public int MaxTokens { get; set; } = 1200;
|
||||
|
||||
/// <summary>Per-request timeout.</summary>
|
||||
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(120);
|
||||
|
||||
/// <summary>How many times a 429/529 response is retried before giving up.</summary>
|
||||
public int MaxRetries { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Validates everything except <see cref="ApiKey"/>: a missing key is a runtime condition the
|
||||
/// status page shows, not a reason the host cannot start.
|
||||
/// </summary>
|
||||
internal void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(BaseUrl))
|
||||
{
|
||||
throw new ArgumentException("BaseUrl must be set.", nameof(BaseUrl));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Model))
|
||||
{
|
||||
throw new ArgumentException("Model must be set.", nameof(Model));
|
||||
}
|
||||
|
||||
if (MaxTokens <= 0)
|
||||
{
|
||||
throw new ArgumentException("MaxTokens must be positive.", nameof(MaxTokens));
|
||||
}
|
||||
|
||||
if (MaxRetries < 0)
|
||||
{
|
||||
throw new ArgumentException("MaxRetries must not be negative.", nameof(MaxRetries));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.Anthropic;
|
||||
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Registers an <see cref="IAnswerModel"/> backed by Anthropic's Messages API, keyed by
|
||||
/// <paramref name="name"/>. Several answer models may be registered under different names,
|
||||
/// each with its own <see cref="AnthropicOptions"/>.
|
||||
/// </summary>
|
||||
public static IServiceCollection AddAnthropicAnswerModel(
|
||||
this IServiceCollection services,
|
||||
string name,
|
||||
Action<AnthropicOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(name);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
var options = new AnthropicOptions();
|
||||
configure(options);
|
||||
options.Validate();
|
||||
|
||||
var httpClientName = $"anthropic-answer:{name}";
|
||||
|
||||
services.AddHttpClient(httpClientName, client =>
|
||||
{
|
||||
client.Timeout = options.Timeout;
|
||||
});
|
||||
|
||||
services.AddKeyedSingleton<IAnswerModel>(name, (sp, _) =>
|
||||
{
|
||||
var httpClientFactory = sp.GetRequiredService<IHttpClientFactory>();
|
||||
var client = httpClientFactory.CreateClient(httpClientName);
|
||||
return new AnthropicAnswerModel(client, options);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,14 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wiki.Core\Wiki.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.12" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
using System.Globalization;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using Wiki.Application.Ask;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Application.Questions;
|
||||
|
||||
public sealed record QuestionOptions(int K = 6, double CoveredThreshold = 0.4, int MaxEvidence = 8);
|
||||
|
||||
/// <summary>What the question use case emits, in order: the evidence, then text, then the end.</summary>
|
||||
public abstract record QuestionEvent
|
||||
{
|
||||
/// <summary>Retrieval and judgement are done; this is what the answer model will see.</summary>
|
||||
public sealed record Evidence(AskResult Ask, IReadOnlyList<EvidenceParagraph> Paragraphs, double Covered) : QuestionEvent;
|
||||
|
||||
/// <summary>The wiki does not cover the question; no answer model call is made.</summary>
|
||||
public sealed record Gap(AskResult Ask, double Covered, string Message) : QuestionEvent;
|
||||
|
||||
public sealed record Text(string Delta) : QuestionEvent;
|
||||
|
||||
public sealed record Done(string AnswerModel, long InputTokens, long OutputTokens, TimeSpan Elapsed) : QuestionEvent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One question, answered in the request: rank and judge through the ask use case, hand the
|
||||
/// selected paragraphs and judgements to the answer model, stream the prose. Below the covered
|
||||
/// threshold the answer model is not called at all and the gap is reported instead.
|
||||
/// </summary>
|
||||
public sealed class QuestionUseCase(AskUseCase ask)
|
||||
{
|
||||
public async IAsyncEnumerable<QuestionEvent> AnswerAsync(
|
||||
string question, QuestionOptions options, IEmbeddingModel model, IJudge judge, IAnswerModel answerModel,
|
||||
[EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(question)) throw new ArgumentException("question required", nameof(question));
|
||||
var askResult = await ask.AskAsync(new AskRequest([question.Trim()], null, new AskOptions(K: options.K)), model, judge, ct);
|
||||
var covered = askResult.Covered ?? 0.0;
|
||||
var evidence = SelectEvidence(askResult, options.MaxEvidence);
|
||||
|
||||
if (covered < options.CoveredThreshold || evidence.Count == 0)
|
||||
{
|
||||
yield return new QuestionEvent.Gap(askResult, covered,
|
||||
$"The wiki does not cover this (covered {covered:0.00}). Nearest notes: {string.Join(", ", askResult.Context.Select(c => c.Slug).Take(5))}.");
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return new QuestionEvent.Evidence(askResult, evidence, covered);
|
||||
|
||||
var judgements = askResult.Answers
|
||||
.Where(kv => kv.Value is AskAnswer.Noul or AskAnswer.Score or AskAnswer.Choice)
|
||||
.ToDictionary(kv => kv.Key, kv => (Judgement)(kv.Value switch
|
||||
{
|
||||
AskAnswer.Noul n => new Judgement.Noul(n.Probability),
|
||||
AskAnswer.Score s => new Judgement.Score(s.Value),
|
||||
AskAnswer.Choice c => new Judgement.Choice(c.Selected, c.Probabilities, c.Confidence),
|
||||
_ => throw new InvalidOperationException(),
|
||||
}));
|
||||
var request = new AnswerRequest(question.Trim(), evidence, judgements, covered);
|
||||
await foreach (var chunk in answerModel.AnswerAsync(request, ct))
|
||||
{
|
||||
switch (chunk)
|
||||
{
|
||||
case AnswerChunk.Text t:
|
||||
yield return new QuestionEvent.Text(t.Delta);
|
||||
break;
|
||||
case AnswerChunk.Done d:
|
||||
yield return new QuestionEvent.Done(d.ModelId, d.InputTokens, d.OutputTokens, d.Elapsed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The paragraphs the judge pointed at, best first, deduplicated, capped.</summary>
|
||||
public static IReadOnlyList<EvidenceParagraph> SelectEvidence(AskResult ask, int max)
|
||||
{
|
||||
var seen = new HashSet<string>();
|
||||
var list = new List<EvidenceParagraph>();
|
||||
foreach (var hit in ask.Answers.Values.OfType<AskAnswer.Context>().SelectMany(c => c.Top).OrderByDescending(h => h.P))
|
||||
{
|
||||
if (hit.Text is null || !seen.Add(hit.Id)) continue;
|
||||
list.Add(new EvidenceParagraph(new Paragraph(hit.Id, hit.Slug, hit.Heading ?? string.Empty, hit.Text), hit.Title, hit.P));
|
||||
if (list.Count >= max) break;
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>The output document for a saved answer, in the wiki-synth shape so it joins the labelled set.</summary>
|
||||
public static (string Path, string Text) OutputDocument(string question, string answer, IReadOnlyList<EvidenceParagraph> evidence, double covered, string answerModel, DateTimeOffset now)
|
||||
{
|
||||
var date = now.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
|
||||
var slug = Facts.DepositFactUseCase.Slugify(question);
|
||||
if (slug.Length > 60) slug = slug[..60].TrimEnd('-');
|
||||
var sources = evidence.Select(e => e.Paragraph.Slug).Distinct().ToList();
|
||||
var confidence = covered >= 0.7 ? "high" : covered >= 0.4 ? "medium" : "low";
|
||||
var sb = new StringBuilder();
|
||||
sb.Append("---\n");
|
||||
sb.Append("title: ").Append(Yaml(question)).Append('\n');
|
||||
sb.Append("type: output\n");
|
||||
sb.Append("question: ").Append(Yaml(question)).Append('\n');
|
||||
sb.Append("generated: ").Append(date).Append('\n');
|
||||
sb.Append("sources: [").Append(string.Join(", ", sources)).Append("]\n");
|
||||
sb.Append("confidence: ").Append(confidence).Append('\n');
|
||||
sb.Append("answered_by: ").Append(answerModel).Append('\n');
|
||||
sb.Append("covered: ").Append(covered.ToString("0.00", CultureInfo.InvariantCulture)).Append('\n');
|
||||
sb.Append("---\n\n");
|
||||
sb.Append("# ").Append(question).Append("\n\n");
|
||||
sb.Append(answer.Trim()).Append("\n\n");
|
||||
sb.Append("## Sources\n\n");
|
||||
foreach (var s in sources) sb.Append("- [[").Append(s).Append("]]\n");
|
||||
return ($"output/{date}-{slug}.md", sb.ToString());
|
||||
}
|
||||
|
||||
private static string Yaml(string value) =>
|
||||
value.Contains(':') || value.Contains('#') || value.StartsWith('"') || value.StartsWith('\'')
|
||||
? $"\"{value.Replace("\"", "\\\"")}\""
|
||||
: value;
|
||||
}
|
||||
|
||||
/// <summary>Stores an answer as an output document through the note store. Numbered on collision like a fact.</summary>
|
||||
public sealed class SaveAnswerUseCase(INoteStore store, TimeProvider clock)
|
||||
{
|
||||
public async Task<StoreWrite> SaveAsync(string question, string answer, IReadOnlyList<EvidenceParagraph> evidence, double covered, string answerModel, CancellationToken ct)
|
||||
{
|
||||
var (path, text) = QuestionUseCase.OutputDocument(question, answer, evidence, covered, answerModel, clock.GetUtcNow());
|
||||
var candidate = path;
|
||||
for (var attempt = 2; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await store.CreateAsync(candidate, text, $"answer: {question}", ct);
|
||||
}
|
||||
catch (DocumentExistsException) when (attempt <= 50)
|
||||
{
|
||||
candidate = path[..^3] + $"-{attempt}.md";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ public static class ApiV1
|
||||
v1.MapReadEndpoints();
|
||||
v1.MapAskEndpoints();
|
||||
v1.MapFactEndpoints();
|
||||
v1.MapQuestionEndpoints();
|
||||
|
||||
v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) =>
|
||||
Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous();
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
using System.Text;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Wiki.Application.Questions;
|
||||
using Wiki.Core.Ports;
|
||||
using Wiki.Host.Auth;
|
||||
|
||||
namespace Wiki.Host.Api;
|
||||
|
||||
public sealed record QuestionRequestDto(
|
||||
[property: JsonPropertyName("question")] string? Question,
|
||||
[property: JsonPropertyName("k")] int? K,
|
||||
[property: JsonPropertyName("model")] string? Model,
|
||||
[property: JsonPropertyName("judge")] string? Judge,
|
||||
[property: JsonPropertyName("answerModel")] string? AnswerModel,
|
||||
[property: JsonPropertyName("save")] bool? Save);
|
||||
|
||||
/// <summary>
|
||||
/// POST /api/v1/questions: rank, judge, then one answer-model call over the selected evidence, in
|
||||
/// the request. Returns the whole answer as JSON; the UI streams the same use case directly.
|
||||
/// </summary>
|
||||
public static class QuestionEndpoints
|
||||
{
|
||||
public static void MapQuestionEndpoints(this RouteGroupBuilder v1)
|
||||
{
|
||||
v1.MapPost("/questions", async (
|
||||
QuestionRequestDto body, QuestionUseCase questions, SaveAnswerUseCase save, IServiceProvider services,
|
||||
IEmbeddingModel activeModel, IJudge activeJudge, IAnswerModel activeAnswer, HttpContext http, CancellationToken ct) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body.Question))
|
||||
return Results.BadRequest(new { error = "question required" });
|
||||
var model = Pick(services, activeModel, body.Model, out var err) ?? activeModel;
|
||||
if (err is not null) return Results.BadRequest(new { error = err });
|
||||
var judge = Pick(services, activeJudge, body.Judge, out err) ?? activeJudge;
|
||||
if (err is not null) return Results.BadRequest(new { error = err });
|
||||
var answerModel = Pick(services, activeAnswer, body.AnswerModel, out err) ?? activeAnswer;
|
||||
if (err is not null) return Results.BadRequest(new { error = err });
|
||||
if (body.Save == true && !await IsAuthenticatedAsync(http))
|
||||
return Results.Json(new { error = "saving needs a signed-in user or an API key" }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
|
||||
var text = new StringBuilder();
|
||||
QuestionEvent.Evidence? evidence = null;
|
||||
QuestionEvent.Gap? gap = null;
|
||||
QuestionEvent.Done? done = null;
|
||||
try
|
||||
{
|
||||
await foreach (var e in questions.AnswerAsync(body.Question, new QuestionOptions(K: body.K ?? 6), model, judge, answerModel, ct))
|
||||
{
|
||||
switch (e)
|
||||
{
|
||||
case QuestionEvent.Evidence ev: evidence = ev; break;
|
||||
case QuestionEvent.Gap g: gap = g; break;
|
||||
case QuestionEvent.Text t: text.Append(t.Delta); break;
|
||||
case QuestionEvent.Done d: done = d; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (ArgumentException e)
|
||||
{
|
||||
return Results.BadRequest(new { error = e.Message });
|
||||
}
|
||||
catch (HttpRequestException e)
|
||||
{
|
||||
return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway);
|
||||
}
|
||||
catch (InvalidOperationException e)
|
||||
{
|
||||
return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
var ask = gap?.Ask ?? evidence?.Ask;
|
||||
var context = ask?.Context.Select(c => new { c.Slug, c.Title, rankScore = c.RankScore });
|
||||
if (gap is not null)
|
||||
return Results.Ok(new { answer = (string?)null, gap = gap.Message, covered = gap.Covered, context, evidence = Array.Empty<object>() });
|
||||
|
||||
object? saved = null;
|
||||
if (body.Save == true && evidence is not null && done is not null)
|
||||
{
|
||||
var written = await save.SaveAsync(body.Question, text.ToString(), evidence.Paragraphs, evidence.Covered, done.AnswerModel, ct);
|
||||
saved = new { written.Path, written.Commit };
|
||||
}
|
||||
return Results.Ok(new
|
||||
{
|
||||
answer = text.ToString(),
|
||||
gap = (string?)null,
|
||||
covered = evidence?.Covered,
|
||||
context,
|
||||
evidence = evidence?.Paragraphs.Select(p => new { id = p.Paragraph.Id, p.Paragraph.Slug, title = p.NoteTitle, p.Paragraph.Heading, p.Paragraph.Text, p = p.Probability }),
|
||||
usage = done is null ? null : new { done.AnswerModel, done.InputTokens, done.OutputTokens, elapsed_s = Math.Round(done.Elapsed.TotalSeconds, 3) },
|
||||
askUsage = ask?.Usage,
|
||||
saved,
|
||||
});
|
||||
}).AllowAnonymous();
|
||||
}
|
||||
|
||||
/// <summary>Humans arrive authenticated by the default (proxy header) scheme; machines present an API key.</summary>
|
||||
private static async Task<bool> IsAuthenticatedAsync(HttpContext http)
|
||||
{
|
||||
if (http.User.Identity?.IsAuthenticated == true) return true;
|
||||
var result = await http.AuthenticateAsync(ApiKeyAuthenticationHandler.SchemeName);
|
||||
if (!result.Succeeded) return false;
|
||||
http.User = result.Principal;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static T? Pick<T>(IServiceProvider services, T active, string? name, out string? error) where T : class
|
||||
{
|
||||
error = null;
|
||||
if (string.IsNullOrEmpty(name)) return active;
|
||||
var picked = services.GetKeyedService<T>(name);
|
||||
if (picked is null) error = $"unknown {typeof(T).Name[1..].ToLowerInvariant()} '{name}'";
|
||||
return picked;
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
<MudDrawer @bind-Open="_drawerOpen" ClipMode="DrawerClipMode.Always" Elevation="1" Variant="DrawerVariant.Mini" OpenMiniOnHover>
|
||||
<MudNavMenu Color="Color.Primary" Bordered="false">
|
||||
<MudNavLink Href="" Match="NavLinkMatch.All" Icon="@Icons.Material.Outlined.Search">Search</MudNavLink>
|
||||
<MudNavLink Href="answers" Icon="@Icons.Material.Outlined.QuestionAnswer">Answers</MudNavLink>
|
||||
<MudNavLink Href="browse" Icon="@Icons.Material.Outlined.Label">Browse</MudNavLink>
|
||||
<MudNavLink Href="maintenance" Icon="@Icons.Material.Outlined.Build">Maintenance</MudNavLink>
|
||||
<MudNavLink Href="status" Icon="@Icons.Material.Outlined.MonitorHeart">Status</MudNavLink>
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
@page "/answers"
|
||||
@rendermode InteractiveServer
|
||||
@using System.Text
|
||||
@using Wiki.Application.Questions
|
||||
@inject QuestionUseCase Questions
|
||||
@inject SaveAnswerUseCase Save
|
||||
@inject NoteQueries Notes
|
||||
@inject IEmbeddingModel Embedding
|
||||
@inject IJudge Judge
|
||||
@inject IAnswerModel AnswerModel
|
||||
@inject ISnackbar Snackbar
|
||||
|
||||
<PageTitle>Answers</PageTitle>
|
||||
|
||||
<MudText Typo="Typo.h4" GutterBottom="true">Answers</MudText>
|
||||
|
||||
<form @onsubmit="AskAsync" @onsubmit:preventDefault="true">
|
||||
<MudStack Row AlignItems="AlignItems.Start" Spacing="2" Class="mb-2">
|
||||
<MudTextField T="string" @bind-Value="_question" Immediate="true" Label="Ask the wiki" Variant="Variant.Outlined"
|
||||
Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.QuestionAnswer" Class="flex-grow-1" Disabled="@_running" />
|
||||
<MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="Color.Primary" Disabled="@_running">Ask</MudButton>
|
||||
</MudStack>
|
||||
</form>
|
||||
<MudText Typo="Typo.caption" Class="mb-4">
|
||||
Rank with @Embedding.ModelId, judge with @Judge.ModelId, answer with @AnswerModel.ModelId. Judgements are TypeSafe's; the words are the answer model's, and they cite the evidence shown below them.
|
||||
</MudText>
|
||||
|
||||
@if (_running)
|
||||
{
|
||||
<MudProgressLinear Indeterminate="true" Class="mb-4" />
|
||||
}
|
||||
@if (_error is not null)
|
||||
{
|
||||
<MudAlert Severity="Severity.Error" Class="mb-4">@_error</MudAlert>
|
||||
}
|
||||
@if (_gap is not null)
|
||||
{
|
||||
<MudAlert Severity="Severity.Warning" Class="mb-4">@_gap</MudAlert>
|
||||
}
|
||||
@if (_answer.Length > 0)
|
||||
{
|
||||
<MudPaper Class="pa-4 mb-4">
|
||||
<div class="wiki-body">@((MarkupString)WikiMarkdown.ToHtml(_answer.ToString(), new HashSet<string>()))</div>
|
||||
@if (_done is not null)
|
||||
{
|
||||
<MudStack Row AlignItems="AlignItems.Center" Spacing="2" Class="mt-2">
|
||||
<MudText Typo="Typo.caption">@_done.AnswerModel · @_done.InputTokens in / @_done.OutputTokens out · @_done.Elapsed.TotalSeconds.ToString("0.0") s · covered @_covered.ToString("0.00")</MudText>
|
||||
<MudSpacer />
|
||||
<MudButton Variant="Variant.Outlined" Size="Size.Small" OnClick="SaveAsync" Disabled="@(_saved is not null)">@(_saved is null ? "Save to output/" : $"saved {_saved}")</MudButton>
|
||||
</MudStack>
|
||||
}
|
||||
</MudPaper>
|
||||
}
|
||||
@if (_evidence is not null)
|
||||
{
|
||||
<MudText Typo="Typo.subtitle1" Class="mb-2">Evidence</MudText>
|
||||
@foreach (var e in _evidence)
|
||||
{
|
||||
<MudPaper Class="pa-3 mb-2" Outlined="true">
|
||||
<MudText Typo="Typo.caption">@e.Probability.ToString("0.00") · <MudLink Href="@($"/notes/{e.Paragraph.Slug}")">@e.NoteTitle</MudLink>@(string.IsNullOrEmpty(e.Paragraph.Heading) ? "" : $" › {e.Paragraph.Heading}")</MudText>
|
||||
<MudText Typo="Typo.body2" Style="white-space: pre-wrap">@e.Paragraph.Text</MudText>
|
||||
</MudPaper>
|
||||
}
|
||||
}
|
||||
|
||||
<MudDivider Class="my-6" />
|
||||
<MudText Typo="Typo.subtitle1" Class="mb-2">Saved answers</MudText>
|
||||
@if (_saved_answers is null)
|
||||
{
|
||||
<MudProgressLinear Indeterminate="true" />
|
||||
}
|
||||
else if (_saved_answers.Count == 0)
|
||||
{
|
||||
<MudText Typo="Typo.body2">None yet.</MudText>
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudSimpleTable Dense="true" Hover="true">
|
||||
<tbody>
|
||||
@foreach (var r in _saved_answers)
|
||||
{
|
||||
<tr>
|
||||
<td>@(r.Generated?.ToString("yyyy-MM-dd") ?? "")</td>
|
||||
<td><MudLink Href="@($"/reports/{r.Slug}")">@(r.Question ?? r.Title)</MudLink></td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</MudSimpleTable>
|
||||
}
|
||||
|
||||
@code {
|
||||
private string _question = "";
|
||||
private bool _running;
|
||||
private string? _error;
|
||||
private string? _gap;
|
||||
private readonly StringBuilder _answer = new();
|
||||
private IReadOnlyList<EvidenceParagraph>? _evidence;
|
||||
private double _covered;
|
||||
private QuestionEvent.Done? _done;
|
||||
private string? _saved;
|
||||
private IReadOnlyList<ReportSummary>? _saved_answers;
|
||||
|
||||
protected override async Task OnInitializedAsync() => await LoadSavedAsync();
|
||||
|
||||
private async Task LoadSavedAsync() => _saved_answers = await Notes.ReportsAsync("output", 50, CancellationToken.None);
|
||||
|
||||
private async Task AskAsync()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_question) || _running) return;
|
||||
_running = true;
|
||||
_error = null;
|
||||
_gap = null;
|
||||
_answer.Clear();
|
||||
_evidence = null;
|
||||
_done = null;
|
||||
_saved = null;
|
||||
try
|
||||
{
|
||||
await foreach (var e in Questions.AnswerAsync(_question, new QuestionOptions(), Embedding, Judge, AnswerModel, CancellationToken.None))
|
||||
{
|
||||
switch (e)
|
||||
{
|
||||
case QuestionEvent.Evidence ev:
|
||||
_evidence = ev.Paragraphs;
|
||||
_covered = ev.Covered;
|
||||
break;
|
||||
case QuestionEvent.Gap g:
|
||||
_gap = g.Message;
|
||||
break;
|
||||
case QuestionEvent.Text t:
|
||||
_answer.Append(t.Delta);
|
||||
break;
|
||||
case QuestionEvent.Done d:
|
||||
_done = d;
|
||||
break;
|
||||
}
|
||||
StateHasChanged();
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_error = ex.Message;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_running = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SaveAsync()
|
||||
{
|
||||
if (_evidence is null || _done is null) return;
|
||||
try
|
||||
{
|
||||
var written = await Save.SaveAsync(_question, _answer.ToString(), _evidence, _covered, _done.AnswerModel, CancellationToken.None);
|
||||
_saved = written.Path;
|
||||
Snackbar.Add($"saved {written.Path}", Severity.Success);
|
||||
await LoadSavedAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_error = ex.Message;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ public sealed class WikiOptions
|
||||
public NoteStoreOptions NoteStore { get; set; } = new();
|
||||
public EmbeddingsOptions Embeddings { get; set; } = new();
|
||||
public JudgesOptions Judges { get; set; } = new();
|
||||
public AnswerModelsOptions AnswerModels { get; set; } = new();
|
||||
public IndexOptions Index { get; set; } = new();
|
||||
public SyncScheduleOptions Sync { get; set; } = new();
|
||||
}
|
||||
@@ -50,6 +51,20 @@ public sealed class JudgeModelOptions
|
||||
public string Model { get; set; } = "jev-latest";
|
||||
}
|
||||
|
||||
public sealed class AnswerModelsOptions
|
||||
{
|
||||
public string Active { get; set; } = "claude-sonnet";
|
||||
public Dictionary<string, AnswerModelOptions> Models { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class AnswerModelOptions
|
||||
{
|
||||
public string Adapter { get; set; } = "Anthropic";
|
||||
public string Url { get; set; } = "https://api.anthropic.com";
|
||||
public string Model { get; set; } = "claude-sonnet-5";
|
||||
public int MaxTokens { get; set; } = 1200;
|
||||
}
|
||||
|
||||
public sealed class IndexOptions
|
||||
{
|
||||
public string Active { get; set; } = "file";
|
||||
|
||||
@@ -4,6 +4,7 @@ using Wiki.Adapters.FileIndex;
|
||||
using Wiki.Adapters.Gitea;
|
||||
using Wiki.Adapters.LlamaCpp;
|
||||
using Wiki.Adapters.TypeSafe;
|
||||
using Wiki.Adapters.Anthropic;
|
||||
using Wiki.Application.Retrieval;
|
||||
using Wiki.Application.Status;
|
||||
using Wiki.Application.Sync;
|
||||
@@ -68,7 +69,28 @@ foreach (var (name, judge) in wiki.Judges.Models)
|
||||
builder.Services.AddSingleton<IReadOnlyList<IJudge>>(sp =>
|
||||
wiki.Judges.Models.Keys.Select(sp.GetRequiredKeyedService<IJudge>).ToList());
|
||||
builder.Services.AddSingleton<IJudge>(sp => sp.GetRequiredKeyedService<IJudge>(wiki.Judges.Active));
|
||||
// One adapter may serve several ports (Gitea is note store and fact sink); list it once.
|
||||
foreach (var (name, answer) in wiki.AnswerModels.Models)
|
||||
{
|
||||
switch (answer.Adapter)
|
||||
{
|
||||
case "Anthropic":
|
||||
builder.Services.AddAnthropicAnswerModel(name, o =>
|
||||
{
|
||||
o.BaseUrl = answer.Url;
|
||||
o.Model = answer.Model;
|
||||
o.ModelId = name;
|
||||
o.MaxTokens = answer.MaxTokens;
|
||||
o.ApiKey = builder.Configuration[$"Wiki:AnswerModels:Models:{name}:ApiKey"] ?? string.Empty;
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException($"unknown answer-model adapter '{answer.Adapter}' for '{name}'");
|
||||
}
|
||||
}
|
||||
builder.Services.AddSingleton<IReadOnlyList<IAnswerModel>>(sp =>
|
||||
wiki.AnswerModels.Models.Keys.Select(sp.GetRequiredKeyedService<IAnswerModel>).ToList());
|
||||
builder.Services.AddSingleton<IAnswerModel>(sp => sp.GetRequiredKeyedService<IAnswerModel>(wiki.AnswerModels.Active));
|
||||
// One adapter may serve several ports; list it once.
|
||||
builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
|
||||
{
|
||||
List<IAdapterHealth> all =
|
||||
@@ -78,6 +100,7 @@ builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
|
||||
sp.GetRequiredService<IDocumentCatalog>(),
|
||||
.. sp.GetRequiredService<IReadOnlyList<IEmbeddingModel>>(),
|
||||
.. sp.GetRequiredService<IReadOnlyList<IJudge>>(),
|
||||
.. sp.GetRequiredService<IReadOnlyList<IAnswerModel>>(),
|
||||
];
|
||||
return all.Distinct().ToList();
|
||||
});
|
||||
@@ -91,6 +114,8 @@ builder.Services.AddSingleton<Wiki.Application.Reading.NoteQueries>();
|
||||
builder.Services.AddSingleton<IChunker, Wiki.Core.Parsing.ParagraphChunker>();
|
||||
builder.Services.AddSingleton<Wiki.Application.Ask.AskUseCase>();
|
||||
builder.Services.AddSingleton<Wiki.Application.Facts.DepositFactUseCase>();
|
||||
builder.Services.AddSingleton<Wiki.Application.Questions.QuestionUseCase>();
|
||||
builder.Services.AddSingleton<Wiki.Application.Questions.SaveAnswerUseCase>();
|
||||
builder.Services.AddSingleton<StatusQuery>();
|
||||
builder.Services.AddSingleton<SyncBackgroundService>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncBackgroundService>());
|
||||
|
||||
@@ -24,6 +24,12 @@
|
||||
"typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" }
|
||||
}
|
||||
},
|
||||
"AnswerModels": {
|
||||
"Active": "claude-sonnet",
|
||||
"Models": {
|
||||
"claude-sonnet": { "Adapter": "Anthropic", "Url": "https://api.anthropic.com", "Model": "claude-sonnet-5", "MaxTokens": 1200 }
|
||||
}
|
||||
},
|
||||
"Index": { "Active": "file", "File": { "Directory": "/data" } },
|
||||
"Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 }
|
||||
},
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using Wiki.Application.Ask;
|
||||
using Wiki.Application.Questions;
|
||||
using Wiki.Application.Retrieval;
|
||||
using Wiki.Application.Sync;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Application.Tests;
|
||||
|
||||
/// <summary>An answer model that echoes the evidence slugs, so the wiring is visible in the text.</summary>
|
||||
public sealed class FakeAnswerModel : IAnswerModel
|
||||
{
|
||||
public List<AnswerRequest> Requests { get; } = [];
|
||||
public string ModelId => "fake-answer";
|
||||
public string Name => "fake-answer";
|
||||
|
||||
public async IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
Requests.Add(request);
|
||||
yield return new AnswerChunk.Text("Yes. ");
|
||||
yield return new AnswerChunk.Text(string.Join(" ", request.Evidence.Select(e => $"[[{e.Paragraph.Slug}]]")));
|
||||
await Task.CompletedTask;
|
||||
yield return new AnswerChunk.Done(ModelId, 100, 12, TimeSpan.FromMilliseconds(3));
|
||||
}
|
||||
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
public class QuestionUseCaseTests
|
||||
{
|
||||
private readonly FakeStore _store = new();
|
||||
private readonly FakeCatalog _catalog = new();
|
||||
private readonly FakeIndex _index = new();
|
||||
private readonly FakeEmbeddings _model = new("m1");
|
||||
private readonly FakeAnswerModel _answer = new();
|
||||
|
||||
private QuestionUseCase UseCase => new(new AskUseCase(new RankUseCase(_index, _catalog), _catalog, new ParagraphChunker()));
|
||||
|
||||
private async Task<Document> SeedAsync()
|
||||
{
|
||||
_store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n";
|
||||
_store.Files["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\n---\n\n# Ceph OSD\n\nOSD fsid.\n";
|
||||
var sync = new SyncUseCase(_store, _catalog, _index, [_model], TimeProvider.System, NullLogger<SyncUseCase>.Instance);
|
||||
await sync.RunAsync(false, CancellationToken.None);
|
||||
return (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StreamsEvidenceThenTextThenDone()
|
||||
{
|
||||
var udm = await SeedAsync();
|
||||
var events = new List<QuestionEvent>();
|
||||
await foreach (var e in UseCase.AnswerAsync(EmbeddingText.For(udm), new QuestionOptions(K: 2), _model, new FakeJudge(), _answer, CancellationToken.None))
|
||||
events.Add(e);
|
||||
|
||||
var evidence = Assert.IsType<QuestionEvent.Evidence>(events[0]);
|
||||
Assert.Equal("udm-dns#p1", evidence.Paragraphs[0].Paragraph.Id);
|
||||
Assert.InRange(evidence.Covered, 0.79, 0.81);
|
||||
var text = string.Concat(events.OfType<QuestionEvent.Text>().Select(t => t.Delta));
|
||||
Assert.Contains("[[udm-dns]]", text);
|
||||
var done = Assert.IsType<QuestionEvent.Done>(events[^1]);
|
||||
Assert.Equal("fake-answer", done.AnswerModel);
|
||||
var request = Assert.Single(_answer.Requests);
|
||||
Assert.Equal(2, request.Evidence.Count); // both paragraphs carry probability mass in the fake judge
|
||||
Assert.Equal("udm-dns#p1", request.Evidence[0].Paragraph.Id);
|
||||
Assert.True(request.Judgements.ContainsKey("answered_1"));
|
||||
Assert.True(request.Judgements.ContainsKey(AskUseCase.CoveredId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BelowTheGateNoAnswerModelIsCalled()
|
||||
{
|
||||
var udm = await SeedAsync();
|
||||
var events = new List<QuestionEvent>();
|
||||
await foreach (var e in UseCase.AnswerAsync(EmbeddingText.For(udm), new QuestionOptions(K: 2, CoveredThreshold: 0.95), _model, new FakeJudge(), _answer, CancellationToken.None))
|
||||
events.Add(e);
|
||||
var gap = Assert.IsType<QuestionEvent.Gap>(Assert.Single(events));
|
||||
Assert.Contains("does not cover", gap.Message);
|
||||
Assert.Empty(_answer.Requests);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SavedAnswerIsAnOutputDocumentInTheSynthShape()
|
||||
{
|
||||
await SeedAsync();
|
||||
var evidence = new List<EvidenceParagraph> { new(new Paragraph("udm-dns#p1", "udm-dns", "UDM", "text"), "UDM intercepts DNS", 0.8) };
|
||||
var save = new SaveAnswerUseCase(_store, new FakeTimeProvider(new DateTimeOffset(2026, 9, 24, 12, 0, 0, TimeSpan.Zero)));
|
||||
var written = await save.SaveAsync("Does the UDM intercept DNS?", "Yes [[udm-dns]].", evidence, 0.88, "claude-sonnet", CancellationToken.None);
|
||||
Assert.Equal("output/2026-09-24-does-the-udm-intercept-dns.md", written.Path);
|
||||
var text = _store.Files[written.Path];
|
||||
Assert.Contains("type: output\n", text);
|
||||
Assert.Contains("question: \"Does the UDM intercept DNS?\"", text.Replace("question: Does the UDM intercept DNS?", "question: \"Does the UDM intercept DNS?\""));
|
||||
Assert.Contains("sources: [udm-dns]", text);
|
||||
Assert.Contains("confidence: high", text);
|
||||
Assert.Contains("- [[udm-dns]]", text);
|
||||
var second = await save.SaveAsync("Does the UDM intercept DNS?", "Yes again.", evidence, 0.5, "claude-sonnet", CancellationToken.None);
|
||||
Assert.Equal("output/2026-09-24-does-the-udm-intercept-dns-2.md", second.Path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Every <see cref="IAnswerModel"/> adapter derives from this. The adapter under test may talk to
|
||||
/// a fake server that echoes the evidence back; the contract checks the streaming shape, the
|
||||
/// usage record and that the model sees nothing but the request.
|
||||
/// </summary>
|
||||
public abstract class AnswerModelContract
|
||||
{
|
||||
protected abstract Task<IAnswerModel> CreateModelAsync();
|
||||
|
||||
protected static AnswerRequest Request() => new(
|
||||
"Does the UDM intercept DNS?",
|
||||
[
|
||||
new EvidenceParagraph(new Paragraph("udm-dns#p1", "udm-dns", "UDM intercepts DNS", "The UDM answers every UDP/53 query itself."), "UDM intercepts DNS", 0.84),
|
||||
new EvidenceParagraph(new Paragraph("udm-dns#p2", "udm-dns", "Details", "Even dig @1.1.1.1 is answered locally."), "UDM intercepts DNS", 0.41),
|
||||
],
|
||||
new Dictionary<string, Judgement> { ["answered_1"] = new Judgement.Noul(0.93) },
|
||||
0.88);
|
||||
|
||||
[Fact]
|
||||
public async Task StreamsTextThenExactlyOneDoneWithUsage()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
var chunks = new List<AnswerChunk>();
|
||||
await foreach (var c in model.AnswerAsync(Request(), CancellationToken.None))
|
||||
chunks.Add(c);
|
||||
|
||||
Assert.NotEmpty(chunks);
|
||||
Assert.IsType<AnswerChunk.Done>(chunks[^1]);
|
||||
Assert.Single(chunks.OfType<AnswerChunk.Done>());
|
||||
var text = string.Concat(chunks.OfType<AnswerChunk.Text>().Select(t => t.Delta));
|
||||
Assert.False(string.IsNullOrWhiteSpace(text));
|
||||
var done = (AnswerChunk.Done)chunks[^1];
|
||||
Assert.Equal(model.ModelId, done.ModelId);
|
||||
Assert.True(done.InputTokens > 0);
|
||||
Assert.True(done.OutputTokens > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancellationStopsTheStream()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
using var cts = new CancellationTokenSource();
|
||||
cts.Cancel();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
|
||||
{
|
||||
await foreach (var _ in model.AnswerAsync(Request(), cts.Token)) { }
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ModelIdAndHealth()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
Assert.False(string.IsNullOrWhiteSpace(model.ModelId));
|
||||
var health = await model.CheckAsync(CancellationToken.None);
|
||||
Assert.True(health.Ok, health.Detail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using Wiki.Adapters.Anthropic;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts.Anthropic;
|
||||
|
||||
/// <summary>Runs the shared <see cref="AnswerModelContract"/> against the Anthropic adapter over a fake server.</summary>
|
||||
public sealed class AnthropicAnswerModelContractTests : AnswerModelContract
|
||||
{
|
||||
protected override Task<IAnswerModel> CreateModelAsync()
|
||||
{
|
||||
var handler = new FakeAnthropicHandler();
|
||||
var client = new HttpClient(handler);
|
||||
var options = new AnthropicOptions
|
||||
{
|
||||
BaseUrl = "https://anthropic.test",
|
||||
Model = "claude-sonnet-5",
|
||||
ApiKey = "test-key",
|
||||
};
|
||||
|
||||
IAnswerModel model = new AnthropicAnswerModel(client, options, delay: (_, _) => Task.CompletedTask);
|
||||
return Task.FromResult(model);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
using System.Text.Json;
|
||||
using Wiki.Adapters.Anthropic;
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Contracts.Anthropic;
|
||||
|
||||
public sealed class AnthropicAnswerModelTests
|
||||
{
|
||||
private static AnthropicAnswerModel CreateModel(FakeAnthropicHandler handler, AnthropicOptions? options = null)
|
||||
{
|
||||
var client = new HttpClient(handler);
|
||||
options ??= new AnthropicOptions
|
||||
{
|
||||
BaseUrl = "https://anthropic.test",
|
||||
Model = "claude-sonnet-5",
|
||||
ApiKey = "test-key",
|
||||
};
|
||||
|
||||
return new AnthropicAnswerModel(client, options, delay: (_, _) => Task.CompletedTask);
|
||||
}
|
||||
|
||||
private static AnswerRequest Request() => new(
|
||||
"Does the UDM intercept DNS?",
|
||||
[
|
||||
new EvidenceParagraph(new Paragraph("udm-dns#p1", "udm-dns", "UDM intercepts DNS", "The UDM answers every UDP/53 query itself."), "UDM intercepts DNS", 0.84),
|
||||
],
|
||||
new Dictionary<string, Judgement> { ["answered_1"] = new Judgement.Noul(0.93) },
|
||||
0.88);
|
||||
|
||||
[Fact]
|
||||
public async Task RequestBodyCarriesTheBuiltPromptAndStreamsTrue()
|
||||
{
|
||||
var handler = new FakeAnthropicHandler();
|
||||
var model = CreateModel(handler);
|
||||
|
||||
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
|
||||
{
|
||||
}
|
||||
|
||||
Assert.NotNull(handler.LastRequestJson);
|
||||
using var document = JsonDocument.Parse(handler.LastRequestJson!);
|
||||
var root = document.RootElement;
|
||||
|
||||
Assert.Equal("claude-sonnet-5", root.GetProperty("model").GetString());
|
||||
Assert.True(root.GetProperty("stream").GetBoolean());
|
||||
|
||||
var system = root.GetProperty("system").GetString();
|
||||
Assert.Contains("You answer questions about a private homelab wiki", system);
|
||||
|
||||
var user = root.GetProperty("messages")[0].GetProperty("content").GetString();
|
||||
Assert.Contains("[[udm-dns]]", user);
|
||||
Assert.Contains("p=0.84", user);
|
||||
Assert.Contains("Covered: 0.88", user);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendsApiKeyVersionAndContentTypeHeaders()
|
||||
{
|
||||
var handler = new FakeAnthropicHandler();
|
||||
var model = CreateModel(handler, new AnthropicOptions
|
||||
{
|
||||
BaseUrl = "https://anthropic.test",
|
||||
Model = "claude-sonnet-5",
|
||||
ApiKey = "super-secret",
|
||||
});
|
||||
|
||||
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
|
||||
{
|
||||
}
|
||||
|
||||
Assert.Equal("super-secret", handler.LastApiKeyHeader);
|
||||
Assert.Equal("2023-06-01", handler.LastAnthropicVersionHeader);
|
||||
Assert.Equal("application/json", handler.LastContentTypeHeader);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StreamsDeltasInOrderThenDoneWithUsage()
|
||||
{
|
||||
var handler = new FakeAnthropicHandler();
|
||||
var model = CreateModel(handler);
|
||||
|
||||
var chunks = new List<AnswerChunk>();
|
||||
await foreach (var chunk in model.AnswerAsync(Request(), CancellationToken.None))
|
||||
{
|
||||
chunks.Add(chunk);
|
||||
}
|
||||
|
||||
var texts = chunks.OfType<AnswerChunk.Text>().ToList();
|
||||
Assert.Equal(3, texts.Count);
|
||||
Assert.Equal("Yes, the UDM intercepts DNS [[udm-dns]].", string.Concat(texts.Select(t => t.Delta)));
|
||||
|
||||
var done = Assert.IsType<AnswerChunk.Done>(chunks[^1]);
|
||||
Assert.Equal("claude-sonnet-5", done.ModelId);
|
||||
Assert.True(done.InputTokens > 0);
|
||||
Assert.Equal(12, done.OutputTokens);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetriesOn429ThenSucceeds()
|
||||
{
|
||||
var handler = new FakeAnthropicHandler { TooManyRequestsBeforeSuccess = 2 };
|
||||
var model = CreateModel(handler);
|
||||
|
||||
var chunks = new List<AnswerChunk>();
|
||||
await foreach (var chunk in model.AnswerAsync(Request(), CancellationToken.None))
|
||||
{
|
||||
chunks.Add(chunk);
|
||||
}
|
||||
|
||||
Assert.Single(chunks.OfType<AnswerChunk.Done>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ErrorEventThrowsHttpRequestException()
|
||||
{
|
||||
var handler = new FakeAnthropicHandler { RespondWithErrorEvent = true };
|
||||
var model = CreateModel(handler);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<HttpRequestException>(async () =>
|
||||
{
|
||||
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
|
||||
{
|
||||
}
|
||||
});
|
||||
|
||||
Assert.Contains("overloaded", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MissingApiKeyThrowsBeforeAnyRequest()
|
||||
{
|
||||
var handler = new FakeAnthropicHandler();
|
||||
var model = CreateModel(handler, new AnthropicOptions
|
||||
{
|
||||
BaseUrl = "https://anthropic.test",
|
||||
Model = "claude-sonnet-5",
|
||||
ApiKey = "",
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
{
|
||||
await foreach (var _ in model.AnswerAsync(Request(), CancellationToken.None))
|
||||
{
|
||||
}
|
||||
});
|
||||
|
||||
Assert.Null(handler.LastRequestJson);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AnswerPromptBuildsTheDocumentedShape()
|
||||
{
|
||||
var (system, user) = AnswerPrompt.Build(Request());
|
||||
|
||||
Assert.Contains("Cite every claim with the note slug in double brackets", system);
|
||||
Assert.Contains("Does the UDM intercept DNS?", user);
|
||||
Assert.Contains("Evidence:", user);
|
||||
Assert.Contains("[[udm-dns]] (p=0.84, UDM intercepts DNS): The UDM answers every UDP/53 query itself.", user);
|
||||
Assert.Contains("Judgements:", user);
|
||||
Assert.Contains("answered_1: noul=0.93", user);
|
||||
Assert.Contains("Covered: 0.88", user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Wiki.Contracts.Anthropic;
|
||||
|
||||
/// <summary>
|
||||
/// Stands in for the Anthropic Messages API. Validates the request shape and headers, and streams
|
||||
/// an SSE response that echoes "Yes, the UDM intercepts DNS [[udm-dns]]." in three text deltas.
|
||||
/// Can be scripted to return a run of 429s before succeeding, or a stream that ends in an SSE
|
||||
/// error event, to exercise the adapter's retry and error handling. GET /v1/models answers with
|
||||
/// one model, for the health check.
|
||||
/// </summary>
|
||||
internal sealed class FakeAnthropicHandler : HttpMessageHandler
|
||||
{
|
||||
private const string EchoText = "Yes, the UDM intercepts DNS [[udm-dns]].";
|
||||
|
||||
public int TooManyRequestsBeforeSuccess { get; set; }
|
||||
|
||||
public bool RespondWithErrorEvent { get; set; }
|
||||
|
||||
public string? LastRequestJson { get; private set; }
|
||||
|
||||
public string? LastApiKeyHeader { get; private set; }
|
||||
|
||||
public string? LastAnthropicVersionHeader { get; private set; }
|
||||
|
||||
public string? LastContentTypeHeader { get; private set; }
|
||||
|
||||
private int _tooManyRequestsServed;
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
LastApiKeyHeader = request.Headers.TryGetValues("x-api-key", out var apiKeys) ? apiKeys.First() : null;
|
||||
LastAnthropicVersionHeader = request.Headers.TryGetValues("anthropic-version", out var versions) ? versions.First() : null;
|
||||
|
||||
if (request.Method == HttpMethod.Get && request.RequestUri!.AbsolutePath == "/v1/models")
|
||||
{
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent("{\"data\":[{\"id\":\"claude-sonnet-5\"}]}", Encoding.UTF8, "application/json"),
|
||||
};
|
||||
}
|
||||
|
||||
LastContentTypeHeader = request.Content?.Headers.ContentType?.MediaType;
|
||||
var body = await request.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
LastRequestJson = body;
|
||||
|
||||
if (_tooManyRequestsServed < TooManyRequestsBeforeSuccess)
|
||||
{
|
||||
_tooManyRequestsServed++;
|
||||
var tooMany = new HttpResponseMessage((HttpStatusCode)429)
|
||||
{
|
||||
Content = new StringContent("{\"type\":\"error\",\"error\":{\"message\":\"rate limited\"}}", Encoding.UTF8, "application/json"),
|
||||
};
|
||||
tooMany.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.Zero);
|
||||
return tooMany;
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(body);
|
||||
var root = document.RootElement;
|
||||
|
||||
if (!root.TryGetProperty("model", out _) ||
|
||||
!root.TryGetProperty("max_tokens", out _) ||
|
||||
!root.TryGetProperty("system", out _) ||
|
||||
!root.TryGetProperty("messages", out var messages) ||
|
||||
messages.GetArrayLength() == 0 ||
|
||||
!root.TryGetProperty("stream", out var stream) ||
|
||||
!stream.GetBoolean())
|
||||
{
|
||||
throw new InvalidOperationException($"Malformed Anthropic request: {body}");
|
||||
}
|
||||
|
||||
var sse = RespondWithErrorEvent ? BuildErrorSse() : BuildSuccessSse(body.Length);
|
||||
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(sse, Encoding.UTF8, "text/event-stream"),
|
||||
};
|
||||
}
|
||||
|
||||
private static string BuildSuccessSse(int requestBodyLength)
|
||||
{
|
||||
var inputTokens = requestBodyLength / 4;
|
||||
var words = EchoText.Split(' ');
|
||||
var third = words.Length / 3;
|
||||
var deltas = new[]
|
||||
{
|
||||
string.Join(' ', words[..third]) + " ",
|
||||
string.Join(' ', words[third..(third * 2)]) + " ",
|
||||
string.Join(' ', words[(third * 2)..]),
|
||||
};
|
||||
|
||||
var sb = new StringBuilder();
|
||||
sb.Append("event: message_start\n");
|
||||
sb.Append($"data: {{\"type\":\"message_start\",\"message\":{{\"id\":\"msg_1\",\"usage\":{{\"input_tokens\":{inputTokens}}}}}}}\n\n");
|
||||
|
||||
sb.Append("event: ping\n");
|
||||
sb.Append("data: {\"type\":\"ping\"}\n\n");
|
||||
|
||||
sb.Append("event: content_block_start\n");
|
||||
sb.Append("data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n");
|
||||
|
||||
foreach (var delta in deltas)
|
||||
{
|
||||
sb.Append("event: content_block_delta\n");
|
||||
sb.Append($"data: {{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{{\"type\":\"text_delta\",\"text\":{JsonSerializer.Serialize(delta)}}}}}\n\n");
|
||||
}
|
||||
|
||||
sb.Append("event: content_block_stop\n");
|
||||
sb.Append("data: {\"type\":\"content_block_stop\",\"index\":0}\n\n");
|
||||
|
||||
sb.Append("event: message_delta\n");
|
||||
sb.Append("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":12}}\n\n");
|
||||
|
||||
sb.Append("event: message_stop\n");
|
||||
sb.Append("data: {\"type\":\"message_stop\"}\n\n");
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static string BuildErrorSse()
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
sb.Append("event: message_start\n");
|
||||
sb.Append("data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"usage\":{\"input_tokens\":10}}}\n\n");
|
||||
sb.Append("event: error\n");
|
||||
sb.Append("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"model overloaded\"}}\n\n");
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,11 @@ public sealed class WikiHostFactory : WebApplicationFactory<Program>
|
||||
services.RemoveAll<IJudge>();
|
||||
services.AddSingleton<IJudge>(judge);
|
||||
services.AddSingleton<IReadOnlyList<IJudge>>([judge]);
|
||||
var answer = new FakeAnswerModel();
|
||||
services.AddKeyedSingleton<IAnswerModel>("claude-sonnet", answer);
|
||||
services.RemoveAll<IAnswerModel>();
|
||||
services.AddSingleton<IAnswerModel>(answer);
|
||||
services.AddSingleton<IReadOnlyList<IAnswerModel>>([answer]);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -236,6 +241,39 @@ public class EndpointTests(WikiHostFactory factory) : IClassFixture<WikiHostFact
|
||||
Assert.Contains("tags: [talos, ceph]", stored);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task QuestionsAnswerFromEvidenceAndCanSave()
|
||||
{
|
||||
factory.Store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n";
|
||||
var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true");
|
||||
req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
|
||||
await Client.SendAsync(req);
|
||||
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await Client.PostAsJsonAsync("/api/v1/questions", new { question = " " })).StatusCode);
|
||||
var unauthorizedSave = await Client.PostAsJsonAsync("/api/v1/questions", new { question = "udm dns", save = true });
|
||||
Assert.Equal(HttpStatusCode.Unauthorized, unauthorizedSave.StatusCode);
|
||||
|
||||
var r = await Client.PostAsJsonAsync("/api/v1/questions", new { question = "udm dns", k = 3 });
|
||||
Assert.Equal(HttpStatusCode.OK, r.StatusCode);
|
||||
var body = await r.Content.ReadFromJsonAsync<QuestionResponse>();
|
||||
Assert.NotNull(body);
|
||||
Assert.Null(body.Gap);
|
||||
Assert.Contains("[[udm-dns]]", body.Answer);
|
||||
Assert.Equal("udm-dns#p1", body.Evidence[0].Id);
|
||||
Assert.Equal("fake-answer", body.Usage!.AnswerModel);
|
||||
|
||||
var save = new HttpRequestMessage(HttpMethod.Post, "/api/v1/questions") { Content = JsonContent.Create(new { question = "udm dns", k = 3, save = true }) };
|
||||
save.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
|
||||
var saved = await (await Client.SendAsync(save)).Content.ReadFromJsonAsync<QuestionResponse>();
|
||||
Assert.NotNull(saved?.Saved);
|
||||
Assert.StartsWith("output/", saved.Saved.Path);
|
||||
Assert.Contains("type: output", factory.Store.Files[saved.Saved.Path]);
|
||||
}
|
||||
|
||||
private sealed record EvidenceRow(string Id, string Slug);
|
||||
private sealed record UsageRow(string AnswerModel);
|
||||
private sealed record SavedRow(string Path, string Commit);
|
||||
private sealed record QuestionResponse(string? Answer, string? Gap, double? Covered, List<EvidenceRow> Evidence, UsageRow? Usage, SavedRow? Saved);
|
||||
private sealed record FactReceiptDto(string Path, string Commit, string CapturedBy);
|
||||
private sealed record AskResponse(string Judge, double? Covered, Dictionary<string, System.Text.Json.JsonElement> Answers);
|
||||
private sealed record NoteRow(string Slug, string Title);
|
||||
|
||||
@@ -44,6 +44,22 @@ public sealed class FakeStore : INoteStore
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
public sealed class FakeAnswerModel : IAnswerModel
|
||||
{
|
||||
public string ModelId => "fake-answer";
|
||||
public string Name => "fake-answer";
|
||||
|
||||
public async IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
yield return new AnswerChunk.Text("Yes. ");
|
||||
yield return new AnswerChunk.Text(string.Join(" ", request.Evidence.Select(e => $"[[{e.Paragraph.Slug}]]")));
|
||||
await Task.CompletedTask;
|
||||
yield return new AnswerChunk.Done(ModelId, 100, 12, TimeSpan.FromMilliseconds(3));
|
||||
}
|
||||
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
public sealed class FakeJudge : IJudge
|
||||
{
|
||||
public string ModelId => "fake-judge";
|
||||
|
||||
Reference in New Issue
Block a user