ask: wiki_ask.py as a use case and POST /api/v1/ask; judges from configuration

Rank every query, unite the top notes into one context under a per-document
and a total budget, expand context choices over paragraph or document ids
plus NONE, add the covered gate, judge context choices alone and the rest
batched, attach the answering paragraphs' text. A missing TypeSafe key is a
call-time error shown in status, not a startup failure.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
2026-09-24 13:57:17 +02:00
co-authored by Claude Fable 5.1
parent e10208ee2c
commit 34cf91ebaf
11 changed files with 623 additions and 5 deletions
@@ -173,6 +173,9 @@ public sealed class TypeSafeJudge : IJudge
var json = body.ToJsonString(JsonOptions);
var delay = TimeSpan.FromSeconds(1);
// A missing key is a runtime condition the status page shows, not a reason the host cannot start.
if (string.IsNullOrWhiteSpace(_options.ApiKey))
throw new InvalidOperationException("TypeSafe API key is not configured.");
for (var attempt = 0; ; attempt++)
{
using var requestMessage = new HttpRequestMessage(HttpMethod.Post, _options.Url)
@@ -39,11 +39,6 @@ public sealed class TypeSafeOptions
throw new ArgumentException("Model must be set.", nameof(Model));
}
if (string.IsNullOrWhiteSpace(ApiKey))
{
throw new ArgumentException("ApiKey must be set.", nameof(ApiKey));
}
if (MaxRetries < 0)
{
throw new ArgumentException("MaxRetries must not be negative.", nameof(MaxRetries));
+253
View File
@@ -0,0 +1,253 @@
using System.Diagnostics;
using Wiki.Application.Retrieval;
using Wiki.Core.Model;
using Wiki.Core.Ports;
namespace Wiki.Application.Ask;
/// <summary>A caller's question, before expansion. Context choices are expanded over the context's paragraph or document ids.</summary>
public abstract record AskQuestion(string Instructions)
{
public sealed record Noul(string Instructions, string TrueCriterion, string FalseCriterion) : AskQuestion(Instructions);
public sealed record Choice(string Instructions, IReadOnlyDictionary<string, string> Options) : AskQuestion(Instructions);
public sealed record ContextChoice(string Instructions, ContextScope Scope) : AskQuestion(Instructions);
public sealed record Score(string Instructions, string Criteria) : AskQuestion(Instructions);
}
public enum ContextScope
{
Paragraphs,
Documents,
}
public sealed record AskOptions(int K = 6, int DocChars = 12_000, int StateChars = 60_000, int MaxParallel = 8);
public sealed record AskRequest(IReadOnlyList<string> Queries, IReadOnlyDictionary<string, AskQuestion>? Questions, AskOptions Options);
public sealed record ContextDocument(string Slug, string Title, IReadOnlyList<Paragraph> Paragraphs, bool Truncated, float RankScore);
public sealed record RetrievalMeta(IReadOnlyList<string> DroppedForBudget, int Chars, int ParagraphCount, IReadOnlyDictionary<string, float> RankScoreBySlug);
/// <summary>A resolved context hit: a paragraph (with its text) or a document (with its title).</summary>
public sealed record ContextHit(string Id, double P, string Slug, string Title, string? Heading, string? Text);
public abstract record AskAnswer
{
public sealed record Noul(double Probability) : AskAnswer;
public sealed record Choice(string Selected, IReadOnlyDictionary<string, double> Probabilities, double Confidence) : AskAnswer;
public sealed record Context(IReadOnlyList<ContextHit> Top, double NoneP, double Confidence) : AskAnswer;
public sealed record Score(double Value) : AskAnswer;
}
public sealed record AskResult(
IReadOnlyList<string> Queries,
IReadOnlyList<ContextDocument> Context,
RetrievalMeta Retrieval,
IReadOnlyDictionary<string, AskAnswer> Answers,
double? Covered,
JudgeUsage Usage,
string JudgeModel,
TimeSpan Elapsed);
/// <summary>
/// wiki_ask.py as a use case: rank every query, unite the top notes into one context (per-document
/// and total budget), expand context choices over the paragraph or document ids plus NONE, add the
/// covered gate, judge (every context choice in its own request, the rest batched), and attach the
/// answering paragraphs' text to the results. Judgements and quotations, never prose.
/// </summary>
public sealed class AskUseCase(RankUseCase rank, IDocumentCatalog catalog, IChunker chunker)
{
public const string None = "NONE";
public const string CoveredId = "_covered";
public async Task<AskResult> AskAsync(AskRequest request, IEmbeddingModel model, IJudge judge, CancellationToken ct)
{
if (request.Queries.Count == 0) throw new ArgumentException("at least one query is required", nameof(request));
var started = Stopwatch.GetTimestamp();
var (context, meta) = await BuildContextAsync(request.Queries, model, request.Options, ct);
var questions = request.Questions ?? DefaultQuestions(request.Queries);
var (expanded, scopes) = Expand(questions, context, request.Queries, judge.MaxChoiceOptions);
var (answers, usage) = await JudgeInParallelAsync(judge, context, expanded, scopes, request.Options.MaxParallel, ct);
var resolved = Resolve(answers, scopes, context);
double? covered = answers.TryGetValue(CoveredId, out var c) && c is Judgement.Noul n ? Math.Round(n.Probability, 3) : null;
return new AskResult(request.Queries, context, meta, resolved, covered, usage, judge.ModelId, Stopwatch.GetElapsedTime(started));
}
public async Task<(IReadOnlyList<ContextDocument> Context, RetrievalMeta Meta)> BuildContextAsync(
IReadOnlyList<string> queries, IEmbeddingModel model, AskOptions options, CancellationToken ct)
{
var order = new Dictionary<string, float>();
foreach (var q in queries)
{
var rows = await rank.RankAsync(model, q, options.K, ct);
for (var pos = 0; pos < rows.Count; pos++)
{
var score = rows[pos].Score > 0 ? rows[pos].Score : 1e-6f / (pos + 1);
order[rows[pos].Slug] = Math.Max(order.GetValueOrDefault(rows[pos].Slug), score);
}
}
var docs = new List<ContextDocument>();
var dropped = new List<string>();
var used = 0;
foreach (var slug in order.Keys.OrderByDescending(s => order[s]).ThenBy(s => s))
{
var document = await catalog.GetAsync(DocumentKind.Wiki, slug, ct);
if (document is null) continue;
var (paragraphs, truncated) = Cap(chunker.Split(document), options.DocChars);
var size = document.Title.Length + paragraphs.Sum(p => p.Text.Length + p.Heading.Length);
if (used + size > options.StateChars && docs.Count > 0)
{
dropped.Add(slug);
continue;
}
docs.Add(new ContextDocument(slug, document.Title, paragraphs, truncated, order[slug]));
used += size;
}
var meta = new RetrievalMeta(dropped, used, docs.Sum(d => d.Paragraphs.Count),
order.ToDictionary(kv => kv.Key, kv => (float)Math.Round(kv.Value, 3)));
return (docs, meta);
}
private static (IReadOnlyList<Paragraph> Paragraphs, bool Truncated) Cap(IReadOnlyList<Paragraph> paragraphs, int docChars)
{
var kept = new List<Paragraph>();
var used = 0;
foreach (var p in paragraphs)
{
var size = p.Text.Length + p.Heading.Length;
if (kept.Count > 0 && used + size > docChars) break;
kept.Add(p);
used += size;
}
return (kept, kept.Count < paragraphs.Count);
}
public static IReadOnlyDictionary<string, AskQuestion> DefaultQuestions(IReadOnlyList<string> queries)
{
var qs = new Dictionary<string, AskQuestion>();
for (var i = 0; i < queries.Count; i++)
{
var q = queries[i];
qs[$"answered_{i + 1}"] = new AskQuestion.Noul(
$"Do the `documents` answer this question: {q}",
"At least one paragraph states what the question asks for.",
"The documents only touch the topic or do not address the question.");
qs[$"where_{i + 1}"] = new AskQuestion.ContextChoice(
$"Which paragraph (by `id`) most directly answers this question: {q} — pick {None} if none does.",
ContextScope.Paragraphs);
}
return qs;
}
public static (IReadOnlyDictionary<string, TypedQuestion> Questions, IReadOnlyDictionary<string, ContextScope> Scopes) Expand(
IReadOnlyDictionary<string, AskQuestion> questions, IReadOnlyList<ContextDocument> context, IReadOnlyList<string> queries, int? maxOptions)
{
var paragraphIds = context.SelectMany(d => d.Paragraphs.Select(p => p.Id)).ToList();
var slugs = context.Select(d => d.Slug).ToList();
var expanded = new Dictionary<string, TypedQuestion>();
var scopes = new Dictionary<string, ContextScope>();
foreach (var (id, q) in questions)
{
switch (q)
{
case AskQuestion.Noul n:
expanded[id] = new TypedQuestion.Noul(n.Instructions, n.TrueCriterion, n.FalseCriterion);
break;
case AskQuestion.Choice c:
expanded[id] = new TypedQuestion.Choice(c.Instructions, c.Options);
break;
case AskQuestion.Score s:
expanded[id] = new TypedQuestion.Score(s.Instructions, s.Criteria);
break;
case AskQuestion.ContextChoice cc:
var ids = cc.Scope == ContextScope.Paragraphs ? paragraphIds : slugs;
if (maxOptions is { } cap && ids.Count + 1 > cap)
throw new ArgumentException($"{id}: {ids.Count} {cc.Scope.ToString().ToLowerInvariant()} exceed the {cap - 1}-option Choice cap; lower k");
var options = ids.ToDictionary(i => i, _ => string.Empty);
options[None] = $"No {(cc.Scope == ContextScope.Paragraphs ? "paragraph" : "document")} in the documents fits";
expanded[id] = new TypedQuestion.Choice(cc.Instructions, options);
scopes[id] = cc.Scope;
break;
}
}
if (!expanded.ContainsKey(CoveredId))
{
expanded[CoveredId] = new TypedQuestion.Noul(
"Do the `documents` together contain enough to answer these retrieval queries: " + string.Join(" | ", queries),
"The queries can be answered from the documents, possibly combining several.",
"The documents only touch the topic; a real answer would need material not here.");
}
return (expanded, scopes);
}
private static async Task<(IReadOnlyDictionary<string, Judgement> Answers, JudgeUsage Usage)> JudgeInParallelAsync(
IJudge judge, IReadOnlyList<ContextDocument> context, IReadOnlyDictionary<string, TypedQuestion> questions,
IReadOnlyDictionary<string, ContextScope> scopes, int maxParallel, CancellationToken ct)
{
var state = new
{
documents = context.Select(d => new
{
slug = d.Slug,
title = d.Title,
paragraphs = d.Paragraphs.Select(p => new { id = p.Id, heading = p.Heading, text = p.Text }),
}),
};
var groups = new List<Dictionary<string, TypedQuestion>>();
foreach (var (id, q) in questions.Where(kv => scopes.ContainsKey(kv.Key)))
groups.Add(new Dictionary<string, TypedQuestion> { [id] = q });
var rest = questions.Where(kv => !scopes.ContainsKey(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value);
if (rest.Count > 0) groups.Add(rest);
using var gate = new SemaphoreSlim(Math.Max(1, maxParallel));
var tasks = groups.Select(async g =>
{
await gate.WaitAsync(ct);
try { return await judge.JudgeAsync(new JudgeRequest(state, g), ct); }
finally { gate.Release(); }
}).ToList();
var results = await Task.WhenAll(tasks);
var answers = new Dictionary<string, Judgement>();
var usage = JudgeUsage.None;
foreach (var r in results)
{
foreach (var (id, a) in r.Answers) answers[id] = a;
usage = usage.Add(r.Usage);
}
return (answers, usage);
}
private static IReadOnlyDictionary<string, AskAnswer> Resolve(
IReadOnlyDictionary<string, Judgement> answers, IReadOnlyDictionary<string, ContextScope> scopes, IReadOnlyList<ContextDocument> context)
{
var byParagraph = context.SelectMany(d => d.Paragraphs.Select(p => (p, d))).ToDictionary(x => x.p.Id, x => x);
var titles = context.ToDictionary(d => d.Slug, d => d.Title);
var resolved = new Dictionary<string, AskAnswer>();
foreach (var (id, a) in answers)
{
if (scopes.TryGetValue(id, out var scope) && a is Judgement.Choice c)
{
var top = c.Probabilities.Where(kv => kv.Key != None && kv.Value >= 0.03)
.OrderByDescending(kv => kv.Value).Take(3)
.Select(kv => scope == ContextScope.Paragraphs && byParagraph.TryGetValue(kv.Key, out var hit)
? new ContextHit(kv.Key, Math.Round(kv.Value, 3), hit.d.Slug, hit.d.Title, hit.p.Heading, hit.p.Text)
: new ContextHit(kv.Key, Math.Round(kv.Value, 3), kv.Key, titles.GetValueOrDefault(kv.Key, kv.Key), null, null))
.ToList();
resolved[id] = new AskAnswer.Context(top, Math.Round(c.Probabilities.GetValueOrDefault(None), 3), c.Confidence);
continue;
}
resolved[id] = a switch
{
Judgement.Noul n => new AskAnswer.Noul(n.Probability),
Judgement.Choice ch => new AskAnswer.Choice(ch.Selected, ch.Probabilities, ch.Confidence),
Judgement.Score s => new AskAnswer.Score(s.Value),
_ => throw new InvalidOperationException($"unknown judgement for {id}"),
};
}
return resolved;
}
}
+1
View File
@@ -10,6 +10,7 @@ public static class ApiV1
{
var v1 = app.MapGroup("/api/v1");
v1.MapReadEndpoints();
v1.MapAskEndpoints();
v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) =>
Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous();
+122
View File
@@ -0,0 +1,122 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Wiki.Application.Ask;
using Wiki.Core.Ports;
namespace Wiki.Host.Api;
/// <summary>Wire shape of a caller's question, as bin/wiki_ask.py's --questions JSON: criteria is a
/// {true,false} object for noul, an options object or the strings "paragraphs"/"documents" for choice, a string for score.</summary>
public sealed record AskQuestionDto(string Type, string Instructions, JsonElement? Criteria);
public sealed record AskRequestDto(
[property: JsonPropertyName("queries")] IReadOnlyList<string>? Queries,
[property: JsonPropertyName("questions")] IReadOnlyDictionary<string, AskQuestionDto>? Questions,
[property: JsonPropertyName("k")] int? K,
[property: JsonPropertyName("model")] string? Model,
[property: JsonPropertyName("judge")] string? Judge);
public static class AskEndpoints
{
public static void MapAskEndpoints(this RouteGroupBuilder v1)
{
v1.MapPost("/ask", async (AskRequestDto body, AskUseCase ask, IEmbeddingModel activeModel, IJudge activeJudge, IServiceProvider services, CancellationToken ct) =>
{
if (body.Queries is null || body.Queries.Count == 0 || body.Queries.All(string.IsNullOrWhiteSpace))
return Results.BadRequest(new { error = "queries required" });
var model = activeModel;
if (!string.IsNullOrEmpty(body.Model))
{
model = services.GetKeyedService<IEmbeddingModel>(body.Model);
if (model is null) return Results.BadRequest(new { error = $"unknown model '{body.Model}'" });
}
var judge = activeJudge;
if (!string.IsNullOrEmpty(body.Judge))
{
judge = services.GetKeyedService<IJudge>(body.Judge);
if (judge is null) return Results.BadRequest(new { error = $"unknown judge '{body.Judge}'" });
}
IReadOnlyDictionary<string, AskQuestion>? questions;
try
{
questions = body.Questions is null ? null : body.Questions.ToDictionary(kv => kv.Key, kv => Parse(kv.Key, kv.Value));
}
catch (ArgumentException e)
{
return Results.BadRequest(new { error = e.Message });
}
try
{
var result = await ask.AskAsync(new AskRequest(body.Queries.Where(q => !string.IsNullOrWhiteSpace(q)).ToList(), questions, new AskOptions(K: body.K ?? 6)), model, judge, ct);
return Results.Ok(new
{
result.Queries,
context = result.Context.Select(d => new { d.Slug, d.Title, paragraphs = d.Paragraphs.Count, d.Truncated, rankScore = d.RankScore }),
retrieval = new { result.Retrieval.DroppedForBudget, result.Retrieval.Chars, result.Retrieval.ParagraphCount },
answers = result.Answers.ToDictionary(kv => kv.Key, kv => (object)(kv.Value switch
{
AskAnswer.Noul n => new { noul = n.Probability },
AskAnswer.Score s => new { score = s.Value },
AskAnswer.Choice c => new { choice = c.Selected, probabilities = c.Probabilities, confidence = c.Confidence },
AskAnswer.Context x => new { top = x.Top, none_p = x.NoneP, confidence = x.Confidence },
_ => new { },
})),
result.Covered,
usage = result.Usage,
judge = result.JudgeModel,
model = model.ModelId,
elapsed_s = Math.Round(result.Elapsed.TotalSeconds, 3),
});
}
catch (ArgumentException e)
{
return Results.BadRequest(new { error = e.Message });
}
catch (HttpRequestException e)
{
return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway);
}
}).AllowAnonymous();
}
public static AskQuestion Parse(string id, AskQuestionDto dto)
{
var criteria = dto.Criteria;
switch (dto.Type?.ToLowerInvariant())
{
case "noul":
{
if (criteria is not { ValueKind: JsonValueKind.Object } o || !o.TryGetProperty("true", out var t) || !o.TryGetProperty("false", out var f))
throw new ArgumentException($"{id}: noul needs criteria {{true, false}}");
return new AskQuestion.Noul(dto.Instructions, t.GetString() ?? string.Empty, f.GetString() ?? string.Empty);
}
case "choice":
{
if (criteria is { ValueKind: JsonValueKind.String } s)
{
return s.GetString() switch
{
"paragraphs" => new AskQuestion.ContextChoice(dto.Instructions, ContextScope.Paragraphs),
"documents" => new AskQuestion.ContextChoice(dto.Instructions, ContextScope.Documents),
var other => throw new ArgumentException($"{id}: unknown choice criteria '{other}'"),
};
}
if (criteria is { ValueKind: JsonValueKind.Object } o)
{
var options = new Dictionary<string, string>();
foreach (var p in o.EnumerateObject()) options[p.Name] = p.Value.ValueKind == JsonValueKind.String ? p.Value.GetString() ?? string.Empty : string.Empty;
if (options.Count == 0) throw new ArgumentException($"{id}: choice needs options");
return new AskQuestion.Choice(dto.Instructions, options);
}
throw new ArgumentException($"{id}: choice needs criteria (options object, \"paragraphs\" or \"documents\")");
}
case "score":
return new AskQuestion.Score(dto.Instructions, criteria is { ValueKind: JsonValueKind.String } sc ? sc.GetString() ?? string.Empty : string.Empty);
default:
throw new ArgumentException($"{id}: unknown question type '{dto.Type}'");
}
}
}
+14
View File
@@ -5,6 +5,7 @@ public sealed class WikiOptions
{
public NoteStoreOptions NoteStore { get; set; } = new();
public EmbeddingsOptions Embeddings { get; set; } = new();
public JudgesOptions Judges { get; set; } = new();
public IndexOptions Index { get; set; } = new();
public SyncScheduleOptions Sync { get; set; } = new();
}
@@ -36,6 +37,19 @@ public sealed class EmbeddingModelOptions
public int Dimension { get; set; } = 1024;
}
public sealed class JudgesOptions
{
public string Active { get; set; } = "typesafe";
public Dictionary<string, JudgeModelOptions> Models { get; set; } = new();
}
public sealed class JudgeModelOptions
{
public string Adapter { get; set; } = "TypeSafe";
public string Url { get; set; } = "https://api.typesafe.ai/v1/systemone";
public string Model { get; set; } = "jev-latest";
}
public sealed class IndexOptions
{
public string Active { get; set; } = "file";
+24
View File
@@ -3,6 +3,7 @@ using MudBlazor.Services;
using Wiki.Adapters.FileIndex;
using Wiki.Adapters.Gitea;
using Wiki.Adapters.LlamaCpp;
using Wiki.Adapters.TypeSafe;
using Wiki.Application.Retrieval;
using Wiki.Application.Status;
using Wiki.Application.Sync;
@@ -47,6 +48,26 @@ foreach (var (name, model) in wiki.Embeddings.Models)
builder.Services.AddSingleton<IReadOnlyList<IEmbeddingModel>>(sp =>
wiki.Embeddings.Models.Keys.Select(sp.GetRequiredKeyedService<IEmbeddingModel>).ToList());
builder.Services.AddSingleton<IEmbeddingModel>(sp => sp.GetRequiredKeyedService<IEmbeddingModel>(wiki.Embeddings.Active));
foreach (var (name, judge) in wiki.Judges.Models)
{
switch (judge.Adapter)
{
case "TypeSafe":
builder.Services.AddTypeSafeJudge(name, o =>
{
o.Url = judge.Url;
o.Model = judge.Model;
o.ModelId = name;
o.ApiKey = builder.Configuration[$"Wiki:Judges:Models:{name}:ApiKey"] ?? string.Empty;
});
break;
default:
throw new InvalidOperationException($"unknown judge adapter '{judge.Adapter}' for '{name}'");
}
}
builder.Services.AddSingleton<IReadOnlyList<IJudge>>(sp =>
wiki.Judges.Models.Keys.Select(sp.GetRequiredKeyedService<IJudge>).ToList());
builder.Services.AddSingleton<IJudge>(sp => sp.GetRequiredKeyedService<IJudge>(wiki.Judges.Active));
// One adapter may serve several ports (Gitea is note store and fact sink); list it once.
builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
{
@@ -56,6 +77,7 @@ builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
sp.GetRequiredService<IVectorIndex>(),
sp.GetRequiredService<IDocumentCatalog>(),
.. sp.GetRequiredService<IReadOnlyList<IEmbeddingModel>>(),
.. sp.GetRequiredService<IReadOnlyList<IJudge>>(),
];
return all.Distinct().ToList();
});
@@ -66,6 +88,8 @@ builder.Services.AddSingleton<SyncUseCase>();
builder.Services.AddSingleton<SyncCoordinator>();
builder.Services.AddSingleton<RankUseCase>();
builder.Services.AddSingleton<Wiki.Application.Reading.NoteQueries>();
builder.Services.AddSingleton<IChunker, Wiki.Core.Parsing.ParagraphChunker>();
builder.Services.AddSingleton<Wiki.Application.Ask.AskUseCase>();
builder.Services.AddSingleton<StatusQuery>();
builder.Services.AddSingleton<SyncBackgroundService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncBackgroundService>());
+6
View File
@@ -18,6 +18,12 @@
"bge-m3": { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc.cluster.local:8080", "Model": "bge-m3", "Dimension": 1024 }
}
},
"Judges": {
"Active": "typesafe",
"Models": {
"typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" }
}
},
"Index": { "Active": "file", "File": { "Directory": "/data" } },
"Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 }
},