From c94afb004c95ef58044f8442b634782074daf7d7 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 25 Sep 2026 09:52:16 +0200 Subject: [PATCH] feat(facts): pictures as part of a statement, kept next to the raw source A statement may carry images (a type label, for instance). The drafter sees them as image blocks and transcribes what is printed; the deposit stores each picture beside the note as raw/-., lists it in the frontmatter and embeds it in the body. INoteStore gains a binary create and a browse URL, so the raw page can link to the pictures. The page takes pictures from a file picker (camera on phones) or from paste, resized in the browser to 1600px. Co-Authored-By: Claude Fable 5.1 --- .../AnthropicFactDrafter.cs | 19 +++++- .../FactDraftPrompt.cs | 3 + .../MessagesTransport.cs | 13 ++-- src/Wiki.Adapters.Gitea/GiteaRepository.cs | 10 ++- .../Facts/DepositFactUseCase.cs | 23 +++++-- .../Facts/RecordStatementUseCase.cs | 7 ++- src/Wiki.Core/Model/Attachment.cs | 20 ++++++ src/Wiki.Core/Model/FactDraft.cs | 6 +- src/Wiki.Core/Model/RawSource.cs | 10 +++ src/Wiki.Core/Ports/Ports.cs | 6 ++ src/Wiki.Host/Components/App.razor | 1 + src/Wiki.Host/Components/Pages/Facts.razor | 63 ++++++++++++++++++- src/Wiki.Host/Components/Pages/Raw.razor | 21 +++++++ src/Wiki.Host/wwwroot/facts.js | 31 +++++++++ test/Wiki.Application.Tests/Fakes.cs | 12 ++++ .../RecordStatementUseCaseTests.cs | 24 ++++++- .../AnthropicFactDrafterContractTests.cs | 19 ++++++ test/Wiki.Contracts/NoteStoreContract.cs | 12 ++++ test/Wiki.Host.Tests/Fakes.cs | 11 ++++ 19 files changed, 290 insertions(+), 21 deletions(-) create mode 100644 src/Wiki.Core/Model/Attachment.cs create mode 100644 src/Wiki.Host/wwwroot/facts.js diff --git a/src/Wiki.Adapters.Anthropic/AnthropicFactDrafter.cs b/src/Wiki.Adapters.Anthropic/AnthropicFactDrafter.cs index 2111460..5106c4b 100644 --- a/src/Wiki.Adapters.Anthropic/AnthropicFactDrafter.cs +++ b/src/Wiki.Adapters.Anthropic/AnthropicFactDrafter.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Text.Json.Nodes; using Wiki.Core.Model; using Wiki.Core.Ports; @@ -26,8 +27,24 @@ public sealed class AnthropicFactDrafter : IFactDrafter throw new ArgumentException("the statement is empty", nameof(statement)); var (system, user) = FactDraftPrompt.Build(statement); + var content = new JsonArray(); + foreach (var a in statement.Attachments) + { + content.Add(new JsonObject + { + ["type"] = "image", + ["source"] = new JsonObject + { + ["type"] = "base64", + ["media_type"] = a.MediaType, + ["data"] = Convert.ToBase64String(a.Bytes), + }, + }); + } + content.Add(new JsonObject { ["type"] = "text", ["text"] = user }); + var text = new StringBuilder(); - await foreach (var chunk in _transport.StreamAsync(system, user, ModelId, ct).ConfigureAwait(false)) + await foreach (var chunk in _transport.StreamAsync(system, content, ModelId, ct).ConfigureAwait(false)) { if (chunk is AnswerChunk.Text t) text.Append(t.Delta); } diff --git a/src/Wiki.Adapters.Anthropic/FactDraftPrompt.cs b/src/Wiki.Adapters.Anthropic/FactDraftPrompt.cs index 01992ac..e915c33 100644 --- a/src/Wiki.Adapters.Anthropic/FactDraftPrompt.cs +++ b/src/Wiki.Adapters.Anthropic/FactDraftPrompt.cs @@ -14,6 +14,7 @@ public static class FactDraftPrompt "title: a short noun phrase in the language of the statement, no trailing period. " + "content: the statement restated faithfully as complete sentences in the language of the statement; keep every detail and every exception, add nothing that was not said, at least 40 characters. " + "tags: one to five lowercase ASCII slugs (a-z, 0-9, hyphen) naming the topics, English. " + + "Images attached to the statement are part of it: transcribe every identifier printed on them exactly as printed (manufacturer, model, type, serial number, ratings, dates), mark characters you cannot read as [unreadable], never guess. " + "No markdown, no code fences, no commentary."; public static (string System, string User) Build(FactStatement statement) @@ -23,6 +24,8 @@ public static class FactDraftPrompt user.Append("Stated by ").Append(statement.By.Name) .Append(" on ").Append(statement.StatedAt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)).Append(":\n\n"); user.Append(statement.Text); + if (statement.Attachments.Count > 0) + user.Append("\n\n(").Append(statement.Attachments.Count.ToString(CultureInfo.InvariantCulture)).Append(" image(s) attached.)"); return (SystemPrompt, user.ToString()); } diff --git a/src/Wiki.Adapters.Anthropic/MessagesTransport.cs b/src/Wiki.Adapters.Anthropic/MessagesTransport.cs index 1d28527..b64efe4 100644 --- a/src/Wiki.Adapters.Anthropic/MessagesTransport.cs +++ b/src/Wiki.Adapters.Anthropic/MessagesTransport.cs @@ -28,11 +28,16 @@ internal sealed class MessagesTransport _delay = delay ?? Task.Delay; } - public async IAsyncEnumerable StreamAsync(string system, string user, string modelId, [EnumeratorCancellation] CancellationToken ct) + /// One user turn of text. + public IAsyncEnumerable StreamAsync(string system, string user, string modelId, CancellationToken ct) => + StreamAsync(system, JsonValue.Create(user), modelId, ct); + + /// One user turn of content blocks (text, images). + public async IAsyncEnumerable StreamAsync(string system, JsonNode userContent, string modelId, [EnumeratorCancellation] CancellationToken ct) { ct.ThrowIfCancellationRequested(); - var body = BuildRequestBody(system, user); + var body = BuildRequestBody(system, userContent); var stopwatch = Stopwatch.StartNew(); using var response = await PostWithRetryAsync(body, ct).ConfigureAwait(false); var stream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); @@ -133,7 +138,7 @@ internal sealed class MessagesTransport } } - private JsonObject BuildRequestBody(string system, string user) => new() + private JsonObject BuildRequestBody(string system, JsonNode userContent) => new() { ["model"] = _options.Model, ["max_tokens"] = _options.MaxTokens, @@ -143,7 +148,7 @@ internal sealed class MessagesTransport new JsonObject { ["role"] = "user", - ["content"] = user, + ["content"] = userContent, }, }, ["stream"] = true, diff --git a/src/Wiki.Adapters.Gitea/GiteaRepository.cs b/src/Wiki.Adapters.Gitea/GiteaRepository.cs index 48676de..693f9b2 100644 --- a/src/Wiki.Adapters.Gitea/GiteaRepository.cs +++ b/src/Wiki.Adapters.Gitea/GiteaRepository.cs @@ -79,13 +79,19 @@ public sealed class GiteaRepository : INoteStore } /// POST contents: Gitea answers 422 when the path already exists on the branch. - public async Task CreateAsync(string path, string text, string message, CancellationToken ct) + public Task CreateAsync(string path, string text, string message, CancellationToken ct) => + CreateAsync(path, Encoding.UTF8.GetBytes(text), message, ct); + + public Uri? BrowseUrl(string path) => + new($"{_options.BaseUrl.TrimEnd('/')}/{_options.Repository}/src/branch/{_options.Branch}/{path}"); + + public async Task CreateAsync(string path, byte[] content, string message, CancellationToken ct) { var repo = GiteaUrls.EncodeSegments(_options.Repository); var encodedPath = GiteaUrls.EncodeSegments(path); var request = new { - content = Convert.ToBase64String(Encoding.UTF8.GetBytes(text)), + content = Convert.ToBase64String(content), message, branch = _options.Branch, }; diff --git a/src/Wiki.Application/Facts/DepositFactUseCase.cs b/src/Wiki.Application/Facts/DepositFactUseCase.cs index 7a7f52d..15729ec 100644 --- a/src/Wiki.Application/Facts/DepositFactUseCase.cs +++ b/src/Wiki.Application/Facts/DepositFactUseCase.cs @@ -24,28 +24,42 @@ public sealed class DepositFactUseCase(INoteStore store, TimeProvider clock) var now = clock.GetUtcNow(); var datePrefix = now.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); var slug = Slugify(fact.Title); - var text = BuildNote(fact, by, datePrefix); var message = $"raw: {fact.Title}"; var path = $"raw/{datePrefix}-{slug}.md"; for (var attempt = 2; ; attempt++) { + // Attachments take the note's stem, so they sort next to it and survive a suffix. + var stem = path[(path.LastIndexOf('/') + 1)..^3]; + var names = fact.Attachments.Select((a, i) => $"{stem}-{i + 1}.{a.Extension}").ToList(); + var text = BuildNote(fact, by, datePrefix, names); + StoreWrite written; try { - var written = await store.CreateAsync(path, text, message, ct); - return new FactReceipt(written.Path, written.Commit, now); + written = await store.CreateAsync(path, text, message, ct); } catch (DocumentExistsException) when (attempt <= MaxSuffixAttempts) { path = $"raw/{datePrefix}-{slug}-{attempt}.md"; + continue; } + for (var i = 0; i < names.Count; i++) + written = await store.CreateAsync($"raw/{names[i]}", fact.Attachments[i].Bytes, $"{message} (attachment {i + 1})", ct); + return new FactReceipt(path, written.Commit, now); } } - public static string BuildNote(RawSource fact, Identity by, string capturedDate) + public static string BuildNote(RawSource fact, Identity by, string capturedDate, IReadOnlyList? attachments = null) { + attachments ??= []; var body = fact.Content; if (!body.EndsWith('\n')) body += "\n"; + if (attachments.Count > 0) + { + body += "\n"; + for (var i = 0; i < attachments.Count; i++) + body += $"![{fact.Attachments[i].Caption ?? $"attachment {i + 1}"}]({attachments[i]})\n"; + } var sb = new StringBuilder(); sb.Append("---\n"); @@ -56,6 +70,7 @@ public sealed class DepositFactUseCase(INoteStore store, TimeProvider clock) sb.Append("captured_by: ").Append(by.Name).Append('\n'); sb.Append("observed: ").Append(fact.ObservedAt.ToString("yyyy-MM-ddTHH:mm:ssK", CultureInfo.InvariantCulture)).Append('\n'); sb.Append("tags: [").Append(string.Join(", ", fact.Tags)).Append("]\n"); + if (attachments.Count > 0) sb.Append("attachments: [").Append(string.Join(", ", attachments)).Append("]\n"); sb.Append("---\n"); sb.Append('\n'); sb.Append(body); diff --git a/src/Wiki.Application/Facts/RecordStatementUseCase.cs b/src/Wiki.Application/Facts/RecordStatementUseCase.cs index 97de53c..d1c56b6 100644 --- a/src/Wiki.Application/Facts/RecordStatementUseCase.cs +++ b/src/Wiki.Application/Facts/RecordStatementUseCase.cs @@ -13,21 +13,22 @@ public sealed record RecordedStatement(FactDraft Draft, RawSource Fact, FactRece /// public sealed class RecordStatementUseCase(DepositFactUseCase deposit, TimeProvider clock) { - public async Task RecordAsync(string statement, Identity by, IFactDrafter drafter, CancellationToken ct) + public async Task RecordAsync(string statement, IReadOnlyList attachments, Identity by, IFactDrafter drafter, CancellationToken ct) { ArgumentException.ThrowIfNullOrWhiteSpace(statement); + ArgumentNullException.ThrowIfNull(attachments); ArgumentNullException.ThrowIfNull(by); ArgumentNullException.ThrowIfNull(drafter); var now = clock.GetUtcNow(); var text = statement.Trim(); - var draft = await drafter.DraftAsync(new FactStatement(text, by, now), ct); + var draft = await drafter.DraftAsync(new FactStatement(text, by, now) { Attachments = attachments }, ct); var fact = new RawSource( draft.Title, $"stated by {by.Name} on {now.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)} via the wiki", Body(draft.Content, text), now, - draft.Tags); + draft.Tags) { Attachments = attachments }; var errors = fact.Validate(); if (errors.Count > 0) throw new InvalidOperationException($"the draft is not a valid raw source: {string.Join("; ", errors)}"); diff --git a/src/Wiki.Core/Model/Attachment.cs b/src/Wiki.Core/Model/Attachment.cs new file mode 100644 index 0000000..8aa34aa --- /dev/null +++ b/src/Wiki.Core/Model/Attachment.cs @@ -0,0 +1,20 @@ +namespace Wiki.Core.Model; + +/// A binary that belongs to a raw source and is kept next to it as evidence, a photo of a type label for instance. +public sealed record Attachment(string MediaType, byte[] Bytes, string? Caption = null) +{ + public const int MaxBytes = 8 * 1024 * 1024; + + private static readonly Dictionary Extensions = new(StringComparer.OrdinalIgnoreCase) + { + ["image/jpeg"] = "jpg", + ["image/png"] = "png", + ["image/webp"] = "webp", + ["image/gif"] = "gif", + }; + + /// The file extension for a supported media type, or null. + public static string? ExtensionFor(string mediaType) => Extensions.TryGetValue(mediaType, out var e) ? e : null; + + public string? Extension => ExtensionFor(MediaType); +} diff --git a/src/Wiki.Core/Model/FactDraft.cs b/src/Wiki.Core/Model/FactDraft.cs index a4cd108..54622b5 100644 --- a/src/Wiki.Core/Model/FactDraft.cs +++ b/src/Wiki.Core/Model/FactDraft.cs @@ -1,7 +1,11 @@ namespace Wiki.Core.Model; /// What a person said, by whom and when. The drafter sees nothing else. -public sealed record FactStatement(string Text, Identity By, DateTimeOffset StatedAt); +public sealed record FactStatement(string Text, Identity By, DateTimeOffset StatedAt) +{ + /// Pictures that are part of the statement, a type label for instance. + public IReadOnlyList Attachments { get; init; } = []; +} /// A statement normalised into the fields of a raw source. Source and provenance are the application's to add. public sealed record FactDraft(string Title, string Content, IReadOnlyList Tags); diff --git a/src/Wiki.Core/Model/RawSource.cs b/src/Wiki.Core/Model/RawSource.cs index c72141f..e015fa5 100644 --- a/src/Wiki.Core/Model/RawSource.cs +++ b/src/Wiki.Core/Model/RawSource.cs @@ -13,6 +13,9 @@ public sealed record RawSource( { public const int MinimumContentLength = 40; + /// Binaries stored next to the note as evidence. Empty for most facts. + public IReadOnlyList Attachments { get; init; } = []; + /// Validation errors, empty when the fact may be stored. public IReadOnlyList Validate() { @@ -21,6 +24,13 @@ public sealed record RawSource( if (string.IsNullOrWhiteSpace(Source)) errors.Add("source is required: where the fact was observed or quoted from"); if (Content is null || Content.Trim().Length < MinimumContentLength) errors.Add($"content must be at least {MinimumContentLength} characters of observation"); + for (var i = 0; i < Attachments.Count; i++) + { + var a = Attachments[i]; + if (a.Extension is null) errors.Add($"attachment {i + 1}: unsupported media type {a.MediaType}"); + if (a.Bytes.Length == 0) errors.Add($"attachment {i + 1}: empty"); + if (a.Bytes.Length > Attachment.MaxBytes) errors.Add($"attachment {i + 1}: larger than {Attachment.MaxBytes / (1024 * 1024)} MiB"); + } return errors; } } diff --git a/src/Wiki.Core/Ports/Ports.cs b/src/Wiki.Core/Ports/Ports.cs index f282172..b2dc5fb 100644 --- a/src/Wiki.Core/Ports/Ports.cs +++ b/src/Wiki.Core/Ports/Ports.cs @@ -26,6 +26,12 @@ public interface INoteStore : IAdapterHealth /// Creates a new document at the path on the current head. Throws when the path is taken. Task CreateAsync(string path, string text, string message, CancellationToken ct); + + /// Creates a new binary (an attachment) at the path. Same rules as the text overload. + Task CreateAsync(string path, byte[] content, string message, CancellationToken ct); + + /// Where a person can look at the path in the store's own UI, or null when the store has none. + Uri? BrowseUrl(string path); } /// Text to vector. One instance per configured model. diff --git a/src/Wiki.Host/Components/App.razor b/src/Wiki.Host/Components/App.razor index d0935ad..6bb833e 100644 --- a/src/Wiki.Host/Components/App.razor +++ b/src/Wiki.Host/Components/App.razor @@ -18,6 +18,7 @@ + diff --git a/src/Wiki.Host/Components/Pages/Facts.razor b/src/Wiki.Host/Components/Pages/Facts.razor index c660005..c889476 100644 --- a/src/Wiki.Host/Components/Pages/Facts.razor +++ b/src/Wiki.Host/Components/Pages/Facts.razor @@ -6,12 +6,13 @@ @inject IFactDrafter Drafter @inject NoteQueries Notes @inject ISnackbar Snackbar +@inject IJSRuntime JS Facts Facts - Say what you observed, in one sentence or a few. @Drafter.ModelId turns it into a raw source under your name, with your words kept verbatim, and the nightly distill turns that into wiki notes. Observations only, never hypotheses. + Say what you observed, in one sentence or a few, and add or paste a picture when the thing itself says it best, a type label for instance. @Drafter.ModelId turns it into a raw source under your name, with your words kept verbatim and the pictures next to the note, and the nightly distill turns that into wiki notes. Observations only, never hypotheses. @if (_user is null) @@ -22,8 +23,24 @@ else {
+