konairius/Wiki
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(core): fail closed where the body scanner and CommonMark disagree

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
claude-bot
2026-10-24 13:45:33 +02:00
co-authored by Claude Opus 5.5
parent 23db9e9240
commit b8e48b7d16
4 changed files with 215 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Wiki.Core/Parsing/ArticleBody.cs
+131 -48
View File
@@ -10,10 +10,18 @@ public sealed record TranslationUnit(int Index, bool Private, string Text);
/// <summary>
/// Article body conventions: <c>::: infobox</c> and <c>::: private</c> containers closed by <c>:::</c>,
/// fenced code (<c>```</c> or <c>~~~</c>, at most three spaces indented), <c>[[slug]]</c> links.
/// Containers are recognised only outside code fences. Fails closed: any <c>:::</c> opener other than
/// <c>::: infobox</c> opens a private block, a private block runs to its matching closer counting
/// nested containers, and an unterminated private block runs to the end of the body.
/// fenced code (<c>```</c> or <c>~~~</c>), <c>[[slug]]</c> links. Line endings are normalised to <c>\n</c>.
/// Fails closed wherever the scanner and a CommonMark renderer could disagree:
/// <list type="bullet">
/// <item>A line that is a run of three or more colons after any indentation and list or quote markers is a
/// container line. It is <c>::: infobox</c> only at column 0, and a closer only with at most three spaces of
/// indentation, at least its opener's colons, and nothing after. Every other container line opens a private block.</item>
/// <item>Private blocks count nesting and run to their matching closer, or to the end of the body.</item>
/// <item>Container lines inside a column-0 code fence are code. Inside a fence opened with indentation or
/// markers (a list item, a quote), a container line ends the fence and opens a private block.</item>
/// <item>A fence still open at the end of the body, or one whose closer is less indented than its opener,
/// makes everything from that line on private.</item>
/// </list>
/// </summary>
public static partial class ArticleBody
{
@@ -40,7 +48,7 @@ public static partial class ArticleBody
}
else
{
output.Add(line.Raw);
output.Add(line.Text);
}
previousPrivate = line.Private;
}
@@ -75,7 +83,7 @@ public static partial class ArticleBody
for (var i = 0; i < lines.Count; i++)
{
if (starts.Contains(i)) output.Add(PrivateOpen);
output.Add(lines[i].Raw);
output.Add(lines[i].Text);
if (ends.Contains(i)) output.Add(Close);
}
return string.Join('\n', output);
@@ -109,7 +117,7 @@ public static partial class ArticleBody
for (var i = 0; i < units.Count; i++)
{
var unit = units[i];
var text = translated[i].Replace("\r", string.Empty).Trim('\n');
var text = translated[i].Replace("\r\n", "\n").Replace('\r', '\n').Trim('\n');
var textLines = text.Split('\n');
foreach (var line in textLines)
{
@@ -132,7 +140,7 @@ public static partial class ArticleBody
i = r.End;
continue;
}
output.Add(lines[i].Raw);
output.Add(lines[i].Text);
}
return string.Join('\n', output);
}
@@ -145,89 +153,163 @@ public static partial class ArticleBody
private enum LineKind { Blank, Text, FenceOpen, FenceBody, FenceClose, ContainerOpen, ContainerClose }
/// <param name="Private">The line lies in a private block, its opener and closer included.</param>
/// <param name="Private">The line lies in a private block, its opener and closer included, or in a forced-private tail.</param>
/// <param name="DepthAfter">Open containers after this line.</param>
/// <param name="Mermaid">A fence line or fence body line of a <c>```mermaid</c> fence.</param>
private sealed record Line(string Raw, string Text, LineKind Kind, bool Private, int DepthAfter, bool Mermaid);
private sealed record Line(string Text, LineKind Kind, bool Private, int DepthAfter, bool Mermaid);
/// <param name="Width">Characters of indentation and markers before the opener's fence marker.</param>
private sealed record OpenFence(int Line, string Marker, int Width, bool Mermaid);
private static List<Line> Scan(string body)
{
var result = new List<Line>();
var containers = new Stack<bool>(); // true: private
var containers = new Stack<(bool Private, int Colons)>();
var privateDepth = 0;
string? fence = null;
var mermaid = false;
foreach (var raw in body.Split('\n'))
OpenFence? fence = null;
int? privateFrom = null;
var lines = body.Replace("\r\n", "\n").Replace('\r', '\n').Split('\n');
for (var n = 0; n < lines.Length; n++)
{
var text = raw.TrimEnd('\r');
var text = lines[n];
var inPrivate = privateDepth > 0;
var (rest, width) = StripPrefix(text);
var isFence = FenceMarker(rest, out var marker, out var info);
var colons = Colons(rest);
if (fence is not null)
{
if (FenceMarker(text, out var marker, out var info) && marker[0] == fence[0] && marker.Length >= fence.Length && info.Length == 0)
var closes = isFence && info.Length == 0 && marker[0] == fence.Marker[0] && marker.Length >= fence.Marker.Length
&& (fence.Width == 0 ? IsPlainIndent(text, width) : width <= fence.Width);
if (closes)
{
result.Add(new Line(raw, text, LineKind.FenceClose, inPrivate, containers.Count, mermaid));
if (width < fence.Width) privateFrom ??= n;
result.Add(new Line(text, LineKind.FenceClose, inPrivate, containers.Count, fence.Mermaid));
fence = null;
mermaid = false;
}
else if (fence.Width > 0 && colons >= 3)
{
fence = null;
containers.Push((true, colons));
privateDepth++;
result.Add(new Line(text, LineKind.ContainerOpen, true, containers.Count, false));
}
else
{
result.Add(new Line(raw, text, LineKind.FenceBody, inPrivate, containers.Count, mermaid));
result.Add(new Line(text, LineKind.FenceBody, inPrivate, containers.Count, fence.Mermaid));
}
continue;
}
if (FenceMarker(text, out var open, out var openInfo))
if (isFence)
{
fence = open;
mermaid = openInfo.Split(' ', 2)[0].Equals("mermaid", StringComparison.OrdinalIgnoreCase);
result.Add(new Line(raw, text, LineKind.FenceOpen, inPrivate, containers.Count, mermaid));
var mermaid = info.Split(' ', 2)[0].Equals("mermaid", StringComparison.OrdinalIgnoreCase);
fence = new OpenFence(n, marker, width, mermaid);
result.Add(new Line(text, LineKind.FenceOpen, inPrivate, containers.Count, mermaid));
continue;
}
var trimmed = text.Trim();
if (trimmed.StartsWith(Close, StringComparison.Ordinal))
if (colons >= 3)
{
var kind = trimmed[Close.Length..].Trim();
if (kind.Length == 0)
var after = rest[colons..];
var closerShape = IsPlainIndent(text, width) && string.IsNullOrWhiteSpace(after);
if (closerShape && containers.Count == 0)
{
if (containers.Count > 0 && containers.Pop()) privateDepth--;
result.Add(new Line(raw, text, LineKind.ContainerClose, inPrivate, containers.Count, false));
result.Add(new Line(text, LineKind.ContainerClose, false, 0, false));
}
else if (closerShape && colons >= containers.Peek().Colons)
{
if (containers.Pop().Private) privateDepth--;
result.Add(new Line(text, LineKind.ContainerClose, inPrivate, containers.Count, false));
}
else if (width == 0 && after.Trim() == "infobox")
{
containers.Push((false, colons));
result.Add(new Line(text, LineKind.ContainerOpen, inPrivate, containers.Count, false));
}
else
{
var isPrivate = !string.Equals(kind, "infobox", StringComparison.Ordinal);
containers.Push(isPrivate);
if (isPrivate) privateDepth++;
result.Add(new Line(raw, text, LineKind.ContainerOpen, privateDepth > 0, containers.Count, false));
containers.Push((true, colons));
privateDepth++;
result.Add(new Line(text, LineKind.ContainerOpen, true, containers.Count, false));
}
continue;
}
result.Add(new Line(raw, text, trimmed.Length == 0 ? LineKind.Blank : LineKind.Text, inPrivate, containers.Count, false));
result.Add(new Line(text, text.Trim().Length == 0 ? LineKind.Blank : LineKind.Text, inPrivate, containers.Count, false));
}
if (fence is not null && (privateFrom is null || fence.Line < privateFrom)) privateFrom = fence.Line;
if (privateFrom is { } from)
{
for (var i = from; i < result.Count; i++)
result[i] = result[i] with { Private = true };
}
return result;
}
/// <summary>A fence line: up to three spaces, then three or more backticks or tildes, then the info string.</summary>
private static bool FenceMarker(string line, out string marker, out string info)
/// <summary>Strips indentation and list or quote markers (<c>&gt;</c>, <c>*</c>, <c>+</c>, <c>-</c>, <c>1.</c>, <c>1)</c>); returns the rest and how many characters went.</summary>
private static (string Content, int Width) StripPrefix(string line)
{
var i = 0;
while (true)
{
var j = i;
while (j < line.Length && (line[j] == ' ' || line[j] == '\t')) j++;
if (j < line.Length && line[j] == '>')
{
i = j + 1;
continue;
}
if (j < line.Length && line[j] is '*' or '+' or '-' && MarkerEnds(line, j + 1))
{
i = j + 1;
continue;
}
var k = j;
while (k < line.Length && k - j < 9 && char.IsAsciiDigit(line[k])) k++;
if (k > j && k < line.Length && line[k] is '.' or ')' && MarkerEnds(line, k + 1))
{
i = k + 1;
continue;
}
return (line[j..], j);
}
static bool MarkerEnds(string line, int at) => at == line.Length || line[at] == ' ' || line[at] == '\t';
}
/// <summary>At most three spaces and nothing else before the content: CommonMark's column-0 indentation.</summary>
private static bool IsPlainIndent(string line, int width) =>
width <= 3 && line.AsSpan(0, width).IndexOfAnyExcept(' ') < 0;
private static int Colons(string rest)
{
var i = 0;
while (i < rest.Length && rest[i] == ':') i++;
return i;
}
/// <summary>A fence marker at the start of <paramref name="rest"/>: three or more backticks or tildes, then the info string, which for backticks contains none.</summary>
private static bool FenceMarker(string rest, out string marker, out string info)
{
marker = string.Empty;
info = string.Empty;
if (rest.Length == 0 || (rest[0] != '`' && rest[0] != '~')) return false;
var ch = rest[0];
var i = 0;
while (i < line.Length && i < 3 && line[i] == ' ') i++;
if (i >= line.Length || (line[i] != '`' && line[i] != '~')) return false;
var ch = line[i];
var start = i;
while (i < line.Length && line[i] == ch) i++;
if (i - start < 3) return false;
marker = line[start..i];
info = line[i..].Trim();
while (i < rest.Length && rest[i] == ch) i++;
if (i < 3) return false;
var tail = rest[i..].Trim();
if (ch == '`' && tail.Contains('`')) return false;
marker = rest[..i];
info = tail;
return true;
}
private static bool IsStructuralLike(string line)
{
var trimmed = line.TrimStart();
return trimmed.StartsWith(Close, StringComparison.Ordinal)
|| trimmed.StartsWith("```", StringComparison.Ordinal)
|| trimmed.StartsWith("~~~", StringComparison.Ordinal);
var (rest, _) = StripPrefix(line);
return rest.StartsWith(Close, StringComparison.Ordinal)
|| rest.StartsWith("```", StringComparison.Ordinal)
|| rest.StartsWith("~~~", StringComparison.Ordinal);
}
private static List<(int Start, int End)> Blocks(List<Line> lines)
@@ -277,6 +359,7 @@ public static partial class ArticleBody
var line = lines[i];
if (line.Kind is LineKind.Text)
{
if (start is not null && lines[start.Value].Private != line.Private) Flush(i);
start ??= i;
continue;
}
src/Wiki.Core/Parsing/ArticleFile.cs
+2
View File
@@ -63,6 +63,8 @@ public static class ArticleFile
var slugSubject = document.Slug[(slash + 1)..];
var lang = Required(document, "lang");
if (lang is not ("de" or "en"))
throw new FormatException($"{document.Path}: unsupported lang '{lang}'.");
if (lang != slugLang)
throw new FormatException($"{document.Path}: lang '{lang}' does not match the path.");
var subject = Required(document, "subject");
test/Wiki.Core.Tests/ArticleBodyTests.cs
+75
View File
@@ -55,6 +55,81 @@ public class ArticleBodyTests
Assert.Contains("B", result);
}
[Theory]
[InlineData("A\n\n::: private\nsecret\n\n :::\nleaked\n:::")]
[InlineData("A\n\n::: private\nsecret\n\t:::\nleaked\n:::")]
[InlineData("A\n\n::: private\nsecret\n:::x\nleaked\n:::")]
[InlineData(":::: private\nsecret\n:::\nleaked\n::::")]
public void RedactPrivate_FailsClosedOnLooseClosers(string body)
{
var result = ArticleBody.RedactPrivate(body);
Assert.DoesNotContain("secret", result);
Assert.DoesNotContain("leaked", result);
}
[Fact]
public void RedactPrivate_IndentedCloserWithinThreeSpaces_Closes()
{
var result = ArticleBody.RedactPrivate("::: private\nsecret\n :::\n\nB");
Assert.DoesNotContain("secret", result);
Assert.EndsWith("\nB", result);
}
[Fact]
public void RedactPrivate_FailsClosedOnBacktickInfoWithBacktick()
{
var result = ArticleBody.RedactPrivate("```ls -la```\n\n::: private\nsecret\n:::");
Assert.DoesNotContain("secret", result);
Assert.StartsWith("```ls -la```", result);
}
[Theory]
[InlineData("- item\n ```\n code\n::: private\nsecret\n:::\n```")]
[InlineData("::: private\n- ```\n :::\n secret\n ```\n:::\n\nB")]
[InlineData("::: private\n ```\n```\n:::\nsecret\n```\n::: private\nsecret\n:::")]
public void RedactPrivate_FailsClosedOnFencesInListItems(string body)
{
Assert.DoesNotContain("secret", ArticleBody.RedactPrivate(body));
}
[Theory]
[InlineData("A\n\n- ::: private\n secret\n :::")]
[InlineData("A\n\n> ::: private\n> secret\n> :::")]
[InlineData("A\n\n1. ::: private\n secret\n :::")]
[InlineData("A\n\n2) ::: private\n secret\n :::")]
[InlineData("A\n\n * > ::: private\n > secret")]
[InlineData("A\n\n ::: infobox\nsecret\n:::")]
public void RedactPrivate_FailsClosedOnContainersInListsAndQuotes(string body)
{
var result = ArticleBody.RedactPrivate(body);
Assert.DoesNotContain("secret", result);
Assert.StartsWith("A\n", result);
}
[Theory]
[InlineData("```\ncode\n\n::: private\nsecret\n:::")]
[InlineData("A\n\n- x\n ```\n code\n\n ::: private\n secret")]
public void RedactPrivate_FailsClosedOnUnterminatedFence(string body)
{
var result = ArticleBody.RedactPrivate(body);
Assert.DoesNotContain("secret", result);
Assert.Contains(ArticleBody.PrivateLocked, result);
}
[Fact]
public void RedactPrivate_FailsClosedOnLoneCarriageReturns()
{
var result = ArticleBody.RedactPrivate("A\r::: private\rsecret\r:::");
Assert.Equal("A\n::: private-locked\n:::", result);
}
[Fact]
public void Reassemble_UnitWithMarkedContainerLine_Throws()
{
Assert.Throws<FormatException>(() => ArticleBody.Reassemble(WithPrivate, ["A", "x\n- :::\nleak", "B"]));
Assert.Throws<FormatException>(() => ArticleBody.Reassemble(WithPrivate, ["A", "x\n> :::\nleak", "B"]));
}
[Fact]
public void PublicBlocks_SkipsPrivateAndNumbersSequentially()
{
test/Wiki.Core.Tests/ArticleFileTests.cs
+7
View File
@@ -55,6 +55,13 @@ public class ArticleFileTests
Assert.Throws<FormatException>(() => ArticleFile.Parse(Load(text)));
}
[Fact]
public void Parse_UnsupportedLang_Throws()
{
var text = ArticleFile.Build(Sample).Replace("lang: de\n", "lang: fr\n");
Assert.Throws<FormatException>(() => ArticleFile.Parse(Load(text, "fr/wall-switches")));
}
[Fact]
public void Parse_NonArticleDocument_Throws()
{