diff --git a/src/Wiki.Core/Parsing/ArticleBody.cs b/src/Wiki.Core/Parsing/ArticleBody.cs
index 075b587..c8711cc 100644
--- a/src/Wiki.Core/Parsing/ArticleBody.cs
+++ b/src/Wiki.Core/Parsing/ArticleBody.cs
@@ -10,10 +10,18 @@ public sealed record TranslationUnit(int Index, bool Private, string Text);
///
/// Article body conventions: ::: infobox and ::: private containers closed by :::,
-/// fenced code (``` or ~~~, at most three spaces indented), [[slug]] links.
-/// Containers are recognised only outside code fences. Fails closed: any ::: opener other than
-/// ::: infobox opens a private block, a private block runs to its matching closer counting
-/// nested containers, and an unterminated private block runs to the end of the body.
+/// fenced code (``` or ~~~), [[slug]] links. Line endings are normalised to \n.
+/// Fails closed wherever the scanner and a CommonMark renderer could disagree:
+///
+/// - A line that is a run of three or more colons after any indentation and list or quote markers is a
+/// container line. It is ::: infobox only at column 0, and a closer only with at most three spaces of
+/// indentation, at least its opener's colons, and nothing after. Every other container line opens a private block.
+/// - Private blocks count nesting and run to their matching closer, or to the end of the body.
+/// - Container lines inside a column-0 code fence are code. Inside a fence opened with indentation or
+/// markers (a list item, a quote), a container line ends the fence and opens a private block.
+/// - A fence still open at the end of the body, or one whose closer is less indented than its opener,
+/// makes everything from that line on private.
+///
///
public static partial class ArticleBody
{
@@ -40,7 +48,7 @@ public static partial class ArticleBody
}
else
{
- output.Add(line.Raw);
+ output.Add(line.Text);
}
previousPrivate = line.Private;
}
@@ -75,7 +83,7 @@ public static partial class ArticleBody
for (var i = 0; i < lines.Count; i++)
{
if (starts.Contains(i)) output.Add(PrivateOpen);
- output.Add(lines[i].Raw);
+ output.Add(lines[i].Text);
if (ends.Contains(i)) output.Add(Close);
}
return string.Join('\n', output);
@@ -109,7 +117,7 @@ public static partial class ArticleBody
for (var i = 0; i < units.Count; i++)
{
var unit = units[i];
- var text = translated[i].Replace("\r", string.Empty).Trim('\n');
+ var text = translated[i].Replace("\r\n", "\n").Replace('\r', '\n').Trim('\n');
var textLines = text.Split('\n');
foreach (var line in textLines)
{
@@ -132,7 +140,7 @@ public static partial class ArticleBody
i = r.End;
continue;
}
- output.Add(lines[i].Raw);
+ output.Add(lines[i].Text);
}
return string.Join('\n', output);
}
@@ -145,89 +153,163 @@ public static partial class ArticleBody
private enum LineKind { Blank, Text, FenceOpen, FenceBody, FenceClose, ContainerOpen, ContainerClose }
- /// The line lies in a private block, its opener and closer included.
+ /// The line lies in a private block, its opener and closer included, or in a forced-private tail.
/// Open containers after this line.
/// A fence line or fence body line of a ```mermaid fence.
- private sealed record Line(string Raw, string Text, LineKind Kind, bool Private, int DepthAfter, bool Mermaid);
+ private sealed record Line(string Text, LineKind Kind, bool Private, int DepthAfter, bool Mermaid);
+
+ /// Characters of indentation and markers before the opener's fence marker.
+ private sealed record OpenFence(int Line, string Marker, int Width, bool Mermaid);
private static List Scan(string body)
{
var result = new List();
- var containers = new Stack(); // true: private
+ var containers = new Stack<(bool Private, int Colons)>();
var privateDepth = 0;
- string? fence = null;
- var mermaid = false;
- foreach (var raw in body.Split('\n'))
+ OpenFence? fence = null;
+ int? privateFrom = null;
+ var lines = body.Replace("\r\n", "\n").Replace('\r', '\n').Split('\n');
+ for (var n = 0; n < lines.Length; n++)
{
- var text = raw.TrimEnd('\r');
+ var text = lines[n];
var inPrivate = privateDepth > 0;
+ var (rest, width) = StripPrefix(text);
+ var isFence = FenceMarker(rest, out var marker, out var info);
+ var colons = Colons(rest);
+
if (fence is not null)
{
- if (FenceMarker(text, out var marker, out var info) && marker[0] == fence[0] && marker.Length >= fence.Length && info.Length == 0)
+ var closes = isFence && info.Length == 0 && marker[0] == fence.Marker[0] && marker.Length >= fence.Marker.Length
+ && (fence.Width == 0 ? IsPlainIndent(text, width) : width <= fence.Width);
+ if (closes)
{
- result.Add(new Line(raw, text, LineKind.FenceClose, inPrivate, containers.Count, mermaid));
+ if (width < fence.Width) privateFrom ??= n;
+ result.Add(new Line(text, LineKind.FenceClose, inPrivate, containers.Count, fence.Mermaid));
fence = null;
- mermaid = false;
+ }
+ else if (fence.Width > 0 && colons >= 3)
+ {
+ fence = null;
+ containers.Push((true, colons));
+ privateDepth++;
+ result.Add(new Line(text, LineKind.ContainerOpen, true, containers.Count, false));
}
else
{
- result.Add(new Line(raw, text, LineKind.FenceBody, inPrivate, containers.Count, mermaid));
+ result.Add(new Line(text, LineKind.FenceBody, inPrivate, containers.Count, fence.Mermaid));
}
continue;
}
- if (FenceMarker(text, out var open, out var openInfo))
+ if (isFence)
{
- fence = open;
- mermaid = openInfo.Split(' ', 2)[0].Equals("mermaid", StringComparison.OrdinalIgnoreCase);
- result.Add(new Line(raw, text, LineKind.FenceOpen, inPrivate, containers.Count, mermaid));
+ var mermaid = info.Split(' ', 2)[0].Equals("mermaid", StringComparison.OrdinalIgnoreCase);
+ fence = new OpenFence(n, marker, width, mermaid);
+ result.Add(new Line(text, LineKind.FenceOpen, inPrivate, containers.Count, mermaid));
continue;
}
- var trimmed = text.Trim();
- if (trimmed.StartsWith(Close, StringComparison.Ordinal))
+ if (colons >= 3)
{
- var kind = trimmed[Close.Length..].Trim();
- if (kind.Length == 0)
+ var after = rest[colons..];
+ var closerShape = IsPlainIndent(text, width) && string.IsNullOrWhiteSpace(after);
+ if (closerShape && containers.Count == 0)
{
- if (containers.Count > 0 && containers.Pop()) privateDepth--;
- result.Add(new Line(raw, text, LineKind.ContainerClose, inPrivate, containers.Count, false));
+ result.Add(new Line(text, LineKind.ContainerClose, false, 0, false));
+ }
+ else if (closerShape && colons >= containers.Peek().Colons)
+ {
+ if (containers.Pop().Private) privateDepth--;
+ result.Add(new Line(text, LineKind.ContainerClose, inPrivate, containers.Count, false));
+ }
+ else if (width == 0 && after.Trim() == "infobox")
+ {
+ containers.Push((false, colons));
+ result.Add(new Line(text, LineKind.ContainerOpen, inPrivate, containers.Count, false));
}
else
{
- var isPrivate = !string.Equals(kind, "infobox", StringComparison.Ordinal);
- containers.Push(isPrivate);
- if (isPrivate) privateDepth++;
- result.Add(new Line(raw, text, LineKind.ContainerOpen, privateDepth > 0, containers.Count, false));
+ containers.Push((true, colons));
+ privateDepth++;
+ result.Add(new Line(text, LineKind.ContainerOpen, true, containers.Count, false));
}
continue;
}
- result.Add(new Line(raw, text, trimmed.Length == 0 ? LineKind.Blank : LineKind.Text, inPrivate, containers.Count, false));
+ result.Add(new Line(text, text.Trim().Length == 0 ? LineKind.Blank : LineKind.Text, inPrivate, containers.Count, false));
+ }
+
+ if (fence is not null && (privateFrom is null || fence.Line < privateFrom)) privateFrom = fence.Line;
+ if (privateFrom is { } from)
+ {
+ for (var i = from; i < result.Count; i++)
+ result[i] = result[i] with { Private = true };
}
return result;
}
- /// A fence line: up to three spaces, then three or more backticks or tildes, then the info string.
- private static bool FenceMarker(string line, out string marker, out string info)
+ /// Strips indentation and list or quote markers (>, *, +, -, 1., 1)); returns the rest and how many characters went.
+ private static (string Content, int Width) StripPrefix(string line)
+ {
+ var i = 0;
+ while (true)
+ {
+ var j = i;
+ while (j < line.Length && (line[j] == ' ' || line[j] == '\t')) j++;
+ if (j < line.Length && line[j] == '>')
+ {
+ i = j + 1;
+ continue;
+ }
+ if (j < line.Length && line[j] is '*' or '+' or '-' && MarkerEnds(line, j + 1))
+ {
+ i = j + 1;
+ continue;
+ }
+ var k = j;
+ while (k < line.Length && k - j < 9 && char.IsAsciiDigit(line[k])) k++;
+ if (k > j && k < line.Length && line[k] is '.' or ')' && MarkerEnds(line, k + 1))
+ {
+ i = k + 1;
+ continue;
+ }
+ return (line[j..], j);
+ }
+
+ static bool MarkerEnds(string line, int at) => at == line.Length || line[at] == ' ' || line[at] == '\t';
+ }
+
+ /// At most three spaces and nothing else before the content: CommonMark's column-0 indentation.
+ private static bool IsPlainIndent(string line, int width) =>
+ width <= 3 && line.AsSpan(0, width).IndexOfAnyExcept(' ') < 0;
+
+ private static int Colons(string rest)
+ {
+ var i = 0;
+ while (i < rest.Length && rest[i] == ':') i++;
+ return i;
+ }
+
+ /// A fence marker at the start of : three or more backticks or tildes, then the info string, which for backticks contains none.
+ private static bool FenceMarker(string rest, out string marker, out string info)
{
marker = string.Empty;
info = string.Empty;
+ if (rest.Length == 0 || (rest[0] != '`' && rest[0] != '~')) return false;
+ var ch = rest[0];
var i = 0;
- while (i < line.Length && i < 3 && line[i] == ' ') i++;
- if (i >= line.Length || (line[i] != '`' && line[i] != '~')) return false;
- var ch = line[i];
- var start = i;
- while (i < line.Length && line[i] == ch) i++;
- if (i - start < 3) return false;
- marker = line[start..i];
- info = line[i..].Trim();
+ while (i < rest.Length && rest[i] == ch) i++;
+ if (i < 3) return false;
+ var tail = rest[i..].Trim();
+ if (ch == '`' && tail.Contains('`')) return false;
+ marker = rest[..i];
+ info = tail;
return true;
}
private static bool IsStructuralLike(string line)
{
- var trimmed = line.TrimStart();
- return trimmed.StartsWith(Close, StringComparison.Ordinal)
- || trimmed.StartsWith("```", StringComparison.Ordinal)
- || trimmed.StartsWith("~~~", StringComparison.Ordinal);
+ var (rest, _) = StripPrefix(line);
+ return rest.StartsWith(Close, StringComparison.Ordinal)
+ || rest.StartsWith("```", StringComparison.Ordinal)
+ || rest.StartsWith("~~~", StringComparison.Ordinal);
}
private static List<(int Start, int End)> Blocks(List lines)
@@ -277,6 +359,7 @@ public static partial class ArticleBody
var line = lines[i];
if (line.Kind is LineKind.Text)
{
+ if (start is not null && lines[start.Value].Private != line.Private) Flush(i);
start ??= i;
continue;
}
diff --git a/src/Wiki.Core/Parsing/ArticleFile.cs b/src/Wiki.Core/Parsing/ArticleFile.cs
index 46449db..17cfd7e 100644
--- a/src/Wiki.Core/Parsing/ArticleFile.cs
+++ b/src/Wiki.Core/Parsing/ArticleFile.cs
@@ -63,6 +63,8 @@ public static class ArticleFile
var slugSubject = document.Slug[(slash + 1)..];
var lang = Required(document, "lang");
+ if (lang is not ("de" or "en"))
+ throw new FormatException($"{document.Path}: unsupported lang '{lang}'.");
if (lang != slugLang)
throw new FormatException($"{document.Path}: lang '{lang}' does not match the path.");
var subject = Required(document, "subject");
diff --git a/test/Wiki.Core.Tests/ArticleBodyTests.cs b/test/Wiki.Core.Tests/ArticleBodyTests.cs
index e0b7bad..788fa32 100644
--- a/test/Wiki.Core.Tests/ArticleBodyTests.cs
+++ b/test/Wiki.Core.Tests/ArticleBodyTests.cs
@@ -55,6 +55,81 @@ public class ArticleBodyTests
Assert.Contains("B", result);
}
+ [Theory]
+ [InlineData("A\n\n::: private\nsecret\n\n :::\nleaked\n:::")]
+ [InlineData("A\n\n::: private\nsecret\n\t:::\nleaked\n:::")]
+ [InlineData("A\n\n::: private\nsecret\n:::x\nleaked\n:::")]
+ [InlineData(":::: private\nsecret\n:::\nleaked\n::::")]
+ public void RedactPrivate_FailsClosedOnLooseClosers(string body)
+ {
+ var result = ArticleBody.RedactPrivate(body);
+ Assert.DoesNotContain("secret", result);
+ Assert.DoesNotContain("leaked", result);
+ }
+
+ [Fact]
+ public void RedactPrivate_IndentedCloserWithinThreeSpaces_Closes()
+ {
+ var result = ArticleBody.RedactPrivate("::: private\nsecret\n :::\n\nB");
+ Assert.DoesNotContain("secret", result);
+ Assert.EndsWith("\nB", result);
+ }
+
+ [Fact]
+ public void RedactPrivate_FailsClosedOnBacktickInfoWithBacktick()
+ {
+ var result = ArticleBody.RedactPrivate("```ls -la```\n\n::: private\nsecret\n:::");
+ Assert.DoesNotContain("secret", result);
+ Assert.StartsWith("```ls -la```", result);
+ }
+
+ [Theory]
+ [InlineData("- item\n ```\n code\n::: private\nsecret\n:::\n```")]
+ [InlineData("::: private\n- ```\n :::\n secret\n ```\n:::\n\nB")]
+ [InlineData("::: private\n ```\n```\n:::\nsecret\n```\n::: private\nsecret\n:::")]
+ public void RedactPrivate_FailsClosedOnFencesInListItems(string body)
+ {
+ Assert.DoesNotContain("secret", ArticleBody.RedactPrivate(body));
+ }
+
+ [Theory]
+ [InlineData("A\n\n- ::: private\n secret\n :::")]
+ [InlineData("A\n\n> ::: private\n> secret\n> :::")]
+ [InlineData("A\n\n1. ::: private\n secret\n :::")]
+ [InlineData("A\n\n2) ::: private\n secret\n :::")]
+ [InlineData("A\n\n * > ::: private\n > secret")]
+ [InlineData("A\n\n ::: infobox\nsecret\n:::")]
+ public void RedactPrivate_FailsClosedOnContainersInListsAndQuotes(string body)
+ {
+ var result = ArticleBody.RedactPrivate(body);
+ Assert.DoesNotContain("secret", result);
+ Assert.StartsWith("A\n", result);
+ }
+
+ [Theory]
+ [InlineData("```\ncode\n\n::: private\nsecret\n:::")]
+ [InlineData("A\n\n- x\n ```\n code\n\n ::: private\n secret")]
+ public void RedactPrivate_FailsClosedOnUnterminatedFence(string body)
+ {
+ var result = ArticleBody.RedactPrivate(body);
+ Assert.DoesNotContain("secret", result);
+ Assert.Contains(ArticleBody.PrivateLocked, result);
+ }
+
+ [Fact]
+ public void RedactPrivate_FailsClosedOnLoneCarriageReturns()
+ {
+ var result = ArticleBody.RedactPrivate("A\r::: private\rsecret\r:::");
+ Assert.Equal("A\n::: private-locked\n:::", result);
+ }
+
+ [Fact]
+ public void Reassemble_UnitWithMarkedContainerLine_Throws()
+ {
+ Assert.Throws(() => ArticleBody.Reassemble(WithPrivate, ["A", "x\n- :::\nleak", "B"]));
+ Assert.Throws(() => ArticleBody.Reassemble(WithPrivate, ["A", "x\n> :::\nleak", "B"]));
+ }
+
[Fact]
public void PublicBlocks_SkipsPrivateAndNumbersSequentially()
{
diff --git a/test/Wiki.Core.Tests/ArticleFileTests.cs b/test/Wiki.Core.Tests/ArticleFileTests.cs
index bf61d87..640b35a 100644
--- a/test/Wiki.Core.Tests/ArticleFileTests.cs
+++ b/test/Wiki.Core.Tests/ArticleFileTests.cs
@@ -55,6 +55,13 @@ public class ArticleFileTests
Assert.Throws(() => ArticleFile.Parse(Load(text)));
}
+ [Fact]
+ public void Parse_UnsupportedLang_Throws()
+ {
+ var text = ArticleFile.Build(Sample).Replace("lang: de\n", "lang: fr\n");
+ Assert.Throws(() => ArticleFile.Parse(Load(text, "fr/wall-switches")));
+ }
+
[Fact]
public void Parse_NonArticleDocument_Throws()
{