fix(review-bot): the agent states its verdict, the bot no longer guesses it from headings
ci/woodpecker/pr/ci-scripts-test Pipeline was successful
ci/woodpecker/pr/review-bot-test Pipeline was successful

A Blocking section that said 'none' but held a sentence turned the review into
REQUEST_CHANGES. The agent now ends its report with 'Verdict: request-changes|comment|approve';
no verdict means a plain comment, and an approval is downgraded when the report still holds
red, yellow or unverified sections.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-11 22:21:38 +02:00
co-authored by Claude Sonnet 5.5
parent aed7f58b3f
commit cdba8b6257
6 changed files with 91 additions and 40 deletions
-1
View File
@@ -149,7 +149,6 @@ type output struct {
Type string `json:"type"`
Name string `json:"name"`
Content json.RawMessage `json:"content"`
Info json.RawMessage `json:"info"`
}
type conversation struct {
+7
View File
@@ -89,6 +89,13 @@ the first heading. Structure it exactly as below. Omit a section entirely when i
## Recommendation
<One line: "Request changes — <n> blocking issue(s)" or "Looks reasonable — no blocking issues; comments above are optional.">
Verdict: <request-changes | comment | approve>
```
The last line of your answer is the verdict, exactly as written above, and nothing follows it. It is what the
bot acts on, so it must agree with your report: `request-changes` when the Blocking section lists at least one
verified defect, `approve` only when you have no blocking issue, suggestion or unverified point, and `comment`
otherwise.
Be specific and concise. Quote only the minimal lines that carry a point.
+28 -12
View File
@@ -58,17 +58,33 @@ func hasHeading(report, prefix string) bool {
return false
}
// reviewEvent derives the Gitea review event from the sections the report contains: a red
// (Blocking) section requests changes; a non-empty change with no red, yellow or unverified
// section and a recommendation is approved; everything else is a comment.
func reviewEvent(report string, nonEmptyChange bool) string {
switch {
case hasHeading(report, "## 🔴"):
return "REQUEST_CHANGES"
case nonEmptyChange && !hasHeading(report, "## 🟡") && !hasHeading(report, "## ⚪") &&
hasHeading(report, "## Recommendation"):
return "APPROVE"
default:
return "COMMENT"
var verdictLine = regexp.MustCompile(`(?i)^verdict:\s*(request[-_ ]changes|comment|approve)\s*$`)
// splitVerdict takes the agent's explicit verdict, its last non-empty line "Verdict: <value>",
// off the answer. The verdict is "" when the line is missing, which later reads as a plain comment.
func splitVerdict(answer string) (rest, verdict string) {
lines := strings.Split(strings.TrimRight(strings.ReplaceAll(answer, "\r\n", "\n"), " \t\n"), "\n")
if len(lines) == 0 {
return answer, ""
}
m := verdictLine.FindStringSubmatch(strings.TrimSpace(lines[len(lines)-1]))
if m == nil {
return answer, ""
}
return strings.Join(lines[:len(lines)-1], "\n"), strings.ToLower(strings.NewReplacer("_", "-", " ", "-").Replace(m[1]))
}
// reviewEvent maps the agent's verdict to the Gitea review event. Changes are requested and
// comments posted as the agent said; an approval is only honoured for a non-empty change whose
// report holds no yellow or unverified section, and anything else is a comment.
func reviewEvent(verdict, report string, nonEmptyChange bool) string {
switch verdict {
case "request-changes":
return "REQUEST_CHANGES"
case "approve":
if nonEmptyChange && !hasHeading(report, "## 🔴") && !hasHeading(report, "## 🟡") && !hasHeading(report, "## ⚪") {
return "APPROVE"
}
}
return "COMMENT"
}
+52 -24
View File
@@ -1,6 +1,9 @@
package main
import "testing"
import (
"strings"
"testing"
)
func TestCleanReport(t *testing.T) {
cases := []struct{ name, in, want string }{
@@ -28,29 +31,6 @@ func TestCleanReport(t *testing.T) {
}
}
func TestReviewEvent(t *testing.T) {
cases := []struct {
name, report string
nonEmpty bool
want string
}{
{"red requests changes", "## Summary\n\nx\n\n## 🔴 Blocking\n\n- y\n\n## Recommendation\n\nz", true, "REQUEST_CHANGES"},
{"nothing to raise approves", "## Summary\n\nx\n\n## Recommendation\n\nz", true, "APPROVE"},
{"nits alone still approve", "## Summary\n\nx\n\n## 🟢 Nits\n\n- a\n\n## Recommendation\n\nz", true, "APPROVE"},
{"yellow comments", "## Summary\n\nx\n\n## 🟡 Suggestions\n\n- a\n\n## Recommendation\n\nz", true, "COMMENT"},
{"unverified comments", "## Summary\n\nx\n\n## ⚪ Unverified\n\n- a\n\n## Recommendation\n\nz", true, "COMMENT"},
{"no recommendation never approves", "## Summary\n\nx", true, "COMMENT"},
{"an empty change never approves", "## Summary\n\nx\n\n## Recommendation\n\nz", false, "COMMENT"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := reviewEvent(c.report, c.nonEmpty); got != c.want {
t.Errorf("got %s, want %s", got, c.want)
}
})
}
}
func TestEligible(t *testing.T) {
pr := func(state, title string, draft bool, sha string) PR {
p := PR{State: state, Title: title, Draft: draft}
@@ -132,3 +112,51 @@ func TestParseAnswer(t *testing.T) {
t.Errorf("an answer without outputs is empty, got %+v, %v", a, err)
}
}
func TestSplitVerdict(t *testing.T) {
cases := []struct {
name, in, wantRest, wantVerdict string
}{
{"request changes", "## A\n\nx\n\nVerdict: request-changes\n", "## A\n\nx\n", "request-changes"},
{"spelled with a space, any case", "## A\n\nVERDICT: Request Changes", "## A\n", "request-changes"},
{"approve", "## A\n\nx\nVerdict: approve", "## A\n\nx", "approve"},
{"missing", "## A\n\nx", "## A\n\nx", ""},
{"not the last line", "Verdict: approve\n\n## A\n\nx", "Verdict: approve\n\n## A\n\nx", ""},
{"unknown value", "## A\n\nVerdict: maybe", "## A\n\nVerdict: maybe", ""},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
rest, verdict := splitVerdict(c.in)
if strings.TrimSpace(rest) != strings.TrimSpace(c.wantRest) || verdict != c.wantVerdict {
t.Errorf("got %q, %q; want %q, %q", rest, verdict, c.wantRest, c.wantVerdict)
}
})
}
}
func TestReviewEvent(t *testing.T) {
const clean = "## Summary\n\nx\n\n## Recommendation\n\nz"
const yellow = "## Summary\n\nx\n\n## 🟡 Suggestions\n\n- a\n\n## Recommendation\n\nz"
const red = "## Summary\n\nx\n\n## 🔴 Blocking\n\nNone at this head; an earlier finding is fixed.\n\n## Recommendation\n\nz"
cases := []struct {
name, verdict, report string
nonEmpty bool
want string
}{
{"the agent requests changes", "request-changes", clean, true, "REQUEST_CHANGES"},
{"a red heading alone does not request changes", "comment", red, true, "COMMENT"},
{"approve", "approve", clean, true, "APPROVE"},
{"approve with a yellow section is only a comment", "approve", yellow, true, "COMMENT"},
{"approve with a red section is only a comment", "approve", red, true, "COMMENT"},
{"an empty change never approves", "approve", clean, false, "COMMENT"},
{"comment", "comment", yellow, true, "COMMENT"},
{"no verdict is a comment", "", clean, true, "COMMENT"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := reviewEvent(c.verdict, c.report, c.nonEmpty); got != c.want {
t.Errorf("got %s, want %s", got, c.want)
}
})
}
}
+3 -2
View File
@@ -74,7 +74,8 @@ func (r *Reviewer) Review(ctx context.Context, slug string, n int) error {
r.metrics.Inc("reviews_total", "result", "violation")
return fmt.Errorf("the agent called tools that are not read-only (%s); nothing is posted", strings.Join(ans.Violations, ", "))
}
report := cleanReport(ans.Text)
text, verdict := splitVerdict(ans.Text)
report := cleanReport(text)
if report == "" {
r.metrics.Inc("reviews_total", "result", "failed")
return errors.New("the agent's answer holds no report")
@@ -95,7 +96,7 @@ func (r *Reviewer) Review(ctx context.Context, slug string, n int) error {
}
}
event := reviewEvent(report, pr.Additions+pr.Deletions+pr.ChangedFiles > 0)
event := reviewEvent(verdict, report, pr.Additions+pr.Deletions+pr.ChangedFiles > 0)
body := fmt.Sprintf("🤖 **Automated review** — model `%s`, commit `%s`.\n\n"+
"> Automated review by the `review-bot`: it approves only when it found nothing red or yellow, "+
"and a human still merges. Push new commits to get a fresh review.\n\n---\n\n%s\n\n<!-- %s -->\n",
+1 -1
View File
@@ -43,7 +43,7 @@ type fixture struct {
const goodAnswer = `{"outputs":[
{"type":"tool.execution","name":"conn_get_file_contents"},
{"type":"message.output","content":[{"type":"thinking","thinking":"x"},{"type":"text","text":"I looked.\n\n## Summary\n\nAdds a field.\n\n## 🔴 Blocking\n\n- a.yaml:3 is wrong.\n\n## 🟡 Suggestions\n\nNone.\n\n## Recommendation\n\nRequest changes — 1 blocking issue(s)"}]}]}`
{"type":"message.output","content":[{"type":"thinking","thinking":"x"},{"type":"text","text":"I looked.\n\n## Summary\n\nAdds a field.\n\n## 🔴 Blocking\n\n- a.yaml:3 is wrong.\n\n## 🟡 Suggestions\n\nNone.\n\n## Recommendation\n\nRequest changes — 1 blocking issue(s)\n\nVerdict: request-changes"}]}]}`
func newFixture(t *testing.T) *fixture {
f := &fixture{t: t, prState: "open", head: "abc123", reviews: "[]", answer: goodAnswer, agents: "[]"}