fix(review-bot): the agent states its verdict, the bot no longer guesses it from headings
A Blocking section that said 'none' but held a sentence turned the review into REQUEST_CHANGES. The agent now ends its report with 'Verdict: request-changes|comment|approve'; no verdict means a plain comment, and an approval is downgraded when the report still holds red, yellow or unverified sections. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -149,7 +149,6 @@ type output struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
Info json.RawMessage `json:"info"`
|
||||
}
|
||||
|
||||
type conversation struct {
|
||||
|
||||
@@ -89,6 +89,13 @@ the first heading. Structure it exactly as below. Omit a section entirely when i
|
||||
## Recommendation
|
||||
|
||||
<One line: "Request changes — <n> blocking issue(s)" or "Looks reasonable — no blocking issues; comments above are optional.">
|
||||
|
||||
Verdict: <request-changes | comment | approve>
|
||||
```
|
||||
|
||||
The last line of your answer is the verdict, exactly as written above, and nothing follows it. It is what the
|
||||
bot acts on, so it must agree with your report: `request-changes` when the Blocking section lists at least one
|
||||
verified defect, `approve` only when you have no blocking issue, suggestion or unverified point, and `comment`
|
||||
otherwise.
|
||||
|
||||
Be specific and concise. Quote only the minimal lines that carry a point.
|
||||
|
||||
+28
-12
@@ -58,17 +58,33 @@ func hasHeading(report, prefix string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// reviewEvent derives the Gitea review event from the sections the report contains: a red
|
||||
// (Blocking) section requests changes; a non-empty change with no red, yellow or unverified
|
||||
// section and a recommendation is approved; everything else is a comment.
|
||||
func reviewEvent(report string, nonEmptyChange bool) string {
|
||||
switch {
|
||||
case hasHeading(report, "## 🔴"):
|
||||
return "REQUEST_CHANGES"
|
||||
case nonEmptyChange && !hasHeading(report, "## 🟡") && !hasHeading(report, "## ⚪") &&
|
||||
hasHeading(report, "## Recommendation"):
|
||||
return "APPROVE"
|
||||
default:
|
||||
return "COMMENT"
|
||||
var verdictLine = regexp.MustCompile(`(?i)^verdict:\s*(request[-_ ]changes|comment|approve)\s*$`)
|
||||
|
||||
// splitVerdict takes the agent's explicit verdict, its last non-empty line "Verdict: <value>",
|
||||
// off the answer. The verdict is "" when the line is missing, which later reads as a plain comment.
|
||||
func splitVerdict(answer string) (rest, verdict string) {
|
||||
lines := strings.Split(strings.TrimRight(strings.ReplaceAll(answer, "\r\n", "\n"), " \t\n"), "\n")
|
||||
if len(lines) == 0 {
|
||||
return answer, ""
|
||||
}
|
||||
m := verdictLine.FindStringSubmatch(strings.TrimSpace(lines[len(lines)-1]))
|
||||
if m == nil {
|
||||
return answer, ""
|
||||
}
|
||||
return strings.Join(lines[:len(lines)-1], "\n"), strings.ToLower(strings.NewReplacer("_", "-", " ", "-").Replace(m[1]))
|
||||
}
|
||||
|
||||
// reviewEvent maps the agent's verdict to the Gitea review event. Changes are requested and
|
||||
// comments posted as the agent said; an approval is only honoured for a non-empty change whose
|
||||
// report holds no yellow or unverified section, and anything else is a comment.
|
||||
func reviewEvent(verdict, report string, nonEmptyChange bool) string {
|
||||
switch verdict {
|
||||
case "request-changes":
|
||||
return "REQUEST_CHANGES"
|
||||
case "approve":
|
||||
if nonEmptyChange && !hasHeading(report, "## 🔴") && !hasHeading(report, "## 🟡") && !hasHeading(report, "## ⚪") {
|
||||
return "APPROVE"
|
||||
}
|
||||
}
|
||||
return "COMMENT"
|
||||
}
|
||||
|
||||
+52
-24
@@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCleanReport(t *testing.T) {
|
||||
cases := []struct{ name, in, want string }{
|
||||
@@ -28,29 +31,6 @@ func TestCleanReport(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewEvent(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, report string
|
||||
nonEmpty bool
|
||||
want string
|
||||
}{
|
||||
{"red requests changes", "## Summary\n\nx\n\n## 🔴 Blocking\n\n- y\n\n## Recommendation\n\nz", true, "REQUEST_CHANGES"},
|
||||
{"nothing to raise approves", "## Summary\n\nx\n\n## Recommendation\n\nz", true, "APPROVE"},
|
||||
{"nits alone still approve", "## Summary\n\nx\n\n## 🟢 Nits\n\n- a\n\n## Recommendation\n\nz", true, "APPROVE"},
|
||||
{"yellow comments", "## Summary\n\nx\n\n## 🟡 Suggestions\n\n- a\n\n## Recommendation\n\nz", true, "COMMENT"},
|
||||
{"unverified comments", "## Summary\n\nx\n\n## ⚪ Unverified\n\n- a\n\n## Recommendation\n\nz", true, "COMMENT"},
|
||||
{"no recommendation never approves", "## Summary\n\nx", true, "COMMENT"},
|
||||
{"an empty change never approves", "## Summary\n\nx\n\n## Recommendation\n\nz", false, "COMMENT"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := reviewEvent(c.report, c.nonEmpty); got != c.want {
|
||||
t.Errorf("got %s, want %s", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEligible(t *testing.T) {
|
||||
pr := func(state, title string, draft bool, sha string) PR {
|
||||
p := PR{State: state, Title: title, Draft: draft}
|
||||
@@ -132,3 +112,51 @@ func TestParseAnswer(t *testing.T) {
|
||||
t.Errorf("an answer without outputs is empty, got %+v, %v", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitVerdict(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, in, wantRest, wantVerdict string
|
||||
}{
|
||||
{"request changes", "## A\n\nx\n\nVerdict: request-changes\n", "## A\n\nx\n", "request-changes"},
|
||||
{"spelled with a space, any case", "## A\n\nVERDICT: Request Changes", "## A\n", "request-changes"},
|
||||
{"approve", "## A\n\nx\nVerdict: approve", "## A\n\nx", "approve"},
|
||||
{"missing", "## A\n\nx", "## A\n\nx", ""},
|
||||
{"not the last line", "Verdict: approve\n\n## A\n\nx", "Verdict: approve\n\n## A\n\nx", ""},
|
||||
{"unknown value", "## A\n\nVerdict: maybe", "## A\n\nVerdict: maybe", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
rest, verdict := splitVerdict(c.in)
|
||||
if strings.TrimSpace(rest) != strings.TrimSpace(c.wantRest) || verdict != c.wantVerdict {
|
||||
t.Errorf("got %q, %q; want %q, %q", rest, verdict, c.wantRest, c.wantVerdict)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewEvent(t *testing.T) {
|
||||
const clean = "## Summary\n\nx\n\n## Recommendation\n\nz"
|
||||
const yellow = "## Summary\n\nx\n\n## 🟡 Suggestions\n\n- a\n\n## Recommendation\n\nz"
|
||||
const red = "## Summary\n\nx\n\n## 🔴 Blocking\n\nNone at this head; an earlier finding is fixed.\n\n## Recommendation\n\nz"
|
||||
cases := []struct {
|
||||
name, verdict, report string
|
||||
nonEmpty bool
|
||||
want string
|
||||
}{
|
||||
{"the agent requests changes", "request-changes", clean, true, "REQUEST_CHANGES"},
|
||||
{"a red heading alone does not request changes", "comment", red, true, "COMMENT"},
|
||||
{"approve", "approve", clean, true, "APPROVE"},
|
||||
{"approve with a yellow section is only a comment", "approve", yellow, true, "COMMENT"},
|
||||
{"approve with a red section is only a comment", "approve", red, true, "COMMENT"},
|
||||
{"an empty change never approves", "approve", clean, false, "COMMENT"},
|
||||
{"comment", "comment", yellow, true, "COMMENT"},
|
||||
{"no verdict is a comment", "", clean, true, "COMMENT"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := reviewEvent(c.verdict, c.report, c.nonEmpty); got != c.want {
|
||||
t.Errorf("got %s, want %s", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,7 +74,8 @@ func (r *Reviewer) Review(ctx context.Context, slug string, n int) error {
|
||||
r.metrics.Inc("reviews_total", "result", "violation")
|
||||
return fmt.Errorf("the agent called tools that are not read-only (%s); nothing is posted", strings.Join(ans.Violations, ", "))
|
||||
}
|
||||
report := cleanReport(ans.Text)
|
||||
text, verdict := splitVerdict(ans.Text)
|
||||
report := cleanReport(text)
|
||||
if report == "" {
|
||||
r.metrics.Inc("reviews_total", "result", "failed")
|
||||
return errors.New("the agent's answer holds no report")
|
||||
@@ -95,7 +96,7 @@ func (r *Reviewer) Review(ctx context.Context, slug string, n int) error {
|
||||
}
|
||||
}
|
||||
|
||||
event := reviewEvent(report, pr.Additions+pr.Deletions+pr.ChangedFiles > 0)
|
||||
event := reviewEvent(verdict, report, pr.Additions+pr.Deletions+pr.ChangedFiles > 0)
|
||||
body := fmt.Sprintf("🤖 **Automated review** — model `%s`, commit `%s`.\n\n"+
|
||||
"> Automated review by the `review-bot`: it approves only when it found nothing red or yellow, "+
|
||||
"and a human still merges. Push new commits to get a fresh review.\n\n---\n\n%s\n\n<!-- %s -->\n",
|
||||
|
||||
@@ -43,7 +43,7 @@ type fixture struct {
|
||||
|
||||
const goodAnswer = `{"outputs":[
|
||||
{"type":"tool.execution","name":"conn_get_file_contents"},
|
||||
{"type":"message.output","content":[{"type":"thinking","thinking":"x"},{"type":"text","text":"I looked.\n\n## Summary\n\nAdds a field.\n\n## 🔴 Blocking\n\n- a.yaml:3 is wrong.\n\n## 🟡 Suggestions\n\nNone.\n\n## Recommendation\n\nRequest changes — 1 blocking issue(s)"}]}]}`
|
||||
{"type":"message.output","content":[{"type":"thinking","thinking":"x"},{"type":"text","text":"I looked.\n\n## Summary\n\nAdds a field.\n\n## 🔴 Blocking\n\n- a.yaml:3 is wrong.\n\n## 🟡 Suggestions\n\nNone.\n\n## Recommendation\n\nRequest changes — 1 blocking issue(s)\n\nVerdict: request-changes"}]}]}`
|
||||
|
||||
func newFixture(t *testing.T) *fixture {
|
||||
f := &fixture{t: t, prState: "open", head: "abc123", reviews: "[]", answer: goodAnswer, agents: "[]"}
|
||||
|
||||
Reference in New Issue
Block a user