Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a14bd5a85a |
@@ -45,10 +45,9 @@ spec:
|
||||
# The first model is the default; a backend without its keys is off.
|
||||
- name: MODELS
|
||||
value: |
|
||||
mistral:mistral-large-4-0 = Mistral Large 4. Solid on routine changes of small to medium size.
|
||||
anthropic:claude-haiku-5-5 = Claude Haiku 5.5. Cheapest and fastest; only for trivial changes such as documentation, comments or version bumps.
|
||||
anthropic:claude-sonnet-5-5 = Claude Sonnet 5.5. Fast and capable; good for most code and configuration changes.
|
||||
anthropic:claude-opus-5-5 = Claude Opus 5.5. The strongest reviewer; for large or cross-cutting changes and anything touching security, secrets, network policy or data.
|
||||
anthropic:claude-sonnet-5-5 = Claude Sonnet 5.5. The standard reviewer; right for almost every code and configuration change.
|
||||
anthropic:claude-opus-5-5 = Claude Opus 5.5. Expensive; only for large cross-cutting changes or changes to security, secrets, network policy or data.
|
||||
mistral:mistral-large-4-0 = Mistral Large 4. Separate budget; for routine changes of small to medium size.
|
||||
- name: SELECTOR
|
||||
value: typesafe
|
||||
- name: ANTHROPIC_API_KEY
|
||||
|
||||
@@ -91,6 +91,10 @@ func (a *Anthropic) Ask(ctx context.Context, model, task string) (Answer, error)
|
||||
if err := stream.Err(); err != nil {
|
||||
return Answer{}, anthropicError(err)
|
||||
}
|
||||
ans.Usage.Input += msg.Usage.InputTokens
|
||||
ans.Usage.Output += msg.Usage.OutputTokens
|
||||
ans.Usage.CacheRead += msg.Usage.CacheReadInputTokens
|
||||
ans.Usage.CacheWrite += msg.Usage.CacheCreationInputTokens
|
||||
for _, block := range msg.Content {
|
||||
switch b := block.AsAny().(type) {
|
||||
case anthropic.BetaMCPToolUseBlock:
|
||||
|
||||
@@ -114,6 +114,9 @@ func TestAnthropicResumesAPausedTurnAndTakesTheFinalText(t *testing.T) {
|
||||
if a.Text != "## Summary\n\nok\n\nVerdict: comment" || a.ToolCalls != 2 || len(a.Violations) != 0 {
|
||||
t.Errorf("answer %+v", a)
|
||||
}
|
||||
if a.Usage.Input != 2 || a.Usage.Output != 10 {
|
||||
t.Errorf("usage must add up over the continued turns: %+v", a.Usage)
|
||||
}
|
||||
if len(f.bodies) != 2 {
|
||||
t.Fatalf("requests %d, want 2", len(f.bodies))
|
||||
}
|
||||
@@ -164,8 +167,8 @@ func TestAnthropicRateLimitIsRetryable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// With the selector on, the review runs on the model TypeSafe picks; when TypeSafe fails, on the
|
||||
// default model.
|
||||
// With the selector on, the review runs on the model TypeSafe picks; when TypeSafe fails or is
|
||||
// unsure, on the default model.
|
||||
func TestSelectorPicksTheBackend(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -175,6 +178,8 @@ func TestSelectorPicksTheBackend(t *testing.T) {
|
||||
}{
|
||||
{"chosen", `{"answers":{"model":{"type":"choice","choice":"anthropic:claude-opus-5-5","confidence":0.8,` +
|
||||
`"probabilities":{"anthropic:claude-opus-5-5":0.9,"mistral:mistral-large-4-0":0.1}}}}`, 200, "claude-opus-5-5"},
|
||||
{"unsure", `{"answers":{"model":{"type":"choice","choice":"anthropic:claude-opus-5-5","confidence":0.4,` +
|
||||
`"probabilities":{"anthropic:claude-opus-5-5":0.7,"mistral:mistral-large-4-0":0.3}}}}`, 200, "mistral-large-4-0"},
|
||||
{"failed", `{"error":"overloaded"}`, 529, "mistral-large-4-0"},
|
||||
{"unknown model", `{"answers":{"model":{"type":"choice","choice":"x:y"}}}`, 200, "mistral-large-4-0"},
|
||||
} {
|
||||
@@ -202,6 +207,7 @@ func TestSelectorPicksTheBackend(t *testing.T) {
|
||||
{Backend: "anthropic", Name: "claude-opus-5-5", Description: "strong"}}
|
||||
f.rev.backends["anthropic"] = fa.backend()
|
||||
f.rev.setReady("anthropic", true)
|
||||
f.rev.cfg.SelectorMinConfidence = 0.6
|
||||
f.rev.selector = &TypeSafe{url: ts.URL, key: "ts-key", model: "jev-latest", http: &http.Client{Timeout: 5 * time.Second}}
|
||||
|
||||
if err := f.rev.Review(context.Background(), "konairius/homelab", 7); err != nil {
|
||||
|
||||
@@ -28,6 +28,13 @@ type Answer struct {
|
||||
Text string
|
||||
ToolCalls int
|
||||
Violations []string
|
||||
Usage Usage
|
||||
}
|
||||
|
||||
// Usage is the token count of one review, summed over its requests. A backend that does not
|
||||
// report it leaves it zero.
|
||||
type Usage struct {
|
||||
Input, Output, CacheRead, CacheWrite int64
|
||||
}
|
||||
|
||||
// apiError is a non-2xx answer of a provider API. Whatever limit headers come back are kept so
|
||||
|
||||
+26
-11
@@ -13,7 +13,7 @@ var defaultMCPTools = []string{
|
||||
"get_commit", "get_dir_contents", "get_file_contents", "get_repository_tree", "get_tag", "get_release",
|
||||
"get_latest_release", "issue_read", "label_read", "list_branches", "list_commits", "list_issues",
|
||||
"list_pull_requests", "list_releases", "list_tags", "milestone_read", "pull_request_read", "search_issues",
|
||||
"search_repos",
|
||||
"search_repos", "compare_commits",
|
||||
}
|
||||
|
||||
// Config is read from the environment. A secret comes from NAME, or from the file named by
|
||||
@@ -40,11 +40,13 @@ type Config struct {
|
||||
MCPToken string // read-only Gitea token Anthropic uses on the MCP server
|
||||
MCPTools []string
|
||||
|
||||
// Selector "typesafe" picks the model per pull request; empty uses the default model.
|
||||
Selector string
|
||||
TypeSafeURL string
|
||||
TypeSafeKey string
|
||||
TypeSafeModel string
|
||||
// Selector "typesafe" picks the model per pull request; empty uses the default model. A pick
|
||||
// other than the default below SelectorMinConfidence falls back to the default.
|
||||
Selector string
|
||||
SelectorMinConfidence float64
|
||||
TypeSafeURL string
|
||||
TypeSafeKey string
|
||||
TypeSafeModel string
|
||||
|
||||
Repos []string
|
||||
Workers int
|
||||
@@ -110,6 +112,18 @@ func loadConfig(getenv func(string) string, readFile func(string) ([]byte, error
|
||||
}
|
||||
return d
|
||||
}
|
||||
fraction := func(name string, def float64) float64 {
|
||||
v := text(name, "")
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
f, err := strconv.ParseFloat(v, 64)
|
||||
if err != nil || f < 0 || f > 1 {
|
||||
errs = append(errs, fmt.Errorf("%s must be a number from 0 to 1, got %q", name, v))
|
||||
return def
|
||||
}
|
||||
return f
|
||||
}
|
||||
list := func(name string, def []string) []string {
|
||||
v := text(name, "")
|
||||
if v == "" {
|
||||
@@ -132,15 +146,16 @@ func loadConfig(getenv func(string) string, readFile func(string) ([]byte, error
|
||||
|
||||
AnthropicURL: strings.TrimRight(text("ANTHROPIC_URL", ""), "/"),
|
||||
AnthropicKey: optional("ANTHROPIC_API_KEY"),
|
||||
AnthropicEffort: text("ANTHROPIC_EFFORT", "high"),
|
||||
AnthropicEffort: text("ANTHROPIC_EFFORT", "medium"),
|
||||
MCPURL: text("MCP_URL", "https://gitea-mcp.renner.uno/mcp"),
|
||||
MCPToken: optional("MCP_TOKEN"),
|
||||
MCPTools: list("MCP_TOOLS", defaultMCPTools),
|
||||
|
||||
Selector: text("SELECTOR", ""),
|
||||
TypeSafeURL: text("TYPESAFE_URL", "https://api.typesafe.ai/v1/systemone"),
|
||||
TypeSafeKey: optional("TYPESAFE_API_KEY"),
|
||||
TypeSafeModel: text("TYPESAFE_MODEL", "jev-latest"),
|
||||
Selector: text("SELECTOR", ""),
|
||||
SelectorMinConfidence: fraction("SELECTOR_MIN_CONFIDENCE", 0.6),
|
||||
TypeSafeURL: text("TYPESAFE_URL", "https://api.typesafe.ai/v1/systemone"),
|
||||
TypeSafeKey: optional("TYPESAFE_API_KEY"),
|
||||
TypeSafeModel: text("TYPESAFE_MODEL", "jev-latest"),
|
||||
|
||||
Workers: number("WORKERS", 1),
|
||||
QueueSize: number("QUEUE_SIZE", 32),
|
||||
|
||||
@@ -25,7 +25,7 @@ func TestLoadConfigDefaults(t *testing.T) {
|
||||
}
|
||||
if c.Listen != ":8080" || c.GiteaURL != "https://gitea.renner.uno" || c.BotLogin != "review-bot" ||
|
||||
c.Workers != 1 || c.Sweep != 30*time.Minute || c.RateLimitRetries != 3 || c.RateLimitWait != time.Minute ||
|
||||
c.Selector != "" || c.MCPURL != "https://gitea-mcp.renner.uno/mcp" || c.AnthropicEffort != "high" {
|
||||
c.Selector != "" || c.MCPURL != "https://gitea-mcp.renner.uno/mcp" || c.AnthropicEffort != "medium" || c.SelectorMinConfidence != 0.6 {
|
||||
t.Errorf("unexpected defaults: %+v", c)
|
||||
}
|
||||
if len(c.Models) != 1 || c.Models[0].ID() != "mistral:mistral-large-4-0" {
|
||||
|
||||
+20
-2
@@ -24,6 +24,20 @@ type PR struct {
|
||||
Head struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"head"`
|
||||
RequestedReviewers []struct {
|
||||
Login string `json:"login"`
|
||||
} `json:"requested_reviewers"`
|
||||
}
|
||||
|
||||
// requested says whether login is a requested reviewer. Gitea drops the request when that
|
||||
// reviewer submits a review.
|
||||
func (p PR) requested(login string) bool {
|
||||
for _, u := range p.RequestedReviewers {
|
||||
if u.Login == login {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Review struct {
|
||||
@@ -119,7 +133,9 @@ var wipTitle = regexp.MustCompile(`^[Ww][Ii][Pp][: ]`)
|
||||
|
||||
func marker(sha string) string { return "pr-review:sha=" + sha }
|
||||
|
||||
// eligible says why a PR needs no review ("closed", "draft", "wip", "reviewed") or "ok".
|
||||
// eligible says why a PR needs no review ("closed", "draft", "wip", "reviewed", "not requested")
|
||||
// or "ok". The bot reviews only while it is a requested reviewer; a request at an already
|
||||
// reviewed head asks for that head again.
|
||||
func eligible(pr PR, reviews []Review, bot string) string {
|
||||
switch {
|
||||
case pr.State != "open":
|
||||
@@ -128,11 +144,13 @@ func eligible(pr PR, reviews []Review, bot string) string {
|
||||
return "draft"
|
||||
case wipTitle.MatchString(pr.Title):
|
||||
return "wip"
|
||||
case pr.requested(bot):
|
||||
return "ok"
|
||||
}
|
||||
if _, ok := headReview(pr, reviews, bot); ok {
|
||||
return "reviewed"
|
||||
}
|
||||
return "ok"
|
||||
return "not requested"
|
||||
}
|
||||
|
||||
// headReview is the bot's last submitted review of the PR's head commit. A pending review is
|
||||
|
||||
@@ -34,10 +34,11 @@ func (m *Metrics) Inc(name string, labels ...string) {
|
||||
m.vals[series(name, labels)]++
|
||||
}
|
||||
|
||||
func (m *Metrics) Add(name string, delta float64) {
|
||||
// Add adds delta to a series; labels are key, value pairs.
|
||||
func (m *Metrics) Add(name string, delta float64, labels ...string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.vals[series(name, nil)] += delta
|
||||
m.vals[series(name, labels)] += delta
|
||||
}
|
||||
|
||||
func (m *Metrics) Set(name string, v float64) {
|
||||
|
||||
+30
-23
@@ -17,24 +17,31 @@ HARD RULES
|
||||
read files at the head sha.
|
||||
|
||||
## How to work
|
||||
1. Read the PR (title, body, author, base, head sha) and its diff, and any issue or comment it links.
|
||||
2. For every changed file, read the full file at the head sha, not only the hunks.
|
||||
3. Read the repo's `CLAUDE.md` or `README` if present, and neighbouring files for conventions (sibling manifests,
|
||||
callers, tests, existing policies, RBAC, naming). Look at recent commits and other PRs when the change depends on
|
||||
history.
|
||||
4. Check each suspicion against what you read before you write it down. A claim that depends on code you did not
|
||||
1. In your first step, read the PR (title, body, author, base, head sha, `mergeable`) together with its reviews
|
||||
and comments. Reviews by `review-bot` carry a marker line `<!-- pr-review:sha=<commit> -->`. If there is one,
|
||||
this is a re-review: follow "Re-reviews" below instead of the rest of this list.
|
||||
2. Read the diff and any issue the PR links to.
|
||||
3. The diff is your main source. Read a whole changed file at the head sha only where the hunks do not show
|
||||
enough to judge the change: the surrounding function, a caller, a schema the hunk depends on. Do not read files
|
||||
whose change the diff already shows in full.
|
||||
4. Read the repo's `CLAUDE.md` or `README`, and neighbouring files (sibling manifests, callers, tests, policies),
|
||||
only when a finding or the change's correctness depends on a convention you have not seen.
|
||||
5. Check each suspicion against what you read before you write it down. A claim that depends on code you did not
|
||||
read, or on the current behaviour of an external product, goes under "Unverified" with what you checked. Your
|
||||
training data is not evidence for how current releases behave.
|
||||
5. Before you write "no other code does X", search for it.
|
||||
6. List the reviews and comments of this pull request. Reviews by `review-bot` carry a marker line
|
||||
`<!-- pr-review:sha=<commit> -->`. If there is one, this is a re-review, and your earlier review is your own
|
||||
earlier judgment: read it first, together with the author's replies, and follow "Re-reviews" below.
|
||||
6. Before you write "no other code does X", search for it.
|
||||
|
||||
## Re-reviews
|
||||
The author may have acted on your earlier review, so judge the pull request against it instead of starting over.
|
||||
- Open the Summary with what changed since the commit in your newest marker (read those commits) and the state of
|
||||
every earlier 🔴 and 🟡 point: fixed, still open, or withdrawn with the reason. An earlier 🔴 that is still open
|
||||
stays in the Blocking section, and the verdict stays `request-changes` until it is fixed.
|
||||
You were asked again, so the author may have acted on your earlier review. Judge the pull request against it
|
||||
instead of starting over, and read only what changed.
|
||||
- Read your newest earlier review and the author's replies after it. Then call `compare_commits` with the sha in
|
||||
your newest marker as base and the head sha as head, and read the diffs of those commits. Do not read the whole
|
||||
pull request diff again, and do not re-read files the new commits do not touch.
|
||||
- If the head sha equals the sha in your newest marker, nothing new was pushed: answer the author's replies and
|
||||
settle the points they dispute, reading only what that needs.
|
||||
- Open the Summary with what changed since that commit and the state of every earlier 🔴 and 🟡 point: fixed,
|
||||
still open, or withdrawn with the reason. An earlier 🔴 that is still open stays in the Blocking section, and the
|
||||
verdict stays `request-changes` until it is fixed.
|
||||
- Review the changes since that commit as carefully as a first review. A new 🔴 on them is fine.
|
||||
- Code and behaviour that were already in the pull request at your last review stay as you judged them. Raise a new
|
||||
point on them only if it is a 🔴 you can verify and you missed it; then say that you missed it. Never add 🟡, ⚪
|
||||
@@ -45,19 +52,19 @@ The author may have acted on your earlier review, so judge the pull request agai
|
||||
- When all earlier 🔴 points are fixed and the new changes have no 🔴, say so in the Recommendation.
|
||||
|
||||
## Mandatory checks
|
||||
Do these for every PR, in addition to reading the diff and the changed files:
|
||||
1. Read the pull request itself for `mergeable`, its base branch and whether the base moved. Read each changed file
|
||||
at the base branch tip as well as at the head, and list recent commits on the base branch that touch those files.
|
||||
State plainly if the change is already on the base branch, superseded by another PR, or conflicts with it. That is
|
||||
a blocking finding.
|
||||
Do these for every PR, in addition to reading the diff:
|
||||
1. Check `mergeable`. Only when it is false, or the PR body or diff suggests the base moved, list recent commits on
|
||||
the base branch that touch the changed files and read those files at the base tip. State plainly if the change is
|
||||
already on the base branch, superseded by another PR, or conflicts with it. That is a blocking finding.
|
||||
2. Read every issue the PR links to.
|
||||
3. Search the repository for other code that does the same thing before you write that nothing else does.
|
||||
|
||||
## Efficiency
|
||||
Request all independent reads in the same step instead of one at a time (the diff, the changed files at head and
|
||||
base, neighbouring files, README or CLAUDE.md together). Do not repeat a call whose result you already have. There is
|
||||
no cap on tool calls; stop reading when every mandatory check is done and every suspicion is settled or listed as
|
||||
unverified.
|
||||
Every tool result stays in the conversation and is paid for again on every later step, so each read costs more the
|
||||
later it comes and the bigger it is. Request all independent reads in the same step instead of one at a time. Do not
|
||||
repeat a call whose result you already have. Aim for about 10 tool calls on a small change and 20 on a large one;
|
||||
go beyond that only to settle a suspected 🔴. Stop reading when the mandatory checks are done and every suspicion is
|
||||
settled or listed as unverified.
|
||||
|
||||
## What to look for, in priority order
|
||||
1. Correctness: real bugs, wrong conditionals, unhandled errors, race conditions, resource leaks, incorrect API
|
||||
|
||||
+17
-10
@@ -32,9 +32,14 @@ func TestCleanReport(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEligible(t *testing.T) {
|
||||
pr := func(state, title string, draft bool, sha string) PR {
|
||||
pr := func(state, title string, draft bool, sha string, requested ...string) PR {
|
||||
p := PR{State: state, Title: title, Draft: draft}
|
||||
p.Head.SHA = sha
|
||||
for _, login := range requested {
|
||||
p.RequestedReviewers = append(p.RequestedReviewers, struct {
|
||||
Login string `json:"login"`
|
||||
}{login})
|
||||
}
|
||||
return p
|
||||
}
|
||||
review := func(login, body string) Review {
|
||||
@@ -51,15 +56,17 @@ func TestEligible(t *testing.T) {
|
||||
pr PR
|
||||
want string
|
||||
}{
|
||||
{"open at a new head", pr("open", "feat: x", false, "ccc333"), "ok"},
|
||||
{"review at an older head does not count", pr("open", "feat: x", false, "ddd444"), "ok"},
|
||||
{"review at this head by the bot", pr("open", "feat: x", false, "aaa111"), "reviewed"},
|
||||
{"marker from another user does not count", pr("open", "feat: x", false, "bbb222"), "ok"},
|
||||
{"closed", pr("closed", "feat: x", false, "ccc333"), "closed"},
|
||||
{"draft", pr("open", "feat: x", true, "ccc333"), "draft"},
|
||||
{"wip with colon", pr("open", "WIP: x", false, "ccc333"), "wip"},
|
||||
{"wip with space", pr("open", "wip x", false, "ccc333"), "wip"},
|
||||
{"title that only starts with wip", pr("open", "wipe the cache", false, "ccc333"), "ok"},
|
||||
{"requested at a new head", pr("open", "feat: x", false, "ccc333", "review-bot"), "ok"},
|
||||
{"requested again at a reviewed head", pr("open", "feat: x", false, "aaa111", "review-bot"), "ok"},
|
||||
{"not requested at a new head", pr("open", "feat: x", false, "ccc333"), "not requested"},
|
||||
{"request for someone else", pr("open", "feat: x", false, "ccc333", "someone"), "not requested"},
|
||||
{"not requested, reviewed at this head", pr("open", "feat: x", false, "aaa111"), "reviewed"},
|
||||
{"marker from another user does not count", pr("open", "feat: x", false, "bbb222"), "not requested"},
|
||||
{"closed", pr("closed", "feat: x", false, "ccc333", "review-bot"), "closed"},
|
||||
{"draft", pr("open", "feat: x", true, "ccc333", "review-bot"), "draft"},
|
||||
{"wip with colon", pr("open", "WIP: x", false, "ccc333", "review-bot"), "wip"},
|
||||
{"wip with space", pr("open", "wip x", false, "ccc333", "review-bot"), "wip"},
|
||||
{"title that only starts with wip", pr("open", "wipe the cache", false, "ccc333", "review-bot"), "ok"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
+15
-2
@@ -77,6 +77,12 @@ func (r *Reviewer) pick(ctx context.Context, slug string, pr PR) (Model, error)
|
||||
r.log.Warn("selector failed, using the default model", "pr", key(slug, pr.Number), "model", def.ID(), "err", err)
|
||||
return def, nil
|
||||
}
|
||||
if c.Model != def.ID() && c.Confidence < r.cfg.SelectorMinConfidence {
|
||||
r.metrics.Inc("selections_total", "result", "unsure")
|
||||
r.log.Info("selector unsure, using the default model", "pr", key(slug, pr.Number), "chosen", c.Model,
|
||||
"confidence", c.Confidence, "model", def.ID(), "probabilities", c.Probabilities)
|
||||
return def, nil
|
||||
}
|
||||
for _, m := range r.cfg.Models {
|
||||
if m.ID() == c.Model && r.isReady(m.Backend) {
|
||||
r.metrics.Inc("selections_total", "result", "chosen")
|
||||
@@ -129,8 +135,15 @@ func (r *Reviewer) Review(ctx context.Context, slug string, n int) error {
|
||||
r.metrics.Inc("reviews_total", "result", "failed")
|
||||
return err
|
||||
}
|
||||
u := ans.Usage
|
||||
r.log.Info("agent answered", "pr", key(slug, n), "model", model.ID(), "tool_calls", ans.ToolCalls,
|
||||
"seconds", int(time.Since(start).Seconds()))
|
||||
"seconds", int(time.Since(start).Seconds()), "input_tokens", u.Input, "output_tokens", u.Output,
|
||||
"cache_read_tokens", u.CacheRead, "cache_write_tokens", u.CacheWrite)
|
||||
for kind, v := range map[string]int64{"input": u.Input, "output": u.Output, "cache_read": u.CacheRead,
|
||||
"cache_write": u.CacheWrite} {
|
||||
r.metrics.Add("tokens_total", float64(v), "model", model.ID(), "kind", kind)
|
||||
}
|
||||
r.metrics.Add("tool_calls_total", float64(ans.ToolCalls), "model", model.ID())
|
||||
if len(ans.Violations) > 0 {
|
||||
r.metrics.Inc("reviews_total", "result", "violation")
|
||||
return fmt.Errorf("the agent called tools that are not read-only (%s); nothing is posted", strings.Join(ans.Violations, ", "))
|
||||
@@ -162,7 +175,7 @@ func (r *Reviewer) Review(ctx context.Context, slug string, n int) error {
|
||||
event := reviewEvent(verdict, report, pr.Additions+pr.Deletions+pr.ChangedFiles > 0)
|
||||
body := fmt.Sprintf("🤖 **Automated review** — model `%s`, commit `%s`.\n\n"+
|
||||
"> Automated review by the `review-bot`: it approves only when it found nothing red or yellow, "+
|
||||
"and a human still merges. Push new commits to get a fresh review.\n\n---\n\n%s\n\n<!-- %s -->\n",
|
||||
"and a human still merges. Request a review from `review-bot` again for a fresh one.\n\n---\n\n%s\n\n<!-- %s -->\n",
|
||||
model.Name, pr.Head.SHA, report, marker(pr.Head.SHA))
|
||||
url, err := r.gitea.PostReview(ctx, slug, n, body, event, pr.Head.SHA)
|
||||
if err != nil {
|
||||
|
||||
+30
-14
@@ -41,6 +41,7 @@ type fixture struct {
|
||||
convCalls int
|
||||
retryAfter string // Retry-After header sent with a simulated 429
|
||||
head string // head commit of PR 7
|
||||
requested bool // the bot is a requested reviewer of PR 7; a posted review clears it
|
||||
pushAfter string // when set, the next conversation answer is followed by a push of this head
|
||||
// breakStreams breaks the next conversation streams off after their first event.
|
||||
breakStreams int
|
||||
@@ -93,12 +94,16 @@ const goodAnswer = `{"outputs":[
|
||||
{"type":"message.output","content":[{"type":"thinking","thinking":"x"},{"type":"text","text":"I looked.\n\n## Summary\n\nAdds a field.\n\n## 🔴 Blocking\n\n- a.yaml:3 is wrong.\n\n## 🟡 Suggestions\n\nNone.\n\n## Recommendation\n\nRequest changes — 1 blocking issue(s)\n\nVerdict: request-changes"}]}]}`
|
||||
|
||||
func newFixture(t *testing.T) *fixture {
|
||||
f := &fixture{t: t, prState: "open", title: "feat: x", head: "abc123", reviews: "[]", answer: goodAnswer, agents: "[]"}
|
||||
f := &fixture{t: t, prState: "open", title: "feat: x", head: "abc123", requested: true, reviews: "[]", answer: goodAnswer, agents: "[]"}
|
||||
pr := func() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
reviewers := "[]"
|
||||
if f.requested {
|
||||
reviewers = `[{"login":"review-bot"}]`
|
||||
}
|
||||
return `{"number":7,"state":"` + f.prState + `","title":"` + f.title + `","additions":3,"deletions":1,"changed_files":1,` +
|
||||
`"html_url":"https://gitea/pr/7","head":{"sha":"` + f.head + `"}}`
|
||||
`"html_url":"https://gitea/pr/7","head":{"sha":"` + f.head + `"},"requested_reviewers":` + reviewers + `}`
|
||||
}
|
||||
f.gitea = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "token review-token" {
|
||||
@@ -119,6 +124,7 @@ func newFixture(t *testing.T) *fixture {
|
||||
var in map[string]string
|
||||
_ = json.NewDecoder(r.Body).Decode(&in)
|
||||
f.posted = append(f.posted, in)
|
||||
f.requested = false
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = io.WriteString(w, `{"html_url":"https://gitea/pr/7#review"}`)
|
||||
case r.Method == "POST" && strings.HasPrefix(r.URL.Path, "/api/v1/repos/konairius/homelab/statuses/"):
|
||||
@@ -271,6 +277,7 @@ func TestReviewPostsTheCleanedReport(t *testing.T) {
|
||||
|
||||
func TestReviewSkipsWhatIsAlreadyReviewed(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.requested = false
|
||||
f.reviews = `[{"body":"<!-- pr-review:sha=abc123 -->","user":{"login":"review-bot"}}]`
|
||||
if err := f.rev.Review(context.Background(), "konairius/homelab", 7); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -440,25 +447,34 @@ func (f *fixture) hook(event, signature string, body string) *httptest.ResponseR
|
||||
|
||||
func TestHook(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
body := `{"action":"opened","number":7,"repository":{"full_name":"konairius/homelab"}}`
|
||||
body := `{"action":"review_requested","number":7,"repository":{"full_name":"konairius/homelab"},` +
|
||||
`"requested_reviewer":{"login":"review-bot"}}`
|
||||
good := sign("hook-secret", []byte(body))
|
||||
signed := func(old, new string) (string, string) {
|
||||
b := strings.Replace(body, old, new, 1)
|
||||
return sign("hook-secret", []byte(b)), b
|
||||
}
|
||||
closedSig, closedBody := signed("review_requested", "closed")
|
||||
openedSig, openedBody := signed("review_requested", "opened")
|
||||
syncSig, syncBody := signed("review_requested", "synchronized")
|
||||
otherSig, otherBody := signed(`"login":"review-bot"`, `"login":"someone"`)
|
||||
repoSig, repoBody := signed("konairius/homelab", "x/y")
|
||||
noNum := `{"action":"review_requested","repository":{"full_name":"konairius/homelab"},"requested_reviewer":{"login":"review-bot"}}`
|
||||
cases := []struct {
|
||||
name, event, sig, body string
|
||||
want int
|
||||
}{
|
||||
{"valid", "pull_request", good, body, 202},
|
||||
{"review requested from the bot", "pull_request", good, body, 202},
|
||||
{"missing signature", "pull_request", "", body, 401},
|
||||
{"wrong signature", "pull_request", sign("other", []byte(body)), body, 401},
|
||||
{"signature that is not hex", "pull_request", "zz", body, 401},
|
||||
{"other event", "push", good, body, 200},
|
||||
{"closed action", "pull_request", sign("hook-secret", []byte(strings.Replace(body, "opened", "closed", 1))),
|
||||
strings.Replace(body, "opened", "closed", 1), 200},
|
||||
{"synchronized", "pull_request", sign("hook-secret", []byte(strings.Replace(body, "opened", "synchronized", 1))),
|
||||
strings.Replace(body, "opened", "synchronized", 1), 202},
|
||||
{"repository that is not configured", "pull_request", sign("hook-secret", []byte(strings.Replace(body, "konairius/homelab", "x/y", 1))),
|
||||
strings.Replace(body, "konairius/homelab", "x/y", 1), 200},
|
||||
{"payload without a number", "pull_request", sign("hook-secret", []byte(`{"action":"opened","repository":{"full_name":"konairius/homelab"}}`)),
|
||||
`{"action":"opened","repository":{"full_name":"konairius/homelab"}}`, 400},
|
||||
{"closed action", "pull_request", closedSig, closedBody, 200},
|
||||
{"opened alone starts nothing", "pull_request", openedSig, openedBody, 200},
|
||||
{"a push alone starts nothing", "pull_request", syncSig, syncBody, 200},
|
||||
{"review requested from someone else", "pull_request", otherSig, otherBody, 200},
|
||||
{"repository that is not configured", "pull_request", repoSig, repoBody, 200},
|
||||
{"payload without a number", "pull_request", sign("hook-secret", []byte(noNum)), noNum, 400},
|
||||
{"payload that is not json", "pull_request", sign("hook-secret", []byte("nope")), "nope", 400},
|
||||
}
|
||||
for _, c := range cases {
|
||||
@@ -498,12 +514,12 @@ func TestSweepQueuesOnlyWhatNeedsAReview(t *testing.T) {
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
f.reviews = `[{"body":"<!-- pr-review:sha=abc123 -->","user":{"login":"review-bot"}}]`
|
||||
f.requested = false
|
||||
f.mu.Unlock()
|
||||
q2 := newQueue(f.rev, 4, f.rev.log, f.metrics)
|
||||
f.rev.Sweep(context.Background(), q2)
|
||||
if len(q2.jobs) != 0 {
|
||||
t.Errorf("queued %d for an already reviewed PR, want 0", len(q2.jobs))
|
||||
t.Errorf("queued %d for a PR that asks for no review, want 0", len(q2.jobs))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,11 +26,11 @@ type Choice struct {
|
||||
|
||||
// selectInstructions is the question the selector answers for every pull request.
|
||||
const selectInstructions = "Pick the model that should review the pull request in `pull_request`. " +
|
||||
"The review reads the repository, checks the change for bugs, security problems and broken behaviour, " +
|
||||
"and must be thorough. Prefer a cheaper or faster model when the change is small and low-risk " +
|
||||
"(documentation, version bumps, simple configuration). Pick a stronger model when the change is large, " +
|
||||
"spans several components, or touches security, authentication, secrets, network policy, concurrency, " +
|
||||
"data migrations or anything that is hard to undo."
|
||||
"The review reads the repository and checks the change for bugs, security problems and broken behaviour. " +
|
||||
"Pick the cheapest model that can review it well; most changes need no more than the cheapest one. " +
|
||||
"Pick a more expensive model only when the change clearly needs it: it is large, spans several " +
|
||||
"components, or touches security, authentication, secrets, network policy, concurrency, data migrations " +
|
||||
"or anything that is hard to undo."
|
||||
|
||||
// TypeSafe picks the review model with a System One Choice question.
|
||||
type TypeSafe struct {
|
||||
|
||||
@@ -74,6 +74,9 @@ type hookPayload struct {
|
||||
Repository struct {
|
||||
FullName string `json:"full_name"`
|
||||
} `json:"repository"`
|
||||
RequestedReviewer struct {
|
||||
Login string `json:"login"`
|
||||
} `json:"requested_reviewer"`
|
||||
}
|
||||
|
||||
// validSignature checks Gitea's X-Gitea-Signature: the hex HMAC-SHA256 of the raw body.
|
||||
@@ -114,11 +117,15 @@ func (s *Server) hook(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "bad payload", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// Gitea says "synchronized" for new commits on an open PR.
|
||||
if !slices.Contains([]string{"opened", "reopened", "synchronize", "synchronized"}, p.Action) {
|
||||
// A review runs only when the bot is asked for one; pushes and new PRs alone start nothing.
|
||||
if p.Action != "review_requested" {
|
||||
ignore("action " + p.Action)
|
||||
return
|
||||
}
|
||||
if p.RequestedReviewer.Login != s.cfg.BotLogin {
|
||||
ignore("review requested from " + p.RequestedReviewer.Login)
|
||||
return
|
||||
}
|
||||
n := p.Number
|
||||
if n == 0 {
|
||||
n = p.PullRequest.Number
|
||||
|
||||
@@ -74,6 +74,7 @@ func TestSkipsResolveTheQueuedStatus(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.title = "feat: x"
|
||||
f.requested = false
|
||||
f.reviews = `[{"body":"<!-- pr-review:sha=abc123 -->","state":"APPROVED","html_url":"https://gitea/r/1","user":{"login":"review-bot"}}]`
|
||||
if err := f.rev.Review(context.Background(), "konairius/homelab", 7); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user