scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 152

fix(triage): bugbot — reconsider dry-run + bot-closed guard + rate limit

Browse source 
Address three Greptile/veria-ai concerns on the @agent-shin reconsider
flow:

1. **Reconsider had no dry-run path.** The previous reconsider mode
   ignored `--close` and always posted comments + reopened on a pass.
   A local operator running
   `python triage_with_llm.py --reconsider --pr N` would silently
   take destructive GitHub actions with no way to preview. Reconsider
   now honors `close=False` the same way regular triage does and
   returns `would-reopen` / `would-reconsider-still-failing` for
   step-summary rendering.

2. **Reconsider could reopen maintainer-closed PRs/issues** (Medium
   security finding from veria-ai). The workflow only checked that the
   commenter was authorized — it did NOT check that the most recent
   close was performed by Agent Shin. A contributor could comment
   `@agent-shin reconsider` on a PR a maintainer closed for non-rubric
   reasons (duplicate, security report, design rejection) and have the
   bot reopen it. Add `was_closed_by_agent_shin()` which inspects the
   issue events API for the most recent `closed` actor and only
   permits reopen when that actor matches the configured bot login
   (default `github-actions[bot]`, overridable via env). Fail-closed
   on missing events.

3. **No rate-limiting on the reconsider trigger.** Every
   `@agent-shin reconsider` comment burns CI minutes + an OpenAI API
   call. Add a 10-minute cooldown via
   `seconds_since_last_reconsider_verdict()` which greps the issue's
   comment list for the bot's own verdict marker
   (`<!-- agent-shin:reconsider-verdict -->`). Inside the window the
   triage returns `skip-rate-limited` and the LLM never runs.

Workflow update:
- `triage_reconsider.yml` now passes `--close` only when
  `AGENT_SHIN_ENABLED=true`, matching the pattern of
  `triage_pr_with_llm.yml`. The script runs in both states so the
  verdict still appears in the step summary for QA.

Tests:
- Add 5 reconsider safety tests: dry-run for pass / fail / linked-issue
  short-circuit, bot-closed-guard refusal on maintainer close,
  rate-limit refusal inside the cooldown window, and cooldown-elapsed
  acceptance.
- Add unit tests for `was_closed_by_agent_shin` (bot / maintainer /
  missing actor / env-override) and
  `seconds_since_last_reconsider_verdict` (no marker / multiple
  markers / non-bot comment with marker / bot comment without marker).
- Pin the `<!-- agent-shin:reconsider-verdict -->` marker in both
  reopen and still-failing comments — dropping it would silently
  break the cooldown.

Existing reconsider tests updated to pass `close=True` (the
production path now) + stub the new guards via
`_stub_reconsider_guards`. 112 tests pass (was 93).

Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-05-18 05:22:25 +00:00
parent 420547f7be
commit 4e0968ad8d
No known key found for this signature in database
3 changed files with 628 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

189
.github/scripts/triage_with_llm.py vendored
View file

@ -30,6 +30,7 @@ Environment:
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import re
@ -42,6 +43,25 @@ DEFAULT_MODEL = "gpt-5.4-mini"
INTERNAL_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"})
# Login of the account that performs Agent Shin's GitHub writes. When the
# workflow uses `secrets.GITHUB_TOKEN` (our default), the closure / reopen
# event's `actor.login` is `github-actions[bot]`. The env override exists
# for local debugging and for repos that wire Agent Shin to a PAT.
AGENT_SHIN_DEFAULT_BOT_LOGIN = "github-actions[bot]"
# HTML marker appended to every reconsider verdict comment. We grep for this
# on subsequent reconsider triggers to enforce a short cooldown so that
# repeated `@agent-shin reconsider` comments don't burn CI/LLM budget.
# Using a unique HTML comment keeps the marker invisible to humans while
# being trivially greppable from a comments-list API response.
RECONSIDER_COMMENT_MARKER = "<!-- agent-shin:reconsider-verdict -->"
# Minimum gap between two reconsider verdicts on the same PR/issue. Set to
# 10 minutes — long enough that a contributor can't trivially spam the
# trigger, short enough that a genuine "I just pushed a fix and reupdated
# the body" iteration loop isn't punished.
RECONSIDER_RATE_LIMIT_SECONDS = 600
# Model families that require `reasoning_effort` to be set, and that reject
# `temperature != 1` unless `reasoning_effort` is "none". For these models we
# pass `reasoning_effort="none"` so a `temperature=0` deterministic judgment
@ -160,6 +180,103 @@ def reopen_issue(repo: str, number: int) -> None:
)
def _iter_paginated_json(*api_args: str) -> Any:
"""Yield JSON objects from `gh api --paginate ... -q '.[]'`.
`gh api --paginate` on a JSON-array endpoint concatenates pages into
one stream; `-q '.[]'` flattens that stream into newline-delimited
objects (jq-style). This keeps memory bounded for chatty endpoints
like issue events/comments on long-lived PRs.
"""
raw = gh("api", "--paginate", *api_args, "-q", ".[]")
for line in raw.splitlines():
line = line.strip()
if not line:
continue
try:
yield json.loads(line)
except json.JSONDecodeError:
# A malformed line should not blow up the whole guard. Skip and
# carry on — at worst the guard fail-closes (returns False /
# None) and the caller treats it as "unknown".
continue
def fetch_last_close_actor(repo: str, number: int) -> str | None:
"""Return the login of the actor who most recently closed this PR/issue.
Returns None if no `closed` event is found (unusual for a closed item,
but possible if the events API returns nothing — in which case the
bot-closed guard should fail-closed, i.e. refuse to reopen).
"""
last: str | None = None
for event in _iter_paginated_json(f"repos/{repo}/issues/{number}/events"):
if event.get("event") == "closed":
last = (event.get("actor") or {}).get("login")
return last
def was_closed_by_agent_shin(
repo: str, number: int, *, bot_login: str | None = None
) -> bool:
"""Return True iff the PR/issue was most-recently closed by Agent Shin.
This is the guard that stops `@agent-shin reconsider` from being used
to override a maintainer's closure for non-rubric reasons (security,
duplicate, design rejection, etc.). The check is intentionally
fail-closed: any uncertainty about who closed the item must be
treated as "not the bot" so the destructive reopen path stays gated.
"""
expected = (
bot_login
or os.environ.get("AGENT_SHIN_BOT_LOGIN")
or AGENT_SHIN_DEFAULT_BOT_LOGIN
).lower()
actor = fetch_last_close_actor(repo, number)
if not actor:
return False
return actor.lower() == expected
def seconds_since_last_reconsider_verdict(
repo: str, number: int, *, bot_login: str | None = None
) -> float | None:
"""Return seconds since the bot's most recent reconsider verdict comment.
Detects comments by matching the HTML marker `RECONSIDER_COMMENT_MARKER`
appended by `format_reopen_comment` and
`format_reconsider_still_failing_comment`. Returns None when the bot
has never posted a reconsider verdict on this PR/issue (or when the
only matching comments are missing a `created_at` timestamp, which
shouldn't happen on a real GitHub response).
"""
expected_login = (
bot_login
or os.environ.get("AGENT_SHIN_BOT_LOGIN")
or AGENT_SHIN_DEFAULT_BOT_LOGIN
).lower()
latest: dt.datetime | None = None
for comment in _iter_paginated_json(f"repos/{repo}/issues/{number}/comments"):
author = ((comment.get("user") or {}).get("login") or "").lower()
if author != expected_login:
continue
body = comment.get("body") or ""
if RECONSIDER_COMMENT_MARKER not in body:
continue
created = comment.get("created_at")
if not created:
continue
try:
ts = dt.datetime.fromisoformat(created.replace("Z", "+00:00"))
except ValueError:
continue
if latest is None or ts > latest:
latest = ts
if latest is None:
return None
return (dt.datetime.now(dt.timezone.utc) - latest).total_seconds()
# ---------------------------------------------------------------------------
# Author classification
@ -458,6 +575,9 @@ def write_step_summary(content: str) -> None:
def format_reopen_comment(kind: str) -> str:
"""Comment posted when Agent Shin reopens after a successful reconsider."""
noun = "PR" if kind == "pr" else "issue"
# The trailing HTML marker is used by `seconds_since_last_reconsider_verdict`
# to enforce a cooldown between repeated `@agent-shin reconsider` triggers.
# Keep the marker on its own line so it doesn't disturb the rendered text.
return (
f"♻️ **Re-evaluated and reopened.** Thanks for updating the {noun}!\n"
"\n"
@ -467,7 +587,9 @@ def format_reopen_comment(kind: str) -> str:
"\n"
"_(If a maintainer ends up closing this for non-rubric reasons, that "
"decision stands; comment `@agent-shin reconsider` again only if you "
"have substantively new information.)_"
"have substantively new information.)_\n"
"\n"
f"{RECONSIDER_COMMENT_MARKER}"
)
@ -476,6 +598,8 @@ def format_reconsider_still_failing_comment(kind: str, verdict: dict) -> str:
missing_lines = _format_missing(verdict.get("missing") or [])
explanation = verdict.get("explanation") or ""
noun = "PR" if kind == "pr" else "issue"
# The trailing HTML marker is used by `seconds_since_last_reconsider_verdict`
# to enforce a cooldown between repeated `@agent-shin reconsider` triggers.
return (
f"⏸️ **Re-evaluated; this {noun} still doesn't meet the rubric.**\n"
"\n"
@ -490,7 +614,9 @@ def format_reconsider_still_failing_comment(kind: str, verdict: dict) -> str:
"`@agent-shin reconsider` again, or ping a maintainer if you think "
"I got this wrong.\n"
"\n"
"_(I'm an LLM and I'm not infallible.)_"
"_(I'm an LLM and I'm not infallible.)_\n"
"\n"
f"{RECONSIDER_COMMENT_MARKER}"
)
@ -514,10 +640,22 @@ def triage(
fail-but-no-comment is replaced with a "still failing" comment + leave
closed; a pass triggers `reopen_pr`/`reopen_issue` plus a reopen comment.
Reconsider mode is intended for the `@agent-shin reconsider` comment
trigger. `close` is forced True implicitly when `reconsider` is set
because the bot has already decided this is a real (non-dry-run)
invocation; it's the caller's responsibility to gate on
AGENT_SHIN_ENABLED before calling reconsider mode.
trigger. Like regular triage, `close=False` keeps reconsider in dry-run
(returns `would-reopen` / `would-reconsider-still-failing` so a local
operator can preview without write side effects); the workflow only
passes `--close` when `AGENT_SHIN_ENABLED=true`.
Reconsider mode adds two extra safety guards on top of the regular
triage skip-internal-author check:
1. **Bot-closed guard.** Only reopens if the most recent close was
performed by the bot identity (default `github-actions[bot]`).
This stops a contributor from using `@agent-shin reconsider` to
override a maintainer's close for non-rubric reasons.
2. **Rate-limit guard.** If the bot has already posted a reconsider
verdict on this PR/issue within `RECONSIDER_RATE_LIMIT_SECONDS`,
skip — repeated triggers from the same contributor shouldn't burn
CI minutes or LLM budget.
"""
fetcher = {"pr": fetch_pr, "issue": fetch_issue}[kind]
item = fetcher(repo, number)
@ -551,6 +689,20 @@ def triage(
if is_internal_contributor(item):
return {**base_result, "action": "skip-internal-author"}
# Reconsider-only guards — these run BEFORE the LLM call so a
# maintainer-closed PR / rate-limited trigger never spends LLM budget.
if reconsider:
if not was_closed_by_agent_shin(repo, number):
return {**base_result, "action": "skip-not-bot-closed"}
age = seconds_since_last_reconsider_verdict(repo, number)
if age is not None and age < RECONSIDER_RATE_LIMIT_SECONDS:
return {
**base_result,
"action": "skip-rate-limited",
"rate_limit_age_seconds": age,
"rate_limit_window_seconds": RECONSIDER_RATE_LIMIT_SECONDS,
}
if kind == "pr":
prompt = build_pr_prompt(title=title, body=body)
# Short-circuit: if body very clearly links a related issue, just pass.
@ -567,6 +719,12 @@ def triage(
if reconsider:
# Pass-on-reconsider -> reopen the PR with a friendly comment.
reopen_body = format_reopen_comment(kind)
if not close:
return {
**base,
"action": "would-reopen",
"comment": reopen_body,
}
post_comment(repo, number, reopen_body)
reopen_pr(repo, number)
return {
@ -607,8 +765,20 @@ def triage(
# Reconsider: pass -> reopen + post reopen comment;
# fail -> leave closed + post a "still failing" comment so the
# contributor can iterate again.
# In dry-run (`close=False`) we return `would-*` actions instead
# of touching GitHub state, mirroring the regular triage flow's
# `would-close`. This lets a local operator preview the outcome
# of `python triage_with_llm.py --reconsider --pr N` without
# risking accidental comments or reopens.
if decision != "fail":
reopen_body = format_reopen_comment(kind)
if not close:
return {
**base_result,
"action": "would-reopen",
"verdict": verdict,
"comment": reopen_body,
}
post_comment(repo, number, reopen_body)
if kind == "pr":
reopen_pr(repo, number)
@ -621,6 +791,13 @@ def triage(
"comment": reopen_body,
}
still_failing = format_reconsider_still_failing_comment(kind, verdict)
if not close:
return {
**base_result,
"action": "would-reconsider-still-failing",
"verdict": verdict,
"comment": still_failing,
}
post_comment(repo, number, still_failing)
return {
**base_result,

24
.github/workflows/triage_reconsider.yml vendored
View file

@ -107,20 +107,24 @@ jobs:
else
ARGS=(--repo "${{ github.repository }}" --issue "${NUMBER}" --reconsider)
fi
# Reconsider IS the destructive path here (it can post comments
# and reopen) — there's no separate `--close` flag because the
# script's reconsider mode handles both pass (reopen) and fail
# (still-failing comment) outcomes itself.
# Reconsider's destructive actions (post comment + reopen) are
# gated on `--close`, mirroring the regular triage workflows.
# When AGENT_SHIN_ENABLED is not the EXACT string "true", we
# still run the script so its verdict + would-X action lands in
# the step summary for QA — but without `--close`, the script
# returns `would-reopen` / `would-reconsider-still-failing`
# instead of touching GitHub state.
#
# Use the positive `= "true"` gate (instead of `!= "true" -> exit`)
# so the workflow guardrails in
# Use the positive `= "true"` gate (not `!= "true" -> exit`) so
# the workflow guardrails in
# tests/test_litellm/test_github_triage_workflows.py see the
# canonical fail-safe enable pattern. Unknown values like "True",
# "yes", "1", or typos will fall through to the dry-run else
# canonical fail-safe enable pattern. Unknown values like
# "True", "yes", "1", or typos fall through to the dry-run
# branch, which is the safe default.
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
echo "::notice::Agent Shin reconsider ENABLED — running real triage."
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"
ARGS+=(--close)
echo "::notice::Agent Shin reconsider ENABLED — running real triage (close=true)."
else
echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> reconsider stays in dry-run (no comment, no reopen)."
fi
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"

436
tests/test_litellm/test_github_triage_with_llm.py
View file

@ -138,6 +138,22 @@ class TestCloseCommentText:
# appear (they can't reopen a PR closed by a bot/maintainer).
assert "Reopen the PR" not in body
def test_reopen_comment_should_carry_reconsider_marker(self, triage_module):
# The marker is what the rate-limit guard greps for to detect a
# prior reconsider verdict on the same PR. If the marker ever
# gets dropped from this comment, the cooldown silently breaks
# and a contributor can spam `@agent-shin reconsider` to burn
# LLM budget.
body = triage_module.format_reopen_comment("pr")
assert triage_module.RECONSIDER_COMMENT_MARKER in body
def test_still_failing_comment_should_carry_reconsider_marker(self, triage_module):
body = triage_module.format_reconsider_still_failing_comment(
"pr",
{"verdict": "fail", "missing": ["QA proof"], "explanation": "thin"},
)
assert triage_module.RECONSIDER_COMMENT_MARKER in body
def test_pr_close_comment_should_not_promise_automatic_reopen_on_open(
self, triage_module
):
@ -158,6 +174,158 @@ class TestCloseCommentText:
assert "Reopen the issue" not in body
class TestWasClosedByAgentShin:
"""Bot-closed guard: only the bot's own closures are reopen candidates."""
def test_should_return_true_when_last_close_actor_is_bot(
self, triage_module, monkeypatch
):
monkeypatch.setattr(
triage_module,
"fetch_last_close_actor",
lambda repo, n: "github-actions[bot]",
)
assert triage_module.was_closed_by_agent_shin("o/r", 1) is True
def test_should_return_false_when_last_close_actor_is_maintainer(
self, triage_module, monkeypatch
):
# A maintainer closed it (e.g. duplicate, security, design). The
# bot must refuse to reopen on @agent-shin reconsider.
monkeypatch.setattr(
triage_module,
"fetch_last_close_actor",
lambda repo, n: "krrishdholakia",
)
assert triage_module.was_closed_by_agent_shin("o/r", 1) is False
def test_should_fail_closed_when_no_close_event(self, triage_module, monkeypatch):
# If the events API returns nothing (network blip, repo permission
# quirk), the guard must fail-closed: refuse to reopen rather than
# assume the bot did it.
monkeypatch.setattr(
triage_module, "fetch_last_close_actor", lambda repo, n: None
)
assert triage_module.was_closed_by_agent_shin("o/r", 1) is False
def test_should_respect_bot_login_override_via_env(
self, triage_module, monkeypatch
):
# Operators wiring Agent Shin to a PAT (instead of GITHUB_TOKEN)
# can override the expected bot login via env. The guard must
# respect the override so non-default deployments still work.
monkeypatch.setenv("AGENT_SHIN_BOT_LOGIN", "my-bot")
monkeypatch.setattr(
triage_module, "fetch_last_close_actor", lambda repo, n: "my-bot"
)
assert triage_module.was_closed_by_agent_shin("o/r", 1) is True
# Default "github-actions[bot]" should NOT match when env is set.
monkeypatch.setattr(
triage_module,
"fetch_last_close_actor",
lambda repo, n: "github-actions[bot]",
)
assert triage_module.was_closed_by_agent_shin("o/r", 1) is False
class TestSecondsSinceLastReconsiderVerdict:
"""Rate-limit guard: detects the bot's own reconsider verdict marker."""
def _make_comment(
self, *, login: str, body: str, created_at: str | None = "2026-05-18T05:00:00Z"
) -> dict:
comment: dict = {"user": {"login": login}, "body": body}
if created_at is not None:
comment["created_at"] = created_at
return comment
def test_should_return_none_when_no_bot_reconsider_comments(
self, triage_module, monkeypatch
):
# An issue with chatter from other users but no bot reconsider
# verdict must not be rate-limited.
comments = [
self._make_comment(login="outside-dev", body="ping?"),
self._make_comment(
login="github-actions[bot]", body="some other bot message"
),
]
monkeypatch.setattr(
triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments)
)
assert triage_module.seconds_since_last_reconsider_verdict("o/r", 1) is None
def test_should_pick_latest_bot_reconsider_marker(self, triage_module, monkeypatch):
# When multiple reconsider verdicts exist, return the AGE of the
# most recent one. Using a frozen reference helps pin the math.
comments = [
self._make_comment(
login="github-actions[bot]",
body="old verdict " + triage_module.RECONSIDER_COMMENT_MARKER,
created_at="2026-05-18T04:00:00Z",
),
self._make_comment(
login="github-actions[bot]",
body="newer verdict " + triage_module.RECONSIDER_COMMENT_MARKER,
created_at="2026-05-18T04:55:00Z",
),
]
monkeypatch.setattr(
triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments)
)
# Freeze "now" via a tiny shim on the module's `dt` import.
import datetime as real_dt
class FrozenDateTime(real_dt.datetime):
@classmethod
def now(cls, tz=None):
return real_dt.datetime(2026, 5, 18, 5, 0, 0, tzinfo=tz)
frozen_module = type(triage_module.dt)("datetime")
frozen_module.datetime = FrozenDateTime
frozen_module.timezone = real_dt.timezone
monkeypatch.setattr(triage_module, "dt", frozen_module)
age = triage_module.seconds_since_last_reconsider_verdict("o/r", 1)
# newer verdict is 5 minutes (300 seconds) before "now"
assert age == 300.0
def test_should_ignore_non_bot_comments_with_marker(
self, triage_module, monkeypatch
):
# A user comment that happens to quote the marker (e.g. in
# a "what does this hidden marker do?" question) must NOT count.
# The rate-limit guard only trusts comments authored by the bot.
comments = [
self._make_comment(
login="curious-user",
body=f"Saw this marker: {triage_module.RECONSIDER_COMMENT_MARKER}",
),
]
monkeypatch.setattr(
triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments)
)
assert triage_module.seconds_since_last_reconsider_verdict("o/r", 1) is None
def test_should_ignore_bot_comments_without_marker(
self, triage_module, monkeypatch
):
# The bot posts other things too (Agent Shin close comments,
# CI status, etc.) — only the reconsider-verdict marker should
# arm the cooldown.
comments = [
self._make_comment(
login="github-actions[bot]",
body="Agent Shin closed this PR (no marker)",
),
]
monkeypatch.setattr(
triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments)
)
assert triage_module.seconds_since_last_reconsider_verdict("o/r", 1) is None
class TestParseVerdict:
def test_should_parse_plain_json(self, triage_module):
raw = '{"verdict": "pass", "missing": []}'
@ -597,13 +765,34 @@ class TestTriageOrchestration:
)
assert result["action"] == "skip-not-closed"
@staticmethod
def _stub_reconsider_guards(triage_module, monkeypatch):
"""Default reconsider-guard stubs: pretend bot closed + no cooldown.
The new safety guards (`was_closed_by_agent_shin`,
`seconds_since_last_reconsider_verdict`) hit the GitHub API in
production. Tests that exercise the reconsider happy path stub
them to "yes the bot closed it, no recent reconsider comment"
so the test stays focused on its actual assertion.
"""
monkeypatch.setattr(
triage_module, "was_closed_by_agent_shin", lambda *a, **kw: True
)
monkeypatch.setattr(
triage_module,
"seconds_since_last_reconsider_verdict",
lambda *a, **kw: None,
)
def test_should_reopen_on_reconsider_pass(self, triage_module, monkeypatch):
# Reconsider on a closed PR with a passing verdict -> reopen + post a
# friendly "re-evaluated" comment.
# friendly "re-evaluated" comment. close=True is the production path
# (the workflow only adds --close when AGENT_SHIN_ENABLED=true).
pr = self._make_pr(
state="closed", body="Updated body with QA proof + screenshots."
)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
self._stub_reconsider_guards(triage_module, monkeypatch)
posted = {}
reopened = {}
monkeypatch.setattr(
@ -627,7 +816,7 @@ class TestTriageOrchestration:
repo="o/r",
kind="pr",
number=42,
close=False,
close=True,
model="m",
judge=lambda p: json.dumps(
{"verdict": "pass", "missing": [], "explanation": "ok now"}
@ -639,11 +828,52 @@ class TestTriageOrchestration:
assert posted["n"] == 42
assert "reopened" in posted["body"].lower()
def test_should_dry_run_reconsider_pass_when_close_false(
self, triage_module, monkeypatch
):
# Reconsider must honor `close=False` (dry-run) just like the
# regular triage flow. A local invocation of
# `python triage_with_llm.py --reconsider --pr N` (no --close)
# must NOT post a comment or reopen the PR — it should return
# `would-reopen` so the operator can preview the outcome.
pr = self._make_pr(
state="closed", body="Updated body with QA proof + screenshots."
)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
self._stub_reconsider_guards(triage_module, monkeypatch)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda *a, **kw: pytest.fail("must not post comment in dry-run reconsider"),
)
monkeypatch.setattr(
triage_module,
"reopen_pr",
lambda *a, **kw: pytest.fail("must not reopen PR in dry-run reconsider"),
)
result = triage_module.triage(
repo="o/r",
kind="pr",
number=42,
close=False,
model="m",
judge=lambda p: json.dumps(
{"verdict": "pass", "missing": [], "explanation": "ok now"}
),
reconsider=True,
)
assert result["action"] == "would-reopen"
# The previewed comment body is still returned so a step-summary
# writer can render exactly what would have been posted.
assert "reopened" in result["comment"].lower()
def test_should_post_still_failing_on_reconsider_fail(
self, triage_module, monkeypatch
):
pr = self._make_pr(state="closed", body="still empty")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
self._stub_reconsider_guards(triage_module, monkeypatch)
posted = {}
monkeypatch.setattr(
triage_module,
@ -671,7 +901,7 @@ class TestTriageOrchestration:
repo="o/r",
kind="pr",
number=42,
close=False,
close=True,
model="m",
judge=lambda p: json.dumps(verdict),
reconsider=True,
@ -680,6 +910,39 @@ class TestTriageOrchestration:
assert posted["n"] == 42
assert "QA proof" in posted["body"]
def test_should_dry_run_reconsider_fail_when_close_false(
self, triage_module, monkeypatch
):
# Mirror dry-run behavior for the FAIL branch — `close=False`
# must NOT post the "still failing" comment.
pr = self._make_pr(state="closed", body="still empty")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
self._stub_reconsider_guards(triage_module, monkeypatch)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda *a, **kw: pytest.fail(
"must not post still-failing comment in dry-run"
),
)
verdict = {
"verdict": "fail",
"missing": ["QA proof"],
"explanation": "Still no QA proof.",
}
result = triage_module.triage(
repo="o/r",
kind="pr",
number=42,
close=False,
model="m",
judge=lambda p: json.dumps(verdict),
reconsider=True,
)
assert result["action"] == "would-reconsider-still-failing"
assert "QA proof" in result["comment"]
def test_should_reopen_on_reconsider_with_linked_issue_short_circuit(
self, triage_module, monkeypatch
):
@ -688,6 +951,7 @@ class TestTriageOrchestration:
# path should reopen the PR without calling the LLM.
pr = self._make_pr(state="closed", body="Fixes #1234\n\nAddresses the bug.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
self._stub_reconsider_guards(triage_module, monkeypatch)
posted = {}
reopened = {}
monkeypatch.setattr(
@ -705,7 +969,7 @@ class TestTriageOrchestration:
repo="o/r",
kind="pr",
number=55,
close=False,
close=True,
model="m",
judge=lambda p: pytest.fail("LLM must not run when linked-issue matches"),
reconsider=True,
@ -714,6 +978,35 @@ class TestTriageOrchestration:
assert reopened["n"] == 55
assert "reopened" in posted["body"].lower()
def test_should_dry_run_reconsider_with_linked_issue_when_close_false(
self, triage_module, monkeypatch
):
# Linked-issue short-circuit must ALSO honor dry-run.
pr = self._make_pr(state="closed", body="Fixes #1234\n\nAddresses the bug.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
self._stub_reconsider_guards(triage_module, monkeypatch)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda *a, **kw: pytest.fail("must not post in dry-run"),
)
monkeypatch.setattr(
triage_module,
"reopen_pr",
lambda *a, **kw: pytest.fail("must not reopen in dry-run"),
)
result = triage_module.triage(
repo="o/r",
kind="pr",
number=55,
close=False,
model="m",
judge=lambda p: pytest.fail("LLM must not run when linked-issue matches"),
reconsider=True,
)
assert result["action"] == "would-reopen"
def test_should_skip_internal_in_reconsider_mode(self, triage_module, monkeypatch):
# Internal authors are exempt from triage in both regular and
# reconsider mode — Agent Shin should never reopen one of their PRs
@ -740,6 +1033,138 @@ class TestTriageOrchestration:
)
assert result["action"] == "skip-internal-author"
def test_should_skip_reconsider_when_not_bot_closed(
self, triage_module, monkeypatch
):
# SECURITY: `@agent-shin reconsider` must NOT reopen a PR/issue
# that a MAINTAINER closed for non-rubric reasons (e.g. duplicate,
# design rejection, security report). Only PRs closed by the bot
# itself should ever be candidates for the reconsider reopen path.
pr = self._make_pr(state="closed", body="something.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
monkeypatch.setattr(
triage_module, "was_closed_by_agent_shin", lambda *a, **kw: False
)
# Even though there's no rate-limit conflict, the bot-closed guard
# alone is sufficient to block. The LLM judge must never run on a
# maintainer-closed PR.
monkeypatch.setattr(
triage_module,
"seconds_since_last_reconsider_verdict",
lambda *a, **kw: None,
)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda *a, **kw: pytest.fail("must not comment on maintainer-closed PR"),
)
monkeypatch.setattr(
triage_module,
"reopen_pr",
lambda *a, **kw: pytest.fail("must not reopen maintainer-closed PR"),
)
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=True,
model="m",
judge=lambda p: pytest.fail("LLM must not run before bot-closed guard"),
reconsider=True,
)
assert result["action"] == "skip-not-bot-closed"
def test_should_rate_limit_repeated_reconsider_triggers(
self, triage_module, monkeypatch
):
# COST CONTROL: each `@agent-shin reconsider` event burns CI
# minutes + an OpenAI API call. If the bot already posted a
# reconsider verdict within the cooldown window
# (RECONSIDER_RATE_LIMIT_SECONDS), refuse to run again. This
# bounds the damage from a contributor spamming the trigger.
pr = self._make_pr(state="closed", body="something with new edits.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
monkeypatch.setattr(
triage_module, "was_closed_by_agent_shin", lambda *a, **kw: True
)
# Pretend the bot posted a reconsider verdict 1 second ago.
monkeypatch.setattr(
triage_module,
"seconds_since_last_reconsider_verdict",
lambda *a, **kw: 1.0,
)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda *a, **kw: pytest.fail("must not comment during cooldown"),
)
monkeypatch.setattr(
triage_module,
"reopen_pr",
lambda *a, **kw: pytest.fail("must not reopen during cooldown"),
)
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=True,
model="m",
judge=lambda p: pytest.fail("LLM must not run during cooldown"),
reconsider=True,
)
assert result["action"] == "skip-rate-limited"
assert result["rate_limit_age_seconds"] == 1.0
assert (
result["rate_limit_window_seconds"]
== triage_module.RECONSIDER_RATE_LIMIT_SECONDS
)
def test_should_allow_reconsider_after_cooldown_window(
self, triage_module, monkeypatch
):
# The cooldown is a window, not a one-shot lock — once
# RECONSIDER_RATE_LIMIT_SECONDS has elapsed since the last bot
# verdict, a fresh `@agent-shin reconsider` is allowed through.
pr = self._make_pr(state="closed", body="updated with screenshots now.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
monkeypatch.setattr(
triage_module, "was_closed_by_agent_shin", lambda *a, **kw: True
)
# Last reconsider was 1 hour ago — well outside the 10-min window.
monkeypatch.setattr(
triage_module,
"seconds_since_last_reconsider_verdict",
lambda *a, **kw: 3600.0,
)
posted = {}
reopened = {}
monkeypatch.setattr(
triage_module,
"post_comment",
lambda repo, n, body: posted.update({"n": n, "body": body}),
)
monkeypatch.setattr(
triage_module,
"reopen_pr",
lambda repo, n: reopened.update({"n": n}),
)
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=True,
model="m",
judge=lambda p: json.dumps(
{"verdict": "pass", "missing": [], "explanation": "ok"}
),
reconsider=True,
)
assert result["action"] == "reopened"
assert reopened["n"] == 1
def test_should_reopen_issue_on_reconsider_pass(self, triage_module, monkeypatch):
issue = {
"number": 7,
@ -750,6 +1175,7 @@ class TestTriageOrchestration:
"user": {"login": "outside"},
}
monkeypatch.setattr(triage_module, "fetch_issue", lambda repo, n: issue)
self._stub_reconsider_guards(triage_module, monkeypatch)
posted = {}
reopened = {}
monkeypatch.setattr(
@ -767,7 +1193,7 @@ class TestTriageOrchestration:
repo="o/r",
kind="issue",
number=7,
close=False,
close=True,
model="m",
judge=lambda p: json.dumps(
{"verdict": "pass", "missing": [], "explanation": "now reproducible"}