mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
fix(triage): anchor reconsider provenance marker to the current open→closed cycle
The previous provenance check returned a false positive when a different workflow that shares the github-actions[bot] identity (e.g. the repo's actions/stale workflow, which uses secrets.GITHUB_TOKEN) re-closes a PR after Agent Shin previously auto-closed it. The check only required ANY historical marker comment by the closer login, so the old Agent Shin marker satisfied it even though the most recent close was stale's. Require the marker comment to be timestamped after the most recent reopened event (if any) and no later than the most recent closed event, so it must belong to the same open→closed cycle that's being reconsidered. Stale's close cycle never posts that marker, so the cycle-anchored check refuses to override it. The legitimate Agent Shin reconsider path still passes because Agent Shin posts the marker comment immediately before calling close_pr / close_issue in the same job.
This commit is contained in:
mateo-berri
parent
7f69ae6755
commit
dc80aa8290
2 changed files with 175 additions and 19 deletions
35
.github/scripts/triage_with_llm.py
vendored
35
.github/scripts/triage_with_llm.py
vendored
|
|
@ -246,16 +246,29 @@ def was_auto_closed_by_agent_shin(repo: str, number: int) -> bool:
|
|||
commenting `@agent-shin reconsider`.
|
||||
2. A comment authored by the same bot login that performed the
|
||||
close contains the Agent Shin auto-close marker
|
||||
(`AGENT_SHIN_AUTO_CLOSE_MARKER`). Matching the comment author
|
||||
to the closer rules out closures by unrelated bots (stale,
|
||||
cla-assistant, etc.) and spoofing via marker text pasted by
|
||||
non-bot accounts.
|
||||
(`AGENT_SHIN_AUTO_CLOSE_MARKER`) AND was posted in the current
|
||||
open→closed cycle (after the most recent `reopened` event, if
|
||||
any, and no later than the most recent `closed` event). This
|
||||
anchors the marker to the close that's actually being
|
||||
reconsidered: a stale-workflow closure that runs as
|
||||
`github-actions[bot]` (because `actions/stale` uses
|
||||
`secrets.GITHUB_TOKEN`, the same identity as Agent Shin) does
|
||||
NOT post a marker in its own cycle, so the cycle-anchored check
|
||||
refuses to override it even though a historical Agent Shin
|
||||
marker comment still exists from an earlier cycle.
|
||||
"""
|
||||
events = fetch_issue_events(repo, number)
|
||||
last_closer: str | None = None
|
||||
last_close_ts = ""
|
||||
last_closer = ""
|
||||
last_reopen_ts = ""
|
||||
for event in events:
|
||||
if (event.get("event") or "").lower() == "closed":
|
||||
kind = (event.get("event") or "").lower()
|
||||
ts = event.get("created_at") or ""
|
||||
if kind == "closed":
|
||||
last_close_ts = ts
|
||||
last_closer = ((event.get("actor") or {}).get("login") or "").lower()
|
||||
elif kind == "reopened":
|
||||
last_reopen_ts = ts
|
||||
if not last_closer or not last_closer.endswith("[bot]"):
|
||||
return False
|
||||
for comment in fetch_issue_comments(repo, number):
|
||||
|
|
@ -263,8 +276,14 @@ def was_auto_closed_by_agent_shin(repo: str, number: int) -> bool:
|
|||
if login != last_closer:
|
||||
continue
|
||||
body = comment.get("body") or ""
|
||||
if AGENT_SHIN_AUTO_CLOSE_MARKER in body:
|
||||
return True
|
||||
if AGENT_SHIN_AUTO_CLOSE_MARKER not in body:
|
||||
continue
|
||||
comment_ts = comment.get("created_at") or ""
|
||||
if last_reopen_ts and comment_ts <= last_reopen_ts:
|
||||
continue
|
||||
if last_close_ts and comment_ts > last_close_ts:
|
||||
continue
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -311,10 +311,17 @@ class TestWasAutoClosedByAgentShin:
|
|||
def test_should_return_true_when_latest_close_was_bot_with_marker(
|
||||
self, triage_module, monkeypatch
|
||||
):
|
||||
events = [{"event": "closed", "actor": {"login": "github-actions[bot]"}}]
|
||||
events = [
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
}
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:05Z",
|
||||
"body": (
|
||||
"👋 Hi, thanks for the PR! I'm **Agent Shin**, the automated "
|
||||
"triage bot for this repository.\n\nThis PR is being **auto-closed**..."
|
||||
|
|
@ -333,10 +340,17 @@ class TestWasAutoClosedByAgentShin:
|
|||
# be treated as proof Agent Shin closed the PR. Even if a bot did
|
||||
# the most recent close, the marker comment must be authored by
|
||||
# that same bot login — not by the human.
|
||||
events = [{"event": "closed", "actor": {"login": "github-actions[bot]"}}]
|
||||
events = [
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
}
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "outside-dev"},
|
||||
"created_at": "2025-01-01T00:00:05Z",
|
||||
"body": "I'm **Agent Shin**, just kidding — please reconsider this.",
|
||||
}
|
||||
]
|
||||
|
|
@ -346,14 +360,22 @@ class TestWasAutoClosedByAgentShin:
|
|||
def test_should_ignore_bot_comment_without_marker(self, triage_module, monkeypatch):
|
||||
# Other bots (codecov, cla-assistant, etc.) post on every PR; their
|
||||
# presence must not satisfy the provenance check.
|
||||
events = [{"event": "closed", "actor": {"login": "github-actions[bot]"}}]
|
||||
events = [
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
}
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "codecov[bot]"},
|
||||
"created_at": "2025-01-01T00:00:01Z",
|
||||
"body": "## Codecov Report ...",
|
||||
},
|
||||
{
|
||||
"user": {"login": "greptile-apps[bot]"},
|
||||
"created_at": "2025-01-01T00:00:02Z",
|
||||
"body": "Confidence Score: 2/5",
|
||||
},
|
||||
]
|
||||
|
|
@ -364,20 +386,31 @@ class TestWasAutoClosedByAgentShin:
|
|||
# Agent Shin auto-closed first, contributor commented after; the
|
||||
# bot-authored marker comment is anywhere in the timeline.
|
||||
events = [
|
||||
{"event": "labeled", "actor": {"login": "krrishdholakia"}},
|
||||
{"event": "closed", "actor": {"login": "github-actions[bot]"}},
|
||||
{
|
||||
"event": "labeled",
|
||||
"actor": {"login": "krrishdholakia"},
|
||||
"created_at": "2025-01-01T00:00:01Z",
|
||||
},
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
},
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "codecov[bot]"},
|
||||
"created_at": "2025-01-01T00:00:02Z",
|
||||
"body": "## Codecov Report",
|
||||
},
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:05Z",
|
||||
"body": "I'm **Agent Shin**, the automated triage bot ...",
|
||||
},
|
||||
{
|
||||
"user": {"login": "outside-dev"},
|
||||
"created_at": "2025-01-01T00:00:15Z",
|
||||
"body": "Replying after auto-close ...",
|
||||
},
|
||||
]
|
||||
|
|
@ -392,13 +425,26 @@ class TestWasAutoClosedByAgentShin:
|
|||
# must NOT override the maintainer's later closure even though the
|
||||
# historical Agent Shin marker comment still exists.
|
||||
events = [
|
||||
{"event": "closed", "actor": {"login": "github-actions[bot]"}},
|
||||
{"event": "reopened", "actor": {"login": "github-actions[bot]"}},
|
||||
{"event": "closed", "actor": {"login": "krrishdholakia"}},
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
},
|
||||
{
|
||||
"event": "reopened",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:20Z",
|
||||
},
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "krrishdholakia"},
|
||||
"created_at": "2025-01-01T00:00:30Z",
|
||||
},
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:05Z",
|
||||
"body": "I'm **Agent Shin**, the automated triage bot ...",
|
||||
}
|
||||
]
|
||||
|
|
@ -413,19 +459,110 @@ class TestWasAutoClosedByAgentShin:
|
|||
# `github-actions[bot]` but the most recent closer is
|
||||
# `stale[bot]`, so the logins don't match -> refuse to reopen.
|
||||
events = [
|
||||
{"event": "closed", "actor": {"login": "github-actions[bot]"}},
|
||||
{"event": "reopened", "actor": {"login": "github-actions[bot]"}},
|
||||
{"event": "closed", "actor": {"login": "stale[bot]"}},
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
},
|
||||
{
|
||||
"event": "reopened",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:20Z",
|
||||
},
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "stale[bot]"},
|
||||
"created_at": "2025-01-01T00:00:30Z",
|
||||
},
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:05Z",
|
||||
"body": "I'm **Agent Shin**, the automated triage bot ...",
|
||||
}
|
||||
]
|
||||
self._install(monkeypatch, triage_module, events, comments)
|
||||
assert triage_module.was_auto_closed_by_agent_shin("o/r", 1) is False
|
||||
|
||||
def test_should_refuse_when_stale_workflow_re_closed_after_agent_shin(
|
||||
self, triage_module, monkeypatch
|
||||
):
|
||||
# The repo's `actions/stale` workflow uses `secrets.GITHUB_TOKEN`,
|
||||
# so its closes are attributed to `github-actions[bot]` — the same
|
||||
# login as Agent Shin. A historical Agent Shin marker comment
|
||||
# from an earlier close cycle must NOT satisfy the provenance check
|
||||
# for a later stale-initiated close (which never posts that marker).
|
||||
events = [
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
},
|
||||
{
|
||||
"event": "reopened",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:20Z",
|
||||
},
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-04-01T00:00:00Z",
|
||||
},
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:05Z",
|
||||
"body": "I'm **Agent Shin**, the automated triage bot ...",
|
||||
},
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-03-25T00:00:00Z",
|
||||
"body": "This pull request has been automatically marked as stale...",
|
||||
},
|
||||
]
|
||||
self._install(monkeypatch, triage_module, events, comments)
|
||||
assert triage_module.was_auto_closed_by_agent_shin("o/r", 1) is False
|
||||
|
||||
def test_should_accept_marker_from_current_cycle_after_reopen(
|
||||
self, triage_module, monkeypatch
|
||||
):
|
||||
# Two clean Agent Shin cycles: closed, reconsider→reopened, closed
|
||||
# again with a new marker comment posted in the current cycle.
|
||||
# The second-cycle marker is what proves provenance.
|
||||
events = [
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:10Z",
|
||||
},
|
||||
{
|
||||
"event": "reopened",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:20Z",
|
||||
},
|
||||
{
|
||||
"event": "closed",
|
||||
"actor": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:40Z",
|
||||
},
|
||||
]
|
||||
comments = [
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:05Z",
|
||||
"body": "I'm **Agent Shin**, the automated triage bot ...",
|
||||
},
|
||||
{
|
||||
"user": {"login": "github-actions[bot]"},
|
||||
"created_at": "2025-01-01T00:00:35Z",
|
||||
"body": "I'm **Agent Shin** — still missing X ...",
|
||||
},
|
||||
]
|
||||
self._install(monkeypatch, triage_module, events, comments)
|
||||
assert triage_module.was_auto_closed_by_agent_shin("o/r", 1) is True
|
||||
|
||||
|
||||
class TestCallLlmJudge:
|
||||
"""call_llm_judge sets gpt-5 specific kwargs correctly."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue