diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index 31cd9421a5e..6989f210569 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -246,16 +246,29 @@ def was_auto_closed_by_agent_shin(repo: str, number: int) -> bool: commenting `@agent-shin reconsider`. 2. A comment authored by the same bot login that performed the close contains the Agent Shin auto-close marker - (`AGENT_SHIN_AUTO_CLOSE_MARKER`). Matching the comment author - to the closer rules out closures by unrelated bots (stale, - cla-assistant, etc.) and spoofing via marker text pasted by - non-bot accounts. + (`AGENT_SHIN_AUTO_CLOSE_MARKER`) AND was posted in the current + open→closed cycle (after the most recent `reopened` event, if + any, and no later than the most recent `closed` event). This + anchors the marker to the close that's actually being + reconsidered: a stale-workflow closure that runs as + `github-actions[bot]` (because `actions/stale` uses + `secrets.GITHUB_TOKEN`, the same identity as Agent Shin) does + NOT post a marker in its own cycle, so the cycle-anchored check + refuses to override it even though a historical Agent Shin + marker comment still exists from an earlier cycle. """ events = fetch_issue_events(repo, number) - last_closer: str | None = None + last_close_ts = "" + last_closer = "" + last_reopen_ts = "" for event in events: - if (event.get("event") or "").lower() == "closed": + kind = (event.get("event") or "").lower() + ts = event.get("created_at") or "" + if kind == "closed": + last_close_ts = ts last_closer = ((event.get("actor") or {}).get("login") or "").lower() + elif kind == "reopened": + last_reopen_ts = ts if not last_closer or not last_closer.endswith("[bot]"): return False for comment in fetch_issue_comments(repo, number): @@ -263,8 +276,14 @@ def was_auto_closed_by_agent_shin(repo: str, number: int) -> bool: if login != last_closer: continue body = comment.get("body") or "" - if AGENT_SHIN_AUTO_CLOSE_MARKER in body: - return True + if AGENT_SHIN_AUTO_CLOSE_MARKER not in body: + continue + comment_ts = comment.get("created_at") or "" + if last_reopen_ts and comment_ts <= last_reopen_ts: + continue + if last_close_ts and comment_ts > last_close_ts: + continue + return True return False diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 88b44ab0500..1776880e1e9 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -311,10 +311,17 @@ class TestWasAutoClosedByAgentShin: def test_should_return_true_when_latest_close_was_bot_with_marker( self, triage_module, monkeypatch ): - events = [{"event": "closed", "actor": {"login": "github-actions[bot]"}}] + events = [ + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + } + ] comments = [ { "user": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:05Z", "body": ( "👋 Hi, thanks for the PR! I'm **Agent Shin**, the automated " "triage bot for this repository.\n\nThis PR is being **auto-closed**..." @@ -333,10 +340,17 @@ class TestWasAutoClosedByAgentShin: # be treated as proof Agent Shin closed the PR. Even if a bot did # the most recent close, the marker comment must be authored by # that same bot login — not by the human. - events = [{"event": "closed", "actor": {"login": "github-actions[bot]"}}] + events = [ + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + } + ] comments = [ { "user": {"login": "outside-dev"}, + "created_at": "2025-01-01T00:00:05Z", "body": "I'm **Agent Shin**, just kidding — please reconsider this.", } ] @@ -346,14 +360,22 @@ class TestWasAutoClosedByAgentShin: def test_should_ignore_bot_comment_without_marker(self, triage_module, monkeypatch): # Other bots (codecov, cla-assistant, etc.) post on every PR; their # presence must not satisfy the provenance check. - events = [{"event": "closed", "actor": {"login": "github-actions[bot]"}}] + events = [ + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + } + ] comments = [ { "user": {"login": "codecov[bot]"}, + "created_at": "2025-01-01T00:00:01Z", "body": "## Codecov Report ...", }, { "user": {"login": "greptile-apps[bot]"}, + "created_at": "2025-01-01T00:00:02Z", "body": "Confidence Score: 2/5", }, ] @@ -364,20 +386,31 @@ class TestWasAutoClosedByAgentShin: # Agent Shin auto-closed first, contributor commented after; the # bot-authored marker comment is anywhere in the timeline. events = [ - {"event": "labeled", "actor": {"login": "krrishdholakia"}}, - {"event": "closed", "actor": {"login": "github-actions[bot]"}}, + { + "event": "labeled", + "actor": {"login": "krrishdholakia"}, + "created_at": "2025-01-01T00:00:01Z", + }, + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + }, ] comments = [ { "user": {"login": "codecov[bot]"}, + "created_at": "2025-01-01T00:00:02Z", "body": "## Codecov Report", }, { "user": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:05Z", "body": "I'm **Agent Shin**, the automated triage bot ...", }, { "user": {"login": "outside-dev"}, + "created_at": "2025-01-01T00:00:15Z", "body": "Replying after auto-close ...", }, ] @@ -392,13 +425,26 @@ class TestWasAutoClosedByAgentShin: # must NOT override the maintainer's later closure even though the # historical Agent Shin marker comment still exists. events = [ - {"event": "closed", "actor": {"login": "github-actions[bot]"}}, - {"event": "reopened", "actor": {"login": "github-actions[bot]"}}, - {"event": "closed", "actor": {"login": "krrishdholakia"}}, + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + }, + { + "event": "reopened", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:20Z", + }, + { + "event": "closed", + "actor": {"login": "krrishdholakia"}, + "created_at": "2025-01-01T00:00:30Z", + }, ] comments = [ { "user": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:05Z", "body": "I'm **Agent Shin**, the automated triage bot ...", } ] @@ -413,19 +459,110 @@ class TestWasAutoClosedByAgentShin: # `github-actions[bot]` but the most recent closer is # `stale[bot]`, so the logins don't match -> refuse to reopen. events = [ - {"event": "closed", "actor": {"login": "github-actions[bot]"}}, - {"event": "reopened", "actor": {"login": "github-actions[bot]"}}, - {"event": "closed", "actor": {"login": "stale[bot]"}}, + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + }, + { + "event": "reopened", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:20Z", + }, + { + "event": "closed", + "actor": {"login": "stale[bot]"}, + "created_at": "2025-01-01T00:00:30Z", + }, ] comments = [ { "user": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:05Z", "body": "I'm **Agent Shin**, the automated triage bot ...", } ] self._install(monkeypatch, triage_module, events, comments) assert triage_module.was_auto_closed_by_agent_shin("o/r", 1) is False + def test_should_refuse_when_stale_workflow_re_closed_after_agent_shin( + self, triage_module, monkeypatch + ): + # The repo's `actions/stale` workflow uses `secrets.GITHUB_TOKEN`, + # so its closes are attributed to `github-actions[bot]` — the same + # login as Agent Shin. A historical Agent Shin marker comment + # from an earlier close cycle must NOT satisfy the provenance check + # for a later stale-initiated close (which never posts that marker). + events = [ + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + }, + { + "event": "reopened", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:20Z", + }, + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-04-01T00:00:00Z", + }, + ] + comments = [ + { + "user": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:05Z", + "body": "I'm **Agent Shin**, the automated triage bot ...", + }, + { + "user": {"login": "github-actions[bot]"}, + "created_at": "2025-03-25T00:00:00Z", + "body": "This pull request has been automatically marked as stale...", + }, + ] + self._install(monkeypatch, triage_module, events, comments) + assert triage_module.was_auto_closed_by_agent_shin("o/r", 1) is False + + def test_should_accept_marker_from_current_cycle_after_reopen( + self, triage_module, monkeypatch + ): + # Two clean Agent Shin cycles: closed, reconsider→reopened, closed + # again with a new marker comment posted in the current cycle. + # The second-cycle marker is what proves provenance. + events = [ + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:10Z", + }, + { + "event": "reopened", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:20Z", + }, + { + "event": "closed", + "actor": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:40Z", + }, + ] + comments = [ + { + "user": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:05Z", + "body": "I'm **Agent Shin**, the automated triage bot ...", + }, + { + "user": {"login": "github-actions[bot]"}, + "created_at": "2025-01-01T00:00:35Z", + "body": "I'm **Agent Shin** — still missing X ...", + }, + ] + self._install(monkeypatch, triage_module, events, comments) + assert triage_module.was_auto_closed_by_agent_shin("o/r", 1) is True + class TestCallLlmJudge: """call_llm_judge sets gpt-5 specific kwargs correctly."""