diff --git a/.github/scripts/_agent_shin_actions.py b/.github/scripts/_agent_shin_actions.py new file mode 100644 index 00000000000..b3d1ff055b3 --- /dev/null +++ b/.github/scripts/_agent_shin_actions.py @@ -0,0 +1,50 @@ +"""Dry-run wrapper(s) around Agent Shin GitHub mutations. + +The rollout scripts currently need only one mutation wrapped, so this module +exposes a single ``maybe_post_comment`` helper. It takes a ``dry_run: bool`` +keyword argument and the body is intentionally trivial: + + if dry_run: + print(...) # log what we would do, return + return + real_mutation(...) # otherwise, actually do it + +That shape means a dry-run preview differs from the real run in exactly one +line per side effect: the call site. So when you `python3 script.py` locally +without ``--close``, you can be confident the actions printed are the ones the +GitHub Action would have performed (modulo ordering on retry/error paths, +which are deliberately simple). Any further mutation a rollout script needs +should get the same ``maybe_*`` treatment instead of calling the raw +``triage_with_llm`` mutation directly. + +Importing from this module pulls in the real mutation from ``triage_with_llm`` +— call sites in the rollout scripts should NEVER import ``post_comment`` +directly; that would skip the dry-run gate and is the bug class this module +exists to prevent. +""" + +from __future__ import annotations + +import sys +import textwrap + +# Import the module itself rather than the bare names so monkeypatching +# `triage_with_llm.post_comment` (or any of the other mutations) in tests is +# reflected here — `from triage_with_llm import post_comment` would bind the +# original function to a local name and bypass the patch, defeating the whole +# point of these wrappers. +import triage_with_llm + + +def _log(line: str) -> None: + """Print a single dry-run line to stdout (one log statement per side effect).""" + print(line, file=sys.stdout, flush=True) + + +def maybe_post_comment(repo: str, number: int, body: str, *, dry_run: bool) -> None: + """Post a comment on ``repo#number`` — or, in dry-run, log what we would post.""" + if dry_run: + _log(f"[DRY RUN] comment {repo}#{number}:") + _log(textwrap.indent(body, " ")) + return + triage_with_llm.post_comment(repo, number, body) diff --git a/.github/scripts/agent_shin_shared.py b/.github/scripts/agent_shin_shared.py new file mode 100644 index 00000000000..8f3dc3c2322 --- /dev/null +++ b/.github/scripts/agent_shin_shared.py @@ -0,0 +1,211 @@ +"""Constants and helpers shared by Agent Shin's triage scripts. + +Both `triage_with_llm.py` (the LLM-judge entrypoint) and +`close_low_quality_prs.py` (the daily Greptile-score sweep) need to +agree on the same notions of: + + * What counts as a Greptile-authored review comment + (``GREPTILE_BOT_LOGINS``) and how to extract a confidence score from + its body (``SCORE_PATTERN`` / :func:`extract_greptile_score`). + * How long the 2-hour grace window is (``GRACE_PERIOD_SECONDS``) and + the HTML marker stamped into a grace-warning comment so the *other* + script can see "Agent Shin already warned" and behave accordingly + (``GRACE_COMMENT_MARKER``). + * Who Agent Shin is on GitHub (``AGENT_SHIN_DEFAULT_BOT_LOGIN``). + * How GitHub-style ISO-8601 timestamps round-trip into timezone-aware + :class:`datetime.datetime` (:func:`parse_iso8601`). + +Keeping these in one module means a future change (new Greptile output +format, a longer grace window, a new allowlisted account) is a single edit +instead of two — the original split version had to call out in comments +that the two copies "must stay in sync" precisely because nothing +enforced it. +""" + +from __future__ import annotations + +import datetime as dt +import json +import os +import re +import subprocess +from typing import Iterable + +GREPTILE_BOT_LOGINS = frozenset({"greptile-apps", "greptile-apps[bot]"}) + +SCORE_PATTERN = re.compile( + r"confidence\s*score\s*[:\-]?\s*(\d+)\s*/\s*5", + re.IGNORECASE, +) + +GRACE_COMMENT_MARKER = "" + +# Hidden HTML marker stamped on every Agent Shin auto-close comment (the LLM +# judge's grace/review-gate close and the daily Greptile sweep's close). +# `was_closed_by_agent_shin` requires this marker — not just the closing actor — +# before `@agent-shin reconsider` may reopen, because the `github-actions[bot]` +# identity is shared with every other workflow in the repo and is not unique to +# Agent Shin. Both close paths must stamp it or the reconsider path silently +# rejects the contributor. +AGENT_SHIN_CLOSE_MARKER = "" + +# 2 hours between the grace warning and the auto-close. Short enough to +# dogfood the "fix it before it closes" loop in one sitting; bump back up +# (e.g. 86400 for a day) for the public rollout. +GRACE_PERIOD_SECONDS = 7200 + +AGENT_SHIN_DEFAULT_BOT_LOGIN = "github-actions[bot]" + + +def _logins(*names: str) -> frozenset[str]: + """Build a login set normalized for case-insensitive membership checks. + + Callers compare via ``login.lower() in ``, so the stored values + must be lowercase. Normalizing here lets the literals keep each + account's canonical GitHub casing (e.g. ``SwiftWinds``) for + readability without breaking the lookup. + """ + return frozenset(name.lower() for name in names) + + +# Dogfood rollout gate. While this set is non-empty, Agent Shin acts ONLY on +# PRs/issues authored by these logins and skips everyone else. For an +# allowlisted author the usual internal/external classification is bypassed, so +# an internal account (e.g. a maintainer's own work login) still gets triaged +# while the bot is being tested on a small set of accounts. Empty the set to +# lift the restriction and restore full triage for the public rollout. Logins +# are compared case-insensitively. +ALLOWLIST_LOGINS = _logins("mateo-berri", "SwiftWinds") + +# `gh {pr,issue} list` has no "fetch everything" flag — `--limit` is the only +# control and it defaults to 30. Pass a ceiling far above any realistic open +# backlog (low thousands today) so gh paginates the API until the queue is +# exhausted rather than silently truncating. The bulk sweeps MUST see the whole +# backlog: gh lists newest-first, so a low cap drops the *oldest* PRs/issues — +# exactly the stale ones a low-quality sweep is meant to catch. +GH_LIST_ALL_LIMIT = 100_000 + + +def extract_greptile_score(comments: Iterable[dict]) -> tuple[int, dict] | None: + """Return (score, comment) for the most recent Greptile-authored comment + that contains a "Confidence Score: X/5". Returns None if no such comment. + + "Most recent" is determined by the comment's `updated_at` (falling back to + `created_at`), so re-reviews override earlier passes. + """ + candidates: list[tuple[str, int, dict]] = [] + for comment in comments: + user = (comment.get("user") or {}).get("login", "") + if user not in GREPTILE_BOT_LOGINS: + continue + body = comment.get("body") or "" + match = SCORE_PATTERN.search(body) + if not match: + continue + score = int(match.group(1)) + timestamp = comment.get("updated_at") or comment.get("created_at") or "" + candidates.append((timestamp, score, comment)) + + if not candidates: + return None + + candidates.sort(key=lambda triple: triple[0]) + _, score, comment = candidates[-1] + return score, comment + + +def parse_iso8601(value: str) -> dt.datetime: + """Parse a GitHub ISO-8601 timestamp into a timezone-aware datetime.""" + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def gh(*args: str) -> str: + """Run a `gh` CLI command and return stdout. Raises on non-zero exit. + + Shared by both Agent Shin entrypoints so a future change here + (timeout handling, logging, retry on transient failures) only needs + to be made once. + """ + result = subprocess.run( + ["gh", *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +def list_open_items(kind: str, *, repo: str | None, fields: str) -> list[dict]: + """Return EVERY open PR (``kind="pr"``) or issue (``kind="issue"``) in ``repo``. + + Wraps ``gh {pr,issue} list`` with ``--limit GH_LIST_ALL_LIMIT`` so the full + backlog is fetched instead of the default 30 (or any other arbitrary cap). + Both bulk sweeps — the daily Greptile closer and the one-shot rollout + heads-up — rely on this seeing the whole queue, including the oldest items. + + ``fields`` is the comma-separated ``--json`` field list the caller needs + (e.g. ``"number"`` for the rollout, the full set for the closer). + """ + if kind not in ("pr", "issue"): + raise ValueError(f"kind must be 'pr' or 'issue', got {kind!r}") + repo_args = ["--repo", repo] if repo else [] + raw = gh( + kind, + "list", + "--state", + "open", + "--limit", + str(GH_LIST_ALL_LIMIT), + "--json", + fields, + *repo_args, + ) + return json.loads(raw) + + +def seconds_since_latest_marker_comment( + comments: Iterable[dict], + *, + marker: str, + bot_login: str | None = None, + now: dt.datetime | None = None, +) -> float | None: + """Return seconds since the bot's most recent comment containing ``marker``. + + Filters comments by author so a contributor who quotes the HTML + marker (e.g. via GitHub's "Quote reply" feature, which preserves + HTML comments in the raw markdown of the quoted text) is not + mistaken for a bot warning — that would silently reset cooldown + timers and suppress legitimate notifications. + + ``bot_login`` defaults to the `AGENT_SHIN_BOT_LOGIN` env override or + ``AGENT_SHIN_DEFAULT_BOT_LOGIN`` so callers normally don't need to + pass it. ``now`` is injectable for tests / callers (like the daily + sweep) that want every age calculation pinned to one snapshot. + """ + expected_login = ( + bot_login + or os.environ.get("AGENT_SHIN_BOT_LOGIN") + or AGENT_SHIN_DEFAULT_BOT_LOGIN + ).lower() + latest: dt.datetime | None = None + for comment in comments: + author = ((comment.get("user") or {}).get("login") or "").lower() + if author != expected_login: + continue + body = comment.get("body") or "" + if marker not in body: + continue + created = comment.get("created_at") + if not created: + continue + try: + ts = parse_iso8601(created) + except ValueError: + continue + if latest is None or ts > latest: + latest = ts + if latest is None: + return None + reference = now if now is not None else dt.datetime.now(dt.timezone.utc) + return (reference - latest).total_seconds() diff --git a/.github/scripts/close_low_quality_prs.py b/.github/scripts/close_low_quality_prs.py new file mode 100644 index 00000000000..7b9bbb579e3 --- /dev/null +++ b/.github/scripts/close_low_quality_prs.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python3 +""" +Auto-close low-quality pull requests. + +Closes open PRs (including drafts, regardless of age) that satisfy ALL of: + 1. Have a Greptile (`greptile-apps`) review comment whose latest + "Confidence Score: X/5" is below the configured threshold (default: 4). + 2. Are authored by an external OSS contributor (internal BerriAI + contributors are exempt). + 3. Do not carry an opt-out label (default: "do not close"). + +`--min-age-days` is retained as an opt-in safety net for one-off backfill +runs (default: 0). The team's intent is that the count of open PRs equals +the count of PRs internal collaborators need to action on, so neither age +nor draft status acts as a free pass. + +For each match, the script posts an explanatory comment and closes the PR. +Because OSS contributors *cannot* reopen a PR closed by the bot/maintainer +(GitHub limitation), the close-comment instructs them to push their fixes +and **open a fresh PR**, or to comment `@agent-shin reconsider` on the +closed PR to have the LLM judge re-evaluate (and reopen on pass). + +Requires the `gh` CLI to be authenticated. + +Usage examples: + # Dry run (default) - prints what would be closed + python3 close_low_quality_prs.py + + # Actually close matching PRs + python3 close_low_quality_prs.py --close + + # Restrict to PRs at least N days old (one-off backfill safety net) + python3 close_low_quality_prs.py --min-age-days 7 --min-score 4 --close +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import subprocess +import sys +from typing import Iterable + +# Add this script's directory to `sys.path` so the sibling +# `agent_shin_shared` module is importable when the script is invoked +# directly (e.g. `python3 .github/scripts/close_low_quality_prs.py ...`). +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from agent_shin_shared import ( # noqa: E402 -- sys.path adjusted above + AGENT_SHIN_CLOSE_MARKER, + ALLOWLIST_LOGINS, + GRACE_COMMENT_MARKER, + GRACE_PERIOD_SECONDS, + GREPTILE_BOT_LOGINS, + SCORE_PATTERN, + extract_greptile_score, + gh, + list_open_items, + parse_iso8601, + seconds_since_latest_marker_comment, +) + +# `GREPTILE_BOT_LOGINS` and `SCORE_PATTERN` (Greptile's GitHub App login +# variants and the "Confidence Score: X/5" regex) are imported from +# `agent_shin_shared` so the LLM judge in `triage_with_llm.py` and this +# daily Greptile sweep read the score through the same set of logins +# and the same regex. + +# `author_association` values for internal BerriAI contributors who should be +# exempt from auto-triage. +INTERNAL_AUTHOR_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"}) + +# Default labels that exempt a PR from auto-close. Defined at module scope (not +# as a mutable argparse default) so that `--optout-label foo` REPLACES the +# defaults instead of appending to them — the argparse `action="append"` + +# `default=[...]` combination silently mutates the shared default list. +DEFAULT_OPTOUT_LABELS = ("do not close", "keep open", "wip") + +# `GRACE_COMMENT_MARKER` (HTML marker appended to grace-period warning +# comments — used by either script to recognize that a warning was +# already posted) and `GRACE_PERIOD_SECONDS` (length of the grace +# period between the warning and the actual auto-close, 2 hours) are +# imported from `agent_shin_shared` so the Agent Shin LLM judge and +# this daily Greptile sweep agree on the same marker and duration. + + +def fetch_open_prs(repo: str | None) -> list[dict]: + """Fetch all open PRs (number, createdAt, isDraft, labels, author). + + Includes drafts: `gh pr list --state open` returns both ready-for-review + and draft PRs by default. This is the desired behavior — drafts are not + a free pass; the internal-collaborator open-PR queue should reflect every + PR that needs human attention regardless of draft status. + """ + fields = "number,title,createdAt,isDraft,labels,author,url" + return list_open_items("pr", repo=repo, fields=fields) + + +def fetch_pr_author_association(pr_number: int, repo: str | None) -> str: + """Return the GitHub `author_association` for a PR, uppercase. + + Values: OWNER, MEMBER, COLLABORATOR, CONTRIBUTOR, FIRST_TIME_CONTRIBUTOR, + FIRST_TIMER, MANNEQUIN, NONE. Returns "" on lookup failure. + """ + endpoint = ( + f"repos/{repo}/pulls/{pr_number}" + if repo + else f"repos/{{owner}}/{{repo}}/pulls/{pr_number}" + ) + try: + data = json.loads(gh("api", endpoint)) + except subprocess.CalledProcessError: + return "" + return (data.get("author_association") or "").upper() + + +def is_external_pr_author(pr: dict, repo: str | None) -> bool: + """Return True if the PR author is an external OSS contributor. + + Internal = `OWNER` / `MEMBER` / `COLLABORATOR` association, or a bot login. + """ + login = ((pr.get("author") or {}).get("login") or "").lower() + if login.endswith("[bot]") or login in {"dependabot", "github-actions"}: + return False + association = fetch_pr_author_association(pr["number"], repo) + # Fail-safe: if the API lookup failed (empty string), treat the author as + # internal so we don't auto-close their PR. Auto-close is destructive, so + # an unknown association should never make a PR eligible for closing. + if not association or association in INTERNAL_AUTHOR_ASSOCIATIONS: + return False + return True + + +def fetch_pr_comments(pr_number: int, repo: str | None) -> list[dict]: + """Fetch issue-level comments on a PR (where Greptile posts its summary).""" + endpoint = ( + f"repos/{repo}/issues/{pr_number}/comments?per_page=100" + if repo + else f"repos/{{owner}}/{{repo}}/issues/{pr_number}/comments?per_page=100" + ) + raw = gh("api", "--paginate", endpoint) + comments: list[dict] = [] + for line in raw.strip().splitlines(): + line = line.strip() + if not line: + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + # A malformed line should not blow up the whole sweep. Skip and + # carry on so the remaining PRs in this run still get evaluated. + continue + if isinstance(parsed, list): + comments.extend(parsed) + else: + comments.append(parsed) + return comments + + +def has_optout_label(pr: dict, optout_labels: set[str]) -> bool: + labels = {label.get("name", "").lower() for label in pr.get("labels", [])} + return bool(labels & {lbl.lower() for lbl in optout_labels}) + + +def seconds_since_last_grace_warning( + comments: Iterable[dict], + *, + bot_login: str | None = None, + now: dt.datetime | None = None, +) -> float | None: + """Return seconds since the bot's most recent grace-period warning, or + None if no such warning has ever been posted on this PR. + + Thin wrapper over + `agent_shin_shared.seconds_since_latest_marker_comment` — the + centralized helper handles the bot-author filter, marker match, + timestamp parsing, and `now` injection. Keeping this wrapper + preserves the closer's "already-fetched comments + injectable now" + interface so callers (and tests) don't need to change. + """ + return seconds_since_latest_marker_comment( + comments, + marker=GRACE_COMMENT_MARKER, + bot_login=bot_login, + now=now, + ) + + +def format_grace_warning_comment(score: int, threshold: int) -> str: + """Comment posted on the FIRST low-Greptile-score detection — gives + the contributor a 2-hour grace window before the auto-close fires on + the next daily cron run. + + Mirrors `format_grace_warning_pr_comment` in + `triage_with_llm.py` in spirit (2-hour grace + escape hatches), but + framed around Greptile's confidence score instead of the LLM judge's + rubric since the close trigger here is the Greptile signal. + """ + return ( + "🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this " + "repository.\n" + "\n" + "Heads up: Greptile's most recent review scored this PR " + f"**{score}/5**, below our merge bar of **{threshold}/5**.\n" + "\n" + "If the score isn't lifted in the next **2 hours**, I'll auto-close this PR. That's " + "**not** us saying the change isn't worthwhile. We want the open-PR list to mirror " + "what a maintainer can act on *right now*, so contributors like you don't get lost in " + "a backlog. Take your time; everything below still works after the close.\n" + "\n" + "**During the grace period:** push fixes that address Greptile's feedback, then comment " + "`@greptileai` to request a fresh review. If " + f"the new score is **{threshold}/5 or higher**, the PR stays open and no further " + "action is needed on your side.\n" + "\n" + "**If the PR does get auto-closed in 2 hours, you still have an easy recovery path:**\n" + "\n" + "- Comment `@greptileai` to request a fresh review. **This still works even after " + f"the PR is closed**, and a score of {threshold}/5 or higher is one of the signals " + "that lifts the PR back into the review queue. A low Greptile score isn't a blocker.\n" + "- Comment `@agent-shin reconsider` after pushing fixes; I'll re-run the rubric and " + "reopen the PR if both gates (description rubric + Greptile score) now pass.\n" + "\n" + f"{GRACE_COMMENT_MARKER}" + ) + + +def post_grace_warning( + pr: dict, + score: int, + threshold: int, + repo: str | None, + dry_run: bool, +) -> None: + """Post the 2-hour grace-period warning comment on `pr`. + + The warning carries `GRACE_COMMENT_MARKER` so subsequent runs can + detect that the contributor has already been told about the + pending close. Does NOT close the PR — the close happens on the + next eligible run after `GRACE_PERIOD_SECONDS` elapses (handled + by `close_pr`). + """ + pr_number = pr["number"] + repo_args = ["--repo", repo] if repo else [] + + if dry_run: + print( + f" [DRY RUN] Would post grace warning to PR #{pr_number} " + f"(greptile={score}/5): {pr['title']}" + ) + return + + comment_body = format_grace_warning_comment(score, threshold) + gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args) + print(f" Posted grace warning on PR #{pr_number} (greptile={score}/5)") + + +def format_close_comment(score: int, threshold: int) -> str: + """Comment posted when a low-Greptile-score PR is auto-closed. + + Carries `AGENT_SHIN_CLOSE_MARKER` so the `@agent-shin reconsider` path + (guarded by `was_closed_by_agent_shin`) recognizes this as an Agent Shin + close and is allowed to reopen the PR once it passes again; without the + marker that recovery path the comment advertises silently rejects the + contributor. + """ + score_sentence = ( + f"Greptile's most recent review scored this PR **{score}/5**, below " + f"our merge bar of **{threshold}/5**, and the 2-hour grace period since " + "the warning has elapsed.\n\n" + ) + return ( + f"Closing as part of automated PR triage.\n\n" + f"{score_sentence}" + "We close low-confidence PRs aggressively to keep the review queue " + "manageable for maintainers and contributors alike. **This is not a " + "rejection of the idea.** To bring this back:\n\n" + "1. Push the fixes that address Greptile's feedback (continue using " + "your existing branch is fine).\n" + "2. **Open a new PR** with the updated branch. Greptile will review " + "it again, and if it scores " + f"**{threshold}/5 or higher** a maintainer will take another look.\n\n" + "_Why open a new PR instead of reopening this one?_ GitHub does not " + "let external contributors reopen a PR that was closed by a bot or " + "maintainer, so a fresh PR is the most reliable path forward. If you " + "would prefer this exact PR re-evaluated, comment " + "`@agent-shin reconsider` once you've pushed the fixes; Agent Shin " + "will re-run triage and reopen this PR if it now meets the bar. " + "You can also comment `@greptileai` to request a fresh Greptile " + "review; that works **even after the PR is closed**.\n\n" + "Thanks for contributing to LiteLLM. We know auto-closures can sting; " + "the goal is to keep the project healthy, not to dismiss your work." + f"\n\n{AGENT_SHIN_CLOSE_MARKER}" + ) + + +def close_pr( + pr: dict, + score: int, + threshold: int, + age_days: int, + repo: str | None, + dry_run: bool, + label: str | None, +) -> None: + """Post the explanatory comment and close the PR.""" + pr_number = pr["number"] + repo_args = ["--repo", repo] if repo else [] + + if dry_run: + print( + f" [DRY RUN] Would close PR #{pr_number} " + f"(age={age_days}d, greptile={score}/5): {pr['title']}" + ) + return + + comment_body = format_close_comment(score, threshold) + gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args) + + if label: + try: + gh("pr", "edit", str(pr_number), "--add-label", label, *repo_args) + except subprocess.CalledProcessError as exc: + stderr = (exc.stderr or "").strip() + print(f" warn: failed to add label '{label}' to #{pr_number}: {stderr}") + + gh("pr", "close", str(pr_number), *repo_args) + print(f" Closed PR #{pr_number} (greptile={score}/5, age={age_days}d)") + + +def evaluate_pr( + pr: dict, + now: dt.datetime, + min_age_days: int, + min_score: int, + repo: str | None, + optout_labels: set[str], + allowlist: frozenset[str] = ALLOWLIST_LOGINS, +) -> tuple[str, int | None, int | None]: + """Decide what to do with `pr` on this triage run. + + Returns (action, score_or_none, age_days_or_none) where action is one of: + "skip-too-young", "skip-optout-label", "skip-not-allowlisted", + "skip-internal", "skip-no-greptile-score", "skip-score-ok", + "warn-grace", "skip-in-grace-period", or "close". + + Drafts are NOT skipped — the goal is "open PR count == PRs internal + collaborators need to action on", and a draft that Greptile scored <4/5 + is still in that queue. Authors can opt out via the `wip` label (see + `DEFAULT_OPTOUT_LABELS`) if they need to keep a long-lived draft open. + + Grace-period semantics: the first time a PR fails the rubric, the + action is `warn-grace` — the caller should post a warning comment but + NOT close the PR. On a subsequent run, if the warning is still less + than `GRACE_PERIOD_SECONDS` old AND the PR still fails, the action is + `skip-in-grace-period`. Once the warning ages out and the rubric is + still failing, the action is `close`. + """ + if has_optout_label(pr, optout_labels): + return ("skip-optout-label", None, None) + + created = parse_iso8601(pr["createdAt"]) + age_days = (now - created).days + # `min_age_days` defaults to 0 (close as soon as Greptile scores low). + # Set a positive value via --min-age-days for one-off backfill runs that + # want to skip very-young PRs. + if min_age_days > 0 and age_days < min_age_days: + return ("skip-too-young", None, age_days) + + # While the allowlist is active it is the sole author gate: only those + # logins are acted on and the external-only restriction is bypassed for + # them. Otherwise auto-close only external OSS contributors — internal + # contributors (BerriAI org members) handle their own backlog. + login = ((pr.get("author") or {}).get("login") or "").lower() + if allowlist: + if login not in allowlist: + return ("skip-not-allowlisted", None, age_days) + elif not is_external_pr_author(pr, repo): + return ("skip-internal", None, age_days) + + comments = fetch_pr_comments(pr["number"], repo) + extraction = extract_greptile_score(comments) + if extraction is None: + return ("skip-no-greptile-score", None, age_days) + + score, _ = extraction + if score >= min_score: + return ("skip-score-ok", score, age_days) + + grace_age = seconds_since_last_grace_warning(comments, now=now) + if grace_age is None: + return ("warn-grace", score, age_days) + if grace_age < GRACE_PERIOD_SECONDS: + return ("skip-in-grace-period", score, age_days) + + return ("close", score, age_days) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repo", + type=str, + default=None, + help="Repository (owner/repo). Auto-detected if omitted.", + ) + parser.add_argument( + "--min-age-days", + type=int, + default=0, + help=( + "Minimum age (in days) before a PR is eligible. Default 0 = " + "close as soon as Greptile flags it. Set a positive value for " + "one-off backfill runs that want to spare very-young PRs." + ), + ) + parser.add_argument( + "--min-score", + type=int, + default=4, + choices=range(1, 6), + help="Greptile score below which a PR is closed (default: 4 -> closes <4/5).", + ) + parser.add_argument( + "--optout-label", + action="append", + default=None, + help=( + "Label(s) that exempt a PR from auto-close. Repeat to add more. " + "Case-insensitive. When omitted, defaults to " + f"{list(DEFAULT_OPTOUT_LABELS)!r}; passing this flag REPLACES the " + "defaults (argparse `append` with a mutable default would append " + "instead, which we explicitly avoid)." + ), + ) + parser.add_argument( + "--close-label", + type=str, + default=None, + help=( + "Optional label to add to PRs that get auto-closed " + "(e.g. 'auto-closed-low-quality'). Must already exist on the repo." + ), + ) + parser.add_argument( + "--close", + action="store_true", + help="Actually close matching PRs (default is dry-run).", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Maximum number of PRs to close in one run (safety net).", + ) + args = parser.parse_args() + + dry_run = not args.close + if dry_run: + print("=== DRY RUN MODE (pass --close to actually close PRs) ===\n") + + print("Fetching open PRs...") + prs = fetch_open_prs(args.repo) + print(f"Found {len(prs)} open PRs.\n") + + now = dt.datetime.now(dt.timezone.utc) + optout_labels = set(args.optout_label or DEFAULT_OPTOUT_LABELS) + + closed = 0 + summary = { + "close": 0, + "warn-grace": 0, + "skip-in-grace-period": 0, + "skip-too-young": 0, + "skip-optout-label": 0, + "skip-not-allowlisted": 0, + "skip-internal": 0, + "skip-no-greptile-score": 0, + "skip-score-ok": 0, + } + + # `warned` tracks grace-warning comments posted in this run so the + # `--limit` safety net bounds *all* destructive write actions, not + # just closures. Without this cap, a backlog of PRs failing the + # threshold simultaneously could flood contributors with comments. + warned = 0 + for pr in sorted(prs, key=lambda p: p["createdAt"]): + try: + action, score, age_days = evaluate_pr( + pr, + now, + args.min_age_days, + args.min_score, + args.repo, + optout_labels, + ) + summary[action] = summary.get(action, 0) + 1 + + if action == "warn-grace": + assert score is not None + print( + f"#{pr['number']}: \"{pr['title']}\" " + f"(age={age_days}d, greptile={score}/5) -> warn-grace" + ) + post_grace_warning( + pr, + score=score, + threshold=args.min_score, + repo=args.repo, + dry_run=dry_run, + ) + if not dry_run: + warned += 1 + if args.limit is not None and (warned + closed) >= args.limit: + print( + f"\nReached --limit={args.limit} " + f"(closed={closed}, warned={warned}); stopping." + ) + break + continue + + if action != "close": + continue + + assert score is not None and age_days is not None + print( + f"#{pr['number']}: \"{pr['title']}\" " + f"(age={age_days}d, greptile={score}/5) -> close" + ) + close_pr( + pr, + score=score, + threshold=args.min_score, + age_days=age_days, + repo=args.repo, + dry_run=dry_run, + label=args.close_label, + ) + + if not dry_run: + closed += 1 + if args.limit is not None and (warned + closed) >= args.limit: + print( + f"\nReached --limit={args.limit} " + f"(closed={closed}, warned={warned}); stopping." + ) + break + except Exception as exc: # noqa: BLE001 - per-PR errors don't abort the sweep + summary["error"] = summary.get("error", 0) + 1 + print( + f"!! PR #{pr.get('number')}: {exc}", + file=sys.stderr, + ) + continue + + print("\n=== Summary ===") + for key, value in summary.items(): + print(f" {key:28s} {value}") + if dry_run: + print(f"\nTotal would close: {summary['close']}") + else: + print(f"\nTotal closed: {closed}") + print( + f"Total {'would warn (grace)' if dry_run else 'warned (grace)'}: " + f"{summary['warn-grace']}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/triage-requirements.txt b/.github/scripts/triage-requirements.txt new file mode 100644 index 00000000000..a18f05fbb95 --- /dev/null +++ b/.github/scripts/triage-requirements.txt @@ -0,0 +1,282 @@ +# Hash-pinned dependency set for the Agent Shin triage scripts. +# Installed in privileged triage workflows, so every package is pinned to an +# exact version with SHA-256 hashes and installed with pip --require-hashes. +# +# Regenerate after bumping openai: +# echo 'openai==' \ +# | uv pip compile - --generate-hashes --python-version 3.12 \ +# --no-annotate --no-header -o .github/scripts/triage-requirements.txt + +annotated-types==0.7.0 \ + --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ + --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 +anyio==4.14.0 \ + --hash=sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89 \ + --hash=sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9 +certifi==2026.6.17 \ + --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ + --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db +distro==1.9.0 \ + --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 +jiter==0.15.0 \ + --hash=sha256:01a8222cf05ab1128e239421156c207949808acaaea2bdfd33130ae666786e86 \ + --hash=sha256:032396229564bca02440396bd327710719f724f5e7b7e9f7a8eb3faa4a2c2281 \ + --hash=sha256:04b400bbf8c9efb03d9bdd976475c919c1d85593b04b9fff7ae234065daf87ae \ + --hash=sha256:05906b93d72f03339e6bb7cf8dc10ebda64a0266126eed6beba79e20abcf5fd4 \ + --hash=sha256:066f8f33f18b2419cd8213b2436fa7fbc9c499f315971cfa3ce1f9820c001b1b \ + --hash=sha256:0ab068bce62a45aa3e7367eceaffb5dde60b7eb853be8dece45132e3d0ff4879 \ + --hash=sha256:0be6f5ad41a809f303f416d17cec92a7a725902fb9b4f3de3d19362ac0ef8554 \ + --hash=sha256:0e90a1c315a0226ec822d973817967f9223b7701546c8c2a7913e7ab0926294d \ + --hash=sha256:0f862193b8696249d22ec433e85fd2ab0ad9596bc3e45e6c0bc55e8aeba97be2 \ + --hash=sha256:1303d4d68a9b051ea90502402063ecf3807da00ad2affa19ca1ae3b90b3c5f67 \ + --hash=sha256:144f8e72cb53dab146347b91cceac01f5481237f2b93b4a339a1ee8f8878b67c \ + --hash=sha256:182226cbc930c9fab81bc2e41a4da672f89539906dadb05e75670ac07b94f71f \ + --hash=sha256:1c11465f97e2abf45a014b83b730222f8f1c5335e802c7055a67d50de6f1f4e3 \ + --hash=sha256:1c15024a3d892223b18f597c86d59387249dc396590844ce6b9f6131d1093bae \ + --hash=sha256:1d54fb5b31dea401a41af3f8a7d2512e9b6a6a005491e6166c7e4ffab9639a9c \ + --hash=sha256:25ffbe229aa8cd98c28879d8aa1a6e34ae77992ab984a65fba800859dab16269 \ + --hash=sha256:2a77aadd57cac1682e4401a72724d2796d89a4ba129b1a5812aa94ee480826eb \ + --hash=sha256:2ae901f3a55bfafdde31d289590fa25e3245735a2b1e8c7cc15871710a002871 \ + --hash=sha256:2b0074e2f56eb2dacca1689760fd2852a068f85a0547a157b82cb4cafeb6768b \ + --hash=sha256:2c8aea7781d2a372227871de4e1a1332aa96f5a89fd76c5e835dafdbad102887 \ + --hash=sha256:2c9cb907439d20bd0c7d7565ca01ee52234203208433749bae5b516907526928 \ + --hash=sha256:2fb6a5d26af81fc0f00f9360a891e05cf755e149bba391c4d563adc54812973d \ + --hash=sha256:2fd73e3da91a0a722d67165e849ce2cdc10de0e0d48738c142be8c6c5f310f4c \ + --hash=sha256:30ce1a5d16b5641dc935d50ef775af6a0871e3d14ab05d6fc54dff371b78e558 \ + --hash=sha256:30ce785d2adb8e32c3f7741442370a74834ec4c01f3c48f0750227a0b4ef27d6 \ + --hash=sha256:30f2218e6a9e5c18bc10fe6d41ac189c442c88eacf11bad9f28ef95a9bef00e6 \ + --hash=sha256:351a341c2105aa430b7047e30f1bf7975f6313b00165d3fc07be2edaf741f279 \ + --hash=sha256:37a10c377ce3a4a85f4a67f28b7afe093154cde77eaf248a72e856aa08b4d865 \ + --hash=sha256:392b8ab019e5502d08aff85c6272209c24bc2cbe706ea82a56368f524236614a \ + --hash=sha256:3e4540b8e74e4268811ac05db226a6a128ff572e7e0ce3f1163b693cadb184cd \ + --hash=sha256:40b2c7e92c44a84d748d21706c68dc6ff8161d80b59c99d774721a0d2317d7c7 \ + --hash=sha256:411fa4dfa5a7ae3d11491027ffb9beadec3996010a986862db70d91abba1c750 \ + --hash=sha256:4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76 \ + --hash=sha256:42bfb257930800cf43e7c62c832402c704ab60797c992faf88d20e903eac8f32 \ + --hash=sha256:4363818355dbc70ae1a8e9eaba9de350d93ede4ff6992b8f8eb8cbb6e5122d42 \ + --hash=sha256:4ab395feec8d249ec4044e228e98a7033f043426a265df439dc3698823f0a4e4 \ + --hash=sha256:50164d7610c00e7cd913a873fce30b6beeebf4b37e53983e33f22de4c900f6b8 \ + --hash=sha256:50e51156192722a9c58db112837d3f8ef96fb3c5ecc14e95f409134b08b158ec \ + --hash=sha256:510c8b3c17a0ed9ac69850c0438dada3c9b82d9c4d589fcb62002a5a9cf3a866 \ + --hash=sha256:5157de9f76eb4bc5ea74a1219366a25f945ad305641d74e04f59c54087091aa9 \ + --hash=sha256:54d5d6090cdc1b7c9e780dfb04949a990adb1e301a2fc0bbcee7de4638d33f9a \ + --hash=sha256:553fcac2ef2cb990877f9fc0833b8b629a3e6a5670b6b5fd58219b41a653ddc4 \ + --hash=sha256:5607e6013ed7e6b0ec9661e467b7ffde0aa7ab36833a04850f26fcf88ed4845b \ + --hash=sha256:5d6a60072b44c3c2b797a7ddcbcbbf2b34ea3cfd4721580fbfd2a09d9d9b84ba \ + --hash=sha256:5f30bae8bc1c2d613e28e5af3e8cceb09b742f1c8a8a5f839fb67afaffc03b61 \ + --hash=sha256:62ebd14e47e9aed9df4472afcb2663668ce4d74891cd54f86bf6e44029d6dc89 \ + --hash=sha256:631f13a3d04e97d4e083993b10f4b99530e3a10d953e2eb5e196b7dc7f812ce0 \ + --hash=sha256:6550fa135c7deb8ead6af49ed7ff648532ea8334a1447fe34a36315ef79c5c29 \ + --hash=sha256:66b1880df2d01e206e8339769d1c7c1753bcb653efd6289e203f6f24ebada0c0 \ + --hash=sha256:6eac374c5c975709b69c10f09afd199df74150172156ad10c8d4fd785b7da995 \ + --hash=sha256:71683c38c825452999b5717fcae07ea708e8c93003e808be4319c1b02e3d176e \ + --hash=sha256:7553333dd0930c104a5a0db8df72bf7219fe663d731383b576bb6ed6351c984d \ + --hash=sha256:75e8a04e91432dde9f1838373cf93d23726c79d3e908d319acf0e796f85592e7 \ + --hash=sha256:773b6eb282ce11ee19f05f6b2d4404fa308e5bbd353b0b80a0262caad6db2cd7 \ + --hash=sha256:774f93f65031856bf14ad9f59bdcab8b8cad501e5ceabd51ba3525f76937a25b \ + --hash=sha256:7c468136b8bd6bb18c8786e4236a1fa27362f24cb23450ba0cb204ab379b8e6f \ + --hash=sha256:7ce8902f939970048b233087082e7bb829db29375811c7ad50687b8624c6fd08 \ + --hash=sha256:7d3d6683288c11cbab50e865f2e2f13950179aa45410e30b2cfbd3fb7b0177bf \ + --hash=sha256:7f6163c0f10b055245f814dcc59f4818da60dfe72f3e72ab89fc24b6bd5e9c52 \ + --hash=sha256:8020c99ec13a7db2b6f96cbe82ef4721c88b426a4892f27478044af0284615ef \ + --hash=sha256:813dfbb17d65328bf86e5f0905dd277ba2265d3ca20556e86c0c7035b7182e5a \ + --hash=sha256:860a74063284a2ae9bfedd694f299cc2c68e2696c5f3d440cc9d18bb81b9dd04 \ + --hash=sha256:8c9004af7c8d67cce7f1aae1026fb55607f4aa600710d08ede3a3ce4aeefe7e0 \ + --hash=sha256:8d2c0c44d569ce0f2850f5c926f8caeb5f245fbc84475aeb36efccc2103e6dbd \ + --hash=sha256:8f7e9bc0f1135039b22ee6eab588d42df1ce55842b30740a352885eb267bd941 \ + --hash=sha256:90c5db5527c221249a876160663ab891ace358c17f7b9c93ec1478b7f0550e5c \ + --hash=sha256:9100ddbec09741cc66feb0fc6773f8bdbd0e3c345689368f260082ff85dcc0cd \ + --hash=sha256:913d02d29c9606643418d9ccfc3b72492ab25a6bf7889934e09a3490f8d3438b \ + --hash=sha256:980c256edb05b78a111b99c4de3b1d32e31634b867fd1fc2cf726e7b7bba9854 \ + --hash=sha256:9f924585cdacf631cd382b657966847bb537bf9ed0a6f9b991da5f05a631480f \ + --hash=sha256:a254e10b593624d230c365b6d616b22ca0ad65e63a16e6631c2b3466022e6ba8 \ + --hash=sha256:a2a438005b6f22d0273413484d6094d7c2c5d10ec1b3a3bf128e0d1d3ba53258 \ + --hash=sha256:a97261f1fccb8e50ecd2890a96e46efdc3f57c80a197324c6777827231eca712 \ + --hash=sha256:ab596fa3837e91e7e6a31b5f639988bfc6a35d1f915ac3932d946062219d588f \ + --hash=sha256:abbf258599526ad0326fe51e252e24f2bd6f24f1852681b4b78feda3808f1d18 \ + --hash=sha256:ac0d9ddea4350974be7a221fc25895f251a8fee748c889bdced2141c0fec1a49 \ + --hash=sha256:acf4ee4d1fc55917239fe72972fb292dd773055d05eb040d36f4326e02cc2c0e \ + --hash=sha256:ae1b0d82ac2d987f9ea512b1c9adfcc71a28de3dea3a6039b54d76cffda9901e \ + --hash=sha256:b15741f501469009ae0ae90b7147958a664a7dede40aa7ff174a8a4645f546d0 \ + --hash=sha256:b15d3ec9b0449c40e85319bdb4caa8b77ab526e74f5532ed94bec15e2f66822c \ + --hash=sha256:b3b3b775e33d3bfaec9899edc526ae97b0da0bf9d071a46124ba419149a414f8 \ + --hash=sha256:b6c0ffae686c39bf3737be60793783267628783ea42545632c10b291105aee45 \ + --hash=sha256:c210f8b35dc6f30aafd4b4365ca89b9d1189f21ab49b8e68fa6322a847aef138 \ + --hash=sha256:c2f6bb8b5216ab9e7873bc08b5d7bef2b8abbb578a3069bf1cd14a45d71d771d \ + --hash=sha256:c60e71b6d10cfc284c9bf36bd885e8d44c46f688ce50aa91b5edd90181dea687 \ + --hash=sha256:c6694a173ecabc12eb60efbc0b474464ead1951ff65cd8b1e72100715c64512b \ + --hash=sha256:c77496cb10bd7549690fbbab3e5ec05857b83e49276f4a9423a766ddd2afcd4c \ + --hash=sha256:c84c1b7be454b0c16f8499b4ebfbfd82ea5cca6527cceefcbbc06a7557b5ed2e \ + --hash=sha256:cc0bc345cf2df9d1c00ac443f50d543c1ccfa8b0422cb85b1ab70d681c0b255b \ + --hash=sha256:ceb8fc27d38793f9c97149be8302720c5b22e5c195a37bf2c45dc36c4600a512 \ + --hash=sha256:cf4bd113a69c0a740e27cb962ce10630c36d2b8f59d759a651b955ee9d18a823 \ + --hash=sha256:d1aa62e277fc1cbd80e6deacae6f4d983b41b3d7728e0645c5d741a6149bba45 \ + --hash=sha256:d1e7b1776f0797956c509e123d0952d10d293a9492dea9f288ab9570ec01d1a5 \ + --hash=sha256:d636d5095155afd364247f65070fab7beda13498d7ff4de331046e704ab9657f \ + --hash=sha256:d726e3ceeb337191324b49de298142f27c3ad10886341555d1d5315b5f252c6a \ + --hash=sha256:d72d8af5c1013656a8870c866660627d1a75bc185814ee022c8533caa1de88ae \ + --hash=sha256:d8d2955167274e15d79a7a020afdd9b39c990eb80b2d89fca695d92dcfdd38ec \ + --hash=sha256:d92a5cd21fdb083931d546c207aa29633787c5dc5b02daab2d32b843f88a2c53 \ + --hash=sha256:e58585a58209d72691ce2d62a9147445f5a87beb0bde97fde284c96ae392a3d1 \ + --hash=sha256:e7196e56f1cd69af1dbb07dff02dcfb260a50b45a82d409d92a06fedb32473b5 \ + --hash=sha256:eda3071db3346334beae1360b46da4606da57bf3528c167b3c38533afaf9f2c5 \ + --hash=sha256:edebcf7d1f601199084bb6e844d7dc67e03e04f6ac786b0332d616635c4ff7a4 \ + --hash=sha256:ef1fd24d9413f6209e00d3d5a453e67acfe004a25cc6c8e8484faed4311ab9e8 \ + --hash=sha256:f0b271b462769543716f92d3a4f90527df6ef5ed05ee95ec4137f513e21e1b77 \ + --hash=sha256:f18f85e4218d1b40f000f42a92239a7a61a902cd42c65e6c360dbd17dcb20894 \ + --hash=sha256:f1e1754960f38ec40613a07e5e372df67acb3b890fb383b6fb3de3e49ddbf3c7 \ + --hash=sha256:f2143ab06181d2b029eedcb6af3cebe95f11bbac62441781860f98ee9330a6a6 \ + --hash=sha256:f3d37768fce7f88dd2a8c6091f2325dea27d30d30d5c6e7a1c0f0af77723b708 \ + --hash=sha256:fa248c9eb220197d363f688818dac2fd4b2f0cd7d843ca7105d652034823427d +openai==2.33.0 \ + --hash=sha256:03ac37d70e8c9e3a8124214e3afa785e2cbc12e627fbd98177a086ef2fd87ad5 \ + --hash=sha256:f850c435e2a4685bba3295bd54912dd26315d9c1b7733068186134d6e0599f9a +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ + --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 +pydantic-core==2.46.4 \ + --hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \ + --hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \ + --hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \ + --hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \ + --hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \ + --hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \ + --hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \ + --hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \ + --hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \ + --hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \ + --hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \ + --hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \ + --hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \ + --hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \ + --hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \ + --hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \ + --hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \ + --hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \ + --hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \ + --hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \ + --hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \ + --hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \ + --hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \ + --hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \ + --hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \ + --hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \ + --hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \ + --hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \ + --hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \ + --hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \ + --hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \ + --hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \ + --hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \ + --hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \ + --hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \ + --hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \ + --hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \ + --hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \ + --hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \ + --hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \ + --hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \ + --hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \ + --hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \ + --hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \ + --hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \ + --hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \ + --hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \ + --hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \ + --hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \ + --hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \ + --hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \ + --hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \ + --hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \ + --hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \ + --hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \ + --hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \ + --hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \ + --hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \ + --hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \ + --hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \ + --hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \ + --hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \ + --hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \ + --hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \ + --hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \ + --hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \ + --hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \ + --hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \ + --hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \ + --hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \ + --hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \ + --hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \ + --hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \ + --hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \ + --hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \ + --hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \ + --hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \ + --hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \ + --hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \ + --hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \ + --hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \ + --hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \ + --hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \ + --hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \ + --hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \ + --hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \ + --hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \ + --hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \ + --hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \ + --hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \ + --hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \ + --hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \ + --hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \ + --hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \ + --hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \ + --hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \ + --hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \ + --hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \ + --hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \ + --hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \ + --hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \ + --hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \ + --hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \ + --hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \ + --hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \ + --hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \ + --hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \ + --hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \ + --hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \ + --hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \ + --hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \ + --hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \ + --hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \ + --hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \ + --hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \ + --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \ + --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc +tqdm==4.68.3 \ + --hash=sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482 \ + --hash=sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03 +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 diff --git a/.github/scripts/triage_rollout_heads_up.py b/.github/scripts/triage_rollout_heads_up.py new file mode 100644 index 00000000000..a5dedb1c9e7 --- /dev/null +++ b/.github/scripts/triage_rollout_heads_up.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python3 +"""One-shot 7-day heads-up sweep for the Agent Shin rollout. + +Posts a friendly "the OSS triage bot kicks in next Monday" comment on every +open external PR/issue that currently *would* fail the new rubric — i.e., +every PR/issue Agent Shin would close once the rollout completes. The point +is to give contributors a full week to fix their description before the bot +ever takes a destructive action, so nobody is surprised by an auto-close. + +The script is designed to run **exactly once** at rollout, fired by a manual +``workflow_dispatch`` (``dry_run=false``) on the heads-up workflow. Re-runs +are safe: every comment is stamped with the hidden ``HEADS_UP_MARKER`` and +PRs/issues that already carry the marker are skipped. + +Dry-run vs. real run +-------------------- +Defaults to dry-run. Passing ``--close`` flips into real mode. Every GitHub +mutation goes through ``_agent_shin_actions``, which has a one-line +``if dry_run: log else: do_it`` per call, so the only difference between a +dry-run preview and the real run is the call site that actually hits the +GitHub API. + +Local preview:: + + python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm + +Real run (the manual rollout dispatch uses this):: + + python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm --close +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import sys +from pathlib import Path +from typing import Any + +# Make the sibling triage_with_llm + _agent_shin_actions importable when this +# script is invoked directly (the GitHub workflow does `python3 .github/scripts/...`). +_SCRIPTS_DIR = Path(__file__).resolve().parent +if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) + +from _agent_shin_actions import maybe_post_comment # noqa: E402 +from agent_shin_shared import ( # noqa: E402 + AGENT_SHIN_DEFAULT_BOT_LOGIN, + ALLOWLIST_LOGINS, + list_open_items, +) +from triage_with_llm import ( # noqa: E402 + DEFAULT_MODEL, + call_llm_judge, + fetch_issue, + fetch_pr, + gh, + is_internal_contributor, + review_gate, + triage, +) + +# Hidden marker so re-runs skip PRs/issues we've already notified. Distinct from +# the within-grace / ready / regressed markers so it can't be confused with the +# steady-state lifecycle comments. +HEADS_UP_MARKER = "" + +# Placeholder until the litellm-docs PR ships. The rollout blog post explains +# the new rubric, the 7-day grace, and how to recover after an auto-close. +# TODO(docs): replace with the canonical URL once the litellm-docs PR merges. +ROLLOUT_BLOG_URL = "https://docs.litellm.ai/docs/agent_shin_triage_rollout" + +# Default cutoff is one week from "now". Computed at runtime so the wording +# stays correct even if the rollout is merged later than planned. The user can +# override with --close-on YYYY-MM-DD when running the script manually. +DEFAULT_GRACE_DAYS = 7 + +# The daily auto-close sweeps (close_low_quality_prs.yml at 09:00 UTC and +# review_gate.yml at 09:30 UTC) are what actually close a still-failing item, +# so the deadline we promise contributors has to name that wall-clock moment. +ACTIVATION_TIME_UTC = "09:00 UTC" + + +def _format_cutoff(cutoff: dt.date) -> str: + """Human-readable, timezone-explicit cutoff, e.g. ``Monday, June 1, 2026 + (09:00 UTC)`` — the moment a still-failing PR/issue gets closed.""" + return ( + f"{cutoff.strftime('%A, %B')} {cutoff.day}, {cutoff.year} " + f"({ACTIVATION_TIME_UTC})" + ) + + +def _rubric_section_pr() -> str: + return ( + "**Going forward, every external PR needs ONE of:**\n" + "\n" + "- A linked GitHub issue using a closing keyword: " + "`Fixes #1234`, `Closes #1234`, or `Resolves #1234`, OR\n" + "- All three of: a clear **problem description**, **expected vs. " + "actual behavior**, and **end-to-end QA proof** (at least one of a " + "short screen recording / video, before/after screenshots, or the " + "exact commands you ran with their real output; mocked or stubbed " + "runs don't count).\n" + "\n" + "PRs also need a **Greptile confidence score of 4/5 or higher** before " + "the bot will tag them `ready for review`. You can `@greptileai` to " + "request a fresh review at any time, including after the PR is closed." + ) + + +def _rubric_section_issue() -> str: + return ( + "**Going forward, every external issue needs:**\n" + "\n" + "- For **bug reports**: end-to-end evidence of the bug (at least one " + "of a screen recording / video, a screenshot, or the exact commands " + "you ran with their real output / traceback) plus expected vs. actual " + "behavior. Written steps with no run output don't count, and mocked " + "or stubbed runs don't count.\n" + "- For **feature requests**: a clear description of the proposed " + "feature plus a use case + concrete example (config, API call, UI " + "flow, or scenario showing what's blocked today)." + ) + + +def _description_only_note(kind: str) -> str: + noun = "PR" if kind == "pr" else "issue" + return ( + f"⚠️ **The requirements must live in the {noun} *description*, not in " + "comments.** Some PRs/issues collect 100+ comments from humans and " + "bots; reading the entire thread on every triage run would balloon " + "GitHub API usage (we'd start getting 429'd) and blow out the LLM " + "judge's context. The bot only reads the description, so anything " + "you add as a comment will be invisible to it." + ) + + +def _missing_section(verdict: dict, greptile_score: int | None) -> str: + """Bullet list of what's currently missing on this PR/issue. + + Combines the LLM judge's `missing` list (rubric items) with a Greptile + shortfall (for PRs) so the contributor sees one list of things to fix. + """ + missing = list(verdict.get("missing") or []) + if greptile_score is not None and greptile_score < 4: + missing.insert( + 0, + f"Greptile's most recent review scored this PR {greptile_score}/5 " + "(below the 4/5 bar Agent Shin will require).", + ) + if not missing: + return ( + "_The bot couldn't articulate a specific missing piece; see the " + "rubric link above and double-check the description includes all " + "of it before the rollout._" + ) + bullets = "\n".join(f"- {m}" for m in missing) + return f"**What this one is currently missing:**\n\n{bullets}" + + +def _recovery_section(kind: str) -> str: + if kind == "pr": + return ( + "**If the bot closes this PR after the rollout:** update the " + "description with the missing pieces, then either open a fresh " + "PR or comment `@agent-shin reconsider` on the closed PR. If " + "Greptile re-scores you at 4/5 or higher I'll reopen and tag " + "the PR `ready for review`. (`@greptileai` works on closed PRs " + "too; a fresh review is one of the signals that lifts you back " + "into the queue.) This is **not** us losing interest in your " + "change; far from it. We just need open PRs to be a list of " + "things a maintainer can act on, so we can get to yours faster." + ) + return ( + "**If the bot closes this issue after the rollout:** edit the issue " + "description to add the missing pieces, then comment `@agent-shin " + "reconsider` on the closed issue. I'll re-evaluate and, if the rubric " + "is met, reopen it. (GitHub doesn't let external authors reopen an " + "issue a maintainer or bot closed, so the comment is the reliable " + "path.) This is **not** us saying the bug isn't real or the request " + "isn't useful; it's so the remaining open issues are a list of things " + "a maintainer can act on." + ) + + +def format_heads_up_comment( + *, kind: str, verdict: dict, greptile_score: int | None, cutoff: dt.date +) -> str: + """Compose the friendly 7-day heads-up comment posted on a failing PR/issue.""" + noun = "PR" if kind == "pr" else "issue" + rubric = _rubric_section_pr() if kind == "pr" else _rubric_section_issue() + cutoff_str = _format_cutoff(cutoff) + explanation = (verdict.get("explanation") or "").strip() + explanation_block = ( + f"> _(The judge's note for this one: {explanation})_\n\n" if explanation else "" + ) + + return ( + "🚅 **Heads-up: we're turning on the OSS triage bot in " + f"{DEFAULT_GRACE_DAYS} days, on {cutoff_str}.**\n" + "\n" + "We're rolling out **Agent Shin**, an LLM-as-judge triage bot for " + f"external {noun}s. Once it's live, the bot reads each open " + f"{noun}'s description, scores it against a small rubric, and " + f"auto-closes any {noun} that's missing the basics, with a single " + f"comment explaining what's missing and how to recover. Full " + f"context: [Agent Shin rollout blog post]({ROLLOUT_BLOG_URL}).\n" + "\n" + f"{rubric}\n" + "\n" + f"{_description_only_note(kind)}\n" + "\n" + f"{_missing_section(verdict, greptile_score)}\n" + "\n" + f"{explanation_block}" + "**Timeline (you have a week):**\n" + "\n" + f"- We turn the bot on in {DEFAULT_GRACE_DAYS} days, on " + f"**{cutoff_str}**. You have until then to update this {noun}'s " + "description with the missing pieces above.\n" + f"- If this {noun} still fails the rubric at **{cutoff_str}**, " + "we'll close it.\n" + f"- From then on the bot runs daily, and every {noun} that fails " + "the rubric gets a **2-hour lifetime**: one warning comment, then " + "auto-close 2 hours later.\n" + "\n" + f"{_recovery_section(kind)}\n" + "\n" + f"{HEADS_UP_MARKER}" + ) + + +def _list_open_numbers(repo: str, kind: str) -> list[int]: + """Return every open PR or issue number in ``repo``. + + Delegates to ``list_open_items`` so the full backlog is fetched (no cap) + and the `gh {pr,issue} list` invocation stays in one shared place. ``gh + issue list`` would include PRs, but ``list_open_items`` uses the dedicated + command per kind, so the two never mix. + """ + return [ + item["number"] for item in list_open_items(kind, repo=repo, fields="number") + ] + + +def _has_heads_up_marker(item: dict) -> bool: + """Cheap fast-path: check the PR/issue body itself for the marker. + + The marker is appended to the *comment* we post, not the body, so this + will only fire if the body literally contains the marker text. We still + do the comment-marker check separately below; this body check just lets + us short-circuit for PRs/issues that quote the marker for any reason. + """ + body = item.get("body") or "" + return HEADS_UP_MARKER in body + + +def _comments_have_marker(repo: str, number: int) -> bool: + """True if the bot already posted a comment carrying the marker. + + Used for idempotency: a re-run skips items the previous run notified. + Filters by author (matching the sibling marker-checks in + ``triage_with_llm._has_marker`` and + ``agent_shin_shared.seconds_since_latest_marker_comment``) so a + contributor who quotes the heads-up via GitHub's "Quote reply" — which + preserves HTML comments in the raw markdown — can't trick the + idempotency check into silently skipping a real heads-up. + + Comments live on the unified issues endpoint regardless of whether the + item is a PR or an issue, so no ``kind`` argument is required here. + """ + expected_login = ( + os.environ.get("AGENT_SHIN_BOT_LOGIN") or AGENT_SHIN_DEFAULT_BOT_LOGIN + ).lower() + raw = gh( + "api", + "--paginate", + f"repos/{repo}/issues/{number}/comments?per_page=100", + ) + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + comments = payload if isinstance(payload, list) else [payload] + for comment in comments: + author = ((comment.get("user") or {}).get("login") or "").lower() + if author != expected_login: + continue + if HEADS_UP_MARKER in (comment.get("body") or ""): + return True + return False + + +def _evaluate_pr(*, repo: str, number: int, model: str, judge: Any = None) -> dict: + """Run the future PR rubric (review_gate) in dry-run and return the result.""" + return review_gate( + repo=repo, + number=number, + close=False, # we only want the verdict, never act here + model=model, + judge=judge, + ) + + +def _evaluate_issue(*, repo: str, number: int, model: str, judge: Any = None) -> dict: + """Run the future issue rubric (triage kind='issue') in dry-run.""" + return triage( + repo=repo, + kind="issue", + number=number, + close=False, + model=model, + judge=judge, + ) + + +def _would_be_closed(kind: str, result: dict) -> bool: + """True if the future triage would auto-close this PR/issue based on the + rubric (regardless of grace-period gating). + + For PRs we trust ``review_gate``'s ``passing`` field — it combines the LLM + verdict and the Greptile score. For issues we read the LLM verdict + directly. Both fields are ``None``/missing on skip paths + (skip-internal-author, skip-llm-error, etc.) where the future bot would + NOT close the item — those return False. + """ + if kind == "pr": + passing = result.get("passing") + if passing is None: + return False # skipped — nothing for the heads-up to warn about + return passing is False + verdict = result.get("verdict") or {} + return (verdict.get("verdict") or "").lower() == "fail" + + +def _process_one( + *, + repo: str, + kind: str, + number: int, + model: str, + cutoff: dt.date, + dry_run: bool, + judge: Any = None, + skip_marker_check: bool = False, + allowlist: frozenset[str] = ALLOWLIST_LOGINS, +) -> dict: + """Evaluate one PR/issue and post a heads-up if it would be auto-closed. + + Returns a per-item dict for the summary table. + """ + base = {"kind": kind, "number": number} + fetcher = fetch_pr if kind == "pr" else fetch_issue + item = fetcher(repo, number) + + if (item.get("state") or "") != "open": + return {**base, "action": "skip-not-open"} + if allowlist: + login = (item.get("user") or {}).get("login") or "" + if login.lower() not in allowlist: + return {**base, "action": "skip-not-allowlisted"} + elif is_internal_contributor(item): + return {**base, "action": "skip-internal-author"} + if not skip_marker_check and _has_heads_up_marker(item): + return {**base, "action": "skip-already-marked-in-body"} + if not skip_marker_check and _comments_have_marker(repo, number): + return {**base, "action": "skip-already-notified"} + + if kind == "pr": + result = _evaluate_pr(repo=repo, number=number, model=model, judge=judge) + else: + result = _evaluate_issue(repo=repo, number=number, model=model, judge=judge) + + if not _would_be_closed(kind, result): + return {**base, "action": "skip-passing", "evaluator": result.get("action")} + + verdict = result.get("verdict") or {} + greptile_score = result.get("greptile_score") if kind == "pr" else None + comment = format_heads_up_comment( + kind=kind, verdict=verdict, greptile_score=greptile_score, cutoff=cutoff + ) + maybe_post_comment(repo, number, comment, dry_run=dry_run) + return { + **base, + "action": "heads-up-posted" if not dry_run else "would-post-heads-up", + "verdict": (verdict.get("verdict") or "").lower(), + "greptile_score": greptile_score, + } + + +def _print_summary(results: list[dict]) -> None: + """Tally per-action counts so a dry-run preview tells you at a glance how + many comments the real run would post.""" + counts: dict[str, int] = {} + for r in results: + counts[r["action"]] = counts.get(r["action"], 0) + 1 + print("\n=== rollout heads-up summary ===") + for action in sorted(counts): + print(f" {action:35s} {counts[action]}") + print(f" total {len(results)}") + + +def run( + *, + repo: str, + close: bool, + cutoff: dt.date, + model: str, + kinds: tuple[str, ...] = ("pr", "issue"), + judge: Any = None, + only_numbers: dict[str, list[int]] | None = None, + skip_marker_check: bool = False, +) -> list[dict]: + """Sweep ``repo`` and post heads-up comments. Returns the per-item results.""" + dry_run = not close + if dry_run: + print( + f"[DRY RUN] sweeping {repo}; --close not passed, no comments will be posted." + ) + else: + print(f"[REAL RUN] sweeping {repo}; comments WILL be posted.") + print(f"Cutoff date in comment body: {cutoff.isoformat()}") + + results: list[dict] = [] + for kind in kinds: + if only_numbers and kind in only_numbers: + numbers = list(only_numbers[kind]) + else: + numbers = _list_open_numbers(repo, kind) + print(f"\n--- {kind}s: {len(numbers)} open ---") + for n in numbers: + try: + result = _process_one( + repo=repo, + kind=kind, + number=n, + model=model, + cutoff=cutoff, + dry_run=dry_run, + judge=judge, + skip_marker_check=skip_marker_check, + ) + except ( + Exception + ) as exc: # noqa: BLE001 - per-item errors don't abort the sweep + result = { + "kind": kind, + "number": n, + "action": "error", + "error": str(exc), + } + print(f"!! {kind}#{n}: {exc}", file=sys.stderr) + print(f" {kind}#{n}: {result['action']}") + results.append(result) + _print_summary(results) + return results + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True, help="owner/repo") + parser.add_argument( + "--close", + action="store_true", + help=( + "Actually post comments. Without this flag the script is in " + "dry-run mode and only logs what it would do." + ), + ) + parser.add_argument( + "--close-on", + type=dt.date.fromisoformat, + default=None, + help=( + "Cutoff date shown in the heads-up comment as the rollout date " + f"(default: today + {DEFAULT_GRACE_DAYS} days)." + ), + ) + parser.add_argument( + "--model", + default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL, + help=f"Model for the rubric LLM judge (default: {DEFAULT_MODEL}).", + ) + parser.add_argument( + "--kind", + choices=("pr", "issue", "both"), + default="both", + help="Restrict the sweep to PRs or issues only (default: both).", + ) + parser.add_argument( + "--only-pr", + type=int, + action="append", + default=[], + help="Limit the PR sweep to these PR numbers (repeat for several).", + ) + parser.add_argument( + "--only-issue", + type=int, + action="append", + default=[], + help="Limit the issue sweep to these issue numbers (repeat for several).", + ) + parser.add_argument( + "--ignore-existing-marker", + action="store_true", + help=( + "Re-post on PRs/issues that already carry the heads-up marker. " + "Useful for testing the comment wording on a known PR." + ), + ) + args = parser.parse_args() + + cutoff = args.close_on or ( + dt.datetime.now(dt.timezone.utc).date() + dt.timedelta(days=DEFAULT_GRACE_DAYS) + ) + + kinds: tuple[str, ...] + if args.kind == "pr": + kinds = ("pr",) + elif args.kind == "issue": + kinds = ("issue",) + else: + kinds = ("pr", "issue") + + only: dict[str, list[int]] = {} + if args.only_pr: + only["pr"] = args.only_pr + if args.only_issue: + only["issue"] = args.only_issue + + # The script must NOT hit the LLM in dry-run if no key is set — we still + # want a useful preview that says "skip-no-llm-key" for items that would + # have been judged. Production runs require OPENAI_API_KEY. + if args.close and not os.environ.get("OPENAI_API_KEY"): + parser.error("OPENAI_API_KEY must be set for --close (real-run) mode.") + + run( + repo=args.repo, + close=args.close, + cutoff=cutoff, + model=args.model, + kinds=kinds, + only_numbers=only or None, + skip_marker_check=args.ignore_existing_marker, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py new file mode 100644 index 00000000000..d2536058e01 --- /dev/null +++ b/.github/scripts/triage_with_llm.py @@ -0,0 +1,1778 @@ +#!/usr/bin/env python3 +""" +Agent Shin — LLM-as-judge triage for external OSS pull requests and issues. + +Evaluates a single PR or issue against the contribution rubric and, when the +LLM judge marks it as failing, posts an explanatory comment + closes the +PR/issue. Re-triggers on `reopened` so contributors can iterate back in by +filling in the missing pieces and reopening. + +Internal BerriAI contributors (`author_association` in {OWNER, MEMBER, +COLLABORATOR}) and bot accounts are skipped entirely. + +Usage: + triage_with_llm.py --repo owner/repo --pr 1234 + triage_with_llm.py --repo owner/repo --issue 5678 + triage_with_llm.py --repo owner/repo --pr 1234 --close # actually close + triage_with_llm.py --repo owner/repo --pr 1234 --print-prompt # show prompt + +Defaults are SAFE: without `--close` the script writes a verdict to stdout (and, +when running in GitHub Actions, to $GITHUB_STEP_SUMMARY) but takes no GitHub +write actions. + +Environment: + GH_TOKEN / GITHUB_TOKEN - for `gh` CLI auth (auto-set in Actions) + OPENAI_API_KEY - required when --close is passed + OPENAI_BASE_URL - optional (route to any OpenAI-compatible API) + TRIAGE_MODEL - optional model override (default: gpt-5.4-mini) +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import re +import subprocess +import sys +import textwrap +import urllib.parse +from typing import Any, Iterable + +# Add this script's directory to `sys.path` so the sibling +# `agent_shin_shared` module is importable when the script is invoked +# directly (e.g. `python3 .github/scripts/triage_with_llm.py ...`) and +# also when the tests load this script via +# `importlib.util.spec_from_file_location`. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from agent_shin_shared import ( # noqa: E402 -- sys.path adjusted above + AGENT_SHIN_CLOSE_MARKER, + AGENT_SHIN_DEFAULT_BOT_LOGIN, + ALLOWLIST_LOGINS, + GRACE_COMMENT_MARKER, + GRACE_PERIOD_SECONDS, + GREPTILE_BOT_LOGINS, + SCORE_PATTERN, + extract_greptile_score, + gh, + parse_iso8601, + seconds_since_latest_marker_comment, +) + +DEFAULT_MODEL = "gpt-5.4-mini" + +INTERNAL_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"}) + +# `AGENT_SHIN_DEFAULT_BOT_LOGIN` is imported from `agent_shin_shared`. +# When the workflow uses the default `secrets.GITHUB_TOKEN`, the +# closure / reopen event's `actor.login` is `github-actions[bot]`. The +# env override `AGENT_SHIN_BOT_LOGIN` exists for local debugging and for +# repos that wire Agent Shin to a PAT. + +# HTML marker appended to every reconsider verdict comment. We grep for this +# on subsequent reconsider triggers to enforce a short cooldown so that +# repeated `@agent-shin reconsider` comments don't burn CI/LLM budget. +# Using a unique HTML comment keeps the marker invisible to humans while +# being trivially greppable from a comments-list API response. +RECONSIDER_COMMENT_MARKER = "" + +# Minimum gap between two reconsider verdicts on the same PR/issue. Set to +# 10 minutes — long enough that a contributor can't trivially spam the +# trigger, short enough that a genuine "I just pushed a fix and reupdated +# the body" iteration loop isn't punished. +RECONSIDER_RATE_LIMIT_SECONDS = 600 + +# `GRACE_COMMENT_MARKER` (HTML marker on the grace-period warning comment +# posted on the first low-quality detection — used on subsequent triage +# runs to detect that a warning was already posted and measure how long +# ago it was posted) and `GRACE_PERIOD_SECONDS` (length of the grace +# period between the warning and the actual auto-close, 2 hours) are +# imported from `agent_shin_shared` so the daily Greptile sweep and the +# LLM judge agree on the same marker and duration. + +# --- Review-gate ("ready for review" label lifecycle) configuration ---------- +# The review gate keeps a single label in sync with whether a PR currently +# clears BOTH quality bars: the LLM rubric (clear problem + expected/actual + +# QA proof, or a linked issue) AND Greptile's most recent confidence score. +READY_FOR_REVIEW_LABEL = "ready for review" +DEFAULT_GRACE_DAYS = 1 # 24h before an un-passing, un-tagged PR is auto-closed +DEFAULT_MIN_GREPTILE_SCORE = 4 # Greptile < 4/5 counts as "not passing" + +# Hidden HTML-comment markers stamped into review-gate comments. They never +# render in the GitHub UI but let the gate detect its own prior actions so it +# (a) posts the within-grace "what's missing" notice at most once and (b) can +# tell a first-time pass ("ready for review") from a recovery after a +# regression ("all clear again"). +READY_MARKER = "" +REGRESSED_MARKER = "" +WITHIN_GRACE_MARKER = "" + +# `GREPTILE_BOT_LOGINS` (Greptile's GitHub App login variants — +# `greptile-apps[bot]` in REST API comments, `greptile-apps` in +# `gh pr view --json` output) and `SCORE_PATTERN` (regex matching lines +# like `Confidence Score: 3/5`) are imported from `agent_shin_shared` +# so the daily sweep and the review gate read the score through the +# same set of logins / patterns. + +# `AGENT_SHIN_CLOSE_MARKER` is imported from `agent_shin_shared` so this LLM +# judge and the daily Greptile sweep stamp the same marker on their close +# comments — `was_closed_by_agent_shin` keys the reconsider reopen path off it. + +# Model families that require `reasoning_effort` to be set, and that reject +# `temperature != 1` unless `reasoning_effort` is "none". For these models we +# pass `reasoning_effort="none"` so a `temperature=0` deterministic judgment +# is still accepted. See litellm/llms/openai/chat/gpt_5_transformation.py for +# the full set of constraints LiteLLM applies to these models. +GPT5_FAMILY_PREFIX = "gpt-5" + +# Regexes for picking off "obvious passes" without burning LLM tokens. +# +# Keep this list to GitHub's documented PR-closing keywords only +# (https://docs.github.com/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue). +# Casual mentions like "see #1234" or "ref #1234" are intentionally NOT +# auto-passed — they should fall through to the LLM judge, which has the +# stricter rubric "a bare issue number without a closing keyword counts only +# if it's clearly the related issue (not a passing mention)". +LINKED_ISSUE_PATTERN = re.compile( + r"\b(?:fixes|fix|fixed|closes|close|closed|resolves|resolve|resolved)\s+" + r"(?:#\d+|https?://github\.com/[\w.-]+/[\w.-]+/issues/\d+)", + re.IGNORECASE, +) +HTML_COMMENT_PATTERN = re.compile(r"", re.DOTALL) + + +# --------------------------------------------------------------------------- +# gh helpers +# +# `gh` is imported from `agent_shin_shared` so a future change (timeout, +# logging, retry) only needs to be made once. + + +def fetch_pr(repo: str, number: int) -> dict: + """Return the full GitHub REST representation of a PR.""" + return json.loads(gh("api", f"repos/{repo}/pulls/{number}")) + + +def fetch_issue(repo: str, number: int) -> dict: + """Return the full GitHub REST representation of an issue.""" + return json.loads(gh("api", f"repos/{repo}/issues/{number}")) + + +def post_comment(repo: str, number: int, body: str) -> None: + """Post an issue-style comment (works for both issues and PRs).""" + gh( + "api", + f"repos/{repo}/issues/{number}/comments", + "-X", + "POST", + "-f", + f"body={body}", + ) + + +def close_pr(repo: str, number: int) -> None: + """Close a pull request (state=closed).""" + gh( + "api", + f"repos/{repo}/pulls/{number}", + "-X", + "PATCH", + "-f", + "state=closed", + ) + + +def reopen_pr(repo: str, number: int) -> None: + """Reopen a previously-closed pull request (state=open). + + Used by the `@agent-shin reconsider` comment-trigger flow: the bot has + write access via GH_TOKEN, so it can reopen on the contributor's behalf + even though GitHub doesn't let the OSS author do it themselves. + """ + gh( + "api", + f"repos/{repo}/pulls/{number}", + "-X", + "PATCH", + "-f", + "state=open", + ) + + +def close_issue(repo: str, number: int, *, not_planned: bool = True) -> None: + """Close an issue, marking state_reason=not_planned by default.""" + args = [ + "api", + f"repos/{repo}/issues/{number}", + "-X", + "PATCH", + "-f", + "state=closed", + ] + if not_planned: + args.extend(["-f", "state_reason=not_planned"]) + gh(*args) + + +def reopen_issue(repo: str, number: int) -> None: + """Reopen a previously-closed issue (state=open, state_reason=reopened).""" + gh( + "api", + f"repos/{repo}/issues/{number}", + "-X", + "PATCH", + "-f", + "state=open", + "-f", + "state_reason=reopened", + ) + + +def add_label(repo: str, number: int, label: str) -> None: + """Add a label to a PR/issue (GitHub creates the label if it's missing).""" + gh( + "api", + f"repos/{repo}/issues/{number}/labels", + "-X", + "POST", + "-f", + f"labels[]={label}", + ) + + +def remove_label(repo: str, number: int, label: str) -> None: + """Remove a label from a PR/issue. A missing label (404) is not an error.""" + encoded = urllib.parse.quote(label, safe="") + try: + gh( + "api", + f"repos/{repo}/issues/{number}/labels/{encoded}", + "-X", + "DELETE", + ) + except subprocess.CalledProcessError as exc: + stderr = (exc.stderr or "").lower() + if "404" in stderr or "not found" in stderr: + return + raise + + +def _iter_paginated_json(*api_args: str) -> Any: + """Yield JSON objects from `gh api --paginate ... -q '.[]'`. + + `gh api --paginate` on a JSON-array endpoint concatenates pages into + one stream; `-q '.[]'` flattens that stream into newline-delimited + objects (jq-style). This keeps memory bounded for chatty endpoints + like issue events/comments on long-lived PRs. + """ + raw = gh("api", "--paginate", *api_args, "-q", ".[]") + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + yield json.loads(line) + except json.JSONDecodeError: + # A malformed line should not blow up the whole guard. Skip and + # carry on — at worst the guard fail-closes (returns False / + # None) and the caller treats it as "unknown". + continue + + +def fetch_last_close_event( + repo: str, number: int +) -> tuple[str | None, dt.datetime | None]: + """Return the actor login and timestamp of the most recent `closed` event. + + Either field may be None: actor when the events API returns nothing + (unusual for a closed item, but possible on transient errors), and + timestamp when the event lacks `created_at` or the value can't be + parsed. `was_closed_by_agent_shin` fail-closes on either. + """ + actor: str | None = None + closed_at: dt.datetime | None = None + for event in _iter_paginated_json(f"repos/{repo}/issues/{number}/events"): + if event.get("event") != "closed": + continue + actor = (event.get("actor") or {}).get("login") + created = event.get("created_at") + if not created: + closed_at = None + continue + try: + closed_at = parse_iso8601(created) + except ValueError: + closed_at = None + return actor, closed_at + + +# How much older than the latest `closed` event the Agent Shin marker +# comment is allowed to be while still counting as "this close was Agent +# Shin's". Agent Shin posts the close comment immediately before closing, +# so the marker timestamp is normally at most a few seconds before the +# close event; the buffer just absorbs clock skew between the comments +# API and the events API. +AGENT_SHIN_CLOSE_MARKER_SKEW_SECONDS = 300 + + +def was_closed_by_agent_shin( + repo: str, number: int, *, bot_login: str | None = None +) -> bool: + """Return True iff Agent Shin itself most-recently closed this PR/issue. + + This is the guard that stops `@agent-shin reconsider` from reopening an + item Agent Shin did not close — a maintainer closing for non-rubric + reasons (security, duplicate, design rejection), or a different workflow + (stale/duplicate sweeps) closing under the shared `github-actions[bot]` + identity. Three independent signals must all hold, because that identity + is not unique to Agent Shin and a marker comment from a prior + closed/reopened cycle would otherwise vouch for an unrelated close: + + 1. The most recent `closed` event's actor is the bot identity. + 2. Agent Shin left one of its auto-close comments, detected via + `AGENT_SHIN_CLOSE_MARKER`. The actor check alone can't tell an + Agent Shin close from any other `github-actions[bot]` close. + 3. That marker comment was posted at (or just before) the latest + close event, not on a previous close in an + Agent-Shin-close -> reconsider-reopen -> other-bot-reclose cycle. + + The check is intentionally fail-closed: any uncertainty about who closed + the item is treated as "not Agent Shin" so the destructive reopen path + stays gated. + """ + expected = ( + bot_login + or os.environ.get("AGENT_SHIN_BOT_LOGIN") + or AGENT_SHIN_DEFAULT_BOT_LOGIN + ).lower() + actor, closed_at = fetch_last_close_event(repo, number) + if not actor or actor.lower() != expected or closed_at is None: + return False + marker_seconds = seconds_since_last_agent_shin_close( + repo, number, bot_login=bot_login + ) + if marker_seconds is None: + return False + close_age_seconds = (dt.datetime.now(dt.timezone.utc) - closed_at).total_seconds() + return marker_seconds <= close_age_seconds + AGENT_SHIN_CLOSE_MARKER_SKEW_SECONDS + + +def _seconds_since_latest_marker_comment( + repo: str, + number: int, + *, + marker: str, + bot_login: str | None = None, +) -> float | None: + """Return seconds since the bot's most recent comment with ``marker``. + + Fetches comments via `_iter_paginated_json` and delegates the + iteration / author-filter / timestamp logic to + `agent_shin_shared.seconds_since_latest_marker_comment` so the daily + Greptile sweep and the LLM judge use one source of truth for the + "bot already posted X" detection. The wall-clock `now` is resolved + against this module's `dt` so tests that freeze time via + `monkeypatch.setattr(triage_module, "dt", ...)` still apply. + """ + return seconds_since_latest_marker_comment( + _iter_paginated_json(f"repos/{repo}/issues/{number}/comments"), + marker=marker, + bot_login=bot_login, + now=dt.datetime.now(dt.timezone.utc), + ) + + +def seconds_since_last_reconsider_verdict( + repo: str, number: int, *, bot_login: str | None = None +) -> float | None: + """Return seconds since the bot's most recent reconsider verdict comment. + + Detects comments by matching the HTML marker `RECONSIDER_COMMENT_MARKER` + appended by `format_reopen_comment` and + `format_reconsider_still_failing_comment`. Returns None when the bot + has never posted a reconsider verdict on this PR/issue (or when the + only matching comments are missing a `created_at` timestamp, which + shouldn't happen on a real GitHub response). + """ + return _seconds_since_latest_marker_comment( + repo, number, marker=RECONSIDER_COMMENT_MARKER, bot_login=bot_login + ) + + +def seconds_since_last_grace_warning( + repo: str, number: int, *, bot_login: str | None = None +) -> float | None: + """Return seconds since the bot's most recent grace-period warning. + + Detects warning comments by matching the HTML marker + `GRACE_COMMENT_MARKER` appended by `format_grace_warning_pr_comment` + and `format_grace_warning_issue_comment`. Returns None when no + grace warning has ever been posted on this PR/issue — that's the + "first low-quality detection" signal that drives the warning path. + """ + return _seconds_since_latest_marker_comment( + repo, number, marker=GRACE_COMMENT_MARKER, bot_login=bot_login + ) + + +def seconds_since_last_agent_shin_close( + repo: str, number: int, *, bot_login: str | None = None +) -> float | None: + """Return seconds since Agent Shin's most recent auto-close comment. + + Detects close comments by matching `AGENT_SHIN_CLOSE_MARKER` (stamped by + `format_pr_close_comment` / `format_issue_close_comment`). Returns None + when Agent Shin has never closed this PR/issue — the signal + `was_closed_by_agent_shin` uses to keep the reconsider reopen path gated + against closures performed by other workflows sharing the bot identity. + """ + return _seconds_since_latest_marker_comment( + repo, number, marker=AGENT_SHIN_CLOSE_MARKER, bot_login=bot_login + ) + + +# --------------------------------------------------------------------------- +# Author classification + + +def is_internal_contributor(item: dict) -> bool: + """Return True if the PR/issue author should be exempted from triage. + + Fail-safe: if `author_association` is missing or empty (which should never + happen on a successful GitHub REST response but is possible on schema + changes or partial responses), treat the author as INTERNAL so the + destructive close path never fires on an unknown contributor. This matches + the sibling `is_external_pr_author` in `close_low_quality_prs.py`. + """ + login = ((item.get("user") or {}).get("login") or "").lower() + if login.endswith("[bot]") or login in {"dependabot", "github-actions"}: + return True + association = (item.get("author_association") or "").upper() + if not association or association in INTERNAL_ASSOCIATIONS: + return True + return False + + +# --------------------------------------------------------------------------- +# Greptile score + age helpers (`extract_greptile_score`, `parse_iso8601`) +# live in `agent_shin_shared` — they're imported at the top of this module +# so both `triage_with_llm.py` and `close_low_quality_prs.py` share a +# single source of truth for the Confidence-Score regex and ISO-8601 +# parsing. + + +# --------------------------------------------------------------------------- +# Prompt construction + + +def strip_html_comments(text: str) -> str: + """Remove HTML comments — template placeholder text shouldn't fool the judge.""" + return HTML_COMMENT_PATTERN.sub("", text or "") + + +def has_linked_issue(text: str) -> bool: + """Heuristic: does this body link to an open issue (Fixes #123 etc.)?""" + return bool(LINKED_ISSUE_PATTERN.search(strip_html_comments(text or ""))) + + +def build_pr_prompt(*, title: str, body: str) -> str: + cleaned_body = strip_html_comments(body or "").strip() or "(empty)" + # Dedent the static template *before* interpolating dynamic fields so that + # multi-line bodies (whose 2nd+ lines start at column 0) don't defeat the + # common-indent computation in textwrap.dedent. + template = textwrap.dedent(""" + You are "Agent Shin", the OSS triage bot for the LiteLLM open-source + repository (BerriAI/litellm). Decide whether this external pull request + meets the project's contribution standards. + + A PR PASSES triage only if BOTH (1) AND (2) are satisfied. A linked + issue alone is NOT enough — it covers context, not proof. + + (1) CONTEXT — the PR provides AT LEAST ONE of: + (a) A link to a related GitHub issue. Acceptable forms: + "Fixes #1234", "Closes #1234", "Resolves #1234", + "Refs https://github.com/BerriAI/litellm/issues/1234". A + bare "#1234" without a closing keyword counts only if it + is clearly the related issue (not a passing mention). + (b) A clear problem description in the body (what bug or + missing feature this addresses, beyond the title) AND + expected vs. actual behavior (or, for features, "what's + possible now vs. with this PR"). + + (2) END-TO-END QA PROOF: the PR body contains AT LEAST ONE of: + (a) A screen recording / video showing the behavior before + and after the change (the bug reproducing, then the fix + working). For a brand-new feature with no meaningful + "before", a recording of it working end-to-end is fine. + (b) A screenshot (or before/after screenshots) showing the + fix or feature working. + (c) Specific commands that were actually run (curl, python, + a CLI invocation, etc.) PAIRED WITH their real + output, demonstrating the change works end-to-end against + the real system. Commands whose external dependencies + (LLM provider, DB, network) are mocked or stubbed do NOT + satisfy (2c); they are not end-to-end. + + `has_qa_proof` must be set to `true` only when (2a), (2b), + or a non-mocked (2c) is actually present in the body. If the + only "proof" is mocked tests, `has_qa_proof` is `false` and + the verdict is "fail". + + The following do NOT count as QA proof: + - Generic claims like "I tested it", "works locally", "all + tests pass", or a checked "I added tests" checkbox with no + output shown. + - A description of what tests exist or were added, without + their actual output in the PR body. + - `pytest` (or any test runner) executed against the + repository's own unit tests. Those mock the LLM provider, + DB, and network, so they are NOT end-to-end and never + satisfy (2), no matter how much passing output is pasted. + - A linked issue. The linked issue is context (1a), never + proof (2). + + FAIL the PR if EITHER (1) or (2) is missing. Do not bias toward PASS: + if QA proof is absent, the verdict is "fail" even when the rest of + the PR is well-written. + + Respond with a single JSON object, no prose: + + {{ + "verdict": "pass" | "fail", + "linked_issue": boolean, + "has_problem_description": boolean, + "has_expected_vs_actual": boolean, + "has_qa_proof": boolean, + "qa_proof_type": "video" | "screenshot" | "commands_with_output" | "none", + "missing": ["plain-english strings naming what is missing"], + "explanation": "1-2 sentence reasoning for the team to skim" + }} + + --- + PR title: {title} + + PR body: + --- + {cleaned_body} + --- + """).strip() + return template.format(title=title, cleaned_body=cleaned_body) + + +def build_issue_prompt(*, title: str, body: str) -> str: + cleaned_body = strip_html_comments(body or "").strip() or "(empty)" + # Dedent the static template *before* interpolating dynamic fields so that + # multi-line bodies (whose 2nd+ lines start at column 0) don't defeat the + # common-indent computation in textwrap.dedent. + template = textwrap.dedent(""" + You are "Agent Shin", the OSS triage bot for the LiteLLM open-source + repository (BerriAI/litellm). Decide whether this GitHub issue meets + the project's reporting standards. + + For a BUG REPORT the issue PASSES triage only when it contains BOTH: + (1) END-TO-END EVIDENCE OF THE BUG (the "before"; set + `has_repro=true` only when this is present): AT LEAST ONE of: + (a) A screen recording / video of the bug happening. + (b) A screenshot of the bug. + (c) The exact command(s) actually run (curl, python, a CLI + invocation, etc.) PAIRED WITH their real output, traceback, + or logs showing the failure against the real system. + Commands whose external dependencies (LLM provider, DB, + network) are mocked or stubbed do NOT count. + Prose-only "steps to reproduce" with no run output, video, or + screenshot do NOT satisfy (1). + (2) Expected vs. actual behavior (`has_expected_vs_actual`). + + FAIL the bug report if either (1) or (2) is missing. Do not bias + toward PASS: if the bug isn't demonstrated end-to-end, the verdict is + "fail" even when the report is well-written. + + For a FEATURE REQUEST the issue PASSES triage only when it contains + ALL of: + - A clear description of the proposed feature (what should LiteLLM do + that it does not today). + - Motivation / use case with a concrete example (config, API call, + UI flow, or scenario showing what's blocked today). + + For an issue that is neither a bug report nor a feature request (a + question, support request, or discussion), PASS as long as it has a + clear, specific ask and is not empty or template placeholder text. + + Respond with a single JSON object, no prose: + + {{ + "verdict": "pass" | "fail", + "kind": "bug" | "feature" | "other", + "has_repro": boolean, + "has_expected_vs_actual": boolean, + "has_motivation_example": boolean, + "missing": ["plain-english strings naming what is missing"], + "explanation": "1-2 sentence reasoning for the team to skim" + }} + + --- + Issue title: {title} + + Issue body: + --- + {cleaned_body} + --- + """).strip() + return template.format(title=title, cleaned_body=cleaned_body) + + +# --------------------------------------------------------------------------- +# LLM call + verdict parsing + + +def call_llm_judge( + prompt: str, *, model: str, api_key: str, base_url: str | None +) -> str: + """Call an OpenAI-compatible chat completions endpoint. Returns raw text.""" + # Import inside the function so unit tests that monkey-patch this never + # need the openai package installed. + from openai import OpenAI + + client = ( + OpenAI(api_key=api_key, base_url=base_url) + if base_url + else OpenAI(api_key=api_key) + ) + kwargs: dict[str, Any] = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0, + "response_format": {"type": "json_object"}, + } + # gpt-5.x reasoning models reject `temperature != 1` unless + # `reasoning_effort` is explicitly "none". Set it via `extra_body` so this + # works across openai SDK versions regardless of whether the SDK natively + # types `reasoning_effort` as a top-level chat-completions param yet. + if model.lower().startswith(GPT5_FAMILY_PREFIX): + kwargs["extra_body"] = {"reasoning_effort": "none"} + response = client.chat.completions.create(**kwargs) + return response.choices[0].message.content or "" + + +def parse_verdict(raw: str) -> dict: + """Parse the LLM's JSON response. Tolerates ```json fences and stray text.""" + if not raw: + raise ValueError("empty LLM response") + text = raw.strip() + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*", "", text) + text = re.sub(r"\s*```$", "", text) + try: + return json.loads(text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", text, re.DOTALL) + if not match: + raise ValueError(f"could not extract JSON from LLM response: {raw[:200]}") + return json.loads(match.group(0)) + + +# --------------------------------------------------------------------------- +# Comment composition + + +def _format_missing(missing: list[str]) -> str: + if not missing: + return "- (see explanation below)" + return "\n".join(f"- {m}" for m in missing) + + +# Rubric items the judge can mark present. The first element of each tuple is +# the verdict-JSON boolean field, the second is the human-readable label we +# render in the "what you got right" section of close / grace-warning comments. +_PR_PRESENT_LABELS: tuple[tuple[str, str], ...] = ( + ("linked_issue", "Linked a related GitHub issue"), + ("has_problem_description", "Clear problem description"), + ("has_expected_vs_actual", "Expected vs. actual behavior"), + ("has_qa_proof", "End-to-end QA proof"), +) + +# Issue rubric labels grouped by `kind`. The judge sets `kind` to one of +# {"bug", "feature", "other"}; when "other" we render both groups so we don't +# silently drop a present-flag the judge actually set to True. +_ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = ( + ( + "has_repro", + "End-to-end evidence of the bug (video, screenshot, or command + real output)", + ), + ("has_expected_vs_actual", "Expected vs. actual behavior"), +) +_ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = ( + ("has_motivation_example", "Motivation and concrete example"), +) + + +def _format_present_for_pr(verdict: dict) -> list[str]: + """Human-readable rubric items the judge confirmed are present on a PR. + + Drives the "what you got right" section in close / grace-warning comments. + The user gave explicit feedback: contributors should see what they nailed + *before* the list of gaps, so the comment doesn't read as pure rejection. + """ + return [label for field, label in _PR_PRESENT_LABELS if verdict.get(field)] + + +def _format_present_for_issue(verdict: dict) -> list[str]: + """Human-readable rubric items the judge confirmed are present on an issue. + + Branches on the judge's `kind` field. For `"other"` (or missing kind) we + render the union so a present-flag isn't dropped just because the judge + couldn't classify the issue cleanly. + """ + kind = (verdict.get("kind") or "").lower() + groups: list[tuple[tuple[str, str], ...]] = [] + if kind in ("bug", "other", ""): + groups.append(_ISSUE_BUG_LABELS) + if kind in ("feature", "other", ""): + groups.append(_ISSUE_FEATURE_LABELS) + out: list[str] = [] + for group in groups: + for field, label in group: + if verdict.get(field) and label not in out: + out.append(label) + return out + + +def _format_present_block(items: list[str]) -> str: + """Render the optional "what you got right" block. Empty string when the + judge didn't confirm anything as present — better to omit the section + entirely than to show "What you got right: (nothing)". + """ + if not items: + return "" + bullets = "\n".join(f"- ✅ {item}" for item in items) + return f"**What you got right:**\n\n{bullets}\n\n" + + +def format_pr_close_comment(verdict: dict) -> str: + missing_lines = _format_missing(verdict.get("missing") or []) + present_block = _format_present_block(_format_present_for_pr(verdict)) + explanation = verdict.get("explanation") or "" + return ( + "🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this " + "repository. " + "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n" + "\n" + "I read the description against our " + "[contribution rubric](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md). " + "Here's how it lined up:\n" + "\n" + f"{present_block}" + "**What's still missing:**\n" + "\n" + f"{missing_lines}\n" + "\n" + f"> {explanation}\n" + "\n" + "**Closing this PR isn't a rejection of the change.** We want the open-PR list to " + "mirror what a maintainer can act on *right now*, so contributors don't get lost in a " + 'backlog. A closed PR is a soft "park this for later"; your work is still here, ' + "the diff is still here, and getting it reopened is one comment away. Take your time.\n" + "\n" + "**To bring this PR back:**\n" + "\n" + "- Update the description with the missing pieces, then comment `@agent-shin reconsider` " + "on this PR. I'll re-evaluate and reopen if it now passes.\n" + "- Or **Open a new PR** with the same fix and the updated description. GitHub doesn't " + "always let external contributors reopen a bot-closed PR, so a fresh PR is the most " + "reliable path back into the review queue.\n" + "- If Greptile's most recent score on this PR was below 4/5, comment `@greptileai` to " + "request a fresh review; that **still works even after the PR is closed**, and a " + "stronger score is one of the signals that lifts the PR back into the queue. A low " + "Greptile score isn't a blocker.\n" + "\n" + '**What "end-to-end QA proof" means**, since it\'s the most common gap: at least one ' + "of a short before/after screen recording / video (the bug reproducing, then the fix " + "working; for a brand-new feature, a recording of it working end-to-end), a screenshot " + "(or before/after screenshots) of it working, or the exact commands you ran paired " + "with their **real output** against the real system. Running `pytest` on the repo's " + "unit tests doesn't count; those mock the LLM provider, DB, and network, so they " + "aren't end-to-end. Output from a real, no-mocks integration run is what we look " + "for. A linked issue alone isn't enough either: it covers context, not proof. See " + "[the full rubric](https://docs.litellm.ai/blog/agent-shin-triage#the-rubric-for-pull-requests).\n" + "\n" + "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n" + "\n" + "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, comment " + "`@agent-shin reconsider` or ping a maintainer; they'll override me.)_" + f"\n\n{AGENT_SHIN_CLOSE_MARKER}" + ) + + +def format_issue_close_comment(verdict: dict) -> str: + missing_lines = _format_missing(verdict.get("missing") or []) + present_block = _format_present_block(_format_present_for_issue(verdict)) + explanation = verdict.get("explanation") or "" + return ( + "🚅 Hi, thanks for filing this! I'm **Agent Shin**, the automated triage bot for this " + "repository. " + "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n" + "\n" + "I read the issue against our reporting checklist. Here's how it lined up:\n" + "\n" + f"{present_block}" + "**What's still missing:**\n" + "\n" + f"{missing_lines}\n" + "\n" + f"> {explanation}\n" + "\n" + "**Closing this isn't us saying the bug isn't real or the request isn't useful.** We " + "want the open-issue list to mirror what a maintainer can act on *right now*, so " + "reports like yours don't get buried in a backlog. A closed issue is a soft \"park " + 'this for later"; your report is still here, and getting it reopened is one comment ' + "away. Take your time.\n" + "\n" + "**To bring this issue back:**\n" + "\n" + "1. Edit the issue description to add the missing pieces:\n" + " - For **bug reports**: end-to-end evidence of the bug (a screen recording / " + "video, a screenshot, or the exact commands you ran with their real output / " + "traceback) plus expected vs. actual behavior. Written steps with no run output, " + "video, or screenshot don't count, and mocked or stubbed runs don't count.\n" + " - For **feature requests**: a concrete description of what should change, plus a " + "use case and example (config / API call / UI flow).\n" + "2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it " + "now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer " + "or bot closed, so the comment-based reconsider is the reliable path.)\n" + "\n" + "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n" + "\n" + "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, comment " + "`@agent-shin reconsider` or ping a maintainer; they'll override me.)_" + f"\n\n{AGENT_SHIN_CLOSE_MARKER}" + ) + + +def format_grace_warning_pr_comment(verdict: dict) -> str: + """Comment posted on the FIRST low-quality detection — gives the + contributor a 2-hour grace window to fix the PR before the next + triage run actually closes it. + + This is the "before-close" warning. On the second triage run, if the + grace marker is older than `GRACE_PERIOD_SECONDS` AND the PR still + fails the rubric, the close path runs (which posts + `format_pr_close_comment` and closes the PR). + """ + missing_lines = _format_missing(verdict.get("missing") or []) + present_block = _format_present_block(_format_present_for_pr(verdict)) + explanation = verdict.get("explanation") or "" + return ( + "🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this " + "repository. " + "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n" + "\n" + "I read the description against our " + "[contribution rubric](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md). " + "Here's how it lined up:\n" + "\n" + f"{present_block}" + "**What's still missing:**\n" + "\n" + f"{missing_lines}\n" + "\n" + f"> {explanation}\n" + "\n" + "If the description isn't updated in the next **2 hours**, I'll auto-close this PR. " + "That's **not** us saying we don't care about the change; we want the open-PR list to " + "mirror what a maintainer can act on *right now*, so contributors don't get lost in a " + 'backlog. A closed PR is a soft "park this for later," not a rejection. Take your ' + "time; everything below still works after the close.\n" + "\n" + "**During the grace period:** just update the PR description with the missing pieces. " + "No need to ping me; I'll re-check on the next sweep and skip the auto-close if it " + "now passes. See " + "[what counts as QA proof](https://docs.litellm.ai/blog/agent-shin-triage#the-rubric-for-pull-requests) " + "for the full rubric (a linked issue alone isn't enough; it covers context, not proof).\n" + "\n" + "**If the PR does get auto-closed in 2 hours, you still have easy recovery paths:**\n" + "\n" + "- Comment `@agent-shin reconsider` after updating the description. I'll re-evaluate " + "and reopen the PR if it now passes.\n" + "- Comment `@greptileai` to request a fresh Greptile review; that **still works even " + "after the PR is closed**, and a stronger score is one of the signals that lifts the " + "PR back into the queue. So a low Greptile score isn't a blocker either.\n" + "\n" + "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n" + "\n" + "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, ping a " + "maintainer; they'll override me.)_\n" + "\n" + f"{GRACE_COMMENT_MARKER}" + ) + + +def format_grace_warning_issue_comment(verdict: dict) -> str: + """Issue analogue of `format_grace_warning_pr_comment`.""" + missing_lines = _format_missing(verdict.get("missing") or []) + present_block = _format_present_block(_format_present_for_issue(verdict)) + explanation = verdict.get("explanation") or "" + return ( + "🚅 Hi, thanks for filing this! I'm **Agent Shin**, the automated triage bot for this " + "repository. " + "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n" + "\n" + "I read the issue against our reporting checklist. Here's how it lined up:\n" + "\n" + f"{present_block}" + "**What's still missing:**\n" + "\n" + f"{missing_lines}\n" + "\n" + f"> {explanation}\n" + "\n" + "If the issue isn't updated in the next **2 hours**, I'll auto-close it. That's **not** us " + "saying the bug isn't real or the request isn't useful; we want the open-issue list " + "to mirror what a maintainer can act on *right now*, so reports like yours don't get " + 'buried in a backlog. A closed issue is a soft "park this for later," not a ' + "rejection. Take your time; reopening is one comment away.\n" + "\n" + "**During the grace period:** just edit the issue description with the missing " + "pieces. No need to ping me; I'll re-check on the next sweep and skip the auto-close " + "if it now passes.\n" + "\n" + "Missing pieces, depending on what this is:\n" + "\n" + "- For **bug reports**: end-to-end evidence of the bug (a screen recording / video, a " + "screenshot, or the exact commands you ran with their real output / traceback) plus " + "expected vs. actual behavior. Written steps with no run output don't count, and " + "mocked or stubbed runs don't count.\n" + "- For **feature requests**: a concrete description of what should change, plus a use " + "case and example (config / API call / UI flow).\n" + "\n" + "**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` " + "and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n" + "\n" + "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n" + "\n" + "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, ping a " + "maintainer; they'll override me.)_\n" + "\n" + f"{GRACE_COMMENT_MARKER}" + ) + + +# --------------------------------------------------------------------------- +# Step-summary helpers + + +def write_step_summary(content: str) -> None: + """When running inside GitHub Actions, append to the step summary file.""" + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + try: + with open(path, "a", encoding="utf-8") as handle: + handle.write(content) + if not content.endswith("\n"): + handle.write("\n") + except OSError as exc: + print(f"warn: failed to write step summary: {exc}", file=sys.stderr) + + +# --------------------------------------------------------------------------- +# Core orchestration + + +def format_reopen_comment(kind: str) -> str: + """Comment posted when Agent Shin reopens after a successful reconsider.""" + noun = "PR" if kind == "pr" else "issue" + # The trailing HTML marker is used by `seconds_since_last_reconsider_verdict` + # to enforce a cooldown between repeated `@agent-shin reconsider` triggers. + # Keep the marker on its own line so it doesn't disturb the rendered text. + return ( + f"♻️ **Re-evaluated and reopened.** Thanks for updating the {noun}!\n" + "\n" + "Agent Shin re-ran triage on the latest description and it now meets " + "the bar. A maintainer will take another look soon; please don't " + f"close this {noun} again unless asked to.\n" + "\n" + "_(If a maintainer ends up closing this for non-rubric reasons, that " + "decision stands; comment `@agent-shin reconsider` again only if you " + "have substantively new information.)_\n" + "\n" + f"{RECONSIDER_COMMENT_MARKER}" + ) + + +def format_reconsider_still_failing_comment(kind: str, verdict: dict) -> str: + """Comment posted when reconsider re-runs triage but the verdict is still fail.""" + missing_lines = _format_missing(verdict.get("missing") or []) + explanation = verdict.get("explanation") or "" + noun = "PR" if kind == "pr" else "issue" + # The trailing HTML marker is used by `seconds_since_last_reconsider_verdict` + # to enforce a cooldown between repeated `@agent-shin reconsider` triggers. + return ( + f"⏸️ **Re-evaluated; this {noun} still doesn't meet the rubric.**\n" + "\n" + "Agent Shin re-ran triage on the current description but is still " + "missing:\n" + "\n" + f"{missing_lines}\n" + "\n" + f"> {explanation}\n" + "\n" + "Update the description with the missing pieces and comment " + "`@agent-shin reconsider` again, or ping a maintainer if you think " + "I got this wrong.\n" + "\n" + "_(I'm an LLM and I'm not infallible.)_\n" + "\n" + f"{RECONSIDER_COMMENT_MARKER}" + ) + + +# --------------------------------------------------------------------------- +# Review gate — "ready for review" label lifecycle + +_UNSET = object() + + +def _combine_missing( + verdict: dict, greptile_score: int | None, min_score: int +) -> list[str]: + """Merge the LLM rubric's `missing` list with a Greptile-score shortfall.""" + missing = list(verdict.get("missing") or []) + if greptile_score is not None and greptile_score < min_score: + missing.insert( + 0, + f"Greptile's most recent review scored this PR {greptile_score}/5 " + f"(below the {min_score}/5 bar)", + ) + return missing or ["(see explanation below)"] + + +def _has_marker( + comments: Iterable[dict], marker: str, *, bot_login: str | None = None +) -> bool: + """Return True iff the bot itself posted a comment containing ``marker``. + + Filters by author so a contributor who quotes the marker (e.g. via + GitHub's "Quote reply" feature, which preserves HTML comments in + raw markdown) is not mistaken for a bot action — that would + silently suppress notifications or change which "recovered" wording + is selected. Matches the author-filter pattern used by the sibling + `_seconds_since_latest_marker_comment` helper. + """ + expected_login = ( + bot_login + or os.environ.get("AGENT_SHIN_BOT_LOGIN") + or AGENT_SHIN_DEFAULT_BOT_LOGIN + ).lower() + for comment in comments: + author = ((comment.get("user") or {}).get("login") or "").lower() + if author != expected_login: + continue + if marker in (comment.get("body") or ""): + return True + return False + + +def format_ready_for_review_comment( + verdict: dict, + greptile_score: int | None, + min_greptile_score: int = DEFAULT_MIN_GREPTILE_SCORE, +) -> str: + """Posted the first time a PR clears the bar (label added).""" + score_line = ( + f" Greptile scored it **{greptile_score}/5**." + if greptile_score is not None + else "" + ) + explanation = verdict.get("explanation") or "" + return ( + "✅ **Triage passed, tagging `ready for review`.**\n" + "\n" + "Agent Shin checked this PR against the " + "[contribution rubric](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md) " + "and it clears the bar (a linked issue, or a clear problem description " + f"+ expected vs. actual + QA proof).{score_line}\n" + "\n" + f"> {explanation}\n" + "\n" + "A maintainer will take it from here. If a later re-check finds the PR " + f"has regressed (Greptile drops below {min_greptile_score}/5, " + "the QA proof is removed, etc.) I'll pull the tag and comment with " + "what's missing; fix it and the tag comes back automatically.\n" + f"{READY_MARKER}" + ) + + +def format_all_clear_comment(verdict: dict, greptile_score: int | None) -> str: + """Posted when a PR recovers after a regression (label re-added).""" + score_line = ( + f" Greptile is back to **{greptile_score}/5**." + if greptile_score is not None + else "" + ) + explanation = verdict.get("explanation") or "" + return ( + "✅ **All clear again, re-adding `ready for review`.**\n" + "\n" + "Thanks for addressing the earlier feedback. On re-check this PR meets " + f"the contribution bar once more.{score_line}\n" + "\n" + f"> {explanation}\n" + "\n" + "A maintainer will take another look.\n" + f"{READY_MARKER}" + ) + + +def format_regression_comment( + missing: list[str], explanation: str, grace_days: int +) -> str: + """Posted when a previously-tagged PR regresses (label removed, PR stays open). + + Discloses the same ``grace_days`` deadline the state machine enforces: + once that window elapses with the PR still failing, the close path fires. + Hiding the deadline behind a bare "stays open" would surprise contributors + with an auto-close they were never warned about. + """ + window = "24 hours" if grace_days == 1 else f"{grace_days} days" + return ( + "⚠️ **Removing the `ready for review` tag.**\n" + "\n" + "On a re-check this PR no longer meets the contribution bar. What's " + "missing now:\n" + "\n" + f"{_format_missing(missing)}\n" + "\n" + f"> {explanation}\n" + "\n" + f"The PR stays open for ~{window}; address the points above and Agent " + 'Shin will post an "all clear" comment and re-add the tag ' + "automatically. If the points still aren't addressed after that " + "window, the PR is auto-closed; that's not a rejection, and you can " + "comment `@agent-shin reconsider` to have it re-evaluated and reopened " + "once it passes.\n" + f"{REGRESSED_MARKER}" + ) + + +def format_within_grace_comment( + missing: list[str], explanation: str, grace_days: int +) -> str: + """Posted once while a failing PR is still inside its grace window.""" + window = "24 hours" if grace_days == 1 else f"{grace_days} days" + return ( + "🚅 Hi, thanks for the PR! This is **Agent Shin**, the automated triage " + "bot. This PR doesn't quite meet the contribution bar yet:\n" + "\n" + f"{_format_missing(missing)}\n" + "\n" + f"> {explanation}\n" + "\n" + f"You have ~{window} from when this PR was opened to add the missing " + "pieces; just update the description and I'll re-check on the next " + "sweep. Once it passes I'll tag it `ready for review`. If it does get " + "auto-closed, that's not a rejection; comment `@agent-shin reconsider` " + "and I'll re-evaluate and reopen if it now passes.\n" + f"{WITHIN_GRACE_MARKER}" + ) + + +def review_gate( + *, + repo: str, + number: int, + close: bool, + model: str, + judge: Any = None, + greptile_score: Any = _UNSET, + comments: Any = _UNSET, + now: dt.datetime | None = None, + grace_days: int = DEFAULT_GRACE_DAYS, + min_greptile_score: int = DEFAULT_MIN_GREPTILE_SCORE, + label: str = READY_FOR_REVIEW_LABEL, + allowlist: frozenset[str] = ALLOWLIST_LOGINS, +) -> dict: + """Reconcile the `ready for review` label with a PR's current quality. + + A PR is *passing* when it clears BOTH gates: the LLM rubric (linked issue, + or problem description + expected/actual + QA proof) AND Greptile's most + recent confidence score (>= ``min_greptile_score``; absence of a score is + not held against the PR). The gate then drives a small state machine, using + the label itself as the persisted state so comments fire only on + transitions (never on every scheduled run): + + passing, untagged -> add label + "ready for review" / "all clear" + passing, tagged -> noop-passing + not passing, tagged -> remove label + regression comment (stays open) + not passing, untagged, old -> close + comment (past the grace window) + not passing, untagged, new -> one-time "what's missing" notice (within grace) + + ``close`` gates every destructive side effect: with ``close=False`` the + function returns a ``would-*`` preview and touches nothing, mirroring the + dry-run contract of :func:`triage`. ``judge``/``greptile_score``/ + ``comments``/``now`` are injectable for tests; in production they are + resolved from the OpenAI judge, the PR's Greptile comment, the live comment + list, and the wall clock respectively. + """ + item = fetch_pr(repo, number) + + title = item.get("title") or "" + body = item.get("body") or "" + login = (item.get("user") or {}).get("login") or "" + association = item.get("author_association") or "" + state = item.get("state") or "" + # GitHub label names are case-insensitive; compare lowercased so a repo + # that already has e.g. "Ready for Review" is recognized as the same + # label as our READY_FOR_REVIEW_LABEL constant ("ready for review"). + labels_now = {(lbl.get("name") or "").lower() for lbl in (item.get("labels") or [])} + label_key = label.lower() + created_raw = item.get("created_at") or "" + + base_result = { + "kind": "pr", + "number": number, + "title": title, + "author": login, + "author_association": association, + "state": state, + "labeled": label_key in labels_now, + "review_gate": True, + } + + if state != "open": + return {**base_result, "action": "skip-not-open"} + + if allowlist: + if login.lower() not in allowlist: + return {**base_result, "action": "skip-not-allowlisted"} + elif is_internal_contributor(item): + return {**base_result, "action": "skip-internal-author"} + + # Resolve the comment list once — used for both the Greptile score and the + # marker-based dedup below. + if comments is _UNSET: + comments = list(_iter_paginated_json(f"repos/{repo}/issues/{number}/comments")) + + # --- rubric verdict: linked-issue short-circuit, else the LLM judge ------- + if has_linked_issue(body): + verdict = { + "verdict": "pass", + "linked_issue": True, + "missing": [], + "explanation": "Linked-issue regex matched; LLM was not called.", + } + rubric_pass = True + else: + prompt = build_pr_prompt(title=title, body=body) + if judge is None: + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + return {**base_result, "action": "skip-no-llm-key"} + base_url = os.environ.get("OPENAI_BASE_URL") or None + + def judge(p: str) -> str: + return call_llm_judge( + p, model=model, api_key=api_key, base_url=base_url + ) + + try: + verdict = parse_verdict(judge(prompt)) + except Exception as exc: # noqa: BLE001 - judge errors must never act + return {**base_result, "action": "skip-llm-error", "error": str(exc)} + rubric_pass = (verdict.get("verdict") or "").lower() == "pass" + + # --- Greptile score ------------------------------------------------------- + if greptile_score is _UNSET: + extraction = extract_greptile_score(comments) + greptile_score = extraction[0] if extraction else None + greptile_ok = greptile_score is None or greptile_score >= min_greptile_score + passing = rubric_pass and greptile_ok + + # --- age ------------------------------------------------------------------ + age_days = None + if created_raw: + reference = now or dt.datetime.now(dt.timezone.utc) + age_days = (reference - parse_iso8601(created_raw)).days + + label_present = label_key in labels_now + explanation = verdict.get("explanation") or "" + # When the rubric short-circuited to pass (linked-issue regex) but + # Greptile dragged the PR below the bar, the synthetic verdict's + # explanation ("LLM was not called") would mislead a contributor reading + # the regression / close comment. Surface the real reason instead. + if rubric_pass and not greptile_ok: + explanation = ( + f"Greptile's most recent review scored this PR " + f"{greptile_score}/5 (below the {min_greptile_score}/5 bar)." + ) + verdict = {**verdict, "explanation": explanation} + base_result = { + **base_result, + "verdict": verdict, + "greptile_score": greptile_score, + "passing": passing, + "age_days": age_days, + } + + if passing: + if label_present: + return {**base_result, "action": "noop-passing"} + recovered = _has_marker(comments, REGRESSED_MARKER) + comment = ( + format_all_clear_comment(verdict, greptile_score) + if recovered + else format_ready_for_review_comment( + verdict, greptile_score, min_greptile_score + ) + ) + if not close: + return {**base_result, "action": "would-label-ready", "comment": comment} + post_comment(repo, number, comment) + add_label(repo, number, label) + return {**base_result, "action": "labeled-ready", "comment": comment} + + missing = _combine_missing(verdict, greptile_score, min_greptile_score) + + if label_present: + comment = format_regression_comment(missing, explanation, grace_days) + if not close: + return {**base_result, "action": "would-remove-label", "comment": comment} + remove_label(repo, number, label) + post_comment(repo, number, comment) + return {**base_result, "action": "label-removed-regressed", "comment": comment} + + # Not passing and not tagged. If the PR was previously tagged and then + # regressed (we removed the label and posted REGRESSED_MARKER), honor the + # "PR stays open — fix it and the tag comes back" promise from + # `format_regression_comment` and skip the close path. Without this guard, + # any PR older than `grace_days` would be closed on the next evaluation, + # giving the contributor no realistic window to address the regression. + # + # The promise has a deliberate expiration: once `grace_days` have elapsed + # since the regression notice, fall through to the close path so a PR that + # was abandoned post-regression doesn't sit open forever. + if _has_marker(comments, REGRESSED_MARKER): + reference = now or dt.datetime.now(dt.timezone.utc) + seconds_since_regression = seconds_since_latest_marker_comment( + comments, marker=REGRESSED_MARKER, now=reference + ) + grace_seconds = grace_days * 86400 + if seconds_since_regression is None or seconds_since_regression < grace_seconds: + return {**base_result, "action": "regressed-already-notified"} + + # Not passing and not tagged: close if past the grace window, else notify once. + if age_days is not None and age_days >= grace_days: + comment = format_pr_close_comment({**verdict, "missing": missing}) + if not close: + return {**base_result, "action": "would-close", "comment": comment} + post_comment(repo, number, comment) + close_pr(repo, number) + return {**base_result, "action": "closed", "comment": comment} + + if _has_marker(comments, WITHIN_GRACE_MARKER): + return {**base_result, "action": "within-grace-already-notified"} + comment = format_within_grace_comment(missing, explanation, grace_days) + if not close: + return { + **base_result, + "action": "would-notify-within-grace", + "comment": comment, + } + post_comment(repo, number, comment) + return {**base_result, "action": "within-grace-notified", "comment": comment} + + +def triage( + *, + repo: str, + kind: str, + number: int, + close: bool, + model: str, + judge: Any = None, + print_prompt: bool = False, + reconsider: bool = False, + allowlist: frozenset[str] = ALLOWLIST_LOGINS, +) -> dict: + """Triage a single PR or issue. Returns a result dict for logging/tests. + + `judge` is an optional callable `(prompt) -> str` for tests / dry-run with + a stub. In production, leave it None and the script uses `call_llm_judge`. + + When `reconsider=True`, the closed-state guard is skipped and a + fail-but-no-comment is replaced with a "still failing" comment + leave + closed; a pass triggers `reopen_pr`/`reopen_issue` plus a reopen comment. + Reconsider mode is intended for the `@agent-shin reconsider` comment + trigger. Like regular triage, `close=False` keeps reconsider in dry-run + (returns `would-reopen` / `would-reconsider-still-failing` so a local + operator can preview without write side effects); the workflow only + passes `--close` when `AGENT_SHIN_ENABLED=true`. + + Reconsider mode adds two extra safety guards on top of the regular + triage skip-internal-author check: + + 1. **Bot-closed guard.** Only reopens if the most recent close was + performed by the bot identity (default `github-actions[bot]`). + This stops a contributor from using `@agent-shin reconsider` to + override a maintainer's close for non-rubric reasons. + 2. **Rate-limit guard.** If the bot has already posted a reconsider + verdict on this PR/issue within `RECONSIDER_RATE_LIMIT_SECONDS`, + skip — repeated triggers from the same contributor shouldn't burn + CI minutes or LLM budget. + """ + fetcher = {"pr": fetch_pr, "issue": fetch_issue}[kind] + item = fetcher(repo, number) + + title = item.get("title") or "" + body = item.get("body") or "" + login = (item.get("user") or {}).get("login") or "" + association = item.get("author_association") or "" + state = item.get("state") or "" + + base_result = { + "kind": kind, + "number": number, + "title": title, + "author": login, + "author_association": association, + "state": state, + "reconsider": reconsider, + } + + # Reconsider only makes sense on a closed PR/issue. A "reconsider on an + # open PR" is a no-op (the regular triage flow already evaluates open + # PRs); return a clear skip so the workflow can short-circuit. + if reconsider: + if state != "closed": + return {**base_result, "action": "skip-not-closed"} + else: + if state != "open": + return {**base_result, "action": "skip-not-open"} + + if allowlist: + if login.lower() not in allowlist: + return {**base_result, "action": "skip-not-allowlisted"} + elif is_internal_contributor(item): + return {**base_result, "action": "skip-internal-author"} + + # Reconsider-only guards — these run BEFORE the LLM call so a + # maintainer-closed PR / rate-limited trigger never spends LLM budget. + if reconsider: + if not was_closed_by_agent_shin(repo, number): + return {**base_result, "action": "skip-not-bot-closed"} + age = seconds_since_last_reconsider_verdict(repo, number) + if age is not None and age < RECONSIDER_RATE_LIMIT_SECONDS: + return { + **base_result, + "action": "skip-rate-limited", + "rate_limit_age_seconds": age, + "rate_limit_window_seconds": RECONSIDER_RATE_LIMIT_SECONDS, + } + + if kind == "pr": + # Short-circuit: if body very clearly links a related issue, just pass. + if has_linked_issue(body): + base = { + **base_result, + "action": "pass-linked-issue", + "verdict": { + "verdict": "pass", + "linked_issue": True, + "explanation": "Linked-issue regex matched; LLM was not called.", + }, + } + if reconsider: + # Pass-on-reconsider -> reopen the PR with a friendly comment. + reopen_body = format_reopen_comment(kind) + if not close: + return { + **base, + "action": "would-reopen", + "comment": reopen_body, + } + post_comment(repo, number, reopen_body) + reopen_pr(repo, number) + return { + **base, + "action": "reopened", + "comment": reopen_body, + } + return base + prompt = build_pr_prompt(title=title, body=body) + else: + prompt = build_issue_prompt(title=title, body=body) + + if print_prompt: + return {**base_result, "action": "print-prompt", "prompt": prompt} + + if judge is None: + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + # No key configured — never take a destructive action. Report skip. + return { + **base_result, + "action": "skip-no-llm-key", + "prompt_preview": prompt[:200], + } + base_url = os.environ.get("OPENAI_BASE_URL") or None + + def judge(p: str) -> str: + return call_llm_judge(p, model=model, api_key=api_key, base_url=base_url) + + try: + raw = judge(prompt) + verdict = parse_verdict(raw) + except Exception as exc: # noqa: BLE001 - judge errors must never close PRs + return {**base_result, "action": "skip-llm-error", "error": str(exc)} + + decision = (verdict.get("verdict") or "").lower() + + if reconsider: + # Reconsider: an explicit `pass` -> reopen + post reopen comment; + # anything else (fail, missing/malformed verdict, typo) -> leave + # closed + post a "still failing" comment so the contributor can + # iterate again. Reopen is destructive, so a flaky/empty verdict + # must not satisfy the gate. + # In dry-run (`close=False`) we return `would-*` actions instead + # of touching GitHub state, mirroring the regular triage flow's + # `would-close`. This lets a local operator preview the outcome + # of `python triage_with_llm.py --reconsider --pr N` without + # risking accidental comments or reopens. + if decision == "pass": + reopen_body = format_reopen_comment(kind) + if not close: + return { + **base_result, + "action": "would-reopen", + "verdict": verdict, + "comment": reopen_body, + } + post_comment(repo, number, reopen_body) + if kind == "pr": + reopen_pr(repo, number) + else: + reopen_issue(repo, number) + return { + **base_result, + "action": "reopened", + "verdict": verdict, + "comment": reopen_body, + } + still_failing = format_reconsider_still_failing_comment(kind, verdict) + if not close: + return { + **base_result, + "action": "would-reconsider-still-failing", + "verdict": verdict, + "comment": still_failing, + } + post_comment(repo, number, still_failing) + return { + **base_result, + "action": "reconsider-still-failing", + "verdict": verdict, + "comment": still_failing, + } + + if decision != "fail": + return {**base_result, "action": "pass-llm", "verdict": verdict} + + # Grace-period flow: on the first low-quality detection, post a warning + # comment instead of closing immediately. On a subsequent triage run + # (manual re-trigger, or the daily `close_low_quality_prs.py` cron + # finding the same PR in its own pass), if `GRACE_PERIOD_SECONDS` has + # elapsed since the warning AND the PR still fails the rubric, close. + grace_age = seconds_since_last_grace_warning(repo, number) + if grace_age is None: + warning_body = ( + format_grace_warning_pr_comment(verdict) + if kind == "pr" + else format_grace_warning_issue_comment(verdict) + ) + if not close: + return { + **base_result, + "action": "would-warn-grace", + "verdict": verdict, + "comment": warning_body, + } + post_comment(repo, number, warning_body) + return { + **base_result, + "action": "warned-grace", + "verdict": verdict, + "comment": warning_body, + } + if grace_age < GRACE_PERIOD_SECONDS: + return { + **base_result, + "action": "skip-in-grace-period", + "verdict": verdict, + "grace_age_seconds": grace_age, + "grace_period_seconds": GRACE_PERIOD_SECONDS, + } + + # The grace window has elapsed. `--close` still gates the destructive + # write so a dry-run preview never posts or closes — the workflow only + # passes `--close` when `AGENT_SHIN_ENABLED=true`, which keeps the bot + # inert by default. + if not close: + return {**base_result, "action": "would-close", "verdict": verdict} + + comment_body = ( + format_pr_close_comment(verdict) + if kind == "pr" + else format_issue_close_comment(verdict) + ) + post_comment(repo, number, comment_body) + if kind == "pr": + close_pr(repo, number) + else: + close_issue(repo, number) + + return { + **base_result, + "action": "closed", + "verdict": verdict, + "comment": comment_body, + } + + +# --------------------------------------------------------------------------- +# CLI + + +def render_summary(result: dict) -> str: + """Render a human-readable summary block (used for stdout + step summary).""" + lines = ["## Agent Shin verdict", ""] + lines.append( + f"- **{result['kind'].upper()} #{result['number']}**: {result.get('title', '')}" + ) + lines.append( + f"- **Author**: `{result.get('author', '')}` ({result.get('author_association', '')})" + ) + lines.append(f"- **State**: {result.get('state', '')}") + lines.append(f"- **Action**: `{result['action']}`") + verdict = result.get("verdict") + if verdict: + lines.append("") + lines.append("```json") + lines.append(json.dumps(verdict, indent=2)) + lines.append("```") + error = result.get("error") + if error: + lines.append("") + lines.append(f"_LLM error: {error}_") + comment = result.get("comment") + if comment: + lines.append("") + lines.append("### Posted comment:") + lines.append("") + lines.append("> " + comment.replace("\n", "\n> ")) + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True, help="Repository (owner/repo).") + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--pr", type=int, help="Pull request number to triage.") + target.add_argument("--issue", type=int, help="Issue number to triage.") + parser.add_argument( + "--close", + action="store_true", + help="Actually post comment + close on fail (default: dry run).", + ) + parser.add_argument( + "--model", + # `os.environ.get("TRIAGE_MODEL", DEFAULT_MODEL)` would return "" when + # GitHub Actions exposes an unset repo variable as an empty-string env + # var, silently bypassing DEFAULT_MODEL and causing every call to fail + # as `skip-llm-error`. The `or` guard collapses empty -> default. + default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL, + help=f"OpenAI-compatible model name (default: {DEFAULT_MODEL}).", + ) + parser.add_argument( + "--print-prompt", + action="store_true", + help="Print the prompt that would be sent to the judge and exit.", + ) + parser.add_argument( + "--reconsider", + action="store_true", + help=( + "Re-run triage on a CLOSED PR/issue and reopen it on pass. " + "Used by the `@agent-shin reconsider` comment-trigger workflow. " + "Only invoke this from a workflow that has already gated on " + "AGENT_SHIN_ENABLED=true and verified the commenter is the " + "PR/issue author or an internal collaborator." + ), + ) + parser.add_argument( + "--review-gate", + action="store_true", + help=( + "Reconcile the `ready for review` label for an OPEN PR: tag on " + "pass, remove the tag + comment on regression, close after the " + "grace window if it never passed. PR-only." + ), + ) + parser.add_argument( + "--grace-days", + type=int, + default=DEFAULT_GRACE_DAYS, + help=( + "Review-gate only: hours/24 a failing, un-tagged PR may stay open " + f"before auto-close (default: {DEFAULT_GRACE_DAYS} = 24h)." + ), + ) + parser.add_argument( + "--min-greptile-score", + type=int, + default=DEFAULT_MIN_GREPTILE_SCORE, + choices=range(1, 6), + help=( + "Review-gate only: Greptile score below which a PR counts as not " + f"passing (default: {DEFAULT_MIN_GREPTILE_SCORE} -> <4/5 regresses)." + ), + ) + args = parser.parse_args() + + kind = "pr" if args.pr is not None else "issue" + number = args.pr if args.pr is not None else args.issue + + if args.review_gate: + if kind != "pr": + parser.error("--review-gate applies to pull requests only (use --pr).") + result = review_gate( + repo=args.repo, + number=number, + close=args.close, + model=args.model, + grace_days=args.grace_days, + min_greptile_score=args.min_greptile_score, + ) + else: + result = triage( + repo=args.repo, + kind=kind, + number=number, + close=args.close, + model=args.model, + print_prompt=args.print_prompt, + reconsider=args.reconsider, + ) + + if result.get("action") == "print-prompt": + print(result["prompt"]) + return 0 + + summary = render_summary(result) + print(summary) + write_step_summary(summary + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/close_low_quality_prs.yml b/.github/workflows/close_low_quality_prs.yml new file mode 100644 index 00000000000..2401be84000 --- /dev/null +++ b/.github/workflows/close_low_quality_prs.yml @@ -0,0 +1,92 @@ +name: Close Low-Quality PRs + +# Auto-close any open PR (including drafts, regardless of age) authored by an +# external OSS contributor that Greptile reviewed with a confidence score +# below 4/5. Closures are explained in a comment that tells the contributor +# to push fixes and open a fresh PR (since OSS authors cannot reopen a PR +# closed by a bot/maintainer) or comment `@agent-shin reconsider` to have +# Agent Shin re-evaluate. +# +# Manual one-off run: +# gh workflow run "Close Low-Quality PRs" -f close=true +# +# Dry-run preview (no PRs are touched): +# gh workflow run "Close Low-Quality PRs" -f close=false + +on: + schedule: + # Daily at 09:00 UTC. Pairs well with the stale-issue workflow at midnight. + - cron: "0 9 * * *" + workflow_dispatch: + inputs: + close: + description: "Actually close matching PRs (false = dry run)." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + min_age_days: + description: "Minimum PR age in days (default 0 = no age filter)." + required: false + default: "0" + min_score: + description: "Greptile score below which a PR is closed (1-5)." + required: false + default: "4" + limit: + description: "Maximum number of PRs to close in a single run." + required: false + default: "25" + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + close-low-quality-prs: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Run low-quality PR closer + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Scheduled runs are ALWAYS dry-run, even when AGENT_SHIN_ENABLED is + # "true", so the team can QA the closer's verdicts in step summaries + # before any contributor sees a PR closed. Real closures only happen + # on manual workflow_dispatch with close=true (and the variable set). + CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + MIN_AGE_DAYS: ${{ github.event.inputs.min_age_days || '0' }} + MIN_SCORE: ${{ github.event.inputs.min_score || '4' }} + LIMIT: ${{ github.event.inputs.limit || '25' }} + run: | + set -euo pipefail + ARGS=( + --repo "${{ github.repository }}" + --min-age-days "${MIN_AGE_DAYS}" + --min-score "${MIN_SCORE}" + --limit "${LIMIT}" + ) + if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then + echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> forcing dry-run regardless of close input." + elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG}" = "true" ]; then + ARGS+=(--close) + echo "::notice::Running in close-on-fail mode." + else + echo "::notice::AGENT_SHIN_ENABLED is true but this trigger is dry-run (scheduled event or close=false)." + fi + python3 .github/scripts/close_low_quality_prs.py "${ARGS[@]}" diff --git a/.github/workflows/review_gate.yml b/.github/workflows/review_gate.yml new file mode 100644 index 00000000000..ba4b488b79d --- /dev/null +++ b/.github/workflows/review_gate.yml @@ -0,0 +1,131 @@ +name: Agent Shin — review gate + +# Keeps the `ready for review` label in sync with whether an external PR +# currently clears BOTH the LLM rubric AND Greptile's confidence score. +# +# pass -> add `ready for review` + a "passed / all clear" comment +# regress -> remove the label + a "what's missing" comment (PR stays open) +# fail, <24h old -> a one-time "what's missing" notice (grace window) +# fail, >24h old -> close + a comment (reopen via `@agent-shin reconsider`) +# +# DRY-RUN BY DEFAULT. Every side effect (label add/remove, comment, close) is +# gated behind `--close`, which is only added when the repo variable +# `AGENT_SHIN_ENABLED == "true"`. Until then runs only write the verdict to the +# workflow step summary. +# +# Manual single PR: gh workflow run "Agent Shin — review gate" -f pr_number=NNN +# Manual dry-run: gh workflow run "Agent Shin — review gate" -f close=false +# +# We use `pull_request_target` so the workflow can read repo secrets and run +# against fork PRs. Fork code is never checked out — only PR metadata is read +# via `gh api`. + +on: + pull_request_target: + types: [opened, reopened, synchronize, ready_for_review] + schedule: + # Daily at 09:30 UTC — re-reconciles labels as Greptile re-reviews land. + - cron: "30 9 * * *" + workflow_dispatch: + inputs: + pr_number: + description: "Single PR to reconcile (omit to sweep all open PRs)." + required: false + close: + description: "If AGENT_SHIN_ENABLED=true, actually act (false = dry run)." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + grace_days: + description: "Hours/24 a failing, un-tagged PR may stay open before close." + required: false + default: "1" + min_greptile_score: + description: "Greptile score below which a PR counts as not passing (1-5)." + required: false + default: "4" + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + review-gate: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt + + - name: Run review gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Mirror the triage workflow: only expose the LLM key when the bot is + # enabled or a collaborator triggers it manually, so an external user + # can't force paid LLM calls by churning a fork PR while the bot is + # still in dry-run. + OPENAI_API_KEY: ${{ (vars.AGENT_SHIN_ENABLED == 'true' || github.event_name == 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }} + GRACE_DAYS: ${{ github.event.inputs.grace_days || '1' }} + MIN_GREPTILE_SCORE: ${{ github.event.inputs.min_greptile_score || '4' }} + EVENT_PR: ${{ github.event.pull_request.number }} + INPUT_PR: ${{ github.event.inputs.pr_number }} + run: | + set -euo pipefail + COMMON=(--review-gate --grace-days "${GRACE_DAYS}" --min-greptile-score "${MIN_GREPTILE_SCORE}") + + # Fail-safe gating, identical philosophy to the Greptile closer: + # - AGENT_SHIN_ENABLED must be the EXACT string "true" to act at all. + # - A manual dispatch can still preview with close=false. + # - Automatic triggers (PR events, schedule) act once enabled — that + # is the whole point of the gate (re-tag / un-tag automatically). + DO_CLOSE="false" + if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then + echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> dry-run (no labels/comments/closes)." + elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG:-false}" = "true" ]; then + DO_CLOSE="true" + echo "::notice::Manual run -> acting for real." + elif [ "${GITHUB_EVENT_NAME:-}" != "workflow_dispatch" ]; then + DO_CLOSE="true" + echo "::notice::Enabled automatic trigger (${GITHUB_EVENT_NAME:-}) -> acting for real." + else + echo "::notice::Manual dispatch with close=false -> dry-run." + fi + if [ "${DO_CLOSE}" = "true" ]; then + COMMON+=(--close) + fi + + # Single PR (PR event or explicit input) vs. sweep over all open PRs. + TARGET_PR="${EVENT_PR:-${INPUT_PR:-}}" + if [ -n "${TARGET_PR}" ]; then + python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${TARGET_PR}" "${COMMON[@]}" + else + echo "::notice::Sweeping all open PRs." + # Match GH_LIST_ALL_LIMIT in agent_shin_shared.py: gh lists newest-first, + # so any cap below the real backlog silently drops the *oldest* PRs — + # exactly the stale ones this daily sweep is meant to reconcile. + mapfile -t NUMBERS < <(gh pr list --repo "${{ github.repository }}" --state open --limit 100000 --json number --jq '.[].number') + for n in "${NUMBERS[@]}"; do + echo "::group::PR #${n}" + python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${n}" "${COMMON[@]}" || echo "::warning::review gate errored on #${n}" + echo "::endgroup::" + done + fi diff --git a/.github/workflows/triage_issue_with_llm.yml b/.github/workflows/triage_issue_with_llm.yml new file mode 100644 index 00000000000..765453cf2c6 --- /dev/null +++ b/.github/workflows/triage_issue_with_llm.yml @@ -0,0 +1,96 @@ +name: Agent Shin — Issue triage + +# LLM-as-judge triage for external GitHub issues. +# +# DRY-RUN BY DEFAULT. See .github/workflows/triage_pr_with_llm.yml for the +# enablement procedure — same repo variable (`AGENT_SHIN_ENABLED=true`) +# unlocks the PR and issue triage flows together. + +on: + issues: + types: [opened, reopened] + workflow_dispatch: + inputs: + issue_number: + description: "Issue number to triage manually." + required: true + close: + description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + +permissions: + contents: read + issues: write + +jobs: + triage: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt + + - name: Run Agent Shin + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Only expose the LLM key when the bot is enabled or a collaborator + # triggers it manually, so an external user can't force paid LLM + # calls by churning issues while the bot is still in dry-run. + # The Python script calls the LLM whenever this var is set + # (regardless of `--close`); stripping `--close` doesn't suppress + # the API call, only the destructive side effects. + OPENAI_API_KEY: ${{ (vars.AGENT_SHIN_ENABLED == 'true' || github.event_name == 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + DISPATCH_CLOSE: ${{ github.event.inputs.close }} + ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }} + run: | + set -euo pipefail + ARGS=(--repo "${{ github.repository }}" --issue "${ISSUE_NUMBER}") + # Fail-safe gating: only the EXACT string "true" enables the + # destructive --close path. The workflow_dispatch input is a + # `choice` dropdown of "true"/"false" so the UI is constrained, + # but the API (`gh workflow run -f close=...`) accepts any + # string, and a `!= "false"` check would treat "True", "yes", + # "1", "TRUE", typos, and accidental whitespace as enabling + # closure. Mirror the Greptile closer's `= "true"` pattern. + if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" = "true" ]; then + ARGS+=(--close) + echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode." + elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then + echo "::notice::Agent Shin is ENABLED but this trigger is dry-run (workflow_dispatch close != 'true')." + else + echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no issues will be closed." + fi + # Automatic `issues` events stay dry-run regardless until the team + # explicitly invokes workflow_dispatch with close=true. + if [ "${GITHUB_EVENT_NAME:-}" = "issues" ]; then + # filter out --close rather than substituting to "" (which would + # leave an empty positional arg that argparse rejects) + FILTERED=() + for arg in "${ARGS[@]}"; do + if [ "${arg}" != "--close" ]; then + FILTERED+=("${arg}") + fi + done + ARGS=("${FILTERED[@]}") + echo "::notice::issues trigger -> forcing dry-run." + fi + python3 .github/scripts/triage_with_llm.py "${ARGS[@]}" diff --git a/.github/workflows/triage_pr_with_llm.yml b/.github/workflows/triage_pr_with_llm.yml new file mode 100644 index 00000000000..936547598fb --- /dev/null +++ b/.github/workflows/triage_pr_with_llm.yml @@ -0,0 +1,110 @@ +name: Agent Shin — PR triage + +# LLM-as-judge triage for external pull requests. +# +# DRY-RUN BY DEFAULT. Closures and public comments are gated on the repo +# variable `AGENT_SHIN_ENABLED` being set to the string `"true"`. Until then, +# every run only writes its verdict to the workflow step summary so the team +# can QA the judge's decisions before flipping it on. +# +# To enable for real: +# 1. Add a repo secret `OPENAI_API_KEY` (or compatible). +# 2. Set repo variable `AGENT_SHIN_ENABLED` to `true` +# (Settings > Secrets and variables > Actions > Variables). +# +# We use `pull_request_target` so the workflow has access to repo secrets +# and runs against PRs from forks. We never check out fork code — only read +# PR metadata via `gh api`, so this is safe. + +on: + pull_request_target: + types: [opened, reopened] + workflow_dispatch: + inputs: + pr_number: + description: "PR number to triage manually." + required: true + close: + description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + triage: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt + + - name: Run Agent Shin + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Only expose the LLM key when the bot is enabled or a collaborator + # triggers it manually, so an external user can't force paid LLM + # calls by churning a fork PR while the bot is still in dry-run. + # The Python script calls the LLM whenever this var is set + # (regardless of `--close`); stripping `--close` doesn't suppress + # the API call, only the destructive side effects. + OPENAI_API_KEY: ${{ (vars.AGENT_SHIN_ENABLED == 'true' || github.event_name == 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + DISPATCH_CLOSE: ${{ github.event.inputs.close }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }} + run: | + set -euo pipefail + ARGS=(--repo "${{ github.repository }}" --pr "${PR_NUMBER}") + # Fail-safe gating: only the EXACT string "true" enables the + # destructive --close path. The workflow_dispatch input is a + # `choice` dropdown of "true"/"false" so the UI is constrained, + # but the API (`gh workflow run -f close=...`) accepts any + # string, and a `!= "false"` check would treat "True", "yes", + # "1", "TRUE", typos, and accidental whitespace as enabling + # closure. Mirror the Greptile closer's `= "true"` pattern. + if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" = "true" ]; then + ARGS+=(--close) + echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode." + elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then + echo "::notice::Agent Shin is ENABLED but this trigger is dry-run (workflow_dispatch close != 'true' or scheduled event)." + else + echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no PRs will be closed." + fi + # On the scheduled/automatic pull_request_target trigger we default to + # dry-run regardless, so the team can review verdicts in the step + # summary before any contributor sees a comment. Only the manual + # workflow_dispatch path (with close=true) closes PRs. + if [ "${GITHUB_EVENT_NAME:-}" = "pull_request_target" ]; then + # strip any --close added above (filter out, don't substitute + # to empty string — that would leave a stray "" positional arg + # that argparse rejects) + FILTERED=() + for arg in "${ARGS[@]}"; do + if [ "${arg}" != "--close" ]; then + FILTERED+=("${arg}") + fi + done + ARGS=("${FILTERED[@]}") + echo "::notice::pull_request_target trigger -> forcing dry-run." + fi + python3 .github/scripts/triage_with_llm.py "${ARGS[@]}" diff --git a/.github/workflows/triage_reconsider.yml b/.github/workflows/triage_reconsider.yml new file mode 100644 index 00000000000..f35f681d09a --- /dev/null +++ b/.github/workflows/triage_reconsider.yml @@ -0,0 +1,172 @@ +name: Agent Shin — reconsider + +# Comment-trigger workflow: when the PR/issue author (or an internal +# collaborator) comments `@agent-shin reconsider` on a CLOSED PR/issue, +# Agent Shin re-runs LLM-judge triage on the current title+body and: +# +# - on PASS: posts a "re-evaluated and reopened" comment + reopens. +# - on FAIL: posts a "still missing X" comment and leaves it closed, +# so the contributor can iterate again. +# +# This exists because GitHub does NOT let an external (non-write-access) +# OSS contributor reopen a PR/issue closed by a bot or maintainer. Without +# this comment trigger, a contributor whose PR Agent Shin auto-closed +# would have no path back into the review queue except opening a fresh PR +# (which loses the original PR's history). The bot, on the other hand, +# has write access via GH_TOKEN and can reopen on their behalf. +# +# DRY-RUN BY DEFAULT — gated on `vars.AGENT_SHIN_ENABLED == 'true'` just +# like the other Agent Shin workflows. The workflow also gates on the +# commenter being either the PR/issue author or an internal collaborator +# (OWNER/MEMBER/COLLABORATOR) so random commenters cannot DOS the LLM +# judge or force a reopen. + +on: + issue_comment: + types: [created] + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + reconsider: + if: | + github.repository == 'BerriAI/litellm' + && contains(github.event.comment.body, '@agent-shin reconsider') + runs-on: ubuntu-latest + steps: + - name: Authorize commenter + # Only the PR/issue author OR an internal collaborator may trigger + # a reconsider. Outside random commenters could otherwise spam the + # phrase to burn LLM budget or, if a fail-open bug were ever + # introduced, force a reopen on someone else's behalf. + # + # We expose the authorization decision as a step output and gate + # every subsequent (potentially destructive) step on it. A `run:` + # step with `exit 0` would NOT stop the job — only `if:` gating + # on a known-true output is safe here. + id: auth + env: + COMMENTER: ${{ github.event.comment.user.login }} + AUTHOR: ${{ github.event.issue.user.login }} + ASSOCIATION: ${{ github.event.comment.author_association }} + run: | + set -euo pipefail + if [ "${COMMENTER}" = "${AUTHOR}" ]; then + echo "::notice::Authorized: commenter is the PR/issue author." + echo "authorized=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + case "${ASSOCIATION}" in + OWNER|MEMBER|COLLABORATOR) + echo "::notice::Authorized: commenter is an internal collaborator (${ASSOCIATION})." + echo "authorized=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "::notice::Commenter '${COMMENTER}' (${ASSOCIATION}) is not authorized to trigger reconsider; skipping subsequent steps." + echo "authorized=false" >> "$GITHUB_OUTPUT" + ;; + esac + + - name: React 👀 to acknowledge the reconsider + # Add an eyes reaction to the triggering comment the moment we accept + # it, so the contributor gets instant feedback that the bot saw their + # `@agent-shin reconsider` before the slower triage steps run. Gated on + # AGENT_SHIN_ENABLED so dry-run leaves no visible trace. Best-effort: + # a reactions API hiccup must never fail the actual reconsider. + if: steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMENT_ID: ${{ github.event.comment.id }} + run: | + set -euo pipefail + gh api --method POST \ + -H "Accept: application/vnd.github+json" \ + "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \ + -f content=eyes \ + || echo "::warning::failed to add 👀 reaction (non-fatal)" + + - name: Checkout triage script + if: steps.auth.outputs.authorized == 'true' + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + if: steps.auth.outputs.authorized == 'true' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + if: steps.auth.outputs.authorized == 'true' + run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt + + - name: Run Agent Shin reconsider + if: steps.auth.outputs.authorized == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Only expose the LLM key when the bot is enabled, so a PR/issue + # author can't force paid LLM calls by spamming `@agent-shin + # reconsider` while the bot is still in dry-run. The Python script + # calls the LLM whenever this var is set (regardless of `--close`); + # stripping `--close` doesn't suppress the API call, only the + # destructive side effects. Mirror the gating used by every other + # Agent Shin workflow (triage_pr_with_llm.yml, review_gate.yml, ...). + OPENAI_API_KEY: ${{ vars.AGENT_SHIN_ENABLED == 'true' && secrets.OPENAI_API_KEY || '' }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + # `issue_comment` events fire for both issues and PR comments. + # `issue.pull_request` is set iff this is a PR comment, so we use + # its presence to decide whether to invoke `--pr N` or `--issue N`. + IS_PR: ${{ github.event.issue.pull_request != null }} + NUMBER: ${{ github.event.issue.number }} + run: | + set -euo pipefail + if [ "${IS_PR}" = "true" ]; then + ARGS=(--repo "${{ github.repository }}" --pr "${NUMBER}" --reconsider) + else + ARGS=(--repo "${{ github.repository }}" --issue "${NUMBER}" --reconsider) + fi + # Reconsider's destructive actions (post comment + reopen) are + # gated on `--close`, mirroring the regular triage workflows. + # When AGENT_SHIN_ENABLED is not the EXACT string "true", we + # still run the script so its verdict + would-X action lands in + # the step summary for QA — but without `--close`, the script + # returns `would-reopen` / `would-reconsider-still-failing` + # instead of touching GitHub state. + # + # Use the positive `= "true"` gate (not `!= "true" -> exit`) so + # the workflow guardrails in + # tests/test_litellm/test_github_triage_workflows.py see the + # canonical fail-safe enable pattern. Unknown values like + # "True", "yes", "1", or typos fall through to the dry-run + # branch, which is the safe default. + if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then + ARGS+=(--close) + echo "::notice::Agent Shin reconsider ENABLED — running real triage (close=true)." + else + echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> reconsider stays in dry-run (no comment, no reopen)." + fi + python3 .github/scripts/triage_with_llm.py "${ARGS[@]}" + + - name: React 👍 when the reconsider finishes + # Once the reconsider run has completed successfully, add a thumbs-up so + # the contributor sees the bot is done (the 👀 stays, signalling + # seen -> handled). `success()` keeps this from firing if the run + # errored, and the AGENT_SHIN_ENABLED gate keeps dry-run inert. + if: success() && steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMENT_ID: ${{ github.event.comment.id }} + run: | + set -euo pipefail + gh api --method POST \ + -H "Accept: application/vnd.github+json" \ + "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \ + -f content=+1 \ + || echo "::warning::failed to add 👍 reaction (non-fatal)" diff --git a/.github/workflows/triage_rollout_heads_up.yml b/.github/workflows/triage_rollout_heads_up.yml new file mode 100644 index 00000000000..903960151e2 --- /dev/null +++ b/.github/workflows/triage_rollout_heads_up.yml @@ -0,0 +1,92 @@ +name: Agent Shin — rollout heads-up (one-shot) + +# Fires the 7-day heads-up comment on every open external PR/issue that the +# new triage bot would auto-close. The real sweep is a deliberate one-shot: +# trigger it at rollout via a manual `workflow_dispatch` with `dry_run=false`. +# The script is idempotent (skips items that already carry the +# `` marker), so a re-run is harmless. +# +# The automatic push trigger runs DRY-RUN only, so merging the script to +# `litellm_internal_staging` never posts a comment; it just confirms the +# workflow is wired up. Posting real comments requires the manual dispatch, +# which is also the only trigger that exposes `OPENAI_API_KEY`. The heads-up +# is intentionally NOT gated on `AGENT_SHIN_ENABLED`: it has to warn +# contributors while that flag is still off, ahead of the flip that turns on +# auto-closing. +# +# The workflow is a thin shell over `.github/scripts/triage_rollout_heads_up.py`. +# Dry-run vs. real run differ in EXACTLY one CLI flag (`--close`), added only +# on a manual dispatch with `dry_run=false`. + +on: + push: + branches: + - litellm_internal_staging + paths: + # The presence of this script on staging IS the rollout merge marker. + # Editing the file later would re-fire the workflow; that's safe because + # the script skips PRs/issues that already have the heads-up marker. + - ".github/scripts/triage_rollout_heads_up.py" + workflow_dispatch: + inputs: + dry_run: + description: "Dry run (true = preview only, false = actually post comments)." + required: false + default: "true" + type: choice + options: + - "true" + - "false" + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + heads-up: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage scripts + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt + + - name: Run heads-up sweep + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Only the manual dispatch (the real-run trigger) needs the LLM key. + # The automatic push trigger runs dry-run and never posts, so it gets + # no key. Mirrors the sibling triage workflows, which expose the key + # only on an enabled/dispatched run rather than unconditionally. + OPENAI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.OPENAI_API_KEY || '' }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + # The real run is a deliberate manual dispatch with dry_run=false. + # Use the EXACT "false" comparison so any unexpected input value + # fail-closes to dry-run (mirrors the AGENT_SHIN_ENABLED pattern in + # the sibling workflows). The automatic push trigger always stays + # dry-run, so merging the script never posts. + DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }} + run: | + set -euo pipefail + ARGS=(--repo "${{ github.repository }}") + if [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${DRY_RUN_INPUT:-true}" = "false" ]; then + ARGS+=(--close) + echo "::notice::Manual rollout dispatch with dry_run=false -> heads-up comments WILL be posted." + elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ]; then + echo "::notice::Manual dispatch in dry-run mode -> previewing only, no comments will be posted." + else + echo "::notice::Automatic push trigger -> dry-run preview only. Fire the real rollout sweep with a manual workflow_dispatch (dry_run=false)." + fi + python3 .github/scripts/triage_rollout_heads_up.py "${ARGS[@]}" diff --git a/tests/test_litellm/test_github_close_low_quality_prs.py b/tests/test_litellm/test_github_close_low_quality_prs.py new file mode 100644 index 00000000000..e3b653dde64 --- /dev/null +++ b/tests/test_litellm/test_github_close_low_quality_prs.py @@ -0,0 +1,856 @@ +"""Unit tests for `.github/scripts/close_low_quality_prs.py`. + +These exercise the pure logic (score extraction and per-PR evaluation) without +hitting GitHub. Network/CLI calls are stubbed via monkeypatch. +""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[2] + / ".github" + / "scripts" + / "close_low_quality_prs.py" +) + + +@pytest.fixture(scope="module") +def closer_module(): + """Load the script as a module via its file path (it lives outside the package).""" + spec = importlib.util.spec_from_file_location("close_low_quality_prs", SCRIPT_PATH) + assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}" + module = importlib.util.module_from_spec(spec) + sys.modules["close_low_quality_prs"] = module + spec.loader.exec_module(module) + return module + + +def _greptile_comment( + body: str, + updated_at: str = "2026-05-10T00:00:00Z", + login: str = "greptile-apps[bot]", +) -> dict: + return { + "user": {"login": login}, + "body": body, + "created_at": updated_at, + "updated_at": updated_at, + } + + +class TestExtractGreptileScore: + def test_should_extract_score_from_html_header(self, closer_module): + comments = [ + _greptile_comment("

Confidence Score: 3/5

\nSome body text.") + ] + result = closer_module.extract_greptile_score(comments) + assert result is not None + score, _ = result + assert score == 3 + + def test_should_accept_both_greptile_login_variants(self, closer_module): + # REST API form ("greptile-apps[bot]") and GraphQL form ("greptile-apps") + for login in ("greptile-apps", "greptile-apps[bot]"): + comments = [ + _greptile_comment("

Confidence Score: 2/5

", login=login) + ] + result = closer_module.extract_greptile_score(comments) + assert result is not None, f"failed to detect score for login={login}" + score, _ = result + assert score == 2 + + def test_should_extract_score_from_plain_text(self, closer_module): + comments = [_greptile_comment("Confidence Score: 5/5 — looks good!")] + result = closer_module.extract_greptile_score(comments) + assert result is not None + score, _ = result + assert score == 5 + + def test_should_tolerate_whitespace_and_case(self, closer_module): + comments = [_greptile_comment("**confidence score : 2 / 5**")] + result = closer_module.extract_greptile_score(comments) + assert result is not None + score, _ = result + assert score == 2 + + def test_should_pick_most_recent_comment_when_rereview_happens(self, closer_module): + comments = [ + _greptile_comment( + "Confidence Score: 2/5", updated_at="2026-05-01T00:00:00Z" + ), + _greptile_comment( + "Confidence Score: 5/5", updated_at="2026-05-12T00:00:00Z" + ), + ] + result = closer_module.extract_greptile_score(comments) + assert result is not None + score, _ = result + assert score == 5 + + def test_should_ignore_non_greptile_authors(self, closer_module): + comments = [ + { + "user": {"login": "some-human"}, + "body": "Confidence Score: 1/5", + "created_at": "2026-05-12T00:00:00Z", + "updated_at": "2026-05-12T00:00:00Z", + } + ] + assert closer_module.extract_greptile_score(comments) is None + + def test_should_return_none_when_no_score_present(self, closer_module): + comments = [_greptile_comment("Greptile summary without a score.")] + assert closer_module.extract_greptile_score(comments) is None + + def test_should_return_none_for_empty_comments(self, closer_module): + assert closer_module.extract_greptile_score([]) is None + + +class TestEvaluatePr: + @pytest.fixture(autouse=True) + def _now(self): + return dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc) + + def _make_pr( + self, + *, + number: int = 1, + created_days_ago: int = 10, + is_draft: bool = False, + labels: list[str] | None = None, + author_login: str = "mateo-berri", + ) -> dict: + created = dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc) - dt.timedelta( + days=created_days_ago + ) + return { + "number": number, + "title": f"PR #{number}", + "createdAt": created.isoformat().replace("+00:00", "Z"), + "isDraft": is_draft, + "labels": [{"name": lbl} for lbl in (labels or [])], + "author": {"login": author_login}, + "url": f"https://example.com/pr/{number}", + } + + @pytest.fixture(autouse=True) + def _external_author(self, closer_module, monkeypatch): + """Treat every test PR as external unless overridden.""" + monkeypatch.setattr( + closer_module, "is_external_pr_author", lambda pr, repo: True + ) + + def test_should_warn_drafts_when_score_low_first_time( + self, closer_module, _now, monkeypatch + ): + # Drafts are NOT a free pass — the open-PR queue should reflect any + # PR that needs human attention regardless of draft status. Authors + # who need a long-lived draft can use the `wip` opt-out label. + # First run: warn the contributor (1-day grace), don't close yet. + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [_greptile_comment("Confidence Score: 2/5")], + ) + action, score, age = closer_module.evaluate_pr( + self._make_pr(is_draft=True, created_days_ago=0), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "warn-grace" + assert score == 2 and age == 0 + + def test_should_warn_brand_new_pr_when_min_age_zero( + self, closer_module, _now, monkeypatch + ): + # `min_age_days=0` means no age filter — a freshly-opened PR is + # eligible the moment Greptile scores it below threshold. The + # first detection still goes through the warn-grace step rather + # than closing immediately, giving the contributor 2 hours to + # respond before the next run actually closes the PR. + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [_greptile_comment("Confidence Score: 1/5")], + ) + action, score, age = closer_module.evaluate_pr( + self._make_pr(created_days_ago=0), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "warn-grace" + assert score == 1 and age == 0 + + def test_should_skip_optout_label_case_insensitive( + self, closer_module, _now, monkeypatch + ): + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: pytest.fail("should not fetch comments for opt-outs"), + ) + action, _, _ = closer_module.evaluate_pr( + self._make_pr(labels=["WIP"]), + now=_now, + min_age_days=7, + min_score=4, + repo=None, + optout_labels={"wip"}, + ) + assert action == "skip-optout-label" + + def test_should_skip_too_young_when_min_age_set( + self, closer_module, _now, monkeypatch + ): + # The min-age-days flag is now opt-in (default 0). When a maintainer + # explicitly passes a positive value (e.g. for a backfill run that + # wants to spare brand-new PRs), the skip-too-young path still works. + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: pytest.fail("should not fetch comments for young PRs"), + ) + action, _, age = closer_module.evaluate_pr( + self._make_pr(created_days_ago=2), + now=_now, + min_age_days=7, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "skip-too-young" + assert age == 2 + + def test_should_not_skip_when_min_age_is_zero( + self, closer_module, _now, monkeypatch + ): + # With the new default min_age_days=0, even a 0-day-old PR is + # evaluated. This test pins that behavior so future refactors don't + # silently restore an age filter. + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [_greptile_comment("Confidence Score: 5/5")], + ) + action, score, age = closer_module.evaluate_pr( + self._make_pr(created_days_ago=0), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "skip-score-ok" + assert score == 5 and age == 0 + + def test_should_skip_when_greptile_has_not_reviewed( + self, closer_module, _now, monkeypatch + ): + monkeypatch.setattr(closer_module, "fetch_pr_comments", lambda *a, **kw: []) + action, score, age = closer_module.evaluate_pr( + self._make_pr(created_days_ago=10), + now=_now, + min_age_days=7, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "skip-no-greptile-score" + assert score is None and age == 10 + + def test_should_skip_when_score_meets_threshold( + self, closer_module, _now, monkeypatch + ): + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [_greptile_comment("Confidence Score: 4/5")], + ) + action, score, age = closer_module.evaluate_pr( + self._make_pr(created_days_ago=10), + now=_now, + min_age_days=7, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "skip-score-ok" + assert score == 4 and age == 10 + + def test_should_warn_when_old_and_low_score_no_prior_warning( + self, closer_module, _now, monkeypatch + ): + # Even an old PR that still has no grace warning gets one on the + # first eligible run — the daily cron is the natural cadence, so + # an existing-but-never-warned PR enters the grace flow normally. + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [_greptile_comment("Confidence Score: 3/5")], + ) + action, score, age = closer_module.evaluate_pr( + self._make_pr(created_days_ago=10), + now=_now, + min_age_days=7, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "warn-grace" + assert score == 3 and age == 10 + + def test_should_close_when_grace_warning_aged_out_and_score_still_low( + self, closer_module, _now, monkeypatch + ): + # Day-1 the closer posted a warning. Day-2 the PR still scores <4 + # AND the warning is older than `GRACE_PERIOD_SECONDS`, so the + # action flips to `close`. This is the "grace expired" path. + old_warning = { + "user": {"login": "github-actions[bot]"}, + "body": ( + "you have 2 hours to fix this\n\n" + closer_module.GRACE_COMMENT_MARKER + ), + "created_at": ( + _now - dt.timedelta(seconds=closer_module.GRACE_PERIOD_SECONDS + 60) + ) + .isoformat() + .replace("+00:00", "Z"), + "updated_at": "2026-05-15T00:00:00Z", + } + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [ + _greptile_comment( + "

Confidence Score: 1/5

", + updated_at="2026-05-15T00:00:00Z", + ), + old_warning, + ], + ) + action, score, _ = closer_module.evaluate_pr( + self._make_pr(created_days_ago=14), + now=_now, + min_age_days=7, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "close" + assert score == 1 + + def test_should_skip_when_grace_warning_within_window( + self, closer_module, _now, monkeypatch + ): + # Within the 2-hour grace window the closer must NOT close the + # PR even if the score is still low. The warning is only an hour + # old; give the contributor time to push fixes before destruction. + recent_warning = { + "user": {"login": "github-actions[bot]"}, + "body": "warning text\n\n" + closer_module.GRACE_COMMENT_MARKER, + "created_at": (_now - dt.timedelta(hours=1)) + .isoformat() + .replace("+00:00", "Z"), + } + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [ + _greptile_comment("Confidence Score: 2/5"), + recent_warning, + ], + ) + action, score, _ = closer_module.evaluate_pr( + self._make_pr(created_days_ago=10), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "skip-in-grace-period" + assert score == 2 + + def test_should_warn_grace_for_swiftwinds_not_close_immediately( + self, closer_module, _now, monkeypatch + ): + # Regression: SwiftWinds (the dogfood account) used to be in a + # now-removed `IMMEDIATE_CLOSE_LOGINS` bypass that closed on first + # detection. It must now follow the SAME grace path as every other + # external author: warn first, close only after the window elapses. + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [_greptile_comment("Confidence Score: 1/5")], + ) + action, score, _ = closer_module.evaluate_pr( + self._make_pr(created_days_ago=0, author_login="SwiftWinds"), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "warn-grace" + assert score == 1 + + def test_should_skip_internal_authors(self, closer_module, _now, monkeypatch): + # Override the fixture for this one test. + monkeypatch.setattr( + closer_module, "is_external_pr_author", lambda pr, repo: False + ) + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: pytest.fail("should not fetch comments for internal"), + ) + action, score, _ = closer_module.evaluate_pr( + self._make_pr(created_days_ago=14, author_login="krrishdholakia"), + now=_now, + min_age_days=7, + min_score=4, + repo=None, + optout_labels=set(), + allowlist=frozenset(), + ) + assert action == "skip-internal" + assert score is None + + +class TestMainOptoutLabelDefault: + """`--optout-label` must REPLACE the canonical defaults, not append.""" + + def _patch_no_op(self, closer_module, monkeypatch): + monkeypatch.setattr(closer_module, "fetch_open_prs", lambda repo: []) + # `optout_labels` is captured indirectly via evaluate_pr; sniff the + # set passed in by stubbing evaluate_pr. + captured: dict = {} + + def fake_evaluate(pr, now, min_age_days, min_score, repo, optout_labels): + captured["optout_labels"] = set(optout_labels) + return ("skip-internal", None, None) + + monkeypatch.setattr(closer_module, "evaluate_pr", fake_evaluate) + return captured + + def test_should_use_canonical_defaults_when_flag_omitted( + self, closer_module, monkeypatch + ): + captured = self._patch_no_op(closer_module, monkeypatch) + # No PRs -> capture won't fire; instead inject one synthetic PR via + # fetch_open_prs so evaluate_pr is invoked at least once. + monkeypatch.setattr( + closer_module, + "fetch_open_prs", + lambda repo: [ + { + "number": 1, + "title": "p", + "createdAt": "2026-05-10T00:00:00Z", + "isDraft": True, + "labels": [], + "author": {"login": "x"}, + } + ], + ) + monkeypatch.setattr(sys, "argv", ["close_low_quality_prs.py"]) + rc = closer_module.main() + assert rc == 0 + assert captured["optout_labels"] == set(closer_module.DEFAULT_OPTOUT_LABELS) + + def test_should_replace_defaults_when_flag_provided( + self, closer_module, monkeypatch + ): + captured = self._patch_no_op(closer_module, monkeypatch) + monkeypatch.setattr( + closer_module, + "fetch_open_prs", + lambda repo: [ + { + "number": 1, + "title": "p", + "createdAt": "2026-05-10T00:00:00Z", + "isDraft": True, + "labels": [], + "author": {"login": "x"}, + } + ], + ) + monkeypatch.setattr( + sys, + "argv", + [ + "close_low_quality_prs.py", + "--optout-label", + "hold", + "--optout-label", + "needs-discussion", + ], + ) + rc = closer_module.main() + assert rc == 0 + # Crucially, none of the canonical defaults leak in. + assert captured["optout_labels"] == {"hold", "needs-discussion"} + for default in closer_module.DEFAULT_OPTOUT_LABELS: + assert default not in captured["optout_labels"], default + + +class TestSecondsSinceLastGraceWarning: + """Grace-period detection: only counts comments by the bot identity + that contain the shared `GRACE_COMMENT_MARKER`.""" + + def _make_marker_comment( + self, + closer_module, + *, + login: str = "github-actions[bot]", + created_at: str = "2026-05-16T00:00:00Z", + include_marker: bool = True, + ) -> dict: + body = "warning text" + if include_marker: + body += "\n\n" + closer_module.GRACE_COMMENT_MARKER + return { + "user": {"login": login}, + "body": body, + "created_at": created_at, + } + + def test_should_return_none_when_no_marker_comment(self, closer_module): + comments = [ + { + "user": {"login": "github-actions[bot]"}, + "body": "Some other bot comment", + "created_at": "2026-05-16T00:00:00Z", + } + ] + assert closer_module.seconds_since_last_grace_warning(comments) is None + + def test_should_return_none_for_empty(self, closer_module): + assert closer_module.seconds_since_last_grace_warning([]) is None + + def test_should_ignore_non_bot_comments_with_marker(self, closer_module): + # If a curious user quotes the marker in a comment, we must NOT + # treat it as a bot warning. The grace timer would then never fire. + comments = [ + self._make_marker_comment(closer_module, login="random-user"), + ] + assert closer_module.seconds_since_last_grace_warning(comments) is None + + def test_should_pick_latest_marker_comment(self, closer_module): + # When multiple grace warnings exist (e.g. a re-open cycle), use + # the most recent one to compute the age. + comments = [ + self._make_marker_comment(closer_module, created_at="2026-05-15T00:00:00Z"), + self._make_marker_comment(closer_module, created_at="2026-05-16T23:00:00Z"), + ] + now = dt.datetime(2026, 5, 17, 0, 0, 0, tzinfo=dt.timezone.utc) + age = closer_module.seconds_since_last_grace_warning(comments, now=now) + # 1h = 3600s + assert age == 3600.0 + + +class TestGraceWarningCommentText: + """Pin the user-facing language in the grace warning comment so the + grace-window and `@greptileai still works after close` promises + don't get accidentally dropped in a future refactor. + """ + + def test_should_state_grace_window(self, closer_module): + body = closer_module.format_grace_warning_comment(score=2, threshold=4) + # The user's PR explicitly said "specify in the comment" — pin + # that the grace window appears in the comment. + assert "2 hours" in body + + def test_should_mention_agent_shin_reconsider(self, closer_module): + body = closer_module.format_grace_warning_comment(score=2, threshold=4) + assert "@agent-shin reconsider" in body + + def test_should_promise_greptileai_works_after_close(self, closer_module): + body = closer_module.format_grace_warning_comment(score=2, threshold=4) + assert "@greptileai" in body + assert "even after the PR is closed" in body + + def test_should_carry_grace_marker(self, closer_module): + # The marker is what `seconds_since_last_grace_warning` greps for + # to detect a prior warning — dropping it would silently break + # the cooldown. + body = closer_module.format_grace_warning_comment(score=2, threshold=4) + assert closer_module.GRACE_COMMENT_MARKER in body + + def test_close_comment_should_mention_greptileai_post_close(self, closer_module): + # The close comment should ALSO point at the @greptileai post-close + # re-review path so contributors see the same options whether they + # read the warning or only catch the close comment. + body = closer_module.format_close_comment(score=2, threshold=4) + assert "@greptileai" in body + assert "even after the PR is closed" in body + + def test_close_comment_should_advertise_reconsider(self, closer_module): + body = closer_module.format_close_comment(score=2, threshold=4) + assert "@agent-shin reconsider" in body + + def test_close_comment_should_carry_agent_shin_close_marker(self, closer_module): + # The close comment advertises `@agent-shin reconsider`, and the + # reconsider reopen guard (`was_closed_by_agent_shin`) only treats a + # PR as Agent-Shin-closed when the close comment carries this marker. + # Dropping it silently breaks the advertised recovery path for every + # PR closed by this daily sweep. + body = closer_module.format_close_comment(score=2, threshold=4) + assert closer_module.AGENT_SHIN_CLOSE_MARKER in body + + def test_close_comment_should_state_score_and_threshold(self, closer_module): + body = closer_module.format_close_comment(score=1, threshold=4) + assert "1/5" in body + assert "4/5" in body + + +class TestHasOptoutLabel: + def test_should_match_label_case_insensitively(self, closer_module): + pr = {"labels": [{"name": "Do Not Close"}, {"name": "bug"}]} + assert closer_module.has_optout_label(pr, {"do not close"}) is True + + def test_should_return_false_when_no_match(self, closer_module): + pr = {"labels": [{"name": "bug"}, {"name": "enhancement"}]} + assert closer_module.has_optout_label(pr, {"wip", "keep open"}) is False + + def test_should_handle_missing_labels(self, closer_module): + assert closer_module.has_optout_label({}, {"wip"}) is False + + +class TestListOpenItemsNoCap: + """The bulk sweeps must fetch the ENTIRE open backlog. + + Regression guard for the old hard-coded ``--limit 1000``: gh lists + newest-first, so a low cap silently dropped the *oldest* PRs/issues — + exactly the stale ones a low-quality sweep exists to catch. + """ + + @staticmethod + def _shared(closer_module): + # `closer_module` loading puts `.github/scripts` on sys.path and + # imports agent_shin_shared, so it's already in sys.modules. + import agent_shin_shared + + return agent_shin_shared + + def _capture_gh_args(self, closer_module, monkeypatch, *, returns="[]"): + shared = self._shared(closer_module) + captured: dict = {} + + def fake_gh(*args): + captured["args"] = args + return returns + + # `list_open_items` looks up `gh` in agent_shin_shared's namespace. + monkeypatch.setattr(shared, "gh", fake_gh) + return shared, captured + + def test_list_open_items_passes_no_cap_limit_not_1000( + self, closer_module, monkeypatch + ): + shared, captured = self._capture_gh_args(closer_module, monkeypatch) + shared.list_open_items("pr", repo="o/r", fields="number,title") + args = captured["args"] + assert "--limit" in args + limit_value = args[args.index("--limit") + 1] + assert limit_value == str(shared.GH_LIST_ALL_LIMIT) + assert limit_value != "1000" + # A meaningful ceiling: comfortably above any realistic open backlog. + assert shared.GH_LIST_ALL_LIMIT >= 100_000 + + def test_list_open_items_uses_dedicated_command_state_and_fields( + self, closer_module, monkeypatch + ): + shared, captured = self._capture_gh_args(closer_module, monkeypatch) + shared.list_open_items("issue", repo="o/r", fields="number") + args = captured["args"] + assert args[0] == "issue" and args[1] == "list" + assert args[args.index("--state") + 1] == "open" + assert args[args.index("--json") + 1] == "number" + assert tuple(args[-2:]) == ("--repo", "o/r") + + def test_list_open_items_omits_repo_when_none(self, closer_module, monkeypatch): + shared, captured = self._capture_gh_args(closer_module, monkeypatch) + shared.list_open_items("pr", repo=None, fields="number") + assert "--repo" not in captured["args"] + + def test_list_open_items_parses_json_array(self, closer_module, monkeypatch): + shared, _ = self._capture_gh_args( + closer_module, monkeypatch, returns='[{"number": 1}, {"number": 2}]' + ) + items = shared.list_open_items("pr", repo=None, fields="number") + assert [i["number"] for i in items] == [1, 2] + + def test_list_open_items_rejects_unknown_kind(self, closer_module): + shared = self._shared(closer_module) + with pytest.raises(ValueError): + shared.list_open_items("both", repo="o/r", fields="number") + + def test_fetch_open_prs_delegates_with_no_cap(self, closer_module, monkeypatch): + shared, captured = self._capture_gh_args(closer_module, monkeypatch) + closer_module.fetch_open_prs("o/r") + args = captured["args"] + assert args[0] == "pr" + assert args[args.index("--limit") + 1] == str(shared.GH_LIST_ALL_LIMIT) + # Still requests every field downstream evaluate_pr / labels logic needs. + assert "createdAt" in args[args.index("--json") + 1] + + +class TestEvaluatePrAllowlist: + """While the dogfood allowlist is active `evaluate_pr` only acts on the + named accounts and bypasses the external-only restriction for them. + Emptying it restores the internal-author skip.""" + + @pytest.fixture(autouse=True) + def _now(self): + return dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc) + + def _make_pr(self, *, author_login: str, created_days_ago: int = 10) -> dict: + created = dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc) - dt.timedelta( + days=created_days_ago + ) + return { + "number": 1, + "title": "PR #1", + "createdAt": created.isoformat().replace("+00:00", "Z"), + "isDraft": False, + "labels": [], + "author": {"login": author_login}, + "url": "https://example.com/pr/1", + } + + def test_should_skip_author_not_on_allowlist( + self, closer_module, _now, monkeypatch + ): + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: pytest.fail("must not fetch comments for non-allowlisted"), + ) + action, score, _ = closer_module.evaluate_pr( + self._make_pr(author_login="random-oss-dev"), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "skip-not-allowlisted" + assert score is None + + def test_should_act_on_allowlisted_internal_author( + self, closer_module, _now, monkeypatch + ): + monkeypatch.setattr( + closer_module, "is_external_pr_author", lambda pr, repo: False + ) + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [_greptile_comment("Confidence Score: 2/5")], + ) + action, score, _ = closer_module.evaluate_pr( + self._make_pr(author_login="mateo-berri", created_days_ago=0), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + ) + assert action == "warn-grace" + assert score == 2 + + def test_empty_allowlist_restores_internal_skip( + self, closer_module, _now, monkeypatch + ): + monkeypatch.setattr( + closer_module, "is_external_pr_author", lambda pr, repo: False + ) + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: pytest.fail("must not fetch comments for internal"), + ) + action, score, _ = closer_module.evaluate_pr( + self._make_pr(author_login="krrishdholakia"), + now=_now, + min_age_days=0, + min_score=4, + repo=None, + optout_labels=set(), + allowlist=frozenset(), + ) + assert action == "skip-internal" + + def test_allowlist_constant_is_the_two_dogfood_accounts(self, closer_module): + assert closer_module.ALLOWLIST_LOGINS == frozenset( + {"mateo-berri", "swiftwinds"} + ) + + +class TestDryRunGateOnClose: + """Regression: the daily sweep is dry-run unless `--close` is passed + (the workflow only adds it when `AGENT_SHIN_ENABLED=true`). A closeable + PR (low score, grace window elapsed) must be DETECTED and reported as + "would close", but the dry run must never make a real GitHub mutation, + so merging Agent Shin stays inert by default.""" + + def _closeable_pr(self) -> dict: + return { + "number": 7, + "title": "thin PR", + "createdAt": "2026-05-10T00:00:00Z", + "isDraft": False, + "labels": [], + "author": {"login": "SwiftWinds"}, + "url": "https://example.com/pr/7", + } + + def test_dry_run_sweep_detects_but_does_not_close( + self, closer_module, monkeypatch, capsys + ): + aged_out_warning = { + "user": {"login": "github-actions[bot]"}, + "body": "warned\n\n" + closer_module.GRACE_COMMENT_MARKER, + # Far enough in the past that it's aged out regardless of + # GRACE_PERIOD_SECONDS, since main() pins `now` to real time. + "created_at": "2020-01-01T00:00:00Z", + } + monkeypatch.setattr( + closer_module, "fetch_open_prs", lambda repo: [self._closeable_pr()] + ) + monkeypatch.setattr( + closer_module, + "fetch_pr_comments", + lambda *a, **kw: [ + _greptile_comment("Confidence Score: 1/5"), + aged_out_warning, + ], + ) + # Any real GitHub mutation during a dry run is the bug under test. + monkeypatch.setattr( + closer_module, + "gh", + lambda *a, **kw: pytest.fail(f"dry run must not call gh: {a}"), + ) + monkeypatch.setattr(sys, "argv", ["close_low_quality_prs.py"]) + + rc = closer_module.main() + + assert rc == 0 + # The PR is detected as closeable, just not acted on. + assert "Total would close: 1" in capsys.readouterr().out diff --git a/tests/test_litellm/test_github_review_gate.py b/tests/test_litellm/test_github_review_gate.py new file mode 100644 index 00000000000..001fa8f43f5 --- /dev/null +++ b/tests/test_litellm/test_github_review_gate.py @@ -0,0 +1,524 @@ +"""Unit tests for the `ready for review` label lifecycle (Agent Shin review gate). + +Exercises `triage_with_llm.review_gate`, the state machine that keeps the +`ready for review` label in sync with whether a PR clears both the LLM rubric +and Greptile's confidence score: + + * pass (untagged) -> add label + "ready for review" comment + * pass (untagged, recovered) -> add label + "all clear again" comment + * pass (already tagged) -> noop + * regress (tagged) -> remove label + "what's missing" comment, stays open + * fail (untagged, within 24h)-> one-time "what's missing" notice + * fail (untagged, >24h) -> close + comment + * dry run (close=False) -> would-* previews, no side effects +""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[2] / ".github" / "scripts" / "triage_with_llm.py" +) + +NOW = dt.datetime(2026, 5, 24, 12, 0, 0, tzinfo=dt.timezone.utc) +JUST_NOW = "2026-05-24T11:00:00Z" # 1h old -> within 24h grace +TWO_DAYS_AGO = "2026-05-22T11:00:00Z" # >24h old -> past grace + + +@pytest.fixture(scope="module") +def triage_module(): + spec = importlib.util.spec_from_file_location("triage_with_llm", SCRIPT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules["triage_with_llm"] = module + spec.loader.exec_module(module) + return module + + +class _Recorder: + """Captures every gh mutation review_gate could fire, and fails loudly + on the ones a given scenario forbids.""" + + def __init__(self, triage_module, monkeypatch): + self.comments: list[str] = [] + self.added: list[str] = [] + self.removed: list[str] = [] + self.closed: list[int] = [] + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: self.comments.append(body), + ) + monkeypatch.setattr( + triage_module, + "add_label", + lambda repo, n, label: self.added.append(label), + ) + monkeypatch.setattr( + triage_module, + "remove_label", + lambda repo, n, label: self.removed.append(label), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda repo, n: self.closed.append(n), + ) + + +def _make_pr(**overrides): + base = { + "number": 7, + "title": "feat: do a thing", + "body": "some body without a linked issue or QA proof", + "state": "open", + "author_association": "NONE", + "user": {"login": "mateo-berri"}, + "labels": [], + "created_at": JUST_NOW, + } + base.update(overrides) + return base + + +def _pass(prompt): + return '{"verdict": "pass", "missing": [], "explanation": "looks good"}' + + +def _fail(prompt): + return ( + '{"verdict": "fail", "missing": ["QA proof", "expected vs. actual"],' + ' "explanation": "thin description"}' + ) + + +def _gate(triage_module, **kwargs): + """Call review_gate with safe defaults for the injectable hooks.""" + params = dict( + repo="o/r", + number=7, + close=True, + model="m", + judge=_pass, + greptile_score=None, + comments=[], + now=NOW, + ) + params.update(kwargs) + return triage_module.review_gate(**params) + + +class TestReviewGatePass: + def test_pass_untagged_adds_label_and_ready_comment( + self, triage_module, monkeypatch + ): + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr()) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate(triage_module, judge=_pass, greptile_score=5) + + assert result["action"] == "labeled-ready" + assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL] + assert rec.removed == [] and rec.closed == [] + assert len(rec.comments) == 1 + assert "ready for review" in rec.comments[0].lower() + assert triage_module.READY_MARKER in rec.comments[0] + assert "5/5" in rec.comments[0] + + def test_pass_already_tagged_is_noop(self, triage_module, monkeypatch): + pr = _make_pr(labels=[{"name": "ready for review"}]) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate(triage_module, judge=_pass, greptile_score=5) + + assert result["action"] == "noop-passing" + assert rec.added == [] and rec.removed == [] and rec.comments == [] + + def test_pass_after_prior_regression_uses_all_clear_wording( + self, triage_module, monkeypatch + ): + # A regression marker in history -> this is a recovery, not a first pass. + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr()) + rec = _Recorder(triage_module, monkeypatch) + prior = [ + { + "user": {"login": "github-actions[bot]"}, + "body": triage_module.REGRESSED_MARKER, + } + ] + + result = _gate(triage_module, judge=_pass, greptile_score=5, comments=prior) + + assert result["action"] == "labeled-ready" + assert "all clear" in rec.comments[0].lower() + + def test_linked_issue_passes_without_calling_judge( + self, triage_module, monkeypatch + ): + pr = _make_pr(body="Fixes #4321\n\nbody") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate( + triage_module, + judge=lambda p: pytest.fail("LLM must not be called for linked issue"), + greptile_score=5, + ) + assert result["action"] == "labeled-ready" + assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL] + + +class TestReviewGateRegression: + def test_regression_removes_label_and_keeps_pr_open( + self, triage_module, monkeypatch + ): + pr = _make_pr(labels=[{"name": "ready for review"}]) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate(triage_module, judge=_fail, greptile_score=5) + + assert result["action"] == "label-removed-regressed" + assert rec.removed == [triage_module.READY_FOR_REVIEW_LABEL] + assert rec.closed == [] # regression NEVER closes the PR + assert triage_module.REGRESSED_MARKER in rec.comments[0] + assert "QA proof" in rec.comments[0] + # The state machine closes a still-failing PR `grace_days` after this + # notice (default 24h); the comment must disclose that deadline rather + # than implying the PR stays open indefinitely. + assert "24 hours" in rec.comments[0] + assert "auto-closed" in rec.comments[0] + + def test_regression_comment_discloses_grace_deadline(self, triage_module): + one_day = triage_module.format_regression_comment( + ["QA proof"], "needs work", grace_days=1 + ) + assert "24 hours" in one_day + assert "auto-closed" in one_day + + three_days = triage_module.format_regression_comment( + ["QA proof"], "needs work", grace_days=3 + ) + assert "3 days" in three_days + assert "auto-closed" in three_days + + def test_greptile_drop_alone_triggers_regression(self, triage_module, monkeypatch): + # Rubric still passes, but Greptile fell to 2/5 -> not passing. + pr = _make_pr(labels=[{"name": "ready for review"}]) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate(triage_module, judge=_pass, greptile_score=2) + + assert result["action"] == "label-removed-regressed" + assert rec.removed == [triage_module.READY_FOR_REVIEW_LABEL] + assert "2/5" in rec.comments[0] + + def test_greptile_score_read_from_comments_when_not_injected( + self, triage_module, monkeypatch + ): + pr = _make_pr(labels=[{"name": "ready for review"}]) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + greptile = [ + { + "user": {"login": "greptile-apps[bot]"}, + "body": "Confidence Score: 2/5", + "created_at": "2026-05-24T10:00:00Z", + } + ] + + result = _gate( + triage_module, + judge=_pass, + greptile_score=triage_module._UNSET, + comments=greptile, + ) + assert result["action"] == "label-removed-regressed" + assert "2/5" in rec.comments[0] + + +class TestReviewGateGraceAndClose: + def test_within_grace_posts_one_time_notice(self, triage_module, monkeypatch): + monkeypatch.setattr( + triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW) + ) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate(triage_module, judge=_fail, greptile_score=None) + + assert result["action"] == "within-grace-notified" + assert rec.closed == [] and rec.added == [] and rec.removed == [] + assert triage_module.WITHIN_GRACE_MARKER in rec.comments[0] + assert "QA proof" in rec.comments[0] + + def test_within_grace_does_not_double_notify(self, triage_module, monkeypatch): + monkeypatch.setattr( + triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW) + ) + rec = _Recorder(triage_module, monkeypatch) + prior = [ + { + "user": {"login": "github-actions[bot]"}, + "body": triage_module.WITHIN_GRACE_MARKER, + } + ] + + result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior) + + assert result["action"] == "within-grace-already-notified" + assert rec.comments == [] + + def test_past_grace_closes_with_comment(self, triage_module, monkeypatch): + monkeypatch.setattr( + triage_module, + "fetch_pr", + lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO), + ) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate(triage_module, judge=_fail, greptile_score=None) + + assert result["action"] == "closed" + assert rec.closed == [7] + assert len(rec.comments) == 1 + # The close comment must carry the reconsider provenance marker so + # `was_closed_by_agent_shin` can later recognize this as an Agent Shin + # close (and not some other workflow's `github-actions[bot]` close). + assert triage_module.AGENT_SHIN_CLOSE_MARKER in rec.comments[0] + + def test_recent_regression_marker_blocks_close(self, triage_module, monkeypatch): + """A failing PR with a fresh regression notice must NOT be closed — + the contributor needs a window to address the regression.""" + monkeypatch.setattr( + triage_module, + "fetch_pr", + lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO), + ) + rec = _Recorder(triage_module, monkeypatch) + prior = [ + { + "user": {"login": "github-actions[bot]"}, + "body": triage_module.REGRESSED_MARKER, + # Posted just an hour before NOW -> well inside grace_days. + "created_at": "2026-05-24T11:00:00Z", + } + ] + + result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior) + + assert result["action"] == "regressed-already-notified" + assert rec.closed == [] and rec.comments == [] + + def test_stale_regression_marker_allows_close(self, triage_module, monkeypatch): + """Once grace_days have elapsed since the regression notice, the + review gate must let the close path fire — otherwise PRs that were + regressed and then abandoned stay open forever.""" + monkeypatch.setattr( + triage_module, + "fetch_pr", + lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO), + ) + rec = _Recorder(triage_module, monkeypatch) + prior = [ + { + "user": {"login": "github-actions[bot]"}, + "body": triage_module.REGRESSED_MARKER, + # Posted 30 days before NOW -> well past the default 1-day grace. + "created_at": "2026-04-24T11:00:00Z", + } + ] + + result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior) + + assert result["action"] == "closed" + assert rec.closed == [7] + assert len(rec.comments) == 1 + + def test_linked_issue_with_greptile_fail_uses_greptile_explanation( + self, triage_module, monkeypatch + ): + """When the rubric short-circuits to pass (linked-issue regex) but + Greptile dragged the PR under the bar, the close comment's + explanation must describe the Greptile shortfall, not the + misleading "LLM was not called" rubric placeholder.""" + pr = _make_pr(body="Fixes #4321\n\nbody", created_at=TWO_DAYS_AGO) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate( + triage_module, + judge=lambda p: pytest.fail("LLM must not be called for linked issue"), + greptile_score=2, + ) + + assert result["action"] == "closed" + assert len(rec.comments) == 1 + body = rec.comments[0] + assert "LLM was not called" not in body + assert "Greptile" in body and "2/5" in body + + +class TestReviewGateDryRun: + @pytest.mark.parametrize( + "scenario,labels,judge,score,created,expected", + [ + ("pass", [], _pass, 5, JUST_NOW, "would-label-ready"), + ( + "regress", + [{"name": "ready for review"}], + _fail, + 5, + JUST_NOW, + "would-remove-label", + ), + ("within-grace", [], _fail, None, JUST_NOW, "would-notify-within-grace"), + ("past-grace", [], _fail, None, TWO_DAYS_AGO, "would-close"), + ], + ) + def test_dry_run_previews_without_side_effects( + self, + triage_module, + monkeypatch, + scenario, + labels, + judge, + score, + created, + expected, + ): + pr = _make_pr(labels=labels, created_at=created) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + + result = _gate(triage_module, close=False, judge=judge, greptile_score=score) + + assert result["action"] == expected + # Dry run touches nothing. + assert rec.added == [] and rec.removed == [] and rec.closed == [] + assert rec.comments == [] + assert "comment" in result # preview body still surfaced + + +class TestReviewGateGuards: + def test_skips_internal_author(self, triage_module, monkeypatch): + pr = _make_pr(author_association="MEMBER", user={"login": "krrish"}) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = _gate( + triage_module, + judge=lambda p: pytest.fail("no LLM for internal"), + allowlist=frozenset(), + ) + assert result["action"] == "skip-internal-author" + + def test_skips_closed_pr(self, triage_module, monkeypatch): + pr = _make_pr(state="closed") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = _gate(triage_module, judge=lambda p: pytest.fail("no LLM for closed")) + assert result["action"] == "skip-not-open" + + def test_llm_error_is_non_destructive(self, triage_module, monkeypatch): + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr()) + rec = _Recorder(triage_module, monkeypatch) + + def boom(prompt): + raise RuntimeError("api down") + + result = _gate(triage_module, judge=boom, greptile_score=None) + + assert result["action"] == "skip-llm-error" + assert rec.closed == [] and rec.added == [] and rec.removed == [] + + def test_full_recovery_cycle(self, triage_module, monkeypatch): + """pass -> regress -> recover, threading labels/comments like GitHub would.""" + state = {"labels": [], "comments": []} + + def fake_fetch(repo, n): + return _make_pr(labels=list(state["labels"]), created_at=JUST_NOW) + + monkeypatch.setattr(triage_module, "fetch_pr", fake_fetch) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: state["comments"].append( + {"user": {"login": "github-actions[bot]"}, "body": body} + ), + ) + monkeypatch.setattr( + triage_module, + "add_label", + lambda repo, n, label: state["labels"].append({"name": label}), + ) + monkeypatch.setattr( + triage_module, + "remove_label", + lambda repo, n, label: state["labels"].clear(), + ) + monkeypatch.setattr( + triage_module, "close_pr", lambda repo, n: pytest.fail("must not close") + ) + + # 1) passes -> tagged + r1 = _gate( + triage_module, judge=_pass, greptile_score=5, comments=state["comments"] + ) + assert r1["action"] == "labeled-ready" + assert any(lbl["name"] == "ready for review" for lbl in state["labels"]) + + # 2) regresses -> tag removed, comment posted, PR still open + r2 = _gate( + triage_module, judge=_fail, greptile_score=2, comments=state["comments"] + ) + assert r2["action"] == "label-removed-regressed" + assert state["labels"] == [] + + # 3) fixed again -> "all clear" + tag back + r3 = _gate( + triage_module, judge=_pass, greptile_score=5, comments=state["comments"] + ) + assert r3["action"] == "labeled-ready" + assert any(lbl["name"] == "ready for review" for lbl in state["labels"]) + assert "all clear" in state["comments"][-1]["body"].lower() + + +class TestReviewGateAllowlist: + """While the dogfood allowlist is active it is the sole author gate: + only the named accounts pass, and for them the internal-author exemption + is bypassed. Emptying it restores the normal internal-author skip.""" + + def test_should_skip_author_not_on_allowlist(self, triage_module, monkeypatch): + pr = _make_pr(user={"login": "random-oss-dev"}) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + result = _gate( + triage_module, judge=lambda p: pytest.fail("no LLM for non-allowlisted") + ) + assert result["action"] == "skip-not-allowlisted" + assert rec.added == [] and rec.comments == [] and rec.closed == [] + + def test_should_act_on_allowlisted_internal_author( + self, triage_module, monkeypatch + ): + pr = _make_pr(author_association="MEMBER", user={"login": "mateo-berri"}) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + rec = _Recorder(triage_module, monkeypatch) + result = _gate(triage_module, judge=_pass, greptile_score=5) + assert result["action"] == "labeled-ready" + assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL] + + def test_empty_allowlist_restores_internal_skip(self, triage_module, monkeypatch): + pr = _make_pr(author_association="MEMBER", user={"login": "krrish"}) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = _gate( + triage_module, + judge=lambda p: pytest.fail("no LLM for internal"), + allowlist=frozenset(), + ) + assert result["action"] == "skip-internal-author" diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py new file mode 100644 index 00000000000..f50cf126c36 --- /dev/null +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -0,0 +1,2073 @@ +"""Unit tests for `.github/scripts/triage_with_llm.py` (Agent Shin).""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[2] / ".github" / "scripts" / "triage_with_llm.py" +) + + +@pytest.fixture(scope="module") +def triage_module(): + spec = importlib.util.spec_from_file_location("triage_with_llm", SCRIPT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules["triage_with_llm"] = module + spec.loader.exec_module(module) + return module + + +class TestIsInternalContributor: + @pytest.mark.parametrize("association", ["OWNER", "MEMBER", "COLLABORATOR"]) + def test_should_mark_org_associations_as_internal(self, triage_module, association): + item = { + "author_association": association, + "user": {"login": "krrishdholakia"}, + } + assert triage_module.is_internal_contributor(item) is True + + @pytest.mark.parametrize( + "association", + ["CONTRIBUTOR", "FIRST_TIME_CONTRIBUTOR", "FIRST_TIMER", "NONE"], + ) + def test_should_mark_outside_associations_as_external( + self, triage_module, association + ): + item = { + "author_association": association, + "user": {"login": "random-oss-dev"}, + } + assert triage_module.is_internal_contributor(item) is False + + @pytest.mark.parametrize( + "item", + [ + {"author_association": "", "user": {"login": "random-oss-dev"}}, + {"user": {"login": "random-oss-dev"}}, # association field absent + ], + ) + def test_should_fail_safe_when_author_association_is_missing( + self, triage_module, item + ): + # Fail-safe: an empty/missing association must never make a PR + # eligible for the destructive close path. Treat as internal (skip). + assert triage_module.is_internal_contributor(item) is True + + @pytest.mark.parametrize( + "login", + ["dependabot[bot]", "greptile-apps[bot]", "dependabot", "github-actions"], + ) + def test_should_skip_bot_accounts_regardless_of_association( + self, triage_module, login + ): + item = {"author_association": "NONE", "user": {"login": login}} + assert triage_module.is_internal_contributor(item) is True + + +class TestHasLinkedIssue: + @pytest.mark.parametrize( + "body", + [ + "Fixes #1234", + "closes #1", + "Resolves #99", + "fix #42 — this addresses the regression", + "Closes https://github.com/BerriAI/litellm/issues/27000", + "Resolved https://github.com/BerriAI/litellm/issues/27001", + ], + ) + def test_should_detect_common_link_phrases(self, triage_module, body): + assert triage_module.has_linked_issue(body) is True + + @pytest.mark.parametrize( + "body", + [ + "", + "Some change", + # Casual mentions must NOT auto-pass — they should fall through to + # the LLM judge so the stricter "not a passing mention" rule fires. + "See #1234", + "see #1234 for context", + "ref #1234", + "Refs https://github.com/BerriAI/litellm/issues/27000", + "this addresses #1234", + ], + ) + def test_should_not_auto_pass_casual_mentions(self, triage_module, body): + assert triage_module.has_linked_issue(body) is False + + def test_should_not_detect_when_only_html_comment_template(self, triage_module): + body = "" + assert triage_module.has_linked_issue(body) is False + + +class TestStripHtmlComments: + def test_should_remove_single_line_comments(self, triage_module): + text = "before after" + assert "placeholder" not in triage_module.strip_html_comments(text) + + def test_should_remove_multiline_comments(self, triage_module): + text = "kept\n\nkept2" + cleaned = triage_module.strip_html_comments(text) + assert "Fixes #1" not in cleaned + assert "kept" in cleaned and "kept2" in cleaned + + def test_should_handle_none(self, triage_module): + assert triage_module.strip_html_comments(None) == "" + + +class TestCloseCommentText: + """Pin the user-facing language in close comments so changes are intentional.""" + + def test_pr_close_comment_should_recommend_new_pr_primarily(self, triage_module): + body = triage_module.format_pr_close_comment( + {"verdict": "fail", "missing": ["QA proof"], "explanation": "thin"} + ) + # Primary path: open a new PR (because OSS authors can't reopen a + # bot-closed PR). Secondary path: `@agent-shin reconsider`. + assert "Open a new PR" in body + assert "@agent-shin reconsider" in body + # Old advice that no longer works for OSS contributors must NOT + # appear (they can't reopen a PR closed by a bot/maintainer). + assert "Reopen the PR" not in body + + def test_reopen_comment_should_carry_reconsider_marker(self, triage_module): + # The marker is what the rate-limit guard greps for to detect a + # prior reconsider verdict on the same PR. If the marker ever + # gets dropped from this comment, the cooldown silently breaks + # and a contributor can spam `@agent-shin reconsider` to burn + # LLM budget. + body = triage_module.format_reopen_comment("pr") + assert triage_module.RECONSIDER_COMMENT_MARKER in body + + def test_still_failing_comment_should_carry_reconsider_marker(self, triage_module): + body = triage_module.format_reconsider_still_failing_comment( + "pr", + {"verdict": "fail", "missing": ["QA proof"], "explanation": "thin"}, + ) + assert triage_module.RECONSIDER_COMMENT_MARKER in body + + def test_pr_close_comment_should_not_promise_automatic_reopen_on_open( + self, triage_module + ): + # The previous comment said "I'll re-evaluate automatically" — that + # only worked because the author could reopen, which they often + # can't. The new wording must point them at the comment trigger or + # a new PR instead. + body = triage_module.format_pr_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert "I'll re-evaluate automatically" not in body + + def test_issue_close_comment_should_use_reconsider_trigger(self, triage_module): + # OSS authors have read access, which only lets them reopen issues + # they closed themselves; they CANNOT reopen an issue a maintainer or + # bot closed. So the recovery path is `@agent-shin reconsider` (the + # bot reopens), exactly like the PR path. If this regresses to "reopen + # it yourself", contributors hit a dead end on bot-closed issues. + body = triage_module.format_issue_close_comment( + {"verdict": "fail", "missing": ["repro"], "explanation": "thin"} + ) + assert "@agent-shin reconsider" in body + + def test_pr_close_comment_should_link_blog_explainer(self, triage_module): + # The blog post is the canonical public explanation of what the bot + # checks and why. Every action-required bot comment must link to it + # so contributors landing on a bot-closed PR can self-serve context + # without pinging a maintainer. + body = triage_module.format_pr_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert "https://docs.litellm.ai/blog/agent-shin-triage" in body + + def test_issue_close_comment_should_link_blog_explainer(self, triage_module): + body = triage_module.format_issue_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert "https://docs.litellm.ai/blog/agent-shin-triage" in body + + def test_pr_close_comment_should_flag_mocked_tests_as_insufficient_proof( + self, triage_module + ): + # The PR rubric was tightened to require end-to-end QA proof and + # explicitly exclude mocked-dependency unit tests. The user-facing + # close comment must say so — otherwise contributors will keep + # re-submitting "pytest passed (mocks)" runs and getting closed + # again with no explanation of why. + body = triage_module.format_pr_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert "end-to-end qa proof" in body.lower() + assert "mock" in body.lower() + + def test_all_agent_shin_comments_should_use_bullet_train_emoji(self, triage_module): + # The bullet train (🚅) is Agent Shin's symbol, matching the LiteLLM + # logo; the previous wave (👋) was generic and didn't match the bot's + # identity. Every action-required comment the bot can post must use the + # bullet train so the contributor recognizes who's writing without + # reading the signoff. + verdict = {"verdict": "fail", "missing": [], "explanation": ""} + comments = { + "pr_close": triage_module.format_pr_close_comment(verdict), + "issue_close": triage_module.format_issue_close_comment(verdict), + "pr_grace": triage_module.format_grace_warning_pr_comment(verdict), + "issue_grace": triage_module.format_grace_warning_issue_comment(verdict), + "within_grace": triage_module.format_within_grace_comment( + [], "", grace_days=1 + ), + } + for name, body in comments.items(): + assert "🚅" in body, f"{name} comment is missing the bullet train emoji" + assert "👋" not in body, f"{name} comment still uses the old wave emoji" + + def test_pr_close_comment_should_show_what_pr_got_right(self, triage_module): + # The user explicitly asked for a "things you got right" section so + # the comment doesn't read as pure rejection. When the judge confirms + # a field is present (e.g. linked_issue), the bullet for it MUST + # appear in the close comment. + body = triage_module.format_pr_close_comment( + { + "verdict": "fail", + "linked_issue": True, + "has_problem_description": True, + "has_expected_vs_actual": False, + "has_qa_proof": False, + "missing": ["QA proof"], + "explanation": "no proof", + } + ) + assert "What you got right" in body + # The two present fields surface as ✅ bullets; the two absent + # fields do not get a ✅ bullet (the QA-proof rubric block still + # mentions the concept, but only the affirmed fields get checkmarks). + assert "- ✅ Linked a related GitHub issue" in body + assert "- ✅ Clear problem description" in body + assert "- ✅ Expected vs. actual behavior" not in body + assert "- ✅ End-to-end QA proof" not in body + + def test_pr_close_comment_should_omit_present_section_when_nothing_present( + self, triage_module + ): + # If the judge says nothing is present (every flag False), the + # "what you got right" block is skipped entirely — better to omit + # than to render "What you got right: (nothing)". + body = triage_module.format_pr_close_comment( + { + "verdict": "fail", + "linked_issue": False, + "has_problem_description": False, + "has_expected_vs_actual": False, + "has_qa_proof": False, + "missing": [], + "explanation": "", + } + ) + assert "What you got right" not in body + + def test_issue_close_comment_should_show_what_issue_got_right(self, triage_module): + # `has_expected_vs_actual` is present, the end-to-end bug evidence is + # not: the "what you got right" block must surface the former and omit + # the latter (no "✅ (nothing)"-style noise for absent items). + body = triage_module.format_issue_close_comment( + { + "verdict": "fail", + "kind": "bug", + "has_repro": False, + "has_expected_vs_actual": True, + "missing": ["end-to-end evidence of the bug"], + "explanation": "no repro shown", + } + ) + assert "What you got right" in body + assert "Expected vs. actual behavior" in body + assert "- ✅ End-to-end evidence of the bug" not in body + + def test_close_comments_should_use_softer_park_for_later_framing( + self, triage_module + ): + # User feedback: the messaging shouldn't feel like punishment. The + # comment must explicitly frame close as a "park this for later," not + # a rejection, and ground that in the queue-hygiene reason. + for body in ( + triage_module.format_pr_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ), + triage_module.format_issue_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ), + ): + assert "park this for later" in body + assert ( + "not a rejection" in body + or "isn't a rejection" in body + or ("isn't us saying" in body) + ) + + def test_only_close_comments_carry_the_agent_shin_close_marker(self, triage_module): + # The reconsider reopen guard keys off AGENT_SHIN_CLOSE_MARKER to tell + # an Agent Shin close from a same-identity close by another workflow. + # That only works if the marker is stamped on the close comments and + # NOT on the grace warnings (which don't close anything). + verdict = {"verdict": "fail", "missing": [], "explanation": ""} + marker = triage_module.AGENT_SHIN_CLOSE_MARKER + assert marker in triage_module.format_pr_close_comment(verdict) + assert marker in triage_module.format_issue_close_comment(verdict) + assert marker not in triage_module.format_grace_warning_pr_comment(verdict) + assert marker not in triage_module.format_grace_warning_issue_comment(verdict) + + +class TestWasClosedByAgentShin: + """Bot-closed guard: only Agent Shin's own closures are reopen candidates.""" + + @staticmethod + def _stub_close_event( + triage_module, + monkeypatch, + *, + actor: str | None, + closed_at: object = "now", + ): + """Stub the most recent `closed` event used by the guard. + + `actor` is the login that closed the item. `closed_at` defaults + to "now" so the marker comment (stubbed at 42s ago) reads as + recent enough relative to the close; tests can pass a concrete + ``datetime`` to simulate older closes (e.g. the stale-marker + regression scenario). + """ + import datetime as real_dt + + if closed_at == "now": + closed_at = real_dt.datetime.now(real_dt.timezone.utc) + monkeypatch.setattr( + triage_module, + "fetch_last_close_event", + lambda repo, n: (actor, closed_at), + ) + + @staticmethod + def _stub_close_marker_present( + triage_module, monkeypatch, *, present: bool, age_seconds: float = 42.0 + ): + """Stub the Agent Shin close-comment marker lookup. + + `was_closed_by_agent_shin` requires the closing actor AND a + recent Agent Shin close comment; these tests pin the latter so + they exercise the actor half in isolation. + """ + monkeypatch.setattr( + triage_module, + "seconds_since_last_agent_shin_close", + lambda *a, **kw: age_seconds if present else None, + ) + + def test_should_return_true_when_bot_closed_and_close_comment_present( + self, triage_module, monkeypatch + ): + self._stub_close_event(triage_module, monkeypatch, actor="github-actions[bot]") + self._stub_close_marker_present(triage_module, monkeypatch, present=True) + assert triage_module.was_closed_by_agent_shin("o/r", 1) is True + + def test_should_return_false_when_bot_closed_but_no_agent_shin_comment( + self, triage_module, monkeypatch + ): + # The `github-actions[bot]` identity is shared across workflows. A + # stale/duplicate sweep closing under that identity must NOT let + # @agent-shin reconsider reopen the item: without an Agent Shin close + # comment the guard fails closed. + self._stub_close_event(triage_module, monkeypatch, actor="github-actions[bot]") + self._stub_close_marker_present(triage_module, monkeypatch, present=False) + assert triage_module.was_closed_by_agent_shin("o/r", 1) is False + + def test_should_return_false_when_last_close_actor_is_maintainer( + self, triage_module, monkeypatch + ): + # A maintainer closed it (e.g. duplicate, security, design). The + # bot must refuse to reopen on @agent-shin reconsider even if an + # earlier Agent Shin close comment is still on the thread. + self._stub_close_event(triage_module, monkeypatch, actor="krrishdholakia") + self._stub_close_marker_present(triage_module, monkeypatch, present=True) + assert triage_module.was_closed_by_agent_shin("o/r", 1) is False + + def test_should_fail_closed_when_no_close_event(self, triage_module, monkeypatch): + # If the events API returns nothing (network blip, repo permission + # quirk), the guard must fail-closed: refuse to reopen rather than + # assume the bot did it. + self._stub_close_event(triage_module, monkeypatch, actor=None, closed_at=None) + self._stub_close_marker_present(triage_module, monkeypatch, present=True) + assert triage_module.was_closed_by_agent_shin("o/r", 1) is False + + def test_should_fail_closed_when_close_event_has_no_timestamp( + self, triage_module, monkeypatch + ): + # Without a usable close timestamp the guard cannot prove the + # marker comment belongs to the latest close; fail-closed. + self._stub_close_event( + triage_module, monkeypatch, actor="github-actions[bot]", closed_at=None + ) + self._stub_close_marker_present(triage_module, monkeypatch, present=True) + assert triage_module.was_closed_by_agent_shin("o/r", 1) is False + + def test_should_return_false_when_marker_predates_latest_close( + self, triage_module, monkeypatch + ): + # Regression for the stale-marker bug: Agent Shin closed once + # (marker stamped), reconsider reopened, and a different workflow + # later closed under the same bot identity without stamping the + # marker. The old marker is still on the thread but does NOT + # belong to the latest close, so reconsider must not reopen. + import datetime as real_dt + + now = real_dt.datetime.now(real_dt.timezone.utc) + # Latest close happened a minute ago. + self._stub_close_event( + triage_module, + monkeypatch, + actor="github-actions[bot]", + closed_at=now - real_dt.timedelta(seconds=60), + ) + # The most recent Agent Shin marker is from an hour ago (a prior + # closed/reopened cycle), which is well outside the skew window. + self._stub_close_marker_present( + triage_module, monkeypatch, present=True, age_seconds=3600.0 + ) + assert triage_module.was_closed_by_agent_shin("o/r", 1) is False + + def test_should_respect_bot_login_override_via_env( + self, triage_module, monkeypatch + ): + # Operators wiring Agent Shin to a PAT (instead of GITHUB_TOKEN) + # can override the expected bot login via env. The guard must + # respect the override so non-default deployments still work. + monkeypatch.setenv("AGENT_SHIN_BOT_LOGIN", "my-bot") + self._stub_close_marker_present(triage_module, monkeypatch, present=True) + self._stub_close_event(triage_module, monkeypatch, actor="my-bot") + assert triage_module.was_closed_by_agent_shin("o/r", 1) is True + # Default "github-actions[bot]" should NOT match when env is set. + self._stub_close_event(triage_module, monkeypatch, actor="github-actions[bot]") + assert triage_module.was_closed_by_agent_shin("o/r", 1) is False + + +class TestSecondsSinceLastAgentShinClose: + """Close-provenance lookup: detects the bot's own auto-close marker.""" + + def _make_comment(self, *, login: str, body: str) -> dict: + return { + "user": {"login": login}, + "body": body, + "created_at": "2026-05-18T05:00:00Z", + } + + def test_should_return_none_when_bot_never_closed(self, triage_module, monkeypatch): + # Comments exist, but none is an Agent Shin close — e.g. only a grace + # warning, or a close by another workflow with no Agent Shin comment. + comments = [ + self._make_comment(login="outside-dev", body="any update?"), + self._make_comment( + login="github-actions[bot]", + body=triage_module.format_grace_warning_pr_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ), + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_agent_shin_close("o/r", 1) is None + + def test_should_detect_bot_close_comment(self, triage_module, monkeypatch): + comments = [ + self._make_comment( + login="github-actions[bot]", + body=triage_module.format_pr_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ), + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_agent_shin_close("o/r", 1) is not None + + def test_should_ignore_non_bot_comment_quoting_marker( + self, triage_module, monkeypatch + ): + # A contributor quoting the hidden marker (GitHub "Quote reply" + # preserves HTML comments) must not be mistaken for a bot close. + comments = [ + self._make_comment( + login="curious-user", + body=f"what is this? {triage_module.AGENT_SHIN_CLOSE_MARKER}", + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_agent_shin_close("o/r", 1) is None + + +class TestSecondsSinceLastReconsiderVerdict: + """Rate-limit guard: detects the bot's own reconsider verdict marker.""" + + def _make_comment( + self, *, login: str, body: str, created_at: str | None = "2026-05-18T05:00:00Z" + ) -> dict: + comment: dict = {"user": {"login": login}, "body": body} + if created_at is not None: + comment["created_at"] = created_at + return comment + + def test_should_return_none_when_no_bot_reconsider_comments( + self, triage_module, monkeypatch + ): + # An issue with chatter from other users but no bot reconsider + # verdict must not be rate-limited. + comments = [ + self._make_comment(login="outside-dev", body="ping?"), + self._make_comment( + login="github-actions[bot]", body="some other bot message" + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_reconsider_verdict("o/r", 1) is None + + def test_should_pick_latest_bot_reconsider_marker(self, triage_module, monkeypatch): + # When multiple reconsider verdicts exist, return the AGE of the + # most recent one. Using a frozen reference helps pin the math. + comments = [ + self._make_comment( + login="github-actions[bot]", + body="old verdict " + triage_module.RECONSIDER_COMMENT_MARKER, + created_at="2026-05-18T04:00:00Z", + ), + self._make_comment( + login="github-actions[bot]", + body="newer verdict " + triage_module.RECONSIDER_COMMENT_MARKER, + created_at="2026-05-18T04:55:00Z", + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + + # Freeze "now" via a tiny shim on the module's `dt` import. + import datetime as real_dt + + class FrozenDateTime(real_dt.datetime): + @classmethod + def now(cls, tz=None): + return real_dt.datetime(2026, 5, 18, 5, 0, 0, tzinfo=tz) + + frozen_module = type(triage_module.dt)("datetime") + frozen_module.datetime = FrozenDateTime + frozen_module.timezone = real_dt.timezone + monkeypatch.setattr(triage_module, "dt", frozen_module) + + age = triage_module.seconds_since_last_reconsider_verdict("o/r", 1) + # newer verdict is 5 minutes (300 seconds) before "now" + assert age == 300.0 + + def test_should_ignore_non_bot_comments_with_marker( + self, triage_module, monkeypatch + ): + # A user comment that happens to quote the marker (e.g. in + # a "what does this hidden marker do?" question) must NOT count. + # The rate-limit guard only trusts comments authored by the bot. + comments = [ + self._make_comment( + login="curious-user", + body=f"Saw this marker: {triage_module.RECONSIDER_COMMENT_MARKER}", + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_reconsider_verdict("o/r", 1) is None + + def test_should_ignore_bot_comments_without_marker( + self, triage_module, monkeypatch + ): + # The bot posts other things too (Agent Shin close comments, + # CI status, etc.) — only the reconsider-verdict marker should + # arm the cooldown. + comments = [ + self._make_comment( + login="github-actions[bot]", + body="Agent Shin closed this PR (no marker)", + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_reconsider_verdict("o/r", 1) is None + + +class TestParseVerdict: + def test_should_parse_plain_json(self, triage_module): + raw = '{"verdict": "pass", "missing": []}' + assert triage_module.parse_verdict(raw)["verdict"] == "pass" + + def test_should_strip_markdown_fence(self, triage_module): + raw = '```json\n{"verdict": "fail", "missing": ["foo"]}\n```' + result = triage_module.parse_verdict(raw) + assert result["verdict"] == "fail" + assert result["missing"] == ["foo"] + + def test_should_extract_embedded_json_from_prose(self, triage_module): + raw = 'Here you go: {"verdict": "pass", "missing": []}\nThanks.' + assert triage_module.parse_verdict(raw)["verdict"] == "pass" + + def test_should_raise_for_unparseable_text(self, triage_module): + with pytest.raises(ValueError): + triage_module.parse_verdict("not even close to json") + + def test_should_raise_for_empty(self, triage_module): + with pytest.raises(ValueError): + triage_module.parse_verdict("") + + +class TestBuildPrompts: + def test_should_include_pr_title_and_body(self, triage_module): + prompt = triage_module.build_pr_prompt( + title="Add foo", body=" Real body" + ) + assert "Add foo" in prompt + assert "Real body" in prompt + assert "comment" not in prompt # HTML comments are stripped + + def test_should_show_empty_marker_for_empty_pr_body(self, triage_module): + prompt = triage_module.build_pr_prompt(title="t", body="") + assert "(empty)" in prompt + + def test_should_include_issue_title_and_body(self, triage_module): + prompt = triage_module.build_issue_prompt(title="Bug", body="repro here") + assert "Bug" in prompt + assert "repro here" in prompt + + def test_issue_bug_rubric_requires_end_to_end_evidence_and_drops_pass_bias( + self, triage_module + ): + # The bug bar was tightened: a report needs the "before" half shown + # end-to-end (video / screenshot / real command output), prose-only + # repro steps no longer pass, and the old "bias toward PASS" leniency + # is gone. If any of these regress, the judge silently goes soft on + # undemonstrated bug reports again. + prompt = triage_module.build_issue_prompt(title="t", body="x") + normalized = " ".join(prompt.split()) + assert "Bias toward PASS when the issue has structure" not in normalized + assert "END-TO-END EVIDENCE OF THE BUG" in normalized + assert "Do not bias toward PASS" in normalized + # The three accepted forms of the "before" demonstration must be named. + assert "screen recording / video" in normalized + assert "screenshot of the bug" in normalized + assert "mocked or stubbed" in normalized + # Prose-only steps are explicitly insufficient now. + assert "steps to reproduce" in normalized + + def test_should_not_crash_when_pr_body_contains_curly_braces(self, triage_module): + """User-supplied content with `{` / `}` must NOT be re-parsed by + `str.format()`. `format` only scans the template literal for + replacement fields; values being substituted in are inserted as + plain strings, so a body like `{"foo": "bar"}` or `{unmatched` + cannot blow up the script. Pinning this here so a future + "improvement" to the templating doesn't reintroduce a crash on + every PR that quotes JSON. + """ + for body in ( + 'Here is some JSON: {"foo": "bar", "n": 1}', + "Half a brace { left dangling, and a stray }", + "Format-spec-looking thing: {0}, {name:>10}, {!r}", + "Nested {a: {b: c}} braces", + ): + pr_prompt = triage_module.build_pr_prompt(title="t", body=body) + issue_prompt = triage_module.build_issue_prompt(title="t", body=body) + assert body in pr_prompt + assert body in issue_prompt + + def test_should_not_crash_when_pr_title_contains_curly_braces(self, triage_module): + title = "Fix bug in {0:>10} format-spec handling" + pr_prompt = triage_module.build_pr_prompt(title=title, body="x") + issue_prompt = triage_module.build_issue_prompt(title=title, body="x") + assert title in pr_prompt + assert title in issue_prompt + + def test_should_preserve_template_indentation_with_multiline_body( + self, triage_module + ): + """`textwrap.dedent` runs on the static template *before* user + content is interpolated, so a multi-line body (whose 2nd+ lines + start at column 0) cannot defeat the common-indent computation + and leave 8-space indentation on every template line. Pin the + dedented shape so the rendered prompt stays consistent for the + LLM judge. + """ + body = "first line\nsecond line at column 0\nthird line at column 0" + for builder in ( + triage_module.build_pr_prompt, + triage_module.build_issue_prompt, + ): + prompt = builder(title="t", body=body) + # Template lines should NOT carry the 8 leading spaces from + # the source-file indentation of the triple-quoted string. + assert " You are " not in prompt + assert 'You are "Agent Shin"' in prompt + assert body in prompt + + +class TestMainModelDefault: + """`--model` falls back to DEFAULT_MODEL even when TRIAGE_MODEL is empty.""" + + def _stub_triage(self, triage_module, monkeypatch): + captured: dict = {} + + def fake_triage(**kwargs): + captured.update(kwargs) + return { + "kind": kwargs["kind"], + "number": kwargs["number"], + "title": "", + "author": "x", + "author_association": "NONE", + "state": "open", + "action": "skip-no-llm-key", + } + + monkeypatch.setattr(triage_module, "triage", fake_triage) + return captured + + def test_should_fall_back_to_default_when_triage_model_env_empty( + self, triage_module, monkeypatch + ): + captured = self._stub_triage(triage_module, monkeypatch) + monkeypatch.setenv("TRIAGE_MODEL", "") + monkeypatch.setattr( + sys, + "argv", + ["triage_with_llm.py", "--repo", "o/r", "--pr", "1"], + ) + rc = triage_module.main() + assert rc == 0 + assert captured["model"] == triage_module.DEFAULT_MODEL + + def test_should_respect_explicit_triage_model_env(self, triage_module, monkeypatch): + captured = self._stub_triage(triage_module, monkeypatch) + monkeypatch.setenv("TRIAGE_MODEL", "gpt-4o-mini") + monkeypatch.setattr( + sys, + "argv", + ["triage_with_llm.py", "--repo", "o/r", "--pr", "1"], + ) + rc = triage_module.main() + assert rc == 0 + assert captured["model"] == "gpt-4o-mini" + + +class TestCallLlmJudge: + """call_llm_judge sets gpt-5 specific kwargs correctly.""" + + def _stub_openai(self, monkeypatch, captured: dict): + """Install a fake `openai.OpenAI` client into sys.modules. + + The fake client records the kwargs passed to chat.completions.create + and returns a minimal response object whose .choices[0].message.content + is "ok". + """ + import types + + class FakeMessage: + content = '{"verdict": "pass"}' + + class FakeChoice: + message = FakeMessage() + + class FakeResponse: + choices = [FakeChoice()] + + class FakeCompletions: + def create(self, **kwargs): + captured.update(kwargs) + return FakeResponse() + + class FakeChat: + completions = FakeCompletions() + + class FakeClient: + def __init__(self, api_key, base_url=None): + captured["__client_kwargs__"] = { + "api_key": api_key, + "base_url": base_url, + } + self.chat = FakeChat() + + fake_module = types.ModuleType("openai") + fake_module.OpenAI = FakeClient + monkeypatch.setitem(sys.modules, "openai", fake_module) + + def test_should_set_reasoning_effort_none_for_gpt5_family( + self, triage_module, monkeypatch + ): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "prompt", model="gpt-5.4-mini", api_key="sk-test", base_url=None + ) + assert captured["model"] == "gpt-5.4-mini" + assert captured["temperature"] == 0 + assert captured["extra_body"] == {"reasoning_effort": "none"} + + def test_should_set_reasoning_effort_for_capitalized_or_dated_gpt5( + self, triage_module, monkeypatch + ): + for model in ("GPT-5.4-mini", "gpt-5.4-mini-2026-03-17", "gpt-5"): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "prompt", model=model, api_key="sk-test", base_url=None + ) + assert captured["extra_body"] == {"reasoning_effort": "none"}, model + + def test_should_omit_reasoning_effort_for_non_gpt5( + self, triage_module, monkeypatch + ): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "prompt", model="gpt-4o-mini", api_key="sk-test", base_url=None + ) + assert "extra_body" not in captured + + def test_should_pass_base_url_when_provided(self, triage_module, monkeypatch): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "p", + model="gpt-5.4-mini", + api_key="sk-test", + base_url="https://proxy.example.com/v1", + ) + assert ( + captured["__client_kwargs__"]["base_url"] == "https://proxy.example.com/v1" + ) + + +class TestTriageOrchestration: + """End-to-end-ish tests that mock both gh fetchers and the LLM.""" + + def _make_pr(self, **overrides): + base = { + "number": 1, + "title": "PR title", + "body": "PR body", + "state": "open", + "author_association": "NONE", + "user": {"login": "mateo-berri"}, + } + base.update(overrides) + return base + + def test_should_skip_internal_author(self, triage_module, monkeypatch): + pr = self._make_pr( + author_association="MEMBER", user={"login": "krrishdholakia"} + ) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + + def boom(*a, **kw): + pytest.fail("LLM should not be called for internal authors") + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=boom, + allowlist=frozenset(), + ) + assert result["action"] == "skip-internal-author" + + def test_should_skip_closed_pr(self, triage_module, monkeypatch): + pr = self._make_pr(state="closed") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("should not run on closed PRs"), + ) + assert result["action"] == "skip-not-open" + + def test_should_short_circuit_on_linked_issue(self, triage_module, monkeypatch): + pr = self._make_pr(body="Fixes #1234\n\nFoo bar") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("LLM should not be called"), + ) + assert result["action"] == "pass-linked-issue" + assert result["verdict"]["verdict"] == "pass" + + def test_should_not_short_circuit_on_casual_mention( + self, triage_module, monkeypatch + ): + # "See #1234" is a passing mention, not a closing keyword. The LLM + # must get a chance to apply the stricter rubric. With no prior + # grace warning, the first failing verdict triggers the warning + # path (`would-warn-grace` in dry-run). + pr = self._make_pr(body="See #1234 for context. No QA proof here.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_grace_no_warning(triage_module, monkeypatch) + called = {"judge": False} + + def judge(prompt): + called["judge"] = True + return json.dumps( + {"verdict": "fail", "missing": ["QA proof"], "explanation": "thin."} + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=False, + model="m", + judge=judge, + ) + assert called["judge"] is True + assert result["action"] == "would-warn-grace" + + def test_should_return_pass_llm_when_judge_passes(self, triage_module, monkeypatch): + pr = self._make_pr(body="Long body, no linked issue.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + captured = {} + + def judge(prompt): + captured["prompt"] = prompt + return json.dumps({"verdict": "pass", "missing": [], "explanation": "ok"}) + + result = triage_module.triage( + repo="o/r", kind="pr", number=1, close=True, model="m", judge=judge + ) + assert result["action"] == "pass-llm" + assert "Long body" in captured["prompt"] + + def test_should_return_would_close_in_dry_run_after_grace_aged_out( + self, triage_module, monkeypatch + ): + # When the grace warning has already aged out (>= GRACE_PERIOD_SECONDS) + # AND the rubric still fails, the dry-run preview returns + # `would-close` so a step-summary writer can render the close + # comment without touching GitHub state. + pr = self._make_pr(body="just a sentence.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_grace_aged_out(triage_module, monkeypatch) + + def fake_post(*a, **kw): + pytest.fail("should not post comments in dry-run") + + def fake_close(*a, **kw): + pytest.fail("should not close in dry-run") + + monkeypatch.setattr(triage_module, "post_comment", fake_post) + monkeypatch.setattr(triage_module, "close_pr", fake_close) + + verdict = { + "verdict": "fail", + "missing": ["problem description", "QA proof"], + "explanation": "Body is one sentence.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=False, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "would-close" + assert result["verdict"]["missing"] == ["problem description", "QA proof"] + + def test_should_post_comment_and_close_after_grace_window( + self, triage_module, monkeypatch + ): + # The "real close" path: --close passed AND the grace warning has + # aged out AND the rubric still fails. The bot posts the close + # comment and closes the PR. + pr = self._make_pr(body="just a sentence.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_grace_aged_out(triage_module, monkeypatch) + posted = {} + closed = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"repo": repo, "n": n, "body": body}), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda repo, n: closed.update({"repo": repo, "n": n}), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "Body too thin.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "closed" + assert posted["n"] == 42 and closed["n"] == 42 + assert "Agent Shin" in posted["body"] + assert "QA proof" in posted["body"] + + def test_should_skip_on_llm_error_in_close_mode(self, triage_module, monkeypatch): + pr = self._make_pr(body="something.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not comment on LLM error"), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda *a, **kw: pytest.fail("must not close on LLM error"), + ) + + def broken_judge(prompt): + raise RuntimeError("upstream 500") + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=broken_judge, + ) + assert result["action"] == "skip-llm-error" + assert "upstream 500" in result["error"] + + def test_should_skip_open_pr_in_reconsider_mode(self, triage_module, monkeypatch): + # Reconsider only makes sense on a CLOSED PR — running it on an open + # one is a no-op (the regular triage flow already evaluated it). + pr = self._make_pr(state="open") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=False, + model="m", + judge=lambda p: pytest.fail("should not run on open PR in reconsider"), + reconsider=True, + ) + assert result["action"] == "skip-not-closed" + + @staticmethod + def _stub_reconsider_guards(triage_module, monkeypatch): + """Default reconsider-guard stubs: pretend bot closed + no cooldown. + + The new safety guards (`was_closed_by_agent_shin`, + `seconds_since_last_reconsider_verdict`) hit the GitHub API in + production. Tests that exercise the reconsider happy path stub + them to "yes the bot closed it, no recent reconsider comment" + so the test stays focused on its actual assertion. + """ + monkeypatch.setattr( + triage_module, "was_closed_by_agent_shin", lambda *a, **kw: True + ) + monkeypatch.setattr( + triage_module, + "seconds_since_last_reconsider_verdict", + lambda *a, **kw: None, + ) + + @staticmethod + def _stub_grace_aged_out(triage_module, monkeypatch): + """Pretend the grace warning has aged out. + + For tests that exercise the post-grace close path. Set the age + to twice the grace window so a future tweak to + `GRACE_PERIOD_SECONDS` doesn't accidentally make the stub fall + back inside the window. + """ + monkeypatch.setattr( + triage_module, + "seconds_since_last_grace_warning", + lambda *a, **kw: triage_module.GRACE_PERIOD_SECONDS * 2, + ) + + @staticmethod + def _stub_grace_no_warning(triage_module, monkeypatch): + """Pretend no grace warning has been posted yet (first detection).""" + monkeypatch.setattr( + triage_module, + "seconds_since_last_grace_warning", + lambda *a, **kw: None, + ) + + def test_should_reopen_on_reconsider_pass(self, triage_module, monkeypatch): + # Reconsider on a closed PR with a passing verdict -> reopen + post a + # friendly "re-evaluated" comment. close=True is the production path + # (the workflow only adds --close when AGENT_SHIN_ENABLED=true). + pr = self._make_pr( + state="closed", body="Updated body with QA proof + screenshots." + ) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_reconsider_guards(triage_module, monkeypatch) + posted = {} + reopened = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"n": n, "body": body}), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda repo, n: reopened.update({"n": n}), + ) + # close_pr / close_issue MUST NOT fire in reconsider mode. + monkeypatch.setattr( + triage_module, + "close_pr", + lambda *a, **kw: pytest.fail("must not close on reconsider pass"), + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=True, + model="m", + judge=lambda p: json.dumps( + {"verdict": "pass", "missing": [], "explanation": "ok now"} + ), + reconsider=True, + ) + assert result["action"] == "reopened" + assert reopened["n"] == 42 + assert posted["n"] == 42 + assert "reopened" in posted["body"].lower() + + def test_should_dry_run_reconsider_pass_when_close_false( + self, triage_module, monkeypatch + ): + # Reconsider must honor `close=False` (dry-run) just like the + # regular triage flow. A local invocation of + # `python triage_with_llm.py --reconsider --pr N` (no --close) + # must NOT post a comment or reopen the PR — it should return + # `would-reopen` so the operator can preview the outcome. + pr = self._make_pr( + state="closed", body="Updated body with QA proof + screenshots." + ) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_reconsider_guards(triage_module, monkeypatch) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not post comment in dry-run reconsider"), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda *a, **kw: pytest.fail("must not reopen PR in dry-run reconsider"), + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=False, + model="m", + judge=lambda p: json.dumps( + {"verdict": "pass", "missing": [], "explanation": "ok now"} + ), + reconsider=True, + ) + assert result["action"] == "would-reopen" + # The previewed comment body is still returned so a step-summary + # writer can render exactly what would have been posted. + assert "reopened" in result["comment"].lower() + + def test_should_post_still_failing_on_reconsider_fail( + self, triage_module, monkeypatch + ): + pr = self._make_pr(state="closed", body="still empty") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_reconsider_guards(triage_module, monkeypatch) + posted = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"n": n, "body": body}), + ) + # Neither reopen nor close should fire when reconsider verdict is fail. + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda *a, **kw: pytest.fail("must not reopen on fail"), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda *a, **kw: pytest.fail("must not close again on reconsider fail"), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "Still no QA proof.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + reconsider=True, + ) + assert result["action"] == "reconsider-still-failing" + assert posted["n"] == 42 + assert "QA proof" in posted["body"] + + def test_should_not_reopen_on_reconsider_with_ambiguous_verdict( + self, triage_module, monkeypatch + ): + # Regression: only an explicit `pass` verdict reopens. Missing, + # empty, or unexpected verdict strings ("failed", "", garbage) + # must fall through to the still-failing branch rather than + # reopen a PR the rubric did not actually clear. + pr = self._make_pr(state="closed", body="still empty") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_reconsider_guards(triage_module, monkeypatch) + posted = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"body": body}), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda *a, **kw: pytest.fail("must not reopen on ambiguous verdict"), + ) + + for ambiguous in ("", "failed", "needs-info", "unknown"): + posted.clear() + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=True, + model="m", + judge=lambda p, v=ambiguous: json.dumps( + {"verdict": v, "missing": [], "explanation": "weird"} + ), + reconsider=True, + ) + assert result["action"] == "reconsider-still-failing", ambiguous + assert "body" in posted, ambiguous + + def test_should_dry_run_reconsider_fail_when_close_false( + self, triage_module, monkeypatch + ): + # Mirror dry-run behavior for the FAIL branch — `close=False` + # must NOT post the "still failing" comment. + pr = self._make_pr(state="closed", body="still empty") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_reconsider_guards(triage_module, monkeypatch) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail( + "must not post still-failing comment in dry-run" + ), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "Still no QA proof.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=False, + model="m", + judge=lambda p: json.dumps(verdict), + reconsider=True, + ) + assert result["action"] == "would-reconsider-still-failing" + assert "QA proof" in result["comment"] + + def test_should_reopen_on_reconsider_with_linked_issue_short_circuit( + self, triage_module, monkeypatch + ): + # The linked-issue short-circuit also has to honor reconsider mode: + # if the contributor edited the body to add `Fixes #1234`, the regex + # path should reopen the PR without calling the LLM. + pr = self._make_pr(state="closed", body="Fixes #1234\n\nAddresses the bug.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_reconsider_guards(triage_module, monkeypatch) + posted = {} + reopened = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"body": body}), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda repo, n: reopened.update({"n": n}), + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=55, + close=True, + model="m", + judge=lambda p: pytest.fail("LLM must not run when linked-issue matches"), + reconsider=True, + ) + assert result["action"] == "reopened" + assert reopened["n"] == 55 + assert "reopened" in posted["body"].lower() + + def test_should_dry_run_reconsider_with_linked_issue_when_close_false( + self, triage_module, monkeypatch + ): + # Linked-issue short-circuit must ALSO honor dry-run. + pr = self._make_pr(state="closed", body="Fixes #1234\n\nAddresses the bug.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_reconsider_guards(triage_module, monkeypatch) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not post in dry-run"), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda *a, **kw: pytest.fail("must not reopen in dry-run"), + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=55, + close=False, + model="m", + judge=lambda p: pytest.fail("LLM must not run when linked-issue matches"), + reconsider=True, + ) + assert result["action"] == "would-reopen" + + def test_should_skip_internal_in_reconsider_mode(self, triage_module, monkeypatch): + # Internal authors are exempt from triage in both regular and + # reconsider mode — Agent Shin should never reopen one of their PRs + # automatically, in case a maintainer closed it intentionally. + pr = self._make_pr( + state="closed", + author_association="MEMBER", + user={"login": "krrishdholakia"}, + ) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda *a, **kw: pytest.fail("must not reopen for internal author"), + ) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=False, + model="m", + judge=lambda p: pytest.fail("LLM must not run for internal author"), + reconsider=True, + allowlist=frozenset(), + ) + assert result["action"] == "skip-internal-author" + + def test_should_skip_reconsider_when_not_bot_closed( + self, triage_module, monkeypatch + ): + # SECURITY: `@agent-shin reconsider` must NOT reopen a PR/issue + # that a MAINTAINER closed for non-rubric reasons (e.g. duplicate, + # design rejection, security report). Only PRs closed by the bot + # itself should ever be candidates for the reconsider reopen path. + pr = self._make_pr(state="closed", body="something.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + monkeypatch.setattr( + triage_module, "was_closed_by_agent_shin", lambda *a, **kw: False + ) + # Even though there's no rate-limit conflict, the bot-closed guard + # alone is sufficient to block. The LLM judge must never run on a + # maintainer-closed PR. + monkeypatch.setattr( + triage_module, + "seconds_since_last_reconsider_verdict", + lambda *a, **kw: None, + ) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not comment on maintainer-closed PR"), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda *a, **kw: pytest.fail("must not reopen maintainer-closed PR"), + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("LLM must not run before bot-closed guard"), + reconsider=True, + ) + assert result["action"] == "skip-not-bot-closed" + + def test_should_rate_limit_repeated_reconsider_triggers( + self, triage_module, monkeypatch + ): + # COST CONTROL: each `@agent-shin reconsider` event burns CI + # minutes + an OpenAI API call. If the bot already posted a + # reconsider verdict within the cooldown window + # (RECONSIDER_RATE_LIMIT_SECONDS), refuse to run again. This + # bounds the damage from a contributor spamming the trigger. + pr = self._make_pr(state="closed", body="something with new edits.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + monkeypatch.setattr( + triage_module, "was_closed_by_agent_shin", lambda *a, **kw: True + ) + # Pretend the bot posted a reconsider verdict 1 second ago. + monkeypatch.setattr( + triage_module, + "seconds_since_last_reconsider_verdict", + lambda *a, **kw: 1.0, + ) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not comment during cooldown"), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda *a, **kw: pytest.fail("must not reopen during cooldown"), + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("LLM must not run during cooldown"), + reconsider=True, + ) + assert result["action"] == "skip-rate-limited" + assert result["rate_limit_age_seconds"] == 1.0 + assert ( + result["rate_limit_window_seconds"] + == triage_module.RECONSIDER_RATE_LIMIT_SECONDS + ) + + def test_should_allow_reconsider_after_cooldown_window( + self, triage_module, monkeypatch + ): + # The cooldown is a window, not a one-shot lock — once + # RECONSIDER_RATE_LIMIT_SECONDS has elapsed since the last bot + # verdict, a fresh `@agent-shin reconsider` is allowed through. + pr = self._make_pr(state="closed", body="updated with screenshots now.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + monkeypatch.setattr( + triage_module, "was_closed_by_agent_shin", lambda *a, **kw: True + ) + # Last reconsider was 1 hour ago — well outside the 10-min window. + monkeypatch.setattr( + triage_module, + "seconds_since_last_reconsider_verdict", + lambda *a, **kw: 3600.0, + ) + posted = {} + reopened = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"n": n, "body": body}), + ) + monkeypatch.setattr( + triage_module, + "reopen_pr", + lambda repo, n: reopened.update({"n": n}), + ) + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: json.dumps( + {"verdict": "pass", "missing": [], "explanation": "ok"} + ), + reconsider=True, + ) + assert result["action"] == "reopened" + assert reopened["n"] == 1 + + def test_should_reopen_issue_on_reconsider_pass(self, triage_module, monkeypatch): + issue = { + "number": 7, + "title": "Bug: now with repro", + "body": "## Repro\n```bash\ncurl ...\n```\n\nExpected X, got Y.", + "state": "closed", + "author_association": "NONE", + "user": {"login": "mateo-berri"}, + } + monkeypatch.setattr(triage_module, "fetch_issue", lambda repo, n: issue) + self._stub_reconsider_guards(triage_module, monkeypatch) + posted = {} + reopened = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"body": body}), + ) + monkeypatch.setattr( + triage_module, + "reopen_issue", + lambda repo, n: reopened.update({"n": n}), + ) + + result = triage_module.triage( + repo="o/r", + kind="issue", + number=7, + close=True, + model="m", + judge=lambda p: json.dumps( + {"verdict": "pass", "missing": [], "explanation": "now reproducible"} + ), + reconsider=True, + ) + assert result["action"] == "reopened" + assert reopened["n"] == 7 + assert "reopened" in posted["body"].lower() + + def test_should_triage_issues_kind(self, triage_module, monkeypatch): + issue = { + "number": 7, + "title": "Bug: X is broken", + "body": "no detail", + "state": "open", + "author_association": "NONE", + "user": {"login": "mateo-berri"}, + } + monkeypatch.setattr(triage_module, "fetch_issue", lambda repo, n: issue) + # Grace already aged out -> close path. (Issues use the same + # GRACE_COMMENT_MARKER detection as PRs.) + self._stub_grace_aged_out(triage_module, monkeypatch) + closed = {} + posted = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update(body=body), + ) + monkeypatch.setattr( + triage_module, "close_issue", lambda repo, n: closed.update(n=n) + ) + + verdict = { + "verdict": "fail", + "kind": "bug", + "has_repro": False, + "missing": ["reproduction", "expected vs. actual"], + "explanation": "No repro provided.", + } + result = triage_module.triage( + repo="o/r", + kind="issue", + number=7, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "closed" + assert closed["n"] == 7 + assert "reproduction" in posted["body"] + + # ---- Grace-period flow ------------------------------------------------ + + def test_should_post_grace_warning_on_first_failing_run_in_close_mode( + self, triage_module, monkeypatch + ): + # First low-quality detection -> bot posts a warning comment with + # the GRACE_COMMENT_MARKER. The PR must NOT be closed yet. + pr = self._make_pr(body="just a sentence.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_grace_no_warning(triage_module, monkeypatch) + posted = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"n": n, "body": body}), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda *a, **kw: pytest.fail("must not close on first detection"), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "Body too thin.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "warned-grace" + assert posted["n"] == 42 + # Pin the user-facing language pieces the user explicitly asked for. + assert "2 hours" in posted["body"] + assert "@agent-shin reconsider" in posted["body"] + assert "@greptileai" in posted["body"] + assert "even after the PR is closed" in posted["body"] + assert triage_module.GRACE_COMMENT_MARKER in posted["body"] + + def test_should_skip_close_inside_grace_window(self, triage_module, monkeypatch): + # A warning was posted recently; do nothing on this run regardless + # of close=True. The next run after `GRACE_PERIOD_SECONDS` elapses + # is the one that flips to actual close. + pr = self._make_pr(body="just a sentence.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + monkeypatch.setattr( + triage_module, + "seconds_since_last_grace_warning", + lambda *a, **kw: 60.0, + ) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not comment during grace window"), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda *a, **kw: pytest.fail("must not close during grace window"), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "Body too thin.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "skip-in-grace-period" + assert result["grace_age_seconds"] == 60.0 + assert result["grace_period_seconds"] == triage_module.GRACE_PERIOD_SECONDS + + def test_should_dry_run_grace_warning_when_close_false( + self, triage_module, monkeypatch + ): + # In dry-run mode the FIRST failing detection returns + # `would-warn-grace` (with the previewed comment body) and never + # touches GitHub state. Lets a local operator preview the + # warning before flipping --close on. + pr = self._make_pr(body="thin") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_grace_no_warning(triage_module, monkeypatch) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not post in dry-run grace warn"), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "thin", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=False, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "would-warn-grace" + assert "2 hours" in result["comment"] + + def test_should_warn_grace_for_swiftwinds_not_close_instantly( + self, triage_module, monkeypatch + ): + # Regression: SwiftWinds (the dogfood account) used to be in a + # now-removed `IMMEDIATE_CLOSE_LOGINS` bypass that skipped the grace + # window and closed on first detection. It must follow the SAME + # grace path as every other author: warn first, close only after the + # window elapses. A re-added instant-close bypass would call + # close_pr here and fail the test. + pr = self._make_pr(body="just a sentence.", user={"login": "SwiftWinds"}) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + self._stub_grace_no_warning(triage_module, monkeypatch) + posted = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"n": n, "body": body}), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda *a, **kw: pytest.fail( + "SwiftWinds must not close on first detection; it gets the grace window" + ), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "Body too thin.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=99, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "warned-grace" + assert "2 hours" in posted["body"] + + +class TestGraceWarningCommentText: + """Pin the user-facing promises in the grace warning so a future + refactor can't silently drop them.""" + + def test_pr_grace_warning_should_state_grace_window(self, triage_module): + body = triage_module.format_grace_warning_pr_comment( + {"verdict": "fail", "missing": ["QA proof"], "explanation": "thin"} + ) + # The user explicitly asked: "specify in the comment" the grace window. + assert "2 hours" in body + + def test_pr_grace_warning_should_mention_reconsider_during_grace( + self, triage_module + ): + body = triage_module.format_grace_warning_pr_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert "@agent-shin reconsider" in body + + def test_pr_grace_warning_should_promise_greptileai_works_post_close( + self, triage_module + ): + body = triage_module.format_grace_warning_pr_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + # Per user: comment should state @greptileai works even after close. + assert "@greptileai" in body + assert "even after the PR is closed" in body + + def test_pr_grace_warning_should_carry_grace_marker(self, triage_module): + # The marker is what `seconds_since_last_grace_warning` greps for + # on subsequent runs to detect that a warning has been posted. + # Dropping it would silently break the close-after-grace path. + body = triage_module.format_grace_warning_pr_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert triage_module.GRACE_COMMENT_MARKER in body + + def test_issue_grace_warning_should_carry_grace_marker(self, triage_module): + body = triage_module.format_grace_warning_issue_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert triage_module.GRACE_COMMENT_MARKER in body + assert "2 hours" in body + # OSS authors can't reopen a bot-closed issue, so recovery is + # `@agent-shin reconsider` (the bot reopens), like the PR path. + assert "@agent-shin reconsider" in body + + def test_pr_close_comment_should_promise_greptileai_works_post_close( + self, triage_module + ): + # The standard close comment must ALSO point at @greptileai so + # contributors see the same options whether they read the warning + # or only catch the close comment. + body = triage_module.format_pr_close_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert "@greptileai" in body + assert "even after the PR is closed" in body + + def test_pr_grace_warning_should_not_prompt_reconsider_during_grace_window( + self, triage_module + ): + # Per user feedback: during the 24h grace window, the contributor + # should just update the PR description. Asking them to also comment + # "@agent-shin reconsider" right away adds a step they don't need — + # the bot re-checks automatically on the next sweep. The reconsider + # trigger is reserved for the post-close recovery path. + # + # We pin this by checking that the grace section explicitly tells + # the contributor they don't need to ping the bot during the grace + # window. The presence of "@agent-shin reconsider" elsewhere in the + # comment (as the post-close path) is fine and required by other + # tests. + body = triage_module.format_grace_warning_pr_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ) + assert "No need to ping" in body or "no need to ping" in body + + def test_grace_warnings_should_show_what_got_right(self, triage_module): + # The "What you got right" section must appear in the grace warning + # too, not only the close comment — the contributor sees the warning + # first and that's their best chance to know what to keep. + pr_body = triage_module.format_grace_warning_pr_comment( + { + "verdict": "fail", + "linked_issue": True, + "has_problem_description": True, + "has_expected_vs_actual": True, + "has_qa_proof": False, + "missing": ["QA proof"], + "explanation": "thin", + } + ) + assert "What you got right" in pr_body + assert "Linked a related GitHub issue" in pr_body + + issue_body = triage_module.format_grace_warning_issue_comment( + { + "verdict": "fail", + "kind": "feature", + "has_motivation_example": True, + "missing": ["concrete description"], + "explanation": "vague", + } + ) + assert "What you got right" in issue_body + assert "Motivation and concrete example" in issue_body + + def test_grace_warnings_should_use_softer_park_for_later_framing( + self, triage_module + ): + # Same softer-framing pin as the close comment, but for the warning + # — the contributor's first contact with the bot must not read as a + # hard deadline / ultimatum. + for body in ( + triage_module.format_grace_warning_pr_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ), + triage_module.format_grace_warning_issue_comment( + {"verdict": "fail", "missing": [], "explanation": ""} + ), + ): + assert "park this for later" in body + assert ( + "not a rejection" in body + or "isn't a rejection" in body + or ("isn't us saying" in body) + ) + + +class TestSecondsSinceLastGraceWarning: + """Mirror of TestSecondsSinceLastReconsiderVerdict for the new helper. + Both helpers share `_seconds_since_latest_marker_comment` underneath + so the parsing logic is exercised either way; these tests pin the + grace-marker-specific behavior.""" + + def _make_comment( + self, + *, + login: str, + body: str, + created_at: str | None = "2026-05-18T05:00:00Z", + ) -> dict: + comment: dict = {"user": {"login": login}, "body": body} + if created_at is not None: + comment["created_at"] = created_at + return comment + + def test_should_return_none_when_no_grace_marker(self, triage_module, monkeypatch): + comments = [ + self._make_comment( + login="github-actions[bot]", + body="Some other bot message", + ), + self._make_comment(login="random-user", body="ping?"), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_grace_warning("o/r", 1) is None + + def test_should_ignore_non_bot_comments_with_marker( + self, triage_module, monkeypatch + ): + # A user who quotes the marker in a question must NOT be treated + # as the bot warning; otherwise the close-after-grace path would + # never fire because the timer keeps resetting. + comments = [ + self._make_comment( + login="random-user", + body=f"What is {triage_module.GRACE_COMMENT_MARKER}?", + ) + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + assert triage_module.seconds_since_last_grace_warning("o/r", 1) is None + + def test_should_pick_latest_grace_marker(self, triage_module, monkeypatch): + comments = [ + self._make_comment( + login="github-actions[bot]", + body="old warning " + triage_module.GRACE_COMMENT_MARKER, + created_at="2026-05-18T03:00:00Z", + ), + self._make_comment( + login="github-actions[bot]", + body="newer warning " + triage_module.GRACE_COMMENT_MARKER, + created_at="2026-05-18T04:55:00Z", + ), + ] + monkeypatch.setattr( + triage_module, "_iter_paginated_json", lambda *a, **kw: iter(comments) + ) + + import datetime as real_dt + + class FrozenDateTime(real_dt.datetime): + @classmethod + def now(cls, tz=None): + return real_dt.datetime(2026, 5, 18, 5, 0, 0, tzinfo=tz) + + frozen_module = type(triage_module.dt)("datetime") + frozen_module.datetime = FrozenDateTime + frozen_module.timezone = real_dt.timezone + monkeypatch.setattr(triage_module, "dt", frozen_module) + + age = triage_module.seconds_since_last_grace_warning("o/r", 1) + # Newer warning is 5 minutes (300s) before "now". + assert age == 300.0 + + +class TestTriageAllowlist: + """The dogfood allowlist gates `triage`: while non-empty it is the sole + author filter (only the named accounts are acted on) and it bypasses the + internal-author exemption for them, so a maintainer can dogfood on their + own org account. Emptying it restores the internal-author skip.""" + + def _make_pr(self, **overrides): + base = { + "number": 1, + "title": "PR title", + "body": "Body with no linked issue and no QA proof.", + "state": "open", + "author_association": "NONE", + "user": {"login": "mateo-berri"}, + } + base.update(overrides) + return base + + def test_should_skip_author_not_on_allowlist(self, triage_module, monkeypatch): + pr = self._make_pr(user={"login": "random-oss-dev"}) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("LLM must not run for non-allowlisted author"), + ) + assert result["action"] == "skip-not-allowlisted" + + def test_should_act_on_allowlisted_internal_author( + self, triage_module, monkeypatch + ): + pr = self._make_pr(author_association="MEMBER", user={"login": "mateo-berri"}) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: json.dumps( + {"verdict": "pass", "missing": [], "explanation": "ok"} + ), + ) + assert result["action"] == "pass-llm" + + def test_empty_allowlist_restores_internal_skip(self, triage_module, monkeypatch): + pr = self._make_pr( + author_association="MEMBER", user={"login": "krrishdholakia"} + ) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("LLM must not run for internal author"), + allowlist=frozenset(), + ) + assert result["action"] == "skip-internal-author" + + def test_allowlist_constant_is_the_two_dogfood_accounts(self, triage_module): + assert triage_module.ALLOWLIST_LOGINS == frozenset( + {"mateo-berri", "swiftwinds"} + ) + for login in triage_module.ALLOWLIST_LOGINS: + assert login == login.lower(), login diff --git a/tests/test_litellm/test_github_triage_workflows.py b/tests/test_litellm/test_github_triage_workflows.py new file mode 100644 index 00000000000..7e718fd0ea8 --- /dev/null +++ b/tests/test_litellm/test_github_triage_workflows.py @@ -0,0 +1,319 @@ +"""Static guardrails for the Agent Shin + Greptile workflow YAML files. + +These workflows can post comments and close PRs/issues on +BerriAI/litellm, so the gating logic that decides "is this a real +close-on-fail run?" must fail-safe on any unexpected input. The risk +is mostly maintenance: someone edits the bash gate, drops a quote, +inverts a comparison, or uses `!= "false"` (which treats "True", +"yes", "1", and typos as enabling closure) and the regression isn't +caught until a real OSS contributor's PR gets auto-closed. + +The tests below pin a set of invariants. The first two apply to every +workflow that gates a destructive `--close`: + + 1. The gate uses the fail-safe `= "true"` comparison — not `!= "false"`, + not `!= ""`. Only the literal string "true" should ever enable + closure. + 2. The gate also requires `AGENT_SHIN_ENABLED = "true"` (or the + scheduled-job equivalent) — disabling the variable must always + force dry-run. + +A third invariant covers every workflow that installs the OpenAI client. +These run with a write-scoped `GITHUB_TOKEN`, so a compromised package +release would execute in that context; the install must therefore come +from the hash-pinned `.github/scripts/triage-requirements.txt` via +`pip --require-hashes`, never a floating `pip install openai>=...`. + +Static parsing of the YAML + bash text is the right level of test here: +the gating logic lives in a `run:` block, not in a Python module we can +import, and end-to-end testing a GitHub Actions workflow from CI is +infeasible. A YAML-level guardrail is exactly what would have caught +the original `!= "false"` regression at PR time. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" + +# Map of workflow file -> the env var name that drives the destructive +# gate inside that workflow's `run:` block. Keeping this table explicit +# (rather than scraping every workflow file) means a new workflow file +# that bypasses the dry-run gating doesn't silently slip past this test. +DESTRUCTIVE_GATE_ENV: dict[str, str] = { + "triage_pr_with_llm.yml": "DISPATCH_CLOSE", + "triage_issue_with_llm.yml": "DISPATCH_CLOSE", + "close_low_quality_prs.yml": "CLOSE_FLAG", + # The reconsider workflow has no per-run "really do it?" knob — its + # only kill switch is `AGENT_SHIN_ENABLED`, which already serves as + # both the destructive gate and the global enablement gate. + "triage_reconsider.yml": "AGENT_SHIN_ENABLED", + # The review gate can add/remove labels, post comments, and close PRs. + # Its per-run knob is `CLOSE_FLAG` (from the workflow_dispatch input), + # gated by an outer `AGENT_SHIN_ENABLED = "true"` check. Listing it + # here ensures the same fail-safe `= "true"` and kill-switch invariants + # we enforce on every other destructive workflow are enforced here too. + "review_gate.yml": "CLOSE_FLAG", +} + + +# Privileged workflows that install the OpenAI client. They run with a +# write-scoped GITHUB_TOKEN, so the install must be hash-pinned: a poisoned +# release would otherwise execute in that context. A new workflow that +# installs the client must be added here and use the same pinned file. +LLM_CLIENT_INSTALLER_WORKFLOWS = ( + "triage_pr_with_llm.yml", + "triage_issue_with_llm.yml", + "review_gate.yml", + "triage_reconsider.yml", + "triage_rollout_heads_up.yml", +) + +PINNED_INSTALL = "--require-hashes -r .github/scripts/triage-requirements.txt" +REQUIREMENTS_FILE = REPO_ROOT / ".github" / "scripts" / "triage-requirements.txt" + + +def _load_workflow(name: str) -> dict: + return yaml.safe_load((WORKFLOWS_DIR / name).read_text()) + + +def _all_run_blocks(workflow: dict) -> list[str]: + """Return every `run:` step's command text, joined.""" + commands: list[str] = [] + jobs = workflow.get("jobs") or {} + for job in jobs.values(): + for step in job.get("steps", []) or []: + if not isinstance(step, dict): + continue + run = step.get("run") + if isinstance(run, str): + commands.append(run) + return commands + + +@pytest.mark.parametrize("workflow_file,env_var", sorted(DESTRUCTIVE_GATE_ENV.items())) +def test_should_use_failsafe_equals_true_comparison(workflow_file: str, env_var: str) -> None: + """The destructive `--close` gate must use `= "true"` (fail-safe), not + `!= "false"` (which would treat "True", "yes", "1", or any typo as + enabling closure). + + Both bare `${ENV_VAR}` and `${ENV_VAR:-false}` (with a default) are + accepted forms — what matters is the comparison operator. The + Greptile closer relies on an outer `AGENT_SHIN_ENABLED` gate so it + can use the bare form; the Agent Shin workflows include `:-false` + for defense in depth. Either is fine. + """ + workflow = _load_workflow(workflow_file) + text = "\n".join(_all_run_blocks(workflow)) + assert env_var in text, ( + f"{workflow_file} no longer references {env_var}; was the gating env var renamed without updating this test?" + ) + accepted_patterns = ( + f'"${{{env_var}}}" = "true"', + f'"${{{env_var}:-false}}" = "true"', + ) + assert any(p in text for p in accepted_patterns), ( + f"{workflow_file} must gate the destructive --close flag on the " + f'EXACT string "true" (one of: {accepted_patterns!r}). Mirror ' + 'the Greptile closer pattern; do NOT use `!= "false"` which ' + 'fail-opens on unknown values like "True", "yes", "1", or typos.' + ) + forbidden_patterns = ( + f'"${{{env_var}}}" != "false"', + f'"${{{env_var}:-false}}" != "false"', + f'"${{{env_var}:-true}}" != "false"', + ) + for forbidden in forbidden_patterns: + assert forbidden not in text, ( + f"{workflow_file} uses the fail-open pattern {forbidden!r}. " + 'Switch to `= "true"` so unknown values stay dry-run.' + ) + + +@pytest.mark.parametrize("workflow_file", sorted(DESTRUCTIVE_GATE_ENV)) +def test_should_require_agent_shin_enabled_for_close(workflow_file: str) -> None: + """Every destructive gate must also gate on the global enablement + variable, so flipping `AGENT_SHIN_ENABLED` off is a kill switch + regardless of any per-run input. + + Two patterns are equally fine: + - Positive: `[ "${AGENT_SHIN_ENABLED:-false}" = "true" ]` to enter + the close branch (Agent Shin workflows). + - Negative: `[ "${AGENT_SHIN_ENABLED:-false}" != "true" ]` then + bail out / force dry-run (Greptile closer). + + What matters is that the comparison value is the literal "true"; + `!= "false"` or `= "1"` etc. would not be a true kill switch. + """ + workflow = _load_workflow(workflow_file) + text = "\n".join(_all_run_blocks(workflow)) + accepted_patterns = ( + '"${AGENT_SHIN_ENABLED:-false}" = "true"', + '"${AGENT_SHIN_ENABLED:-false}" != "true"', + ) + assert any(p in text for p in accepted_patterns), ( + f"{workflow_file} must gate destructive actions on " + '`AGENT_SHIN_ENABLED = "true"` (or the inverted `!= "true"` ' + "guard that forces dry-run). Without this, an unset repo " + "variable would not be treated as a kill switch." + ) + + +@pytest.mark.parametrize("workflow_file", LLM_CLIENT_INSTALLER_WORKFLOWS) +def test_llm_client_install_is_hash_pinned(workflow_file: str) -> None: + """Every privileged workflow installs the OpenAI client from the + hash-pinned requirements file, never by floating version. + + A bare `pip install "openai>=1.40.0"` resolves to whatever PyPI serves + at run time and executes during install/import while a write-scoped + `GITHUB_TOKEN` is in scope, so a compromised release runs in a + privileged context. This test fails if that floating form comes back or + if the `--require-hashes` install is loosened. + """ + blocks = _all_run_blocks(_load_workflow(workflow_file)) + assert PINNED_INSTALL in "\n".join(blocks), ( + f"{workflow_file} must install the client via `pip install " + f"{PINNED_INSTALL}`; a floating install runs unverified code with a " + "write-scoped token." + ) + offenders = [b for b in blocks if "pip install" in b and "openai" in b] + assert not offenders, ( + f"{workflow_file} installs openai by name ({offenders!r}); pin it " + "through the hash-locked requirements file so the version and " + "checksum are fixed." + ) + + +def test_triage_requirements_are_fully_hash_pinned() -> None: + """The shared requirements file pins every package to an exact version + with a sha256 hash, which is what `pip --require-hashes` enforces at + install time. A loosened pin or a missing hash here would silently widen + the supply-chain surface for all the installer workflows. + """ + assert REQUIREMENTS_FILE.exists(), ( + f"the hash-pinned requirements file the triage workflows install from is missing at {REQUIREMENTS_FILE}" + ) + joined = REQUIREMENTS_FILE.read_text().replace("\\\n", " ") + entries = [line.strip() for line in joined.splitlines() if line.strip() and not line.strip().startswith("#")] + assert any(e.split()[0].startswith("openai==") for e in entries), ( + "openai must be pinned to an exact version in the triage requirements" + ) + for entry in entries: + spec = entry.split()[0] + assert "==" in spec, ( + f"requirement {spec!r} is not pinned to an exact version; " + "--require-hashes needs every package pinned with ==" + ) + assert "--hash=sha256:" in entry, ( + f"requirement {spec!r} has no sha256 hash; every pin must carry " + "checksums so --require-hashes can verify the download" + ) + + +def _heads_up_run_step() -> dict: + workflow = _load_workflow("triage_rollout_heads_up.yml") + for step in workflow["jobs"]["heads-up"]["steps"]: + if isinstance(step.get("run"), str) and "triage_rollout_heads_up.py" in step["run"]: + return step + raise AssertionError("no run step invokes triage_rollout_heads_up.py") + + +def test_rollout_heads_up_push_trigger_never_posts() -> None: + """Merging the heads-up script to staging must stay inert: the automatic + push trigger only ever runs dry-run. The real one-shot sweep is a + deliberate manual `workflow_dispatch` with `dry_run=false`, the sole path + that adds `--close`. + + This guards the "inert by default" invariant for the one workflow that is + intentionally not gated on AGENT_SHIN_ENABLED (it has to warn contributors + before that flag flips on). A regression to auto-`--close`-on-push would + post real comments on every push that touches the script. + """ + run = _heads_up_run_step()["run"] + assert '"${GITHUB_EVENT_NAME:-}" = "workflow_dispatch"' in run, ( + "the real (--close) run must be a manual workflow_dispatch, not the automatic push trigger" + ) + assert '"${DRY_RUN_INPUT:-true}" = "false"' in run, ( + "the real run must require the dry_run input to be the exact string 'false' (fail-safe); any other value stays dry-run" + ) + assert run.count("ARGS+=(--close)") == 1, ( + "--close must appear once, inside the manual real-run branch; a second occurrence means the push path posts real comments on merge" + ) + + +def test_rollout_heads_up_key_is_dispatch_gated() -> None: + """OPENAI_API_KEY is exposed only on the manual dispatch (the real-run + trigger), never unconditionally. The sibling triage workflows gate the key + the same way; an unconditional `secrets.OPENAI_API_KEY` here would hand the + key to the automatic push run, which must stay a no-op dry-run preview. + """ + key_expr = (_heads_up_run_step().get("env") or {}).get("OPENAI_API_KEY", "") + assert "github.event_name == 'workflow_dispatch'" in key_expr, ( + f"OPENAI_API_KEY must be gated on workflow_dispatch so the automatic push trigger gets no key; found: {key_expr!r}" + ) + + +def _reconsider_steps() -> list[dict]: + workflow = _load_workflow("triage_reconsider.yml") + return workflow["jobs"]["reconsider"]["steps"] + + +def _index_of_run_step(steps: list[dict], needle: str) -> int: + for i, step in enumerate(steps): + run = step.get("run") + if isinstance(run, str) and needle in run: + return i + raise AssertionError(f"no run step contains {needle!r}") + + +def _reaction_steps(steps: list[dict], content: str) -> list[tuple[int, dict]]: + return [ + (i, s) + for i, s in enumerate(steps) + if isinstance(s.get("run"), str) and f"content={content}" in s["run"] and "/reactions" in s["run"] + ] + + +class TestReconsiderReactions: + """The reconsider workflow acknowledges the triggering comment with a 👀 + reaction the moment it accepts the trigger, and a 👍 once the run finishes, + so the contributor gets feedback immediately instead of waiting on a cron. + + Both reactions are gated on `AGENT_SHIN_ENABLED == 'true'` so a dry-run + leaves no visible trace, and both target the comment that fired the event + (`github.event.comment.id`). The ordering (👀 before the triage run, 👍 + after) is the whole point — these tests fail if a refactor reorders the + steps, drops a reaction, or stops gating them. + """ + + def test_eyes_reaction_is_posted_before_the_triage_run(self) -> None: + steps = _reconsider_steps() + run_idx = _index_of_run_step(steps, "triage_with_llm.py") + eyes = _reaction_steps(steps, "eyes") + assert len(eyes) == 1, "expected exactly one 👀 (eyes) reaction step" + idx, step = eyes[0] + assert idx < run_idx, "👀 must be posted BEFORE the slow triage run, not after" + assert "github.event.comment.id" in (step.get("env") or {}).get("COMMENT_ID", ""), ( + "👀 must react to the comment that triggered the workflow" + ) + assert "${COMMENT_ID}" in step["run"], "👀 must react to the triggering comment, not a hardcoded id" + assert "vars.AGENT_SHIN_ENABLED == 'true'" in step["if"], ( + "👀 must be gated on AGENT_SHIN_ENABLED so dry-run stays inert" + ) + + def test_thumbs_up_reaction_is_posted_after_a_successful_run(self) -> None: + steps = _reconsider_steps() + run_idx = _index_of_run_step(steps, "triage_with_llm.py") + thumbs = _reaction_steps(steps, "+1") + assert len(thumbs) == 1, "expected exactly one 👍 (+1) reaction step" + idx, step = thumbs[0] + assert idx > run_idx, "👍 must come AFTER the triage run" + assert "success()" in step["if"], "👍 must only fire when the reconsider run succeeded" + assert "vars.AGENT_SHIN_ENABLED == 'true'" in step["if"], ( + "👍 must be gated on AGENT_SHIN_ENABLED so dry-run stays inert" + ) diff --git a/tests/test_litellm/test_triage_rollout_heads_up.py b/tests/test_litellm/test_triage_rollout_heads_up.py new file mode 100644 index 00000000000..535590fd2d6 --- /dev/null +++ b/tests/test_litellm/test_triage_rollout_heads_up.py @@ -0,0 +1,612 @@ +"""Unit tests for the one-shot 7-day heads-up sweep. + +Exercises: + + * The ``_agent_shin_actions`` dry-run wrappers — each ``maybe_*`` helper + must call the real underlying mutation iff ``dry_run=False``, and log to + stdout otherwise. + * ``triage_rollout_heads_up._would_be_closed`` — the predicate that + decides "would the future bot close this?" for both PRs and issues. + * ``triage_rollout_heads_up._process_one`` — the per-item processor: + skip when state != open, skip internal authors, skip already-notified + items, post heads-up on failing items, leave passing items alone. + * ``triage_rollout_heads_up.run`` — the sweep loop end-to-end, in both + dry-run and real modes, with the comment-posting injected so we never + talk to GitHub. + +Every test stubs out ``gh()`` and the GitHub mutations; nothing in this file +ever shells out. +""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +import sys +from pathlib import Path + +import pytest + +_SCRIPTS_DIR = Path(__file__).resolve().parents[2] / ".github" / "scripts" + + +@pytest.fixture(scope="module") +def triage_module(): + """Load triage_with_llm under its canonical name so the sibling modules + can `from triage_with_llm import ...`.""" + spec = importlib.util.spec_from_file_location( + "triage_with_llm", _SCRIPTS_DIR / "triage_with_llm.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules["triage_with_llm"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def actions_module(triage_module): + spec = importlib.util.spec_from_file_location( + "_agent_shin_actions", _SCRIPTS_DIR / "_agent_shin_actions.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules["_agent_shin_actions"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def heads_up_module(triage_module, actions_module): + spec = importlib.util.spec_from_file_location( + "triage_rollout_heads_up", _SCRIPTS_DIR / "triage_rollout_heads_up.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules["triage_rollout_heads_up"] = module + spec.loader.exec_module(module) + return module + + +# --------------------------------------------------------------------------- +# _agent_shin_actions: the dry-run wrappers + + +class TestActionsDryRun: + """Each maybe_* helper must NOT hit GitHub in dry-run, and MUST hit it + in real mode. The whole rollout's safety story rests on this.""" + + def test_maybe_post_comment_dry_run_logs_only( + self, actions_module, triage_module, monkeypatch, capsys + ): + called = [] + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **k: called.append((a, k)), + ) + actions_module.maybe_post_comment("o/r", 7, "hello", dry_run=True) + assert called == [] + assert "[DRY RUN] comment o/r#7" in capsys.readouterr().out + + def test_maybe_post_comment_real_run_calls_through( + self, actions_module, triage_module, monkeypatch + ): + called = [] + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: called.append((repo, n, body)), + ) + actions_module.maybe_post_comment("o/r", 7, "hello", dry_run=False) + assert called == [("o/r", 7, "hello")] + + +# --------------------------------------------------------------------------- +# _would_be_closed predicate + + +class TestWouldBeClosed: + def test_pr_passing_returns_false(self, heads_up_module): + assert ( + heads_up_module._would_be_closed( + "pr", {"passing": True, "action": "noop-passing"} + ) + is False + ) + + def test_pr_failing_returns_true(self, heads_up_module): + assert ( + heads_up_module._would_be_closed( + "pr", + { + "passing": False, + "action": "would-close", + "verdict": {"verdict": "fail"}, + }, + ) + is True + ) + + def test_pr_skipped_returns_false(self, heads_up_module): + # passing is None for skip paths (internal-author, llm-error, etc.) + assert ( + heads_up_module._would_be_closed("pr", {"action": "skip-internal-author"}) + is False + ) + + def test_issue_pass_returns_false(self, heads_up_module): + assert ( + heads_up_module._would_be_closed( + "issue", {"action": "pass-llm", "verdict": {"verdict": "pass"}} + ) + is False + ) + + def test_issue_fail_returns_true(self, heads_up_module): + assert ( + heads_up_module._would_be_closed( + "issue", {"action": "would-close", "verdict": {"verdict": "fail"}} + ) + is True + ) + + def test_issue_missing_verdict_returns_false(self, heads_up_module): + # Skip paths don't surface a verdict; treat as "won't close". + assert ( + heads_up_module._would_be_closed("issue", {"action": "skip-not-open"}) + is False + ) + + +# --------------------------------------------------------------------------- +# Comment formatter — wording sanity checks + + +class TestHeadsUpCommentBody: + def test_pr_comment_contains_cutoff_rubric_marker(self, heads_up_module): + body = heads_up_module.format_heads_up_comment( + kind="pr", + verdict={"verdict": "fail", "missing": ["QA proof"], "explanation": "thin"}, + greptile_score=3, + cutoff=dt.date(2026, 6, 1), + ) + assert "Monday, June 1, 2026" in body # cutoff readable + assert "09:00 UTC" in body # deadline is timezone-explicit + assert "we'll close it" in body # hard deadline, not a passive notice + assert "2-hour lifetime" in body # post-rollout steady state + assert "Greptile" in body and "3/5" in body # specific shortfall + assert "QA proof" in body # missing piece surfaced + assert "PR *description*" in body # description-only note + assert heads_up_module.HEADS_UP_MARKER in body # idempotency marker + + def test_issue_comment_uses_reconsider_recovery_path(self, heads_up_module): + # OSS authors can't reopen an issue the bot closed (read access only + # lets them reopen issues they closed themselves), so the heads-up + # recovery path is `@agent-shin reconsider`, not self-reopen. + body = heads_up_module.format_heads_up_comment( + kind="issue", + verdict={"verdict": "fail", "missing": ["repro"], "explanation": ""}, + greptile_score=None, + cutoff=dt.date(2026, 6, 1), + ) + assert "@agent-shin reconsider" in body + assert heads_up_module.HEADS_UP_MARKER in body + + def test_empty_missing_uses_fallback_copy(self, heads_up_module): + body = heads_up_module.format_heads_up_comment( + kind="pr", + verdict={"verdict": "fail", "missing": [], "explanation": ""}, + greptile_score=None, + cutoff=dt.date(2026, 6, 1), + ) + assert "couldn't articulate" in body + # Make sure the fallback didn't leave us with a broken sentence. + assert "specific missing piece" in body + + +# --------------------------------------------------------------------------- +# _process_one — per-item dispatch + + +def _stub_fetchers(heads_up_module, triage_module, *, item): + """Monkeypatch fetch_pr and fetch_issue (both in triage_with_llm and the + re-imported names in heads_up_module) to return `item`.""" + return [ + (triage_module, "fetch_pr", lambda repo, n: item), + (triage_module, "fetch_issue", lambda repo, n: item), + (heads_up_module, "fetch_pr", lambda repo, n: item), + (heads_up_module, "fetch_issue", lambda repo, n: item), + ] + + +class TestProcessOne: + """Per-item processing: the right skip reason fires for each scenario, + and the heads-up only goes out when the rubric is genuinely failing.""" + + @pytest.fixture + def patch_env(self, heads_up_module, triage_module, monkeypatch): + """Helper that returns a callable to install a PR/issue body, suppress + marker checks, and stub the comment poster.""" + posts = [] + monkeypatch.setattr( + heads_up_module, + "maybe_post_comment", + lambda repo, n, body, *, dry_run: posts.append((repo, n, body, dry_run)), + ) + monkeypatch.setattr(heads_up_module, "_has_heads_up_marker", lambda item: False) + monkeypatch.setattr( + heads_up_module, "_comments_have_marker", lambda repo, n: False + ) + + def _install(item): + for mod, name, fn in _stub_fetchers( + heads_up_module, triage_module, item=item + ): + monkeypatch.setattr(mod, name, fn) + + return _install, posts + + def test_skip_closed_pr(self, heads_up_module, patch_env): + install, posts = patch_env + install( + {"state": "closed", "user": {"login": "ext"}, "author_association": "NONE"} + ) + r = heads_up_module._process_one( + repo="o/r", + kind="pr", + number=7, + model="m", + cutoff=dt.date(2026, 6, 1), + dry_run=True, + ) + assert r["action"] == "skip-not-open" + assert posts == [] + + def test_skip_internal_pr(self, heads_up_module, patch_env): + install, posts = patch_env + install( + { + "state": "open", + "user": {"login": "krrishdholakia"}, + "author_association": "MEMBER", + "body": "", + "labels": [], + "created_at": "2026-05-25T00:00:00Z", + } + ) + r = heads_up_module._process_one( + repo="o/r", + kind="pr", + number=7, + model="m", + cutoff=dt.date(2026, 6, 1), + dry_run=True, + allowlist=frozenset(), + ) + assert r["action"] == "skip-internal-author" + assert posts == [] + + def test_skip_passing_pr(self, heads_up_module, patch_env, monkeypatch): + install, posts = patch_env + install( + { + "state": "open", + "user": {"login": "mateo-berri"}, + "author_association": "NONE", + "body": "Fixes #123 — clean fix with a passing rubric.", + "labels": [], + "created_at": "2026-05-25T00:00:00Z", + } + ) + monkeypatch.setattr( + heads_up_module, + "_evaluate_pr", + lambda **kwargs: { + "action": "noop-passing", + "passing": True, + "verdict": {"verdict": "pass"}, + "greptile_score": 5, + }, + ) + r = heads_up_module._process_one( + repo="o/r", + kind="pr", + number=7, + model="m", + cutoff=dt.date(2026, 6, 1), + dry_run=True, + ) + assert r["action"] == "skip-passing" + assert posts == [] + + def test_failing_pr_posts_heads_up_dry_run( + self, heads_up_module, patch_env, monkeypatch, capsys + ): + install, posts = patch_env + install( + { + "state": "open", + "user": {"login": "mateo-berri"}, + "author_association": "NONE", + "body": "thin", + "labels": [], + "created_at": "2026-05-25T00:00:00Z", + } + ) + monkeypatch.setattr( + heads_up_module, + "_evaluate_pr", + lambda **kwargs: { + "action": "would-close", + "passing": False, + "verdict": { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "PR body is one line.", + }, + "greptile_score": 3, + }, + ) + r = heads_up_module._process_one( + repo="o/r", + kind="pr", + number=7, + model="m", + cutoff=dt.date(2026, 6, 1), + dry_run=True, + ) + assert r["action"] == "would-post-heads-up" + assert posts == [("o/r", 7, posts[0][2], True)] # tuple shape preserved + assert "QA proof" in posts[0][2] + assert heads_up_module.HEADS_UP_MARKER in posts[0][2] + + def test_failing_issue_posts_heads_up_real_run( + self, heads_up_module, patch_env, monkeypatch + ): + install, posts = patch_env + install( + { + "state": "open", + "user": {"login": "mateo-berri"}, + "author_association": "NONE", + "body": "X is broken", + "labels": [], + "created_at": "2026-05-25T00:00:00Z", + } + ) + monkeypatch.setattr( + heads_up_module, + "_evaluate_issue", + lambda **kwargs: { + "action": "would-close", + "verdict": { + "verdict": "fail", + "missing": ["reproduction"], + "explanation": "too thin", + }, + }, + ) + r = heads_up_module._process_one( + repo="o/r", + kind="issue", + number=42, + model="m", + cutoff=dt.date(2026, 6, 1), + dry_run=False, + ) + assert r["action"] == "heads-up-posted" + assert len(posts) == 1 + _, n, _, dry = posts[0] + assert n == 42 and dry is False + + def test_already_notified_is_skipped(self, heads_up_module, patch_env, monkeypatch): + install, posts = patch_env + install( + { + "state": "open", + "user": {"login": "mateo-berri"}, + "author_association": "NONE", + "body": "thin", + "labels": [], + "created_at": "2026-05-25T00:00:00Z", + } + ) + # Override the marker check for this scenario only. + monkeypatch.setattr( + heads_up_module, "_comments_have_marker", lambda repo, n: True + ) + r = heads_up_module._process_one( + repo="o/r", + kind="pr", + number=7, + model="m", + cutoff=dt.date(2026, 6, 1), + dry_run=True, + ) + assert r["action"] == "skip-already-notified" + assert posts == [] + + def test_ignore_existing_marker_forces_post( + self, heads_up_module, patch_env, monkeypatch + ): + install, posts = patch_env + install( + { + "state": "open", + "user": {"login": "mateo-berri"}, + "author_association": "NONE", + "body": "thin", + "labels": [], + "created_at": "2026-05-25T00:00:00Z", + } + ) + monkeypatch.setattr( + heads_up_module, "_comments_have_marker", lambda repo, n: True + ) + monkeypatch.setattr( + heads_up_module, + "_evaluate_pr", + lambda **kwargs: { + "action": "would-close", + "passing": False, + "verdict": {"verdict": "fail", "missing": ["X"], "explanation": ""}, + "greptile_score": None, + }, + ) + r = heads_up_module._process_one( + repo="o/r", + kind="pr", + number=7, + model="m", + cutoff=dt.date(2026, 6, 1), + dry_run=True, + skip_marker_check=True, + ) + assert r["action"] == "would-post-heads-up" + + +# --------------------------------------------------------------------------- +# run() — sweep loop + + +class TestRun: + """End-to-end the sweep loop with a tiny fake repo: 1 passing PR, 1 + failing PR, 1 passing issue, 1 failing issue.""" + + @pytest.fixture + def configured(self, heads_up_module, triage_module, monkeypatch): + posts = [] + monkeypatch.setattr( + heads_up_module, + "maybe_post_comment", + lambda repo, n, body, *, dry_run: posts.append((n, dry_run, body)), + ) + monkeypatch.setattr(heads_up_module, "_has_heads_up_marker", lambda item: False) + monkeypatch.setattr( + heads_up_module, "_comments_have_marker", lambda repo, n: False + ) + + def fake_list(repo, kind): + return [1, 2] if kind == "pr" else [101, 102] + + monkeypatch.setattr(heads_up_module, "_list_open_numbers", fake_list) + + def make_item(login="mateo-berri"): + return { + "state": "open", + "user": {"login": login}, + "author_association": "NONE", + "body": "thin", + "labels": [], + "created_at": "2026-05-25T00:00:00Z", + } + + monkeypatch.setattr(heads_up_module, "fetch_pr", lambda repo, n: make_item()) + monkeypatch.setattr(heads_up_module, "fetch_issue", lambda repo, n: make_item()) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: make_item()) + monkeypatch.setattr(triage_module, "fetch_issue", lambda repo, n: make_item()) + + def pr_eval(*, number, **kwargs): + if number == 1: + return { + "action": "noop-passing", + "passing": True, + "verdict": {"verdict": "pass"}, + } + return { + "action": "would-close", + "passing": False, + "verdict": {"verdict": "fail", "missing": ["m"], "explanation": ""}, + "greptile_score": 2, + } + + def issue_eval(*, number, **kwargs): + if number == 101: + return {"action": "pass-llm", "verdict": {"verdict": "pass"}} + return { + "action": "would-close", + "verdict": {"verdict": "fail", "missing": ["repro"], "explanation": ""}, + } + + monkeypatch.setattr(heads_up_module, "_evaluate_pr", pr_eval) + monkeypatch.setattr(heads_up_module, "_evaluate_issue", issue_eval) + return posts + + def test_dry_run_posts_nothing_but_logs_both_would_posts( + self, heads_up_module, configured, capsys + ): + results = heads_up_module.run( + repo="o/r", + close=False, + cutoff=dt.date(2026, 6, 1), + model="m", + ) + actions = [r["action"] for r in results] + assert actions.count("would-post-heads-up") == 2 + assert actions.count("skip-passing") == 2 + assert all(dry for _, dry, _ in configured) # every post was dry-run + + def test_real_run_posts_two_comments(self, heads_up_module, configured): + results = heads_up_module.run( + repo="o/r", + close=True, + cutoff=dt.date(2026, 6, 1), + model="m", + ) + assert sum(1 for r in results if r["action"] == "heads-up-posted") == 2 + # Two real-run posts: one failing PR (#2), one failing issue (#102). + real_posts = [n for n, dry, _ in configured if dry is False] + assert sorted(real_posts) == [2, 102] + + def test_kinds_filter_skips_issues(self, heads_up_module, configured): + results = heads_up_module.run( + repo="o/r", + close=False, + cutoff=dt.date(2026, 6, 1), + model="m", + kinds=("pr",), + ) + assert {r["kind"] for r in results} == {"pr"} + + def test_only_numbers_restricts_sweep(self, heads_up_module, configured): + results = heads_up_module.run( + repo="o/r", + close=False, + cutoff=dt.date(2026, 6, 1), + model="m", + only_numbers={"pr": [2], "issue": [101]}, + ) + assert sorted((r["kind"], r["number"]) for r in results) == [ + ("issue", 101), + ("pr", 2), + ] + + +class TestListOpenNumbersNoCap: + """`_list_open_numbers` must sweep the WHOLE backlog, not a capped page. + + Regression guard: the rollout sweep is one-shot, so any item it misses + here never gets a heads-up before the bot starts auto-closing. + """ + + def test_delegates_to_list_open_items_with_no_cap( + self, heads_up_module, monkeypatch + ): + import agent_shin_shared + + captured: dict = {} + + def fake_gh(*args): + captured["args"] = args + return '[{"number": 5}, {"number": 9}]' + + monkeypatch.setattr(agent_shin_shared, "gh", fake_gh) + numbers = heads_up_module._list_open_numbers("o/r", "issue") + assert numbers == [5, 9] + args = captured["args"] + assert args[0] == "issue" + assert args[args.index("--limit") + 1] == str( + agent_shin_shared.GH_LIST_ALL_LIMIT + ) + assert "1000" not in args