scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 73

ci: remove the dead Agent Shin triage workflows and scripts

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ryan 2026-09-18 22:59:32 +00:00
parent bc6b540205
commit 139445179a
12 changed files with 0 additions and 6983 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
.github/scripts/_agent_shin_actions.py vendored
View file

@ -1,50 +0,0 @@
"""Dry-run wrapper(s) around Agent Shin GitHub mutations.
The rollout scripts currently need only one mutation wrapped, so this module
exposes a single ``maybe_post_comment`` helper. It takes a ``dry_run: bool``
keyword argument and the body is intentionally trivial:
if dry_run:
print(...) # log what we would do, return
return
real_mutation(...) # otherwise, actually do it
That shape means a dry-run preview differs from the real run in exactly one
line per side effect: the call site. So when you `python3 script.py` locally
without ``--close``, you can be confident the actions printed are the ones the
GitHub Action would have performed (modulo ordering on retry/error paths,
which are deliberately simple). Any further mutation a rollout script needs
should get the same ``maybe_*`` treatment instead of calling the raw
``triage_with_llm`` mutation directly.
Importing from this module pulls in the real mutation from ``triage_with_llm``
— call sites in the rollout scripts should NEVER import ``post_comment``
directly; that would skip the dry-run gate and is the bug class this module
exists to prevent.
"""
from __future__ import annotations
import sys
import textwrap
# Import the module itself rather than the bare names so monkeypatching
# `triage_with_llm.post_comment` (or any of the other mutations) in tests is
# reflected here — `from triage_with_llm import post_comment` would bind the
# original function to a local name and bypass the patch, defeating the whole
# point of these wrappers.
import triage_with_llm
def _log(line: str) -> None:
"""Print a single dry-run line to stdout (one log statement per side effect)."""
print(line, file=sys.stdout, flush=True)
def maybe_post_comment(repo: str, number: int, body: str, *, dry_run: bool) -> None:
"""Post a comment on ``repo#number`` — or, in dry-run, log what we would post."""
if dry_run:
_log(f"[DRY RUN] comment {repo}#{number}:")
_log(textwrap.indent(body, " "))
return
triage_with_llm.post_comment(repo, number, body)

211
.github/scripts/agent_shin_shared.py vendored
View file

@ -1,211 +0,0 @@
"""Constants and helpers shared by Agent Shin's triage scripts.
Both `triage_with_llm.py` (the LLM-judge entrypoint) and
`close_low_quality_prs.py` (the daily Greptile-score sweep) need to
agree on the same notions of:
* What counts as a Greptile-authored review comment
(``GREPTILE_BOT_LOGINS``) and how to extract a confidence score from
its body (``SCORE_PATTERN`` / :func:`extract_greptile_score`).
* How long the 2-hour grace window is (``GRACE_PERIOD_SECONDS``) and
the HTML marker stamped into a grace-warning comment so the *other*
script can see "Agent Shin already warned" and behave accordingly
(``GRACE_COMMENT_MARKER``).
* Who Agent Shin is on GitHub (``AGENT_SHIN_DEFAULT_BOT_LOGIN``).
* How GitHub-style ISO-8601 timestamps round-trip into timezone-aware
:class:`datetime.datetime` (:func:`parse_iso8601`).
Keeping these in one module means a future change (new Greptile output
format, a longer grace window, a new allowlisted account) is a single edit
instead of two — the original split version had to call out in comments
that the two copies "must stay in sync" precisely because nothing
enforced it.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import re
import subprocess
from typing import Iterable
GREPTILE_BOT_LOGINS = frozenset({"greptile-apps", "greptile-apps[bot]"})
SCORE_PATTERN = re.compile(
r"confidence\s*score\s*[:\-]?\s*(\d+)\s*/\s*5",
re.IGNORECASE,
)
GRACE_COMMENT_MARKER = "<!-- agent-shin:grace-warning -->"
# Hidden HTML marker stamped on every Agent Shin auto-close comment (the LLM
# judge's grace/review-gate close and the daily Greptile sweep's close).
# `was_closed_by_agent_shin` requires this marker — not just the closing actor —
# before `@agent-shin reconsider` may reopen, because the `github-actions[bot]`
# identity is shared with every other workflow in the repo and is not unique to
# Agent Shin. Both close paths must stamp it or the reconsider path silently
# rejects the contributor.
AGENT_SHIN_CLOSE_MARKER = "<!-- agent-shin:closed -->"
# 2 hours between the grace warning and the auto-close. Short enough to
# dogfood the "fix it before it closes" loop in one sitting; bump back up
# (e.g. 86400 for a day) for the public rollout.
GRACE_PERIOD_SECONDS = 7200
AGENT_SHIN_DEFAULT_BOT_LOGIN = "github-actions[bot]"
def _logins(*names: str) -> frozenset[str]:
"""Build a login set normalized for case-insensitive membership checks.
Callers compare via ``login.lower() in <set>``, so the stored values
must be lowercase. Normalizing here lets the literals keep each
account's canonical GitHub casing (e.g. ``SwiftWinds``) for
readability without breaking the lookup.
"""
return frozenset(name.lower() for name in names)
# Dogfood rollout gate. While this set is non-empty, Agent Shin acts ONLY on
# PRs/issues authored by these logins and skips everyone else. For an
# allowlisted author the usual internal/external classification is bypassed, so
# an internal account (e.g. a maintainer's own work login) still gets triaged
# while the bot is being tested on a small set of accounts. Empty the set to
# lift the restriction and restore full triage for the public rollout. Logins
# are compared case-insensitively.
ALLOWLIST_LOGINS = _logins("mateo-berri", "SwiftWinds")
# `gh {pr,issue} list` has no "fetch everything" flag — `--limit` is the only
# control and it defaults to 30. Pass a ceiling far above any realistic open
# backlog (low thousands today) so gh paginates the API until the queue is
# exhausted rather than silently truncating. The bulk sweeps MUST see the whole
# backlog: gh lists newest-first, so a low cap drops the *oldest* PRs/issues —
# exactly the stale ones a low-quality sweep is meant to catch.
GH_LIST_ALL_LIMIT = 100_000
def extract_greptile_score(comments: Iterable[dict]) -> tuple[int, dict] | None:
"""Return (score, comment) for the most recent Greptile-authored comment
that contains a "Confidence Score: X/5". Returns None if no such comment.
"Most recent" is determined by the comment's `updated_at` (falling back to
`created_at`), so re-reviews override earlier passes.
"""
candidates: list[tuple[str, int, dict]] = []
for comment in comments:
user = (comment.get("user") or {}).get("login", "")
if user not in GREPTILE_BOT_LOGINS:
continue
body = comment.get("body") or ""
match = SCORE_PATTERN.search(body)
if not match:
continue
score = int(match.group(1))
timestamp = comment.get("updated_at") or comment.get("created_at") or ""
candidates.append((timestamp, score, comment))
if not candidates:
return None
candidates.sort(key=lambda triple: triple[0])
_, score, comment = candidates[-1]
return score, comment
def parse_iso8601(value: str) -> dt.datetime:
"""Parse a GitHub ISO-8601 timestamp into a timezone-aware datetime."""
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
def gh(*args: str) -> str:
"""Run a `gh` CLI command and return stdout. Raises on non-zero exit.
Shared by both Agent Shin entrypoints so a future change here
(timeout handling, logging, retry on transient failures) only needs
to be made once.
"""
result = subprocess.run(
["gh", *args],
capture_output=True,
text=True,
check=True,
)
return result.stdout
def list_open_items(kind: str, *, repo: str | None, fields: str) -> list[dict]:
"""Return EVERY open PR (``kind="pr"``) or issue (``kind="issue"``) in ``repo``.
Wraps ``gh {pr,issue} list`` with ``--limit GH_LIST_ALL_LIMIT`` so the full
backlog is fetched instead of the default 30 (or any other arbitrary cap).
Both bulk sweeps — the daily Greptile closer and the one-shot rollout
heads-up — rely on this seeing the whole queue, including the oldest items.
``fields`` is the comma-separated ``--json`` field list the caller needs
(e.g. ``"number"`` for the rollout, the full set for the closer).
"""
if kind not in ("pr", "issue"):
raise ValueError(f"kind must be 'pr' or 'issue', got {kind!r}")
repo_args = ["--repo", repo] if repo else []
raw = gh(
kind,
"list",
"--state",
"open",
"--limit",
str(GH_LIST_ALL_LIMIT),
"--json",
fields,
*repo_args,
)
return json.loads(raw)
def seconds_since_latest_marker_comment(
comments: Iterable[dict],
*,
marker: str,
bot_login: str | None = None,
now: dt.datetime | None = None,
) -> float | None:
"""Return seconds since the bot's most recent comment containing ``marker``.
Filters comments by author so a contributor who quotes the HTML
marker (e.g. via GitHub's "Quote reply" feature, which preserves
HTML comments in the raw markdown of the quoted text) is not
mistaken for a bot warning — that would silently reset cooldown
timers and suppress legitimate notifications.
``bot_login`` defaults to the `AGENT_SHIN_BOT_LOGIN` env override or
``AGENT_SHIN_DEFAULT_BOT_LOGIN`` so callers normally don't need to
pass it. ``now`` is injectable for tests / callers (like the daily
sweep) that want every age calculation pinned to one snapshot.
"""
expected_login = (
bot_login
or os.environ.get("AGENT_SHIN_BOT_LOGIN")
or AGENT_SHIN_DEFAULT_BOT_LOGIN
).lower()
latest: dt.datetime | None = None
for comment in comments:
author = ((comment.get("user") or {}).get("login") or "").lower()
if author != expected_login:
continue
body = comment.get("body") or ""
if marker not in body:
continue
created = comment.get("created_at")
if not created:
continue
try:
ts = parse_iso8601(created)
except ValueError:
continue
if latest is None or ts > latest:
latest = ts
if latest is None:
return None
reference = now if now is not None else dt.datetime.now(dt.timezone.utc)
return (reference - latest).total_seconds()

573
.github/scripts/close_low_quality_prs.py vendored
View file

@ -1,573 +0,0 @@
#!/usr/bin/env python3
"""
Auto-close low-quality pull requests.
Closes open PRs (including drafts, regardless of age) that satisfy ALL of:
1. Have a Greptile (`greptile-apps`) review comment whose latest
"Confidence Score: X/5" is below the configured threshold (default: 4).
2. Are authored by an external OSS contributor (internal BerriAI
contributors are exempt).
3. Do not carry an opt-out label (default: "do not close").
`--min-age-days` is retained as an opt-in safety net for one-off backfill
runs (default: 0). The team's intent is that the count of open PRs equals
the count of PRs internal collaborators need to action on, so neither age
nor draft status acts as a free pass.
For each match, the script posts an explanatory comment and closes the PR.
Because OSS contributors *cannot* reopen a PR closed by the bot/maintainer
(GitHub limitation), the close-comment instructs them to push their fixes
and **open a fresh PR**, or to comment `@agent-shin reconsider` on the
closed PR to have the LLM judge re-evaluate (and reopen on pass).
Requires the `gh` CLI to be authenticated.
Usage examples:
# Dry run (default) - prints what would be closed
python3 close_low_quality_prs.py
# Actually close matching PRs
python3 close_low_quality_prs.py --close
# Restrict to PRs at least N days old (one-off backfill safety net)
python3 close_low_quality_prs.py --min-age-days 7 --min-score 4 --close
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import subprocess
import sys
from typing import Iterable
# Add this script's directory to `sys.path` so the sibling
# `agent_shin_shared` module is importable when the script is invoked
# directly (e.g. `python3 .github/scripts/close_low_quality_prs.py ...`).
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from agent_shin_shared import ( # noqa: E402 -- sys.path adjusted above
AGENT_SHIN_CLOSE_MARKER,
ALLOWLIST_LOGINS,
GRACE_COMMENT_MARKER,
GRACE_PERIOD_SECONDS,
GREPTILE_BOT_LOGINS,
SCORE_PATTERN,
extract_greptile_score,
gh,
list_open_items,
parse_iso8601,
seconds_since_latest_marker_comment,
)
# `GREPTILE_BOT_LOGINS` and `SCORE_PATTERN` (Greptile's GitHub App login
# variants and the "Confidence Score: X/5" regex) are imported from
# `agent_shin_shared` so the LLM judge in `triage_with_llm.py` and this
# daily Greptile sweep read the score through the same set of logins
# and the same regex.
# `author_association` values for internal BerriAI contributors who should be
# exempt from auto-triage.
INTERNAL_AUTHOR_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"})
# Default labels that exempt a PR from auto-close. Defined at module scope (not
# as a mutable argparse default) so that `--optout-label foo` REPLACES the
# defaults instead of appending to them — the argparse `action="append"` +
# `default=[...]` combination silently mutates the shared default list.
DEFAULT_OPTOUT_LABELS = ("do not close", "keep open", "wip")
# `GRACE_COMMENT_MARKER` (HTML marker appended to grace-period warning
# comments — used by either script to recognize that a warning was
# already posted) and `GRACE_PERIOD_SECONDS` (length of the grace
# period between the warning and the actual auto-close, 2 hours) are
# imported from `agent_shin_shared` so the Agent Shin LLM judge and
# this daily Greptile sweep agree on the same marker and duration.
def fetch_open_prs(repo: str | None) -> list[dict]:
"""Fetch all open PRs (number, createdAt, isDraft, labels, author).
Includes drafts: `gh pr list --state open` returns both ready-for-review
and draft PRs by default. This is the desired behavior — drafts are not
a free pass; the internal-collaborator open-PR queue should reflect every
PR that needs human attention regardless of draft status.
"""
fields = "number,title,createdAt,isDraft,labels,author,url"
return list_open_items("pr", repo=repo, fields=fields)
def fetch_pr_author_association(pr_number: int, repo: str | None) -> str:
"""Return the GitHub `author_association` for a PR, uppercase.
Values: OWNER, MEMBER, COLLABORATOR, CONTRIBUTOR, FIRST_TIME_CONTRIBUTOR,
FIRST_TIMER, MANNEQUIN, NONE. Returns "" on lookup failure.
"""
endpoint = (
f"repos/{repo}/pulls/{pr_number}"
if repo
else f"repos/{{owner}}/{{repo}}/pulls/{pr_number}"
)
try:
data = json.loads(gh("api", endpoint))
except subprocess.CalledProcessError:
return ""
return (data.get("author_association") or "").upper()
def is_external_pr_author(pr: dict, repo: str | None) -> bool:
"""Return True if the PR author is an external OSS contributor.
Internal = `OWNER` / `MEMBER` / `COLLABORATOR` association, or a bot login.
"""
login = ((pr.get("author") or {}).get("login") or "").lower()
if login.endswith("[bot]") or login in {"dependabot", "github-actions"}:
return False
association = fetch_pr_author_association(pr["number"], repo)
# Fail-safe: if the API lookup failed (empty string), treat the author as
# internal so we don't auto-close their PR. Auto-close is destructive, so
# an unknown association should never make a PR eligible for closing.
if not association or association in INTERNAL_AUTHOR_ASSOCIATIONS:
return False
return True
def fetch_pr_comments(pr_number: int, repo: str | None) -> list[dict]:
"""Fetch issue-level comments on a PR (where Greptile posts its summary)."""
endpoint = (
f"repos/{repo}/issues/{pr_number}/comments?per_page=100"
if repo
else f"repos/{{owner}}/{{repo}}/issues/{pr_number}/comments?per_page=100"
)
raw = gh("api", "--paginate", endpoint)
comments: list[dict] = []
for line in raw.strip().splitlines():
line = line.strip()
if not line:
continue
try:
parsed = json.loads(line)
except json.JSONDecodeError:
# A malformed line should not blow up the whole sweep. Skip and
# carry on so the remaining PRs in this run still get evaluated.
continue
if isinstance(parsed, list):
comments.extend(parsed)
else:
comments.append(parsed)
return comments
def has_optout_label(pr: dict, optout_labels: set[str]) -> bool:
labels = {label.get("name", "").lower() for label in pr.get("labels", [])}
return bool(labels & {lbl.lower() for lbl in optout_labels})
def seconds_since_last_grace_warning(
comments: Iterable[dict],
*,
bot_login: str | None = None,
now: dt.datetime | None = None,
) -> float | None:
"""Return seconds since the bot's most recent grace-period warning, or
None if no such warning has ever been posted on this PR.
Thin wrapper over
`agent_shin_shared.seconds_since_latest_marker_comment` — the
centralized helper handles the bot-author filter, marker match,
timestamp parsing, and `now` injection. Keeping this wrapper
preserves the closer's "already-fetched comments + injectable now"
interface so callers (and tests) don't need to change.
"""
return seconds_since_latest_marker_comment(
comments,
marker=GRACE_COMMENT_MARKER,
bot_login=bot_login,
now=now,
)
def format_grace_warning_comment(score: int, threshold: int) -> str:
"""Comment posted on the FIRST low-Greptile-score detection — gives
the contributor a 2-hour grace window before the auto-close fires on
the next daily cron run.
Mirrors `format_grace_warning_pr_comment` in
`triage_with_llm.py` in spirit (2-hour grace + escape hatches), but
framed around Greptile's confidence score instead of the LLM judge's
rubric since the close trigger here is the Greptile signal.
"""
return (
"🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this "
"repository.\n"
"\n"
"Heads up: Greptile's most recent review scored this PR "
f"**{score}/5**, below our merge bar of **{threshold}/5**.\n"
"\n"
"If the score isn't lifted in the next **2 hours**, I'll auto-close this PR. That's "
"**not** us saying the change isn't worthwhile. We want the open-PR list to mirror "
"what a maintainer can act on *right now*, so contributors like you don't get lost in "
"a backlog. Take your time; everything below still works after the close.\n"
"\n"
"**During the grace period:** push fixes that address Greptile's feedback, then comment "
"`@greptileai` to request a fresh review. If "
f"the new score is **{threshold}/5 or higher**, the PR stays open and no further "
"action is needed on your side.\n"
"\n"
"**If the PR does get auto-closed in 2 hours, you still have an easy recovery path:**\n"
"\n"
"- Comment `@greptileai` to request a fresh review. **This still works even after "
f"the PR is closed**, and a score of {threshold}/5 or higher is one of the signals "
"that lifts the PR back into the review queue. A low Greptile score isn't a blocker.\n"
"- Comment `@agent-shin reconsider` after pushing fixes; I'll re-run the rubric and "
"reopen the PR if both gates (description rubric + Greptile score) now pass.\n"
"\n"
f"{GRACE_COMMENT_MARKER}"
)
def post_grace_warning(
pr: dict,
score: int,
threshold: int,
repo: str | None,
dry_run: bool,
) -> None:
"""Post the 2-hour grace-period warning comment on `pr`.
The warning carries `GRACE_COMMENT_MARKER` so subsequent runs can
detect that the contributor has already been told about the
pending close. Does NOT close the PR — the close happens on the
next eligible run after `GRACE_PERIOD_SECONDS` elapses (handled
by `close_pr`).
"""
pr_number = pr["number"]
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(
f" [DRY RUN] Would post grace warning to PR #{pr_number} "
f"(greptile={score}/5): {pr['title']}"
)
return
comment_body = format_grace_warning_comment(score, threshold)
gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args)
print(f" Posted grace warning on PR #{pr_number} (greptile={score}/5)")
def format_close_comment(score: int, threshold: int) -> str:
"""Comment posted when a low-Greptile-score PR is auto-closed.
Carries `AGENT_SHIN_CLOSE_MARKER` so the `@agent-shin reconsider` path
(guarded by `was_closed_by_agent_shin`) recognizes this as an Agent Shin
close and is allowed to reopen the PR once it passes again; without the
marker that recovery path the comment advertises silently rejects the
contributor.
"""
score_sentence = (
f"Greptile's most recent review scored this PR **{score}/5**, below "
f"our merge bar of **{threshold}/5**, and the 2-hour grace period since "
"the warning has elapsed.\n\n"
)
return (
f"Closing as part of automated PR triage.\n\n"
f"{score_sentence}"
"We close low-confidence PRs aggressively to keep the review queue "
"manageable for maintainers and contributors alike. **This is not a "
"rejection of the idea.** To bring this back:\n\n"
"1. Push the fixes that address Greptile's feedback (continue using "
"your existing branch is fine).\n"
"2. **Open a new PR** with the updated branch. Greptile will review "
"it again, and if it scores "
f"**{threshold}/5 or higher** a maintainer will take another look.\n\n"
"_Why open a new PR instead of reopening this one?_ GitHub does not "
"let external contributors reopen a PR that was closed by a bot or "
"maintainer, so a fresh PR is the most reliable path forward. If you "
"would prefer this exact PR re-evaluated, comment "
"`@agent-shin reconsider` once you've pushed the fixes; Agent Shin "
"will re-run triage and reopen this PR if it now meets the bar. "
"You can also comment `@greptileai` to request a fresh Greptile "
"review; that works **even after the PR is closed**.\n\n"
"Thanks for contributing to LiteLLM. We know auto-closures can sting; "
"the goal is to keep the project healthy, not to dismiss your work."
f"\n\n{AGENT_SHIN_CLOSE_MARKER}"
)
def close_pr(
pr: dict,
score: int,
threshold: int,
age_days: int,
repo: str | None,
dry_run: bool,
label: str | None,
) -> None:
"""Post the explanatory comment and close the PR."""
pr_number = pr["number"]
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(
f" [DRY RUN] Would close PR #{pr_number} "
f"(age={age_days}d, greptile={score}/5): {pr['title']}"
)
return
comment_body = format_close_comment(score, threshold)
gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args)
if label:
try:
gh("pr", "edit", str(pr_number), "--add-label", label, *repo_args)
except subprocess.CalledProcessError as exc:
stderr = (exc.stderr or "").strip()
print(f" warn: failed to add label '{label}' to #{pr_number}: {stderr}")
gh("pr", "close", str(pr_number), *repo_args)
print(f" Closed PR #{pr_number} (greptile={score}/5, age={age_days}d)")
def evaluate_pr(
pr: dict,
now: dt.datetime,
min_age_days: int,
min_score: int,
repo: str | None,
optout_labels: set[str],
allowlist: frozenset[str] = ALLOWLIST_LOGINS,
) -> tuple[str, int | None, int | None]:
"""Decide what to do with `pr` on this triage run.
Returns (action, score_or_none, age_days_or_none) where action is one of:
"skip-too-young", "skip-optout-label", "skip-not-allowlisted",
"skip-internal", "skip-no-greptile-score", "skip-score-ok",
"warn-grace", "skip-in-grace-period", or "close".
Drafts are NOT skipped — the goal is "open PR count == PRs internal
collaborators need to action on", and a draft that Greptile scored <4/5
is still in that queue. Authors can opt out via the `wip` label (see
`DEFAULT_OPTOUT_LABELS`) if they need to keep a long-lived draft open.
Grace-period semantics: the first time a PR fails the rubric, the
action is `warn-grace` — the caller should post a warning comment but
NOT close the PR. On a subsequent run, if the warning is still less
than `GRACE_PERIOD_SECONDS` old AND the PR still fails, the action is
`skip-in-grace-period`. Once the warning ages out and the rubric is
still failing, the action is `close`.
"""
if has_optout_label(pr, optout_labels):
return ("skip-optout-label", None, None)
created = parse_iso8601(pr["createdAt"])
age_days = (now - created).days
# `min_age_days` defaults to 0 (close as soon as Greptile scores low).
# Set a positive value via --min-age-days for one-off backfill runs that
# want to skip very-young PRs.
if min_age_days > 0 and age_days < min_age_days:
return ("skip-too-young", None, age_days)
# While the allowlist is active it is the sole author gate: only those
# logins are acted on and the external-only restriction is bypassed for
# them. Otherwise auto-close only external OSS contributors — internal
# contributors (BerriAI org members) handle their own backlog.
login = ((pr.get("author") or {}).get("login") or "").lower()
if allowlist:
if login not in allowlist:
return ("skip-not-allowlisted", None, age_days)
elif not is_external_pr_author(pr, repo):
return ("skip-internal", None, age_days)
comments = fetch_pr_comments(pr["number"], repo)
extraction = extract_greptile_score(comments)
if extraction is None:
return ("skip-no-greptile-score", None, age_days)
score, _ = extraction
if score >= min_score:
return ("skip-score-ok", score, age_days)
grace_age = seconds_since_last_grace_warning(comments, now=now)
if grace_age is None:
return ("warn-grace", score, age_days)
if grace_age < GRACE_PERIOD_SECONDS:
return ("skip-in-grace-period", score, age_days)
return ("close", score, age_days)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--repo",
type=str,
default=None,
help="Repository (owner/repo). Auto-detected if omitted.",
)
parser.add_argument(
"--min-age-days",
type=int,
default=0,
help=(
"Minimum age (in days) before a PR is eligible. Default 0 = "
"close as soon as Greptile flags it. Set a positive value for "
"one-off backfill runs that want to spare very-young PRs."
),
)
parser.add_argument(
"--min-score",
type=int,
default=4,
choices=range(1, 6),
help="Greptile score below which a PR is closed (default: 4 -> closes <4/5).",
)
parser.add_argument(
"--optout-label",
action="append",
default=None,
help=(
"Label(s) that exempt a PR from auto-close. Repeat to add more. "
"Case-insensitive. When omitted, defaults to "
f"{list(DEFAULT_OPTOUT_LABELS)!r}; passing this flag REPLACES the "
"defaults (argparse `append` with a mutable default would append "
"instead, which we explicitly avoid)."
),
)
parser.add_argument(
"--close-label",
type=str,
default=None,
help=(
"Optional label to add to PRs that get auto-closed "
"(e.g. 'auto-closed-low-quality'). Must already exist on the repo."
),
)
parser.add_argument(
"--close",
action="store_true",
help="Actually close matching PRs (default is dry-run).",
)
parser.add_argument(
"--limit",
type=int,
default=None,
help="Maximum number of PRs to close in one run (safety net).",
)
args = parser.parse_args()
dry_run = not args.close
if dry_run:
print("=== DRY RUN MODE (pass --close to actually close PRs) ===\n")
print("Fetching open PRs...")
prs = fetch_open_prs(args.repo)
print(f"Found {len(prs)} open PRs.\n")
now = dt.datetime.now(dt.timezone.utc)
optout_labels = set(args.optout_label or DEFAULT_OPTOUT_LABELS)
closed = 0
summary = {
"close": 0,
"warn-grace": 0,
"skip-in-grace-period": 0,
"skip-too-young": 0,
"skip-optout-label": 0,
"skip-not-allowlisted": 0,
"skip-internal": 0,
"skip-no-greptile-score": 0,
"skip-score-ok": 0,
}
# `warned` tracks grace-warning comments posted in this run so the
# `--limit` safety net bounds *all* destructive write actions, not
# just closures. Without this cap, a backlog of PRs failing the
# threshold simultaneously could flood contributors with comments.
warned = 0
for pr in sorted(prs, key=lambda p: p["createdAt"]):
try:
action, score, age_days = evaluate_pr(
pr,
now,
args.min_age_days,
args.min_score,
args.repo,
optout_labels,
)
summary[action] = summary.get(action, 0) + 1
if action == "warn-grace":
assert score is not None
print(
f"#{pr['number']}: \"{pr['title']}\" "
f"(age={age_days}d, greptile={score}/5) -> warn-grace"
)
post_grace_warning(
pr,
score=score,
threshold=args.min_score,
repo=args.repo,
dry_run=dry_run,
)
if not dry_run:
warned += 1
if args.limit is not None and (warned + closed) >= args.limit:
print(
f"\nReached --limit={args.limit} "
f"(closed={closed}, warned={warned}); stopping."
)
break
continue
if action != "close":
continue
assert score is not None and age_days is not None
print(
f"#{pr['number']}: \"{pr['title']}\" "
f"(age={age_days}d, greptile={score}/5) -> close"
)
close_pr(
pr,
score=score,
threshold=args.min_score,
age_days=age_days,
repo=args.repo,
dry_run=dry_run,
label=args.close_label,
)
if not dry_run:
closed += 1
if args.limit is not None and (warned + closed) >= args.limit:
print(
f"\nReached --limit={args.limit} "
f"(closed={closed}, warned={warned}); stopping."
)
break
except Exception as exc: # noqa: BLE001 - per-PR errors don't abort the sweep
summary["error"] = summary.get("error", 0) + 1
print(
f"!! PR #{pr.get('number')}: {exc}",
file=sys.stderr,
)
continue
print("\n=== Summary ===")
for key, value in summary.items():
print(f" {key:28s} {value}")
if dry_run:
print(f"\nTotal would close: {summary['close']}")
else:
print(f"\nTotal closed: {closed}")
print(
f"Total {'would warn (grace)' if dry_run else 'warned (grace)'}: "
f"{summary['warn-grace']}"
)
return 0
if __name__ == "__main__":
sys.exit(main())

282
.github/scripts/triage-requirements.txt vendored
View file

@ -1,282 +0,0 @@
# Hash-pinned dependency set for the Agent Shin triage scripts.
# Installed in privileged triage workflows, so every package is pinned to an
# exact version with SHA-256 hashes and installed with pip --require-hashes.
#
# Regenerate after bumping openai:
# echo 'openai==<version>' \
# | uv pip compile - --generate-hashes --python-version 3.12 \
# --no-annotate --no-header -o .github/scripts/triage-requirements.txt
annotated-types==0.7.0 \
--hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
--hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
anyio==4.14.0 \
--hash=sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89 \
--hash=sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9
certifi==2026.6.17 \
--hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \
--hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db
distro==1.9.0 \
--hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \
--hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2
h11==0.16.0 \
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
httpcore==1.0.9 \
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
httpx==0.28.1 \
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
idna==3.18 \
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
jiter==0.15.0 \
--hash=sha256:01a8222cf05ab1128e239421156c207949808acaaea2bdfd33130ae666786e86 \
--hash=sha256:032396229564bca02440396bd327710719f724f5e7b7e9f7a8eb3faa4a2c2281 \
--hash=sha256:04b400bbf8c9efb03d9bdd976475c919c1d85593b04b9fff7ae234065daf87ae \
--hash=sha256:05906b93d72f03339e6bb7cf8dc10ebda64a0266126eed6beba79e20abcf5fd4 \
--hash=sha256:066f8f33f18b2419cd8213b2436fa7fbc9c499f315971cfa3ce1f9820c001b1b \
--hash=sha256:0ab068bce62a45aa3e7367eceaffb5dde60b7eb853be8dece45132e3d0ff4879 \
--hash=sha256:0be6f5ad41a809f303f416d17cec92a7a725902fb9b4f3de3d19362ac0ef8554 \
--hash=sha256:0e90a1c315a0226ec822d973817967f9223b7701546c8c2a7913e7ab0926294d \
--hash=sha256:0f862193b8696249d22ec433e85fd2ab0ad9596bc3e45e6c0bc55e8aeba97be2 \
--hash=sha256:1303d4d68a9b051ea90502402063ecf3807da00ad2affa19ca1ae3b90b3c5f67 \
--hash=sha256:144f8e72cb53dab146347b91cceac01f5481237f2b93b4a339a1ee8f8878b67c \
--hash=sha256:182226cbc930c9fab81bc2e41a4da672f89539906dadb05e75670ac07b94f71f \
--hash=sha256:1c11465f97e2abf45a014b83b730222f8f1c5335e802c7055a67d50de6f1f4e3 \
--hash=sha256:1c15024a3d892223b18f597c86d59387249dc396590844ce6b9f6131d1093bae \
--hash=sha256:1d54fb5b31dea401a41af3f8a7d2512e9b6a6a005491e6166c7e4ffab9639a9c \
--hash=sha256:25ffbe229aa8cd98c28879d8aa1a6e34ae77992ab984a65fba800859dab16269 \
--hash=sha256:2a77aadd57cac1682e4401a72724d2796d89a4ba129b1a5812aa94ee480826eb \
--hash=sha256:2ae901f3a55bfafdde31d289590fa25e3245735a2b1e8c7cc15871710a002871 \
--hash=sha256:2b0074e2f56eb2dacca1689760fd2852a068f85a0547a157b82cb4cafeb6768b \
--hash=sha256:2c8aea7781d2a372227871de4e1a1332aa96f5a89fd76c5e835dafdbad102887 \
--hash=sha256:2c9cb907439d20bd0c7d7565ca01ee52234203208433749bae5b516907526928 \
--hash=sha256:2fb6a5d26af81fc0f00f9360a891e05cf755e149bba391c4d563adc54812973d \
--hash=sha256:2fd73e3da91a0a722d67165e849ce2cdc10de0e0d48738c142be8c6c5f310f4c \
--hash=sha256:30ce1a5d16b5641dc935d50ef775af6a0871e3d14ab05d6fc54dff371b78e558 \
--hash=sha256:30ce785d2adb8e32c3f7741442370a74834ec4c01f3c48f0750227a0b4ef27d6 \
--hash=sha256:30f2218e6a9e5c18bc10fe6d41ac189c442c88eacf11bad9f28ef95a9bef00e6 \
--hash=sha256:351a341c2105aa430b7047e30f1bf7975f6313b00165d3fc07be2edaf741f279 \
--hash=sha256:37a10c377ce3a4a85f4a67f28b7afe093154cde77eaf248a72e856aa08b4d865 \
--hash=sha256:392b8ab019e5502d08aff85c6272209c24bc2cbe706ea82a56368f524236614a \
--hash=sha256:3e4540b8e74e4268811ac05db226a6a128ff572e7e0ce3f1163b693cadb184cd \
--hash=sha256:40b2c7e92c44a84d748d21706c68dc6ff8161d80b59c99d774721a0d2317d7c7 \
--hash=sha256:411fa4dfa5a7ae3d11491027ffb9beadec3996010a986862db70d91abba1c750 \
--hash=sha256:4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76 \
--hash=sha256:42bfb257930800cf43e7c62c832402c704ab60797c992faf88d20e903eac8f32 \
--hash=sha256:4363818355dbc70ae1a8e9eaba9de350d93ede4ff6992b8f8eb8cbb6e5122d42 \
--hash=sha256:4ab395feec8d249ec4044e228e98a7033f043426a265df439dc3698823f0a4e4 \
--hash=sha256:50164d7610c00e7cd913a873fce30b6beeebf4b37e53983e33f22de4c900f6b8 \
--hash=sha256:50e51156192722a9c58db112837d3f8ef96fb3c5ecc14e95f409134b08b158ec \
--hash=sha256:510c8b3c17a0ed9ac69850c0438dada3c9b82d9c4d589fcb62002a5a9cf3a866 \
--hash=sha256:5157de9f76eb4bc5ea74a1219366a25f945ad305641d74e04f59c54087091aa9 \
--hash=sha256:54d5d6090cdc1b7c9e780dfb04949a990adb1e301a2fc0bbcee7de4638d33f9a \
--hash=sha256:553fcac2ef2cb990877f9fc0833b8b629a3e6a5670b6b5fd58219b41a653ddc4 \
--hash=sha256:5607e6013ed7e6b0ec9661e467b7ffde0aa7ab36833a04850f26fcf88ed4845b \
--hash=sha256:5d6a60072b44c3c2b797a7ddcbcbbf2b34ea3cfd4721580fbfd2a09d9d9b84ba \
--hash=sha256:5f30bae8bc1c2d613e28e5af3e8cceb09b742f1c8a8a5f839fb67afaffc03b61 \
--hash=sha256:62ebd14e47e9aed9df4472afcb2663668ce4d74891cd54f86bf6e44029d6dc89 \
--hash=sha256:631f13a3d04e97d4e083993b10f4b99530e3a10d953e2eb5e196b7dc7f812ce0 \
--hash=sha256:6550fa135c7deb8ead6af49ed7ff648532ea8334a1447fe34a36315ef79c5c29 \
--hash=sha256:66b1880df2d01e206e8339769d1c7c1753bcb653efd6289e203f6f24ebada0c0 \
--hash=sha256:6eac374c5c975709b69c10f09afd199df74150172156ad10c8d4fd785b7da995 \
--hash=sha256:71683c38c825452999b5717fcae07ea708e8c93003e808be4319c1b02e3d176e \
--hash=sha256:7553333dd0930c104a5a0db8df72bf7219fe663d731383b576bb6ed6351c984d \
--hash=sha256:75e8a04e91432dde9f1838373cf93d23726c79d3e908d319acf0e796f85592e7 \
--hash=sha256:773b6eb282ce11ee19f05f6b2d4404fa308e5bbd353b0b80a0262caad6db2cd7 \
--hash=sha256:774f93f65031856bf14ad9f59bdcab8b8cad501e5ceabd51ba3525f76937a25b \
--hash=sha256:7c468136b8bd6bb18c8786e4236a1fa27362f24cb23450ba0cb204ab379b8e6f \
--hash=sha256:7ce8902f939970048b233087082e7bb829db29375811c7ad50687b8624c6fd08 \
--hash=sha256:7d3d6683288c11cbab50e865f2e2f13950179aa45410e30b2cfbd3fb7b0177bf \
--hash=sha256:7f6163c0f10b055245f814dcc59f4818da60dfe72f3e72ab89fc24b6bd5e9c52 \
--hash=sha256:8020c99ec13a7db2b6f96cbe82ef4721c88b426a4892f27478044af0284615ef \
--hash=sha256:813dfbb17d65328bf86e5f0905dd277ba2265d3ca20556e86c0c7035b7182e5a \
--hash=sha256:860a74063284a2ae9bfedd694f299cc2c68e2696c5f3d440cc9d18bb81b9dd04 \
--hash=sha256:8c9004af7c8d67cce7f1aae1026fb55607f4aa600710d08ede3a3ce4aeefe7e0 \
--hash=sha256:8d2c0c44d569ce0f2850f5c926f8caeb5f245fbc84475aeb36efccc2103e6dbd \
--hash=sha256:8f7e9bc0f1135039b22ee6eab588d42df1ce55842b30740a352885eb267bd941 \
--hash=sha256:90c5db5527c221249a876160663ab891ace358c17f7b9c93ec1478b7f0550e5c \
--hash=sha256:9100ddbec09741cc66feb0fc6773f8bdbd0e3c345689368f260082ff85dcc0cd \
--hash=sha256:913d02d29c9606643418d9ccfc3b72492ab25a6bf7889934e09a3490f8d3438b \
--hash=sha256:980c256edb05b78a111b99c4de3b1d32e31634b867fd1fc2cf726e7b7bba9854 \
--hash=sha256:9f924585cdacf631cd382b657966847bb537bf9ed0a6f9b991da5f05a631480f \
--hash=sha256:a254e10b593624d230c365b6d616b22ca0ad65e63a16e6631c2b3466022e6ba8 \
--hash=sha256:a2a438005b6f22d0273413484d6094d7c2c5d10ec1b3a3bf128e0d1d3ba53258 \
--hash=sha256:a97261f1fccb8e50ecd2890a96e46efdc3f57c80a197324c6777827231eca712 \
--hash=sha256:ab596fa3837e91e7e6a31b5f639988bfc6a35d1f915ac3932d946062219d588f \
--hash=sha256:abbf258599526ad0326fe51e252e24f2bd6f24f1852681b4b78feda3808f1d18 \
--hash=sha256:ac0d9ddea4350974be7a221fc25895f251a8fee748c889bdced2141c0fec1a49 \
--hash=sha256:acf4ee4d1fc55917239fe72972fb292dd773055d05eb040d36f4326e02cc2c0e \
--hash=sha256:ae1b0d82ac2d987f9ea512b1c9adfcc71a28de3dea3a6039b54d76cffda9901e \
--hash=sha256:b15741f501469009ae0ae90b7147958a664a7dede40aa7ff174a8a4645f546d0 \
--hash=sha256:b15d3ec9b0449c40e85319bdb4caa8b77ab526e74f5532ed94bec15e2f66822c \
--hash=sha256:b3b3b775e33d3bfaec9899edc526ae97b0da0bf9d071a46124ba419149a414f8 \
--hash=sha256:b6c0ffae686c39bf3737be60793783267628783ea42545632c10b291105aee45 \
--hash=sha256:c210f8b35dc6f30aafd4b4365ca89b9d1189f21ab49b8e68fa6322a847aef138 \
--hash=sha256:c2f6bb8b5216ab9e7873bc08b5d7bef2b8abbb578a3069bf1cd14a45d71d771d \
--hash=sha256:c60e71b6d10cfc284c9bf36bd885e8d44c46f688ce50aa91b5edd90181dea687 \
--hash=sha256:c6694a173ecabc12eb60efbc0b474464ead1951ff65cd8b1e72100715c64512b \
--hash=sha256:c77496cb10bd7549690fbbab3e5ec05857b83e49276f4a9423a766ddd2afcd4c \
--hash=sha256:c84c1b7be454b0c16f8499b4ebfbfd82ea5cca6527cceefcbbc06a7557b5ed2e \
--hash=sha256:cc0bc345cf2df9d1c00ac443f50d543c1ccfa8b0422cb85b1ab70d681c0b255b \
--hash=sha256:ceb8fc27d38793f9c97149be8302720c5b22e5c195a37bf2c45dc36c4600a512 \
--hash=sha256:cf4bd113a69c0a740e27cb962ce10630c36d2b8f59d759a651b955ee9d18a823 \
--hash=sha256:d1aa62e277fc1cbd80e6deacae6f4d983b41b3d7728e0645c5d741a6149bba45 \
--hash=sha256:d1e7b1776f0797956c509e123d0952d10d293a9492dea9f288ab9570ec01d1a5 \
--hash=sha256:d636d5095155afd364247f65070fab7beda13498d7ff4de331046e704ab9657f \
--hash=sha256:d726e3ceeb337191324b49de298142f27c3ad10886341555d1d5315b5f252c6a \
--hash=sha256:d72d8af5c1013656a8870c866660627d1a75bc185814ee022c8533caa1de88ae \
--hash=sha256:d8d2955167274e15d79a7a020afdd9b39c990eb80b2d89fca695d92dcfdd38ec \
--hash=sha256:d92a5cd21fdb083931d546c207aa29633787c5dc5b02daab2d32b843f88a2c53 \
--hash=sha256:e58585a58209d72691ce2d62a9147445f5a87beb0bde97fde284c96ae392a3d1 \
--hash=sha256:e7196e56f1cd69af1dbb07dff02dcfb260a50b45a82d409d92a06fedb32473b5 \
--hash=sha256:eda3071db3346334beae1360b46da4606da57bf3528c167b3c38533afaf9f2c5 \
--hash=sha256:edebcf7d1f601199084bb6e844d7dc67e03e04f6ac786b0332d616635c4ff7a4 \
--hash=sha256:ef1fd24d9413f6209e00d3d5a453e67acfe004a25cc6c8e8484faed4311ab9e8 \
--hash=sha256:f0b271b462769543716f92d3a4f90527df6ef5ed05ee95ec4137f513e21e1b77 \
--hash=sha256:f18f85e4218d1b40f000f42a92239a7a61a902cd42c65e6c360dbd17dcb20894 \
--hash=sha256:f1e1754960f38ec40613a07e5e372df67acb3b890fb383b6fb3de3e49ddbf3c7 \
--hash=sha256:f2143ab06181d2b029eedcb6af3cebe95f11bbac62441781860f98ee9330a6a6 \
--hash=sha256:f3d37768fce7f88dd2a8c6091f2325dea27d30d30d5c6e7a1c0f0af77723b708 \
--hash=sha256:fa248c9eb220197d363f688818dac2fd4b2f0cd7d843ca7105d652034823427d
openai==2.33.0 \
--hash=sha256:03ac37d70e8c9e3a8124214e3afa785e2cbc12e627fbd98177a086ef2fd87ad5 \
--hash=sha256:f850c435e2a4685bba3295bd54912dd26315d9c1b7733068186134d6e0599f9a
pydantic==2.13.4 \
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
pydantic-core==2.46.4 \
--hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \
--hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \
--hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \
--hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \
--hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \
--hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \
--hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \
--hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \
--hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \
--hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \
--hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \
--hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \
--hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \
--hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \
--hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \
--hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \
--hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \
--hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \
--hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \
--hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \
--hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \
--hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \
--hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \
--hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \
--hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \
--hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \
--hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \
--hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \
--hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \
--hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \
--hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \
--hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \
--hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \
--hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \
--hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \
--hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \
--hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \
--hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \
--hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \
--hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \
--hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \
--hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \
--hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \
--hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \
--hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \
--hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \
--hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \
--hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \
--hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \
--hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \
--hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \
--hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \
--hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \
--hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \
--hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \
--hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \
--hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \
--hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \
--hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \
--hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \
--hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \
--hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \
--hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \
--hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \
--hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \
--hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \
--hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \
--hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \
--hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \
--hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \
--hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \
--hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \
--hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \
--hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \
--hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \
--hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \
--hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \
--hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \
--hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \
--hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \
--hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \
--hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \
--hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \
--hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \
--hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \
--hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \
--hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \
--hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \
--hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \
--hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \
--hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \
--hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \
--hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \
--hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \
--hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \
--hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \
--hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \
--hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \
--hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \
--hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \
--hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \
--hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \
--hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \
--hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \
--hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \
--hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \
--hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \
--hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \
--hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \
--hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \
--hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \
--hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \
--hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \
--hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \
--hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \
--hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \
--hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \
--hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \
--hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \
--hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae
sniffio==1.3.1 \
--hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
--hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
tqdm==4.68.3 \
--hash=sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482 \
--hash=sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03
typing-extensions==4.15.0 \
--hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \
--hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548
typing-inspection==0.4.2 \
--hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \
--hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464

1797
.github/scripts/triage_with_llm.py vendored
View file

File diff suppressed because it is too large Load diff

92
.github/workflows/close_low_quality_prs.yml vendored
View file

@ -1,92 +0,0 @@
name: Close Low-Quality PRs
# Auto-close any open PR (including drafts, regardless of age) authored by an
# external OSS contributor that Greptile reviewed with a confidence score
# below 4/5. Closures are explained in a comment that tells the contributor
# to push fixes and open a fresh PR (since OSS authors cannot reopen a PR
# closed by a bot/maintainer) or comment `@agent-shin reconsider` to have
# Agent Shin re-evaluate.
#
# Manual one-off run:
# gh workflow run "Close Low-Quality PRs" -f close=true
#
# Dry-run preview (no PRs are touched):
# gh workflow run "Close Low-Quality PRs" -f close=false
on:
schedule:
# Daily at 09:00 UTC. Pairs well with the stale-issue workflow at midnight.
- cron: "0 9 * * *"
workflow_dispatch:
inputs:
close:
description: "Actually close matching PRs (false = dry run)."
required: false
default: "false"
type: choice
options:
- "true"
- "false"
min_age_days:
description: "Minimum PR age in days (default 0 = no age filter)."
required: false
default: "0"
min_score:
description: "Greptile score below which a PR is closed (1-5)."
required: false
default: "4"
limit:
description: "Maximum number of PRs to close in a single run."
required: false
default: "25"
permissions:
contents: read
pull-requests: write
issues: write
jobs:
close-low-quality-prs:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Run low-quality PR closer
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Scheduled runs are ALWAYS dry-run, even when AGENT_SHIN_ENABLED is
# "true", so the team can QA the closer's verdicts in step summaries
# before any contributor sees a PR closed. Real closures only happen
# on manual workflow_dispatch with close=true (and the variable set).
CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
MIN_AGE_DAYS: ${{ github.event.inputs.min_age_days || '0' }}
MIN_SCORE: ${{ github.event.inputs.min_score || '4' }}
LIMIT: ${{ github.event.inputs.limit || '25' }}
run: |
set -euo pipefail
ARGS=(
--repo "${{ github.repository }}"
--min-age-days "${MIN_AGE_DAYS}"
--min-score "${MIN_SCORE}"
--limit "${LIMIT}"
)
if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then
echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> forcing dry-run regardless of close input."
elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG}" = "true" ]; then
ARGS+=(--close)
echo "::notice::Running in close-on-fail mode."
else
echo "::notice::AGENT_SHIN_ENABLED is true but this trigger is dry-run (scheduled event or close=false)."
fi
python3 .github/scripts/close_low_quality_prs.py "${ARGS[@]}"

28
.github/workflows/create_daily_oss_agent_shin_branch.yml vendored
View file

@ -1,28 +0,0 @@
name: Create Daily oss-agent-shin Branch
on:
schedule:
- cron: "0 0 * * *" # Runs every day at midnight UTC
workflow_dispatch: # Allow manual trigger
jobs:
create-oss-agent-shin-branch:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Create daily oss-agent-shin branch
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
BRANCH_NAME="litellm_oss_agent_shin_$(date +'%m_%d_%Y')"
echo "Creating branch: $BRANCH_NAME"
if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
echo "Branch $BRANCH_NAME already exists. Skipping creation."
exit 0
fi
MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

172
.github/workflows/triage_reconsider.yml vendored
View file

@ -1,172 +0,0 @@
name: Agent Shin — reconsider
# Comment-trigger workflow: when the PR/issue author (or an internal
# collaborator) comments `@agent-shin reconsider` on a CLOSED PR/issue,
# Agent Shin re-runs LLM-judge triage on the current title+body and:
#
# - on PASS: posts a "re-evaluated and reopened" comment + reopens.
# - on FAIL: posts a "still missing X" comment and leaves it closed,
# so the contributor can iterate again.
#
# This exists because GitHub does NOT let an external (non-write-access)
# OSS contributor reopen a PR/issue closed by a bot or maintainer. Without
# this comment trigger, a contributor whose PR Agent Shin auto-closed
# would have no path back into the review queue except opening a fresh PR
# (which loses the original PR's history). The bot, on the other hand,
# has write access via GH_TOKEN and can reopen on their behalf.
#
# DRY-RUN BY DEFAULT — gated on `vars.AGENT_SHIN_ENABLED == 'true'` just
# like the other Agent Shin workflows. The workflow also gates on the
# commenter being either the PR/issue author or an internal collaborator
# (OWNER/MEMBER/COLLABORATOR) so random commenters cannot DOS the LLM
# judge or force a reopen.
on:
issue_comment:
types: [created]
permissions:
contents: read
issues: write
pull-requests: write
jobs:
reconsider:
if: |
github.repository == 'BerriAI/litellm'
&& contains(github.event.comment.body, '@agent-shin reconsider')
runs-on: ubuntu-latest
steps:
- name: Authorize commenter
# Only the PR/issue author OR an internal collaborator may trigger
# a reconsider. Outside random commenters could otherwise spam the
# phrase to burn LLM budget or, if a fail-open bug were ever
# introduced, force a reopen on someone else's behalf.
#
# We expose the authorization decision as a step output and gate
# every subsequent (potentially destructive) step on it. A `run:`
# step with `exit 0` would NOT stop the job — only `if:` gating
# on a known-true output is safe here.
id: auth
env:
COMMENTER: ${{ github.event.comment.user.login }}
AUTHOR: ${{ github.event.issue.user.login }}
ASSOCIATION: ${{ github.event.comment.author_association }}
run: |
set -euo pipefail
if [ "${COMMENTER}" = "${AUTHOR}" ]; then
echo "::notice::Authorized: commenter is the PR/issue author."
echo "authorized=true" >> "$GITHUB_OUTPUT"
exit 0
fi
case "${ASSOCIATION}" in
OWNER|MEMBER|COLLABORATOR)
echo "::notice::Authorized: commenter is an internal collaborator (${ASSOCIATION})."
echo "authorized=true" >> "$GITHUB_OUTPUT"
;;
*)
echo "::notice::Commenter '${COMMENTER}' (${ASSOCIATION}) is not authorized to trigger reconsider; skipping subsequent steps."
echo "authorized=false" >> "$GITHUB_OUTPUT"
;;
esac
- name: React 👀 to acknowledge the reconsider
# Add an eyes reaction to the triggering comment the moment we accept
# it, so the contributor gets instant feedback that the bot saw their
# `@agent-shin reconsider` before the slower triage steps run. Gated on
# AGENT_SHIN_ENABLED so dry-run leaves no visible trace. Best-effort:
# a reactions API hiccup must never fail the actual reconsider.
if: steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMENT_ID: ${{ github.event.comment.id }}
run: |
set -euo pipefail
gh api --method POST \
-H "Accept: application/vnd.github+json" \
"repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \
-f content=eyes \
|| echo "::warning::failed to add 👀 reaction (non-fatal)"
- name: Checkout triage script
if: steps.auth.outputs.authorized == 'true'
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
if: steps.auth.outputs.authorized == 'true'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
if: steps.auth.outputs.authorized == 'true'
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run Agent Shin reconsider
if: steps.auth.outputs.authorized == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Only expose the LLM key when the bot is enabled, so a PR/issue
# author can't force paid LLM calls by spamming `@agent-shin
# reconsider` while the bot is still in dry-run. The Python script
# calls the LLM whenever this var is set (regardless of `--close`);
# stripping `--close` doesn't suppress the API call, only the
# destructive side effects. Mirror the gating used by every other
# Agent Shin workflow (triage_pr_with_llm.yml, review_gate.yml, ...).
OPENAI_API_KEY: ${{ vars.AGENT_SHIN_ENABLED == 'true' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
# `issue_comment` events fire for both issues and PR comments.
# `issue.pull_request` is set iff this is a PR comment, so we use
# its presence to decide whether to invoke `--pr N` or `--issue N`.
IS_PR: ${{ github.event.issue.pull_request != null }}
NUMBER: ${{ github.event.issue.number }}
run: |
set -euo pipefail
if [ "${IS_PR}" = "true" ]; then
ARGS=(--repo "${{ github.repository }}" --pr "${NUMBER}" --reconsider)
else
ARGS=(--repo "${{ github.repository }}" --issue "${NUMBER}" --reconsider)
fi
# Reconsider's destructive actions (post comment + reopen) are
# gated on `--close`, mirroring the regular triage workflows.
# When AGENT_SHIN_ENABLED is not the EXACT string "true", we
# still run the script so its verdict + would-X action lands in
# the step summary for QA — but without `--close`, the script
# returns `would-reopen` / `would-reconsider-still-failing`
# instead of touching GitHub state.
#
# Use the positive `= "true"` gate (not `!= "true" -> exit`) so
# the workflow guardrails in
# tests/test_litellm/test_github_triage_workflows.py see the
# canonical fail-safe enable pattern. Unknown values like
# "True", "yes", "1", or typos fall through to the dry-run
# branch, which is the safe default.
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
ARGS+=(--close)
echo "::notice::Agent Shin reconsider ENABLED — running real triage (close=true)."
else
echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> reconsider stays in dry-run (no comment, no reopen)."
fi
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"
- name: React 👍 when the reconsider finishes
# Once the reconsider run has completed successfully, add a thumbs-up so
# the contributor sees the bot is done (the 👀 stays, signalling
# seen -> handled). `success()` keeps this from firing if the run
# errored, and the AGENT_SHIN_ENABLED gate keeps dry-run inert.
if: success() && steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMENT_ID: ${{ github.event.comment.id }}
run: |
set -euo pipefail
gh api --method POST \
-H "Accept: application/vnd.github+json" \
"repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \
-f content=+1 \
|| echo "::warning::failed to add 👍 reaction (non-fatal)"

856
tests/test_litellm/test_github_close_low_quality_prs.py
View file

@ -1,856 +0,0 @@
"""Unit tests for `.github/scripts/close_low_quality_prs.py`.
These exercise the pure logic (score extraction and per-PR evaluation) without
hitting GitHub. Network/CLI calls are stubbed via monkeypatch.
"""
from __future__ import annotations
import datetime as dt
import importlib.util
import sys
from pathlib import Path
import pytest
SCRIPT_PATH = (
Path(__file__).resolve().parents[2]
/ ".github"
/ "scripts"
/ "close_low_quality_prs.py"
)
@pytest.fixture(scope="module")
def closer_module():
"""Load the script as a module via its file path (it lives outside the package)."""
spec = importlib.util.spec_from_file_location("close_low_quality_prs", SCRIPT_PATH)
assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}"
module = importlib.util.module_from_spec(spec)
sys.modules["close_low_quality_prs"] = module
spec.loader.exec_module(module)
return module
def _greptile_comment(
body: str,
updated_at: str = "2026-05-10T00:00:00Z",
login: str = "greptile-apps[bot]",
) -> dict:
return {
"user": {"login": login},
"body": body,
"created_at": updated_at,
"updated_at": updated_at,
}
class TestExtractGreptileScore:
def test_should_extract_score_from_html_header(self, closer_module):
comments = [
_greptile_comment("<h3>Confidence Score: 3/5</h3>\nSome body text.")
]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 3
def test_should_accept_both_greptile_login_variants(self, closer_module):
# REST API form ("greptile-apps[bot]") and GraphQL form ("greptile-apps")
for login in ("greptile-apps", "greptile-apps[bot]"):
comments = [
_greptile_comment("<h3>Confidence Score: 2/5</h3>", login=login)
]
result = closer_module.extract_greptile_score(comments)
assert result is not None, f"failed to detect score for login={login}"
score, _ = result
assert score == 2
def test_should_extract_score_from_plain_text(self, closer_module):
comments = [_greptile_comment("Confidence Score: 5/5 — looks good!")]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 5
def test_should_tolerate_whitespace_and_case(self, closer_module):
comments = [_greptile_comment("**confidence score : 2 / 5**")]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 2
def test_should_pick_most_recent_comment_when_rereview_happens(self, closer_module):
comments = [
_greptile_comment(
"Confidence Score: 2/5", updated_at="2026-05-01T00:00:00Z"
),
_greptile_comment(
"Confidence Score: 5/5", updated_at="2026-05-12T00:00:00Z"
),
]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 5
def test_should_ignore_non_greptile_authors(self, closer_module):
comments = [
{
"user": {"login": "some-human"},
"body": "Confidence Score: 1/5",
"created_at": "2026-05-12T00:00:00Z",
"updated_at": "2026-05-12T00:00:00Z",
}
]
assert closer_module.extract_greptile_score(comments) is None
def test_should_return_none_when_no_score_present(self, closer_module):
comments = [_greptile_comment("Greptile summary without a score.")]
assert closer_module.extract_greptile_score(comments) is None
def test_should_return_none_for_empty_comments(self, closer_module):
assert closer_module.extract_greptile_score([]) is None
class TestEvaluatePr:
@pytest.fixture(autouse=True)
def _now(self):
return dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc)
def _make_pr(
self,
*,
number: int = 1,
created_days_ago: int = 10,
is_draft: bool = False,
labels: list[str] | None = None,
author_login: str = "mateo-berri",
) -> dict:
created = dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc) - dt.timedelta(
days=created_days_ago
)
return {
"number": number,
"title": f"PR #{number}",
"createdAt": created.isoformat().replace("+00:00", "Z"),
"isDraft": is_draft,
"labels": [{"name": lbl} for lbl in (labels or [])],
"author": {"login": author_login},
"url": f"https://example.com/pr/{number}",
}
@pytest.fixture(autouse=True)
def _external_author(self, closer_module, monkeypatch):
"""Treat every test PR as external unless overridden."""
monkeypatch.setattr(
closer_module, "is_external_pr_author", lambda pr, repo: True
)
def test_should_warn_drafts_when_score_low_first_time(
self, closer_module, _now, monkeypatch
):
# Drafts are NOT a free pass — the open-PR queue should reflect any
# PR that needs human attention regardless of draft status. Authors
# who need a long-lived draft can use the `wip` opt-out label.
# First run: warn the contributor (1-day grace), don't close yet.
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 2/5")],
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(is_draft=True, created_days_ago=0),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "warn-grace"
assert score == 2 and age == 0
def test_should_warn_brand_new_pr_when_min_age_zero(
self, closer_module, _now, monkeypatch
):
# `min_age_days=0` means no age filter — a freshly-opened PR is
# eligible the moment Greptile scores it below threshold. The
# first detection still goes through the warn-grace step rather
# than closing immediately, giving the contributor 2 hours to
# respond before the next run actually closes the PR.
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 1/5")],
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=0),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "warn-grace"
assert score == 1 and age == 0
def test_should_skip_optout_label_case_insensitive(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("should not fetch comments for opt-outs"),
)
action, _, _ = closer_module.evaluate_pr(
self._make_pr(labels=["WIP"]),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels={"wip"},
)
assert action == "skip-optout-label"
def test_should_skip_too_young_when_min_age_set(
self, closer_module, _now, monkeypatch
):
# The min-age-days flag is now opt-in (default 0). When a maintainer
# explicitly passes a positive value (e.g. for a backfill run that
# wants to spare brand-new PRs), the skip-too-young path still works.
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("should not fetch comments for young PRs"),
)
action, _, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=2),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-too-young"
assert age == 2
def test_should_not_skip_when_min_age_is_zero(
self, closer_module, _now, monkeypatch
):
# With the new default min_age_days=0, even a 0-day-old PR is
# evaluated. This test pins that behavior so future refactors don't
# silently restore an age filter.
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 5/5")],
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=0),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-score-ok"
assert score == 5 and age == 0
def test_should_skip_when_greptile_has_not_reviewed(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(closer_module, "fetch_pr_comments", lambda *a, **kw: [])
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=10),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-no-greptile-score"
assert score is None and age == 10
def test_should_skip_when_score_meets_threshold(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 4/5")],
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=10),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-score-ok"
assert score == 4 and age == 10
def test_should_warn_when_old_and_low_score_no_prior_warning(
self, closer_module, _now, monkeypatch
):
# Even an old PR that still has no grace warning gets one on the
# first eligible run — the daily cron is the natural cadence, so
# an existing-but-never-warned PR enters the grace flow normally.
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 3/5")],
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=10),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "warn-grace"
assert score == 3 and age == 10
def test_should_close_when_grace_warning_aged_out_and_score_still_low(
self, closer_module, _now, monkeypatch
):
# Day-1 the closer posted a warning. Day-2 the PR still scores <4
# AND the warning is older than `GRACE_PERIOD_SECONDS`, so the
# action flips to `close`. This is the "grace expired" path.
old_warning = {
"user": {"login": "github-actions[bot]"},
"body": (
"you have 2 hours to fix this\n\n" + closer_module.GRACE_COMMENT_MARKER
),
"created_at": (
_now - dt.timedelta(seconds=closer_module.GRACE_PERIOD_SECONDS + 60)
)
.isoformat()
.replace("+00:00", "Z"),
"updated_at": "2026-05-15T00:00:00Z",
}
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [
_greptile_comment(
"<h3>Confidence Score: 1/5</h3>",
updated_at="2026-05-15T00:00:00Z",
),
old_warning,
],
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(created_days_ago=14),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "close"
assert score == 1
def test_should_skip_when_grace_warning_within_window(
self, closer_module, _now, monkeypatch
):
# Within the 2-hour grace window the closer must NOT close the
# PR even if the score is still low. The warning is only an hour
# old; give the contributor time to push fixes before destruction.
recent_warning = {
"user": {"login": "github-actions[bot]"},
"body": "warning text\n\n" + closer_module.GRACE_COMMENT_MARKER,
"created_at": (_now - dt.timedelta(hours=1))
.isoformat()
.replace("+00:00", "Z"),
}
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [
_greptile_comment("Confidence Score: 2/5"),
recent_warning,
],
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(created_days_ago=10),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-in-grace-period"
assert score == 2
def test_should_warn_grace_for_swiftwinds_not_close_immediately(
self, closer_module, _now, monkeypatch
):
# Regression: SwiftWinds (the dogfood account) used to be in a
# now-removed `IMMEDIATE_CLOSE_LOGINS` bypass that closed on first
# detection. It must now follow the SAME grace path as every other
# external author: warn first, close only after the window elapses.
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 1/5")],
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(created_days_ago=0, author_login="SwiftWinds"),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "warn-grace"
assert score == 1
def test_should_skip_internal_authors(self, closer_module, _now, monkeypatch):
# Override the fixture for this one test.
monkeypatch.setattr(
closer_module, "is_external_pr_author", lambda pr, repo: False
)
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("should not fetch comments for internal"),
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(created_days_ago=14, author_login="krrishdholakia"),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
allowlist=frozenset(),
)
assert action == "skip-internal"
assert score is None
class TestMainOptoutLabelDefault:
"""`--optout-label` must REPLACE the canonical defaults, not append."""
def _patch_no_op(self, closer_module, monkeypatch):
monkeypatch.setattr(closer_module, "fetch_open_prs", lambda repo: [])
# `optout_labels` is captured indirectly via evaluate_pr; sniff the
# set passed in by stubbing evaluate_pr.
captured: dict = {}
def fake_evaluate(pr, now, min_age_days, min_score, repo, optout_labels):
captured["optout_labels"] = set(optout_labels)
return ("skip-internal", None, None)
monkeypatch.setattr(closer_module, "evaluate_pr", fake_evaluate)
return captured
def test_should_use_canonical_defaults_when_flag_omitted(
self, closer_module, monkeypatch
):
captured = self._patch_no_op(closer_module, monkeypatch)
# No PRs -> capture won't fire; instead inject one synthetic PR via
# fetch_open_prs so evaluate_pr is invoked at least once.
monkeypatch.setattr(
closer_module,
"fetch_open_prs",
lambda repo: [
{
"number": 1,
"title": "p",
"createdAt": "2026-05-10T00:00:00Z",
"isDraft": True,
"labels": [],
"author": {"login": "x"},
}
],
)
monkeypatch.setattr(sys, "argv", ["close_low_quality_prs.py"])
rc = closer_module.main()
assert rc == 0
assert captured["optout_labels"] == set(closer_module.DEFAULT_OPTOUT_LABELS)
def test_should_replace_defaults_when_flag_provided(
self, closer_module, monkeypatch
):
captured = self._patch_no_op(closer_module, monkeypatch)
monkeypatch.setattr(
closer_module,
"fetch_open_prs",
lambda repo: [
{
"number": 1,
"title": "p",
"createdAt": "2026-05-10T00:00:00Z",
"isDraft": True,
"labels": [],
"author": {"login": "x"},
}
],
)
monkeypatch.setattr(
sys,
"argv",
[
"close_low_quality_prs.py",
"--optout-label",
"hold",
"--optout-label",
"needs-discussion",
],
)
rc = closer_module.main()
assert rc == 0
# Crucially, none of the canonical defaults leak in.
assert captured["optout_labels"] == {"hold", "needs-discussion"}
for default in closer_module.DEFAULT_OPTOUT_LABELS:
assert default not in captured["optout_labels"], default
class TestSecondsSinceLastGraceWarning:
"""Grace-period detection: only counts comments by the bot identity
that contain the shared `GRACE_COMMENT_MARKER`."""
def _make_marker_comment(
self,
closer_module,
*,
login: str = "github-actions[bot]",
created_at: str = "2026-05-16T00:00:00Z",
include_marker: bool = True,
) -> dict:
body = "warning text"
if include_marker:
body += "\n\n" + closer_module.GRACE_COMMENT_MARKER
return {
"user": {"login": login},
"body": body,
"created_at": created_at,
}
def test_should_return_none_when_no_marker_comment(self, closer_module):
comments = [
{
"user": {"login": "github-actions[bot]"},
"body": "Some other bot comment",
"created_at": "2026-05-16T00:00:00Z",
}
]
assert closer_module.seconds_since_last_grace_warning(comments) is None
def test_should_return_none_for_empty(self, closer_module):
assert closer_module.seconds_since_last_grace_warning([]) is None
def test_should_ignore_non_bot_comments_with_marker(self, closer_module):
# If a curious user quotes the marker in a comment, we must NOT
# treat it as a bot warning. The grace timer would then never fire.
comments = [
self._make_marker_comment(closer_module, login="random-user"),
]
assert closer_module.seconds_since_last_grace_warning(comments) is None
def test_should_pick_latest_marker_comment(self, closer_module):
# When multiple grace warnings exist (e.g. a re-open cycle), use
# the most recent one to compute the age.
comments = [
self._make_marker_comment(closer_module, created_at="2026-05-15T00:00:00Z"),
self._make_marker_comment(closer_module, created_at="2026-05-16T23:00:00Z"),
]
now = dt.datetime(2026, 5, 17, 0, 0, 0, tzinfo=dt.timezone.utc)
age = closer_module.seconds_since_last_grace_warning(comments, now=now)
# 1h = 3600s
assert age == 3600.0
class TestGraceWarningCommentText:
"""Pin the user-facing language in the grace warning comment so the
grace-window and `@greptileai still works after close` promises
don't get accidentally dropped in a future refactor.
"""
def test_should_state_grace_window(self, closer_module):
body = closer_module.format_grace_warning_comment(score=2, threshold=4)
# The user's PR explicitly said "specify in the comment" — pin
# that the grace window appears in the comment.
assert "2 hours" in body
def test_should_mention_agent_shin_reconsider(self, closer_module):
body = closer_module.format_grace_warning_comment(score=2, threshold=4)
assert "@agent-shin reconsider" in body
def test_should_promise_greptileai_works_after_close(self, closer_module):
body = closer_module.format_grace_warning_comment(score=2, threshold=4)
assert "@greptileai" in body
assert "even after the PR is closed" in body
def test_should_carry_grace_marker(self, closer_module):
# The marker is what `seconds_since_last_grace_warning` greps for
# to detect a prior warning — dropping it would silently break
# the cooldown.
body = closer_module.format_grace_warning_comment(score=2, threshold=4)
assert closer_module.GRACE_COMMENT_MARKER in body
def test_close_comment_should_mention_greptileai_post_close(self, closer_module):
# The close comment should ALSO point at the @greptileai post-close
# re-review path so contributors see the same options whether they
# read the warning or only catch the close comment.
body = closer_module.format_close_comment(score=2, threshold=4)
assert "@greptileai" in body
assert "even after the PR is closed" in body
def test_close_comment_should_advertise_reconsider(self, closer_module):
body = closer_module.format_close_comment(score=2, threshold=4)
assert "@agent-shin reconsider" in body
def test_close_comment_should_carry_agent_shin_close_marker(self, closer_module):
# The close comment advertises `@agent-shin reconsider`, and the
# reconsider reopen guard (`was_closed_by_agent_shin`) only treats a
# PR as Agent-Shin-closed when the close comment carries this marker.
# Dropping it silently breaks the advertised recovery path for every
# PR closed by this daily sweep.
body = closer_module.format_close_comment(score=2, threshold=4)
assert closer_module.AGENT_SHIN_CLOSE_MARKER in body
def test_close_comment_should_state_score_and_threshold(self, closer_module):
body = closer_module.format_close_comment(score=1, threshold=4)
assert "1/5" in body
assert "4/5" in body
class TestHasOptoutLabel:
def test_should_match_label_case_insensitively(self, closer_module):
pr = {"labels": [{"name": "Do Not Close"}, {"name": "bug"}]}
assert closer_module.has_optout_label(pr, {"do not close"}) is True
def test_should_return_false_when_no_match(self, closer_module):
pr = {"labels": [{"name": "bug"}, {"name": "enhancement"}]}
assert closer_module.has_optout_label(pr, {"wip", "keep open"}) is False
def test_should_handle_missing_labels(self, closer_module):
assert closer_module.has_optout_label({}, {"wip"}) is False
class TestListOpenItemsNoCap:
"""The bulk sweeps must fetch the ENTIRE open backlog.
Regression guard for the old hard-coded ``--limit 1000``: gh lists
newest-first, so a low cap silently dropped the *oldest* PRs/issues —
exactly the stale ones a low-quality sweep exists to catch.
"""
@staticmethod
def _shared(closer_module):
# `closer_module` loading puts `.github/scripts` on sys.path and
# imports agent_shin_shared, so it's already in sys.modules.
import agent_shin_shared
return agent_shin_shared
def _capture_gh_args(self, closer_module, monkeypatch, *, returns="[]"):
shared = self._shared(closer_module)
captured: dict = {}
def fake_gh(*args):
captured["args"] = args
return returns
# `list_open_items` looks up `gh` in agent_shin_shared's namespace.
monkeypatch.setattr(shared, "gh", fake_gh)
return shared, captured
def test_list_open_items_passes_no_cap_limit_not_1000(
self, closer_module, monkeypatch
):
shared, captured = self._capture_gh_args(closer_module, monkeypatch)
shared.list_open_items("pr", repo="o/r", fields="number,title")
args = captured["args"]
assert "--limit" in args
limit_value = args[args.index("--limit") + 1]
assert limit_value == str(shared.GH_LIST_ALL_LIMIT)
assert limit_value != "1000"
# A meaningful ceiling: comfortably above any realistic open backlog.
assert shared.GH_LIST_ALL_LIMIT >= 100_000
def test_list_open_items_uses_dedicated_command_state_and_fields(
self, closer_module, monkeypatch
):
shared, captured = self._capture_gh_args(closer_module, monkeypatch)
shared.list_open_items("issue", repo="o/r", fields="number")
args = captured["args"]
assert args[0] == "issue" and args[1] == "list"
assert args[args.index("--state") + 1] == "open"
assert args[args.index("--json") + 1] == "number"
assert tuple(args[-2:]) == ("--repo", "o/r")
def test_list_open_items_omits_repo_when_none(self, closer_module, monkeypatch):
shared, captured = self._capture_gh_args(closer_module, monkeypatch)
shared.list_open_items("pr", repo=None, fields="number")
assert "--repo" not in captured["args"]
def test_list_open_items_parses_json_array(self, closer_module, monkeypatch):
shared, _ = self._capture_gh_args(
closer_module, monkeypatch, returns='[{"number": 1}, {"number": 2}]'
)
items = shared.list_open_items("pr", repo=None, fields="number")
assert [i["number"] for i in items] == [1, 2]
def test_list_open_items_rejects_unknown_kind(self, closer_module):
shared = self._shared(closer_module)
with pytest.raises(ValueError, match="kind must be 'pr' or 'issue', got 'both"):
shared.list_open_items("both", repo="o/r", fields="number")
def test_fetch_open_prs_delegates_with_no_cap(self, closer_module, monkeypatch):
shared, captured = self._capture_gh_args(closer_module, monkeypatch)
closer_module.fetch_open_prs("o/r")
args = captured["args"]
assert args[0] == "pr"
assert args[args.index("--limit") + 1] == str(shared.GH_LIST_ALL_LIMIT)
# Still requests every field downstream evaluate_pr / labels logic needs.
assert "createdAt" in args[args.index("--json") + 1]
class TestEvaluatePrAllowlist:
"""While the dogfood allowlist is active `evaluate_pr` only acts on the
named accounts and bypasses the external-only restriction for them.
Emptying it restores the internal-author skip."""
@pytest.fixture(autouse=True)
def _now(self):
return dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc)
def _make_pr(self, *, author_login: str, created_days_ago: int = 10) -> dict:
created = dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc) - dt.timedelta(
days=created_days_ago
)
return {
"number": 1,
"title": "PR #1",
"createdAt": created.isoformat().replace("+00:00", "Z"),
"isDraft": False,
"labels": [],
"author": {"login": author_login},
"url": "https://example.com/pr/1",
}
def test_should_skip_author_not_on_allowlist(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("must not fetch comments for non-allowlisted"),
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(author_login="random-oss-dev"),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-not-allowlisted"
assert score is None
def test_should_act_on_allowlisted_internal_author(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module, "is_external_pr_author", lambda pr, repo: False
)
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 2/5")],
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(author_login="mateo-berri", created_days_ago=0),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "warn-grace"
assert score == 2
def test_empty_allowlist_restores_internal_skip(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module, "is_external_pr_author", lambda pr, repo: False
)
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("must not fetch comments for internal"),
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(author_login="krrishdholakia"),
now=_now,
min_age_days=0,
min_score=4,
repo=None,
optout_labels=set(),
allowlist=frozenset(),
)
assert action == "skip-internal"
def test_allowlist_constant_is_the_two_dogfood_accounts(self, closer_module):
assert closer_module.ALLOWLIST_LOGINS == frozenset(
{"mateo-berri", "swiftwinds"}
)
class TestDryRunGateOnClose:
"""Regression: the daily sweep is dry-run unless `--close` is passed
(the workflow only adds it when `AGENT_SHIN_ENABLED=true`). A closeable
PR (low score, grace window elapsed) must be DETECTED and reported as
"would close", but the dry run must never make a real GitHub mutation,
so merging Agent Shin stays inert by default."""
def _closeable_pr(self) -> dict:
return {
"number": 7,
"title": "thin PR",
"createdAt": "2026-05-10T00:00:00Z",
"isDraft": False,
"labels": [],
"author": {"login": "SwiftWinds"},
"url": "https://example.com/pr/7",
}
def test_dry_run_sweep_detects_but_does_not_close(
self, closer_module, monkeypatch, capsys
):
aged_out_warning = {
"user": {"login": "github-actions[bot]"},
"body": "warned\n\n" + closer_module.GRACE_COMMENT_MARKER,
# Far enough in the past that it's aged out regardless of
# GRACE_PERIOD_SECONDS, since main() pins `now` to real time.
"created_at": "2020-01-01T00:00:00Z",
}
monkeypatch.setattr(
closer_module, "fetch_open_prs", lambda repo: [self._closeable_pr()]
)
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [
_greptile_comment("Confidence Score: 1/5"),
aged_out_warning,
],
)
# Any real GitHub mutation during a dry run is the bug under test.
monkeypatch.setattr(
closer_module,
"gh",
lambda *a, **kw: pytest.fail(f"dry run must not call gh: {a}"),
)
monkeypatch.setattr(sys, "argv", ["close_low_quality_prs.py"])
rc = closer_module.main()
assert rc == 0
# The PR is detected as closeable, just not acted on.
assert "Total would close: 1" in capsys.readouterr().out

524
tests/test_litellm/test_github_review_gate.py
View file

@ -1,524 +0,0 @@
"""Unit tests for the `ready for review` label lifecycle (Agent Shin review gate).
Exercises `triage_with_llm.review_gate`, the state machine that keeps the
`ready for review` label in sync with whether a PR clears both the LLM rubric
and Greptile's confidence score:
* pass (untagged) -> add label + "ready for review" comment
* pass (untagged, recovered) -> add label + "all clear again" comment
* pass (already tagged) -> noop
* regress (tagged) -> remove label + "what's missing" comment, stays open
* fail (untagged, within 24h)-> one-time "what's missing" notice
* fail (untagged, >24h) -> close + comment
* dry run (close=False) -> would-* previews, no side effects
"""
from __future__ import annotations
import datetime as dt
import importlib.util
import sys
from pathlib import Path
import pytest
SCRIPT_PATH = (
Path(__file__).resolve().parents[2] / ".github" / "scripts" / "triage_with_llm.py"
)
NOW = dt.datetime(2026, 5, 24, 12, 0, 0, tzinfo=dt.timezone.utc)
JUST_NOW = "2026-05-24T11:00:00Z" # 1h old -> within 24h grace
TWO_DAYS_AGO = "2026-05-22T11:00:00Z" # >24h old -> past grace
@pytest.fixture(scope="module")
def triage_module():
spec = importlib.util.spec_from_file_location("triage_with_llm", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules["triage_with_llm"] = module
spec.loader.exec_module(module)
return module
class _Recorder:
"""Captures every gh mutation review_gate could fire, and fails loudly
on the ones a given scenario forbids."""
def __init__(self, triage_module, monkeypatch):
self.comments: list[str] = []
self.added: list[str] = []
self.removed: list[str] = []
self.closed: list[int] = []
monkeypatch.setattr(
triage_module,
"post_comment",
lambda repo, n, body: self.comments.append(body),
)
monkeypatch.setattr(
triage_module,
"add_label",
lambda repo, n, label: self.added.append(label),
)
monkeypatch.setattr(
triage_module,
"remove_label",
lambda repo, n, label: self.removed.append(label),
)
monkeypatch.setattr(
triage_module,
"close_pr",
lambda repo, n: self.closed.append(n),
)
def _make_pr(**overrides):
base = {
"number": 7,
"title": "feat: do a thing",
"body": "some body without a linked issue or QA proof",
"state": "open",
"author_association": "NONE",
"user": {"login": "mateo-berri"},
"labels": [],
"created_at": JUST_NOW,
}
base.update(overrides)
return base
def _pass(prompt):
return '{"verdict": "pass", "missing": [], "explanation": "looks good"}'
def _fail(prompt):
return (
'{"verdict": "fail", "missing": ["QA proof", "expected vs. actual"],'
' "explanation": "thin description"}'
)
def _gate(triage_module, **kwargs):
"""Call review_gate with safe defaults for the injectable hooks."""
params = dict(
repo="o/r",
number=7,
close=True,
model="m",
judge=_pass,
greptile_score=None,
comments=[],
now=NOW,
)
params.update(kwargs)
return triage_module.review_gate(**params)
class TestReviewGatePass:
def test_pass_untagged_adds_label_and_ready_comment(
self, triage_module, monkeypatch
):
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr())
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
assert rec.removed == [] and rec.closed == []
assert len(rec.comments) == 1
assert "ready for review" in rec.comments[0].lower()
assert triage_module.READY_MARKER in rec.comments[0]
assert "5/5" in rec.comments[0]
def test_pass_already_tagged_is_noop(self, triage_module, monkeypatch):
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "noop-passing"
assert rec.added == [] and rec.removed == [] and rec.comments == []
def test_pass_after_prior_regression_uses_all_clear_wording(
self, triage_module, monkeypatch
):
# A regression marker in history -> this is a recovery, not a first pass.
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr())
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.REGRESSED_MARKER,
}
]
result = _gate(triage_module, judge=_pass, greptile_score=5, comments=prior)
assert result["action"] == "labeled-ready"
assert "all clear" in rec.comments[0].lower()
def test_linked_issue_passes_without_calling_judge(
self, triage_module, monkeypatch
):
pr = _make_pr(body="Fixes #4321\n\nbody")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("LLM must not be called for linked issue"),
greptile_score=5,
)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
class TestReviewGateRegression:
def test_regression_removes_label_and_keeps_pr_open(
self, triage_module, monkeypatch
):
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=5)
assert result["action"] == "label-removed-regressed"
assert rec.removed == [triage_module.READY_FOR_REVIEW_LABEL]
assert rec.closed == [] # regression NEVER closes the PR
assert triage_module.REGRESSED_MARKER in rec.comments[0]
assert "QA proof" in rec.comments[0]
# The state machine closes a still-failing PR `grace_days` after this
# notice (default 24h); the comment must disclose that deadline rather
# than implying the PR stays open indefinitely.
assert "24 hours" in rec.comments[0]
assert "auto-closed" in rec.comments[0]
def test_regression_comment_discloses_grace_deadline(self, triage_module):
one_day = triage_module.format_regression_comment(
["QA proof"], "needs work", grace_days=1
)
assert "24 hours" in one_day
assert "auto-closed" in one_day
three_days = triage_module.format_regression_comment(
["QA proof"], "needs work", grace_days=3
)
assert "3 days" in three_days
assert "auto-closed" in three_days
def test_greptile_drop_alone_triggers_regression(self, triage_module, monkeypatch):
# Rubric still passes, but Greptile fell to 2/5 -> not passing.
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=2)
assert result["action"] == "label-removed-regressed"
assert rec.removed == [triage_module.READY_FOR_REVIEW_LABEL]
assert "2/5" in rec.comments[0]
def test_greptile_score_read_from_comments_when_not_injected(
self, triage_module, monkeypatch
):
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
greptile = [
{
"user": {"login": "greptile-apps[bot]"},
"body": "Confidence Score: 2/5",
"created_at": "2026-05-24T10:00:00Z",
}
]
result = _gate(
triage_module,
judge=_pass,
greptile_score=triage_module._UNSET,
comments=greptile,
)
assert result["action"] == "label-removed-regressed"
assert "2/5" in rec.comments[0]
class TestReviewGateGraceAndClose:
def test_within_grace_posts_one_time_notice(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW)
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=None)
assert result["action"] == "within-grace-notified"
assert rec.closed == [] and rec.added == [] and rec.removed == []
assert triage_module.WITHIN_GRACE_MARKER in rec.comments[0]
assert "QA proof" in rec.comments[0]
def test_within_grace_does_not_double_notify(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW)
)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.WITHIN_GRACE_MARKER,
}
]
result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior)
assert result["action"] == "within-grace-already-notified"
assert rec.comments == []
def test_past_grace_closes_with_comment(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=None)
assert result["action"] == "closed"
assert rec.closed == [7]
assert len(rec.comments) == 1
# The close comment must carry the reconsider provenance marker so
# `was_closed_by_agent_shin` can later recognize this as an Agent Shin
# close (and not some other workflow's `github-actions[bot]` close).
assert triage_module.AGENT_SHIN_CLOSE_MARKER in rec.comments[0]
def test_recent_regression_marker_blocks_close(self, triage_module, monkeypatch):
"""A failing PR with a fresh regression notice must NOT be closed —
the contributor needs a window to address the regression."""
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.REGRESSED_MARKER,
# Posted just an hour before NOW -> well inside grace_days.
"created_at": "2026-05-24T11:00:00Z",
}
]
result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior)
assert result["action"] == "regressed-already-notified"
assert rec.closed == [] and rec.comments == []
def test_stale_regression_marker_allows_close(self, triage_module, monkeypatch):
"""Once grace_days have elapsed since the regression notice, the
review gate must let the close path fire — otherwise PRs that were
regressed and then abandoned stay open forever."""
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.REGRESSED_MARKER,
# Posted 30 days before NOW -> well past the default 1-day grace.
"created_at": "2026-04-24T11:00:00Z",
}
]
result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior)
assert result["action"] == "closed"
assert rec.closed == [7]
assert len(rec.comments) == 1
def test_linked_issue_with_greptile_fail_uses_greptile_explanation(
self, triage_module, monkeypatch
):
"""When the rubric short-circuits to pass (linked-issue regex) but
Greptile dragged the PR under the bar, the close comment's
explanation must describe the Greptile shortfall, not the
misleading "LLM was not called" rubric placeholder."""
pr = _make_pr(body="Fixes #4321\n\nbody", created_at=TWO_DAYS_AGO)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("LLM must not be called for linked issue"),
greptile_score=2,
)
assert result["action"] == "closed"
assert len(rec.comments) == 1
body = rec.comments[0]
assert "LLM was not called" not in body
assert "Greptile" in body and "2/5" in body
class TestReviewGateDryRun:
@pytest.mark.parametrize(
"scenario,labels,judge,score,created,expected",
[
("pass", [], _pass, 5, JUST_NOW, "would-label-ready"),
(
"regress",
[{"name": "ready for review"}],
_fail,
5,
JUST_NOW,
"would-remove-label",
),
("within-grace", [], _fail, None, JUST_NOW, "would-notify-within-grace"),
("past-grace", [], _fail, None, TWO_DAYS_AGO, "would-close"),
],
)
def test_dry_run_previews_without_side_effects(
self,
triage_module,
monkeypatch,
scenario,
labels,
judge,
score,
created,
expected,
):
pr = _make_pr(labels=labels, created_at=created)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, close=False, judge=judge, greptile_score=score)
assert result["action"] == expected
# Dry run touches nothing.
assert rec.added == [] and rec.removed == [] and rec.closed == []
assert rec.comments == []
assert "comment" in result # preview body still surfaced
class TestReviewGateGuards:
def test_skips_internal_author(self, triage_module, monkeypatch):
pr = _make_pr(author_association="MEMBER", user={"login": "krrish"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("no LLM for internal"),
allowlist=frozenset(),
)
assert result["action"] == "skip-internal-author"
def test_skips_closed_pr(self, triage_module, monkeypatch):
pr = _make_pr(state="closed")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = _gate(triage_module, judge=lambda p: pytest.fail("no LLM for closed"))
assert result["action"] == "skip-not-open"
def test_llm_error_is_non_destructive(self, triage_module, monkeypatch):
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr())
rec = _Recorder(triage_module, monkeypatch)
def boom(prompt):
raise RuntimeError("api down")
result = _gate(triage_module, judge=boom, greptile_score=None)
assert result["action"] == "skip-llm-error"
assert rec.closed == [] and rec.added == [] and rec.removed == []
def test_full_recovery_cycle(self, triage_module, monkeypatch):
"""pass -> regress -> recover, threading labels/comments like GitHub would."""
state = {"labels": [], "comments": []}
def fake_fetch(repo, n):
return _make_pr(labels=list(state["labels"]), created_at=JUST_NOW)
monkeypatch.setattr(triage_module, "fetch_pr", fake_fetch)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda repo, n, body: state["comments"].append(
{"user": {"login": "github-actions[bot]"}, "body": body}
),
)
monkeypatch.setattr(
triage_module,
"add_label",
lambda repo, n, label: state["labels"].append({"name": label}),
)
monkeypatch.setattr(
triage_module,
"remove_label",
lambda repo, n, label: state["labels"].clear(),
)
monkeypatch.setattr(
triage_module, "close_pr", lambda repo, n: pytest.fail("must not close")
)
# 1) passes -> tagged
r1 = _gate(
triage_module, judge=_pass, greptile_score=5, comments=state["comments"]
)
assert r1["action"] == "labeled-ready"
assert any(lbl["name"] == "ready for review" for lbl in state["labels"])
# 2) regresses -> tag removed, comment posted, PR still open
r2 = _gate(
triage_module, judge=_fail, greptile_score=2, comments=state["comments"]
)
assert r2["action"] == "label-removed-regressed"
assert state["labels"] == []
# 3) fixed again -> "all clear" + tag back
r3 = _gate(
triage_module, judge=_pass, greptile_score=5, comments=state["comments"]
)
assert r3["action"] == "labeled-ready"
assert any(lbl["name"] == "ready for review" for lbl in state["labels"])
assert "all clear" in state["comments"][-1]["body"].lower()
class TestReviewGateAllowlist:
"""While the dogfood allowlist is active it is the sole author gate:
only the named accounts pass, and for them the internal-author exemption
is bypassed. Emptying it restores the normal internal-author skip."""
def test_should_skip_author_not_on_allowlist(self, triage_module, monkeypatch):
pr = _make_pr(user={"login": "random-oss-dev"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(
triage_module, judge=lambda p: pytest.fail("no LLM for non-allowlisted")
)
assert result["action"] == "skip-not-allowlisted"
assert rec.added == [] and rec.comments == [] and rec.closed == []
def test_should_act_on_allowlisted_internal_author(
self, triage_module, monkeypatch
):
pr = _make_pr(author_association="MEMBER", user={"login": "mateo-berri"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
def test_empty_allowlist_restores_internal_skip(self, triage_module, monkeypatch):
pr = _make_pr(author_association="MEMBER", user={"login": "krrish"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("no LLM for internal"),
allowlist=frozenset(),
)
assert result["action"] == "skip-internal-author"

2134
tests/test_litellm/test_github_triage_with_llm.py
View file

File diff suppressed because it is too large Load diff

264
tests/test_litellm/test_github_triage_workflows.py
View file

@ -1,264 +0,0 @@
"""Static guardrails for the Agent Shin + Greptile workflow YAML files.
These workflows can post comments and close PRs/issues on
BerriAI/litellm, so the gating logic that decides "is this a real
close-on-fail run?" must fail-safe on any unexpected input. The risk
is mostly maintenance: someone edits the bash gate, drops a quote,
inverts a comparison, or uses `!= "false"` (which treats "True",
"yes", "1", and typos as enabling closure) and the regression isn't
caught until a real OSS contributor's PR gets auto-closed.
The tests below pin a set of invariants. The first two apply to every
workflow that gates a destructive `--close`:
1. The gate uses the fail-safe `= "true"` comparison — not `!= "false"`,
not `!= ""`. Only the literal string "true" should ever enable
closure.
2. The gate also requires `AGENT_SHIN_ENABLED = "true"` (or the
scheduled-job equivalent) — disabling the variable must always
force dry-run.
A third invariant covers every workflow that installs the OpenAI client.
These run with a write-scoped `GITHUB_TOKEN`, so a compromised package
release would execute in that context; the install must therefore come
from the hash-pinned `.github/scripts/triage-requirements.txt` via
`pip --require-hashes`, never a floating `pip install openai>=...`.
Static parsing of the YAML + bash text is the right level of test here:
the gating logic lives in a `run:` block, not in a Python module we can
import, and end-to-end testing a GitHub Actions workflow from CI is
infeasible. A YAML-level guardrail is exactly what would have caught
the original `!= "false"` regression at PR time.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows"
# Map of workflow file -> the env var name that drives the destructive
# gate inside that workflow's `run:` block. Keeping this table explicit
# (rather than scraping every workflow file) means a new workflow file
# that bypasses the dry-run gating doesn't silently slip past this test.
DESTRUCTIVE_GATE_ENV: dict[str, str] = {
"close_low_quality_prs.yml": "CLOSE_FLAG",
# The reconsider workflow has no per-run "really do it?" knob — its
# only kill switch is `AGENT_SHIN_ENABLED`, which already serves as
# both the destructive gate and the global enablement gate.
"triage_reconsider.yml": "AGENT_SHIN_ENABLED",
}
# Privileged workflows that install the OpenAI client. They run with a
# write-scoped GITHUB_TOKEN, so the install must be hash-pinned: a poisoned
# release would otherwise execute in that context. A new workflow that
# installs the client must be added here and use the same pinned file.
LLM_CLIENT_INSTALLER_WORKFLOWS = (
"triage_reconsider.yml",
)
PINNED_INSTALL = "--require-hashes -r .github/scripts/triage-requirements.txt"
REQUIREMENTS_FILE = REPO_ROOT / ".github" / "scripts" / "triage-requirements.txt"
def _load_workflow(name: str) -> dict:
return yaml.safe_load((WORKFLOWS_DIR / name).read_text())
def _all_run_blocks(workflow: dict) -> list[str]:
"""Return every `run:` step's command text, joined."""
commands: list[str] = []
jobs = workflow.get("jobs") or {}
for job in jobs.values():
for step in job.get("steps", []) or []:
if not isinstance(step, dict):
continue
run = step.get("run")
if isinstance(run, str):
commands.append(run)
return commands
@pytest.mark.parametrize("workflow_file,env_var", sorted(DESTRUCTIVE_GATE_ENV.items()))
def test_should_use_failsafe_equals_true_comparison(workflow_file: str, env_var: str) -> None:
"""The destructive `--close` gate must use `= "true"` (fail-safe), not
`!= "false"` (which would treat "True", "yes", "1", or any typo as
enabling closure).
Both bare `${ENV_VAR}` and `${ENV_VAR:-false}` (with a default) are
accepted forms — what matters is the comparison operator. The
Greptile closer relies on an outer `AGENT_SHIN_ENABLED` gate so it
can use the bare form; the Agent Shin workflows include `:-false`
for defense in depth. Either is fine.
"""
workflow = _load_workflow(workflow_file)
text = "\n".join(_all_run_blocks(workflow))
assert env_var in text, (
f"{workflow_file} no longer references {env_var}; was the gating env var renamed without updating this test?"
)
accepted_patterns = (
f'"${{{env_var}}}" = "true"',
f'"${{{env_var}:-false}}" = "true"',
)
assert any(p in text for p in accepted_patterns), (
f"{workflow_file} must gate the destructive --close flag on the "
f'EXACT string "true" (one of: {accepted_patterns!r}). Mirror '
'the Greptile closer pattern; do NOT use `!= "false"` which '
'fail-opens on unknown values like "True", "yes", "1", or typos.'
)
forbidden_patterns = (
f'"${{{env_var}}}" != "false"',
f'"${{{env_var}:-false}}" != "false"',
f'"${{{env_var}:-true}}" != "false"',
)
for forbidden in forbidden_patterns:
assert forbidden not in text, (
f"{workflow_file} uses the fail-open pattern {forbidden!r}. "
'Switch to `= "true"` so unknown values stay dry-run.'
)
@pytest.mark.parametrize("workflow_file", sorted(DESTRUCTIVE_GATE_ENV))
def test_should_require_agent_shin_enabled_for_close(workflow_file: str) -> None:
"""Every destructive gate must also gate on the global enablement
variable, so flipping `AGENT_SHIN_ENABLED` off is a kill switch
regardless of any per-run input.
Two patterns are equally fine:
- Positive: `[ "${AGENT_SHIN_ENABLED:-false}" = "true" ]` to enter
the close branch (Agent Shin workflows).
- Negative: `[ "${AGENT_SHIN_ENABLED:-false}" != "true" ]` then
bail out / force dry-run (Greptile closer).
What matters is that the comparison value is the literal "true";
`!= "false"` or `= "1"` etc. would not be a true kill switch.
"""
workflow = _load_workflow(workflow_file)
text = "\n".join(_all_run_blocks(workflow))
accepted_patterns = (
'"${AGENT_SHIN_ENABLED:-false}" = "true"',
'"${AGENT_SHIN_ENABLED:-false}" != "true"',
)
assert any(p in text for p in accepted_patterns), (
f"{workflow_file} must gate destructive actions on "
'`AGENT_SHIN_ENABLED = "true"` (or the inverted `!= "true"` '
"guard that forces dry-run). Without this, an unset repo "
"variable would not be treated as a kill switch."
)
@pytest.mark.parametrize("workflow_file", LLM_CLIENT_INSTALLER_WORKFLOWS)
def test_llm_client_install_is_hash_pinned(workflow_file: str) -> None:
"""Every privileged workflow installs the OpenAI client from the
hash-pinned requirements file, never by floating version.
A bare `pip install "openai>=1.40.0"` resolves to whatever PyPI serves
at run time and executes during install/import while a write-scoped
`GITHUB_TOKEN` is in scope, so a compromised release runs in a
privileged context. This test fails if that floating form comes back or
if the `--require-hashes` install is loosened.
"""
blocks = _all_run_blocks(_load_workflow(workflow_file))
assert PINNED_INSTALL in "\n".join(blocks), (
f"{workflow_file} must install the client via `pip install "
f"{PINNED_INSTALL}`; a floating install runs unverified code with a "
"write-scoped token."
)
offenders = [b for b in blocks if "pip install" in b and "openai" in b]
assert not offenders, (
f"{workflow_file} installs openai by name ({offenders!r}); pin it "
"through the hash-locked requirements file so the version and "
"checksum are fixed."
)
def test_triage_requirements_are_fully_hash_pinned() -> None:
"""The shared requirements file pins every package to an exact version
with a sha256 hash, which is what `pip --require-hashes` enforces at
install time. A loosened pin or a missing hash here would silently widen
the supply-chain surface for all the installer workflows.
"""
assert REQUIREMENTS_FILE.exists(), (
f"the hash-pinned requirements file the triage workflows install from is missing at {REQUIREMENTS_FILE}"
)
joined = REQUIREMENTS_FILE.read_text().replace("\\\n", " ")
entries = [line.strip() for line in joined.splitlines() if line.strip() and not line.strip().startswith("#")]
assert any(e.split()[0].startswith("openai==") for e in entries), (
"openai must be pinned to an exact version in the triage requirements"
)
for entry in entries:
spec = entry.split()[0]
assert "==" in spec, (
f"requirement {spec!r} is not pinned to an exact version; "
"--require-hashes needs every package pinned with =="
)
assert "--hash=sha256:" in entry, (
f"requirement {spec!r} has no sha256 hash; every pin must carry "
"checksums so --require-hashes can verify the download"
)
def _reconsider_steps() -> list[dict]:
workflow = _load_workflow("triage_reconsider.yml")
return workflow["jobs"]["reconsider"]["steps"]
def _index_of_run_step(steps: list[dict], needle: str) -> int:
for i, step in enumerate(steps):
run = step.get("run")
if isinstance(run, str) and needle in run:
return i
raise AssertionError(f"no run step contains {needle!r}")
def _reaction_steps(steps: list[dict], content: str) -> list[tuple[int, dict]]:
return [
(i, s)
for i, s in enumerate(steps)
if isinstance(s.get("run"), str) and f"content={content}" in s["run"] and "/reactions" in s["run"]
]
class TestReconsiderReactions:
"""The reconsider workflow acknowledges the triggering comment with a 👀
reaction the moment it accepts the trigger, and a 👍 once the run finishes,
so the contributor gets feedback immediately instead of waiting on a cron.
Both reactions are gated on `AGENT_SHIN_ENABLED == 'true'` so a dry-run
leaves no visible trace, and both target the comment that fired the event
(`github.event.comment.id`). The ordering (👀 before the triage run, 👍
after) is the whole point — these tests fail if a refactor reorders the
steps, drops a reaction, or stops gating them.
"""
def test_eyes_reaction_is_posted_before_the_triage_run(self) -> None:
steps = _reconsider_steps()
run_idx = _index_of_run_step(steps, "triage_with_llm.py")
eyes = _reaction_steps(steps, "eyes")
assert len(eyes) == 1, "expected exactly one 👀 (eyes) reaction step"
idx, step = eyes[0]
assert idx < run_idx, "👀 must be posted BEFORE the slow triage run, not after"
assert "github.event.comment.id" in (step.get("env") or {}).get("COMMENT_ID", ""), (
"👀 must react to the comment that triggered the workflow"
)
assert "${COMMENT_ID}" in step["run"], "👀 must react to the triggering comment, not a hardcoded id"
assert "vars.AGENT_SHIN_ENABLED == 'true'" in step["if"], (
"👀 must be gated on AGENT_SHIN_ENABLED so dry-run stays inert"
)
def test_thumbs_up_reaction_is_posted_after_a_successful_run(self) -> None:
steps = _reconsider_steps()
run_idx = _index_of_run_step(steps, "triage_with_llm.py")
thumbs = _reaction_steps(steps, "+1")
assert len(thumbs) == 1, "expected exactly one 👍 (+1) reaction step"
idx, step = thumbs[0]
assert idx > run_idx, "👍 must come AFTER the triage run"
assert "success()" in step["if"], "👍 must only fire when the reconsider run succeeded"
assert "vars.AGENT_SHIN_ENABLED == 'true'" in step["if"], (
"👍 must be gated on AGENT_SHIN_ENABLED so dry-run stays inert"
)