feat(triage): auto-close stale PRs with Greptile score <4/5

Adds .github/scripts/close_low_quality_prs.py and a daily workflow that
closes PRs which:
  - are open for at least 7 days, and
  - carry a most-recent greptile-apps review with Confidence Score <4/5,
  - and are not drafts or opt-out-labeled ('do not close', 'wip', etc.).

Each closure posts an explanatory comment telling the contributor how to
bring the PR back (rebase, re-request greptile, reopen at 4+/5). The
4/5 bar is already documented in the PR template
(.github/pull_request_template.md), so this just enforces it.

Tested with a dry run against the live BerriAI/litellm backlog of 1000
open PRs: 100 candidates identified, 598 PRs pass the bar (4+/5), 186
are too young, 97 are drafts, 19 lack any Greptile review and are left
alone.

Workflow defaults to closing 25 PRs/run as a safety net and supports
workflow_dispatch with overrides (close=false for a dry run, custom
min_age_days/min_score/limit).

18 unit tests cover score extraction (HTML/markdown/plain text, login
variants, multi-review picks latest) and per-PR evaluation (drafts,
opt-out labels, age, missing/passing/failing scores).

Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-05-17 09:24:59 +00:00
parent cf9b5e4fa7
commit 2e7fdc6f9c
No known key found for this signature in database
3 changed files with 705 additions and 0 deletions

348
.github/scripts/close_low_quality_prs.py vendored Normal file
View file

@ -0,0 +1,348 @@
#!/usr/bin/env python3
"""
Auto-close stale, low-quality pull requests.
Closes open PRs that satisfy ALL of the following:
1. Are at least N days old (default: 7) since creation.
2. Have a Greptile (`greptile-apps`) review comment whose latest
"Confidence Score: X/5" is below the configured threshold (default: 4).
3. Are not drafts.
4. Do not carry an opt-out label (default: "do not close").
For each match, the script posts an explanatory comment and closes the PR.
Contributors are invited to rebase and request a new Greptile review; if
Greptile then scores 4/5 or higher the PR can be reopened by anyone with
push access.
Requires the `gh` CLI to be authenticated.
Usage examples:
# Dry run (default) - prints what would be closed
python3 close_low_quality_prs.py
# Actually close matching PRs
python3 close_low_quality_prs.py --close
# Tweak thresholds
python3 close_low_quality_prs.py --min-age-days 7 --min-score 4 --close
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import re
import subprocess
import sys
from typing import Any, Iterable
# Greptile's GitHub App appears as `greptile-apps[bot]` in REST API comments
# and `greptile-apps` in `gh pr view --json` output. Accept either form.
GREPTILE_BOT_LOGINS = frozenset({"greptile-apps", "greptile-apps[bot]"})
# Matches lines like:
# <h3>Confidence Score: 3/5</h3>
# **Confidence Score: 4/5**
# Confidence Score: 5 / 5
SCORE_PATTERN = re.compile(
r"confidence\s*score\s*[:\-]?\s*(\d+)\s*/\s*5",
re.IGNORECASE,
)
def gh(*args: str) -> str:
"""Run a `gh` CLI command and return stdout. Raises on non-zero exit."""
result = subprocess.run(
["gh", *args],
capture_output=True,
text=True,
check=True,
)
return result.stdout
def gh_json(*args: str) -> Any:
"""Run a `gh` CLI command that emits JSON and return the parsed value."""
return json.loads(gh(*args))
def fetch_open_prs(repo: str | None) -> list[dict]:
"""Fetch all open PRs (number, createdAt, isDraft, labels, author)."""
repo_args = ["--repo", repo] if repo else []
fields = "number,title,createdAt,isDraft,labels,author,url"
raw = gh(
"pr",
"list",
"--state",
"open",
"--limit",
"1000",
"--json",
fields,
*repo_args,
)
return json.loads(raw)
def fetch_pr_comments(pr_number: int, repo: str | None) -> list[dict]:
"""Fetch issue-level comments on a PR (where Greptile posts its summary)."""
endpoint = (
f"repos/{repo}/issues/{pr_number}/comments?per_page=100"
if repo
else f"repos/{{owner}}/{{repo}}/issues/{pr_number}/comments?per_page=100"
)
raw = gh("api", "--paginate", endpoint)
comments: list[dict] = []
for line in raw.strip().splitlines():
line = line.strip()
if not line:
continue
parsed = json.loads(line)
if isinstance(parsed, list):
comments.extend(parsed)
else:
comments.append(parsed)
return comments
def extract_greptile_score(comments: Iterable[dict]) -> tuple[int, dict] | None:
"""Return (score, comment) for the most recent Greptile-authored comment
that contains a "Confidence Score: X/5". Returns None if no such comment.
"Most recent" is determined by the comment's `updated_at` (falling back to
`created_at`), so re-reviews override earlier passes.
"""
candidates: list[tuple[str, int, dict]] = []
for comment in comments:
user = (comment.get("user") or {}).get("login", "")
if user not in GREPTILE_BOT_LOGINS:
continue
body = comment.get("body") or ""
match = SCORE_PATTERN.search(body)
if not match:
continue
score = int(match.group(1))
timestamp = comment.get("updated_at") or comment.get("created_at") or ""
candidates.append((timestamp, score, comment))
if not candidates:
return None
candidates.sort(key=lambda triple: triple[0])
_, score, comment = candidates[-1]
return score, comment
def parse_iso8601(value: str) -> dt.datetime:
"""Parse a GitHub ISO-8601 timestamp into a timezone-aware datetime."""
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
def has_optout_label(pr: dict, optout_labels: set[str]) -> bool:
labels = {label.get("name", "").lower() for label in pr.get("labels", [])}
return bool(labels & {lbl.lower() for lbl in optout_labels})
def close_pr(
pr: dict,
score: int,
threshold: int,
age_days: int,
repo: str | None,
dry_run: bool,
label: str | None,
) -> None:
"""Post the explanatory comment and close the PR."""
pr_number = pr["number"]
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(
f" [DRY RUN] Would close PR #{pr_number} "
f"(age={age_days}d, greptile={score}/5): {pr['title']}"
)
return
comment_body = (
f"Closing as part of automated PR triage.\n\n"
f"This PR has been open for **{age_days} day(s)** and Greptile's most "
f"recent review scored it **{score}/5**, below our merge bar of "
f"**{threshold}/5**.\n\n"
"We close low-confidence PRs aggressively to keep the review queue "
"manageable for maintainers and contributors alike. **This is not a "
"rejection of the idea** — to bring this back:\n\n"
"1. Rebase on the latest `main` and address the points Greptile raised.\n"
f"2. Re-request a review from `@greptileai` once you've pushed the fixes.\n"
f"3. If Greptile returns a score of **{threshold}/5 or higher**, reopen "
"this PR (or open a new one) — a maintainer will take another look.\n\n"
"Thanks for contributing to LiteLLM. We know auto-closures can sting; "
"the goal is to keep the project healthy, not to dismiss your work."
)
gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args)
if label:
try:
gh("pr", "edit", str(pr_number), "--add-label", label, *repo_args)
except subprocess.CalledProcessError as exc:
stderr = (exc.stderr or "").strip()
print(f" warn: failed to add label '{label}' to #{pr_number}: {stderr}")
gh("pr", "close", str(pr_number), *repo_args)
print(f" Closed PR #{pr_number} (greptile={score}/5, age={age_days}d)")
def evaluate_pr(
pr: dict,
now: dt.datetime,
min_age_days: int,
min_score: int,
repo: str | None,
optout_labels: set[str],
) -> tuple[str, int | None, int | None]:
"""Decide whether to close `pr`.
Returns (action, score_or_none, age_days_or_none) where action is one of:
"skip-draft", "skip-too-young", "skip-optout-label",
"skip-no-greptile-score", "skip-score-ok", or "close".
"""
if pr.get("isDraft"):
return ("skip-draft", None, None)
if has_optout_label(pr, optout_labels):
return ("skip-optout-label", None, None)
created = parse_iso8601(pr["createdAt"])
age_days = (now - created).days
if age_days < min_age_days:
return ("skip-too-young", None, age_days)
comments = fetch_pr_comments(pr["number"], repo)
extraction = extract_greptile_score(comments)
if extraction is None:
return ("skip-no-greptile-score", None, age_days)
score, _ = extraction
if score >= min_score:
return ("skip-score-ok", score, age_days)
return ("close", score, age_days)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--repo",
type=str,
default=None,
help="Repository (owner/repo). Auto-detected if omitted.",
)
parser.add_argument(
"--min-age-days",
type=int,
default=7,
help="Minimum age (in days) before a PR is eligible (default: 7).",
)
parser.add_argument(
"--min-score",
type=int,
default=4,
choices=range(1, 6),
help="Greptile score below which a PR is closed (default: 4 -> closes <4/5).",
)
parser.add_argument(
"--optout-label",
action="append",
default=["do not close", "keep open", "wip"],
help=(
"Label(s) that exempt a PR from auto-close. "
"Repeat to add more. Case-insensitive."
),
)
parser.add_argument(
"--close-label",
type=str,
default=None,
help=(
"Optional label to add to PRs that get auto-closed "
"(e.g. 'auto-closed-low-quality'). Must already exist on the repo."
),
)
parser.add_argument(
"--close",
action="store_true",
help="Actually close matching PRs (default is dry-run).",
)
parser.add_argument(
"--limit",
type=int,
default=None,
help="Maximum number of PRs to close in one run (safety net).",
)
args = parser.parse_args()
dry_run = not args.close
if dry_run:
print("=== DRY RUN MODE (pass --close to actually close PRs) ===\n")
print("Fetching open PRs...")
prs = fetch_open_prs(args.repo)
print(f"Found {len(prs)} open PRs.\n")
now = dt.datetime.now(dt.timezone.utc)
optout_labels = set(args.optout_label)
closed = 0
summary = {
"close": 0,
"skip-draft": 0,
"skip-too-young": 0,
"skip-optout-label": 0,
"skip-no-greptile-score": 0,
"skip-score-ok": 0,
}
for pr in sorted(prs, key=lambda p: p["createdAt"]):
action, score, age_days = evaluate_pr(
pr,
now,
args.min_age_days,
args.min_score,
args.repo,
optout_labels,
)
summary[action] = summary.get(action, 0) + 1
if action != "close":
continue
assert score is not None and age_days is not None
print(
f"#{pr['number']}: \"{pr['title']}\" "
f"(age={age_days}d, greptile={score}/5) -> close"
)
close_pr(
pr,
score=score,
threshold=args.min_score,
age_days=age_days,
repo=args.repo,
dry_run=dry_run,
label=args.close_label,
)
if not dry_run:
closed += 1
if args.limit is not None and closed >= args.limit:
print(f"\nReached --limit={args.limit}; stopping.")
break
print("\n=== Summary ===")
for key, value in summary.items():
print(f" {key:28s} {value}")
print(f"\nTotal {'would close' if dry_run else 'closed'}: {summary['close']}")
return 0
if __name__ == "__main__":
sys.exit(main())

View file

@ -0,0 +1,80 @@
name: Close Low-Quality Stale PRs
# Auto-close PRs that have been open for a week or more and that Greptile has
# reviewed with a confidence score below 4/5. Closures are explained in a
# comment so contributors know how to bring the PR back (rebase, request a new
# Greptile review, reopen on a 4+/5).
#
# Manual one-off run:
# gh workflow run "Close Low-Quality Stale PRs" -f close=true
#
# Dry-run preview (no PRs are touched):
# gh workflow run "Close Low-Quality Stale PRs" -f close=false
on:
schedule:
# Daily at 09:00 UTC. Pairs well with the stale-issue workflow at midnight.
- cron: "0 9 * * *"
workflow_dispatch:
inputs:
close:
description: "Actually close matching PRs (false = dry run)."
required: false
default: "true"
type: choice
options:
- "true"
- "false"
min_age_days:
description: "Minimum PR age in days before it is eligible."
required: false
default: "7"
min_score:
description: "Greptile score below which a PR is closed (1-5)."
required: false
default: "4"
limit:
description: "Maximum number of PRs to close in a single run."
required: false
default: "25"
permissions:
contents: read
pull-requests: write
issues: write
jobs:
close-low-quality-prs:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Run low-quality PR closer
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CLOSE_FLAG: ${{ github.event.inputs.close || 'true' }}
MIN_AGE_DAYS: ${{ github.event.inputs.min_age_days || '7' }}
MIN_SCORE: ${{ github.event.inputs.min_score || '4' }}
LIMIT: ${{ github.event.inputs.limit || '25' }}
run: |
set -euo pipefail
ARGS=(
--repo "${{ github.repository }}"
--min-age-days "${MIN_AGE_DAYS}"
--min-score "${MIN_SCORE}"
--limit "${LIMIT}"
)
if [ "${CLOSE_FLAG}" = "true" ]; then
ARGS+=(--close)
fi
python3 .github/scripts/close_low_quality_prs.py "${ARGS[@]}"

View file

@ -0,0 +1,277 @@
"""Unit tests for `.github/scripts/close_low_quality_prs.py`.
These exercise the pure logic (score extraction and per-PR evaluation) without
hitting GitHub. Network/CLI calls are stubbed via monkeypatch.
"""
from __future__ import annotations
import datetime as dt
import importlib.util
import sys
from pathlib import Path
import pytest
SCRIPT_PATH = (
Path(__file__).resolve().parents[2]
/ ".github"
/ "scripts"
/ "close_low_quality_prs.py"
)
@pytest.fixture(scope="module")
def closer_module():
"""Load the script as a module via its file path (it lives outside the package)."""
spec = importlib.util.spec_from_file_location("close_low_quality_prs", SCRIPT_PATH)
assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}"
module = importlib.util.module_from_spec(spec)
sys.modules["close_low_quality_prs"] = module
spec.loader.exec_module(module)
return module
def _greptile_comment(
body: str,
updated_at: str = "2026-05-10T00:00:00Z",
login: str = "greptile-apps[bot]",
) -> dict:
return {
"user": {"login": login},
"body": body,
"created_at": updated_at,
"updated_at": updated_at,
}
class TestExtractGreptileScore:
def test_should_extract_score_from_html_header(self, closer_module):
comments = [
_greptile_comment("<h3>Confidence Score: 3/5</h3>\nSome body text.")
]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 3
def test_should_accept_both_greptile_login_variants(self, closer_module):
# REST API form ("greptile-apps[bot]") and GraphQL form ("greptile-apps")
for login in ("greptile-apps", "greptile-apps[bot]"):
comments = [
_greptile_comment("<h3>Confidence Score: 2/5</h3>", login=login)
]
result = closer_module.extract_greptile_score(comments)
assert result is not None, f"failed to detect score for login={login}"
score, _ = result
assert score == 2
def test_should_extract_score_from_plain_text(self, closer_module):
comments = [_greptile_comment("Confidence Score: 5/5 — looks good!")]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 5
def test_should_tolerate_whitespace_and_case(self, closer_module):
comments = [_greptile_comment("**confidence score : 2 / 5**")]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 2
def test_should_pick_most_recent_comment_when_rereview_happens(self, closer_module):
comments = [
_greptile_comment(
"Confidence Score: 2/5", updated_at="2026-05-01T00:00:00Z"
),
_greptile_comment(
"Confidence Score: 5/5", updated_at="2026-05-12T00:00:00Z"
),
]
result = closer_module.extract_greptile_score(comments)
assert result is not None
score, _ = result
assert score == 5
def test_should_ignore_non_greptile_authors(self, closer_module):
comments = [
{
"user": {"login": "some-human"},
"body": "Confidence Score: 1/5",
"created_at": "2026-05-12T00:00:00Z",
"updated_at": "2026-05-12T00:00:00Z",
}
]
assert closer_module.extract_greptile_score(comments) is None
def test_should_return_none_when_no_score_present(self, closer_module):
comments = [_greptile_comment("Greptile summary without a score.")]
assert closer_module.extract_greptile_score(comments) is None
def test_should_return_none_for_empty_comments(self, closer_module):
assert closer_module.extract_greptile_score([]) is None
class TestEvaluatePr:
@pytest.fixture(autouse=True)
def _now(self):
return dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc)
def _make_pr(
self,
*,
number: int = 1,
created_days_ago: int = 10,
is_draft: bool = False,
labels: list[str] | None = None,
) -> dict:
created = dt.datetime(2026, 5, 17, tzinfo=dt.timezone.utc) - dt.timedelta(
days=created_days_ago
)
return {
"number": number,
"title": f"PR #{number}",
"createdAt": created.isoformat().replace("+00:00", "Z"),
"isDraft": is_draft,
"labels": [{"name": lbl} for lbl in (labels or [])],
"author": {"login": "someone"},
"url": f"https://example.com/pr/{number}",
}
def test_should_skip_drafts(self, closer_module, _now, monkeypatch):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("should not fetch comments for drafts"),
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(is_draft=True),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-draft"
assert score is None and age is None
def test_should_skip_optout_label_case_insensitive(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("should not fetch comments for opt-outs"),
)
action, _, _ = closer_module.evaluate_pr(
self._make_pr(labels=["WIP"]),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels={"wip"},
)
assert action == "skip-optout-label"
def test_should_skip_too_young(self, closer_module, _now, monkeypatch):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: pytest.fail("should not fetch comments for young PRs"),
)
action, _, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=2),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-too-young"
assert age == 2
def test_should_skip_when_greptile_has_not_reviewed(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(closer_module, "fetch_pr_comments", lambda *a, **kw: [])
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=10),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-no-greptile-score"
assert score is None and age == 10
def test_should_skip_when_score_meets_threshold(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 4/5")],
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=10),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "skip-score-ok"
assert score == 4 and age == 10
def test_should_close_when_old_and_low_score(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("Confidence Score: 3/5")],
)
action, score, age = closer_module.evaluate_pr(
self._make_pr(created_days_ago=10),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "close"
assert score == 3 and age == 10
def test_should_close_when_old_and_very_low_score(
self, closer_module, _now, monkeypatch
):
monkeypatch.setattr(
closer_module,
"fetch_pr_comments",
lambda *a, **kw: [_greptile_comment("<h3>Confidence Score: 1/5</h3>")],
)
action, score, _ = closer_module.evaluate_pr(
self._make_pr(created_days_ago=14),
now=_now,
min_age_days=7,
min_score=4,
repo=None,
optout_labels=set(),
)
assert action == "close"
assert score == 1
class TestHasOptoutLabel:
def test_should_match_label_case_insensitively(self, closer_module):
pr = {"labels": [{"name": "Do Not Close"}, {"name": "bug"}]}
assert closer_module.has_optout_label(pr, {"do not close"}) is True
def test_should_return_false_when_no_match(self, closer_module):
pr = {"labels": [{"name": "bug"}, {"name": "enhancement"}]}
assert closer_module.has_optout_label(pr, {"wip", "keep open"}) is False
def test_should_handle_missing_labels(self, closer_module):
assert closer_module.has_optout_label({}, {"wip"}) is False