scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

feat(triage): Agent Shin LLM-as-judge for external PRs and issues

Browse source 
Adds a new triage flow that evaluates external pull requests and issues
against the project's contribution rubric and, when configured to do so,
auto-closes non-conforming ones with an explanatory comment. Contributors
can update + reopen to be re-evaluated.

Scope:
- Internal BerriAI contributors (author_association OWNER/MEMBER/COLLABORATOR)
  and bot accounts are skipped entirely.
- 'Fixes #1234' / 'Resolves https://github.com/.../issues/N' in the PR body
  short-circuits to PASS without burning LLM tokens.
- LLM judge returns structured JSON (verdict, missing[], explanation);
  parser tolerates markdown fences and embedded JSON.
- LLM errors NEVER close PRs/issues — failure surfaces as 'skip-llm-error'.

Safety:
- pull_request_target / issues triggers are FORCED dry-run in the workflow;
  only manual workflow_dispatch with close=true (and AGENT_SHIN_ENABLED=true)
  takes destructive action.
- Default mode writes verdicts to GITHUB_STEP_SUMMARY only — no public
  comments until the team flips the AGENT_SHIN_ENABLED repo variable.
- LLM uses an OpenAI-compatible endpoint (model and base URL configurable
  via repo variables; key via OPENAI_API_KEY secret).

Files:
- .github/scripts/triage_with_llm.py   - judge orchestrator + CLI
- .github/workflows/triage_pr_with_llm.yml
- .github/workflows/triage_issue_with_llm.yml
- tests/test_litellm/test_github_triage_with_llm.py - 33 unit tests

End-to-end validated against four real PRs (#28117 internal collaborator,
#28108 bot, #28129 'Fixes #28128', #28116 no linked issue) and issue
#28132 with a stubbed LLM judge: each path produces the expected action.

Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-05-17 16:25:23 +00:00
parent ac18f1407e
commit 7b4a09353e
No known key found for this signature in database
4 changed files with 1088 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

567
.github/scripts/triage_with_llm.py vendored Normal file
View file

@ -0,0 +1,567 @@
#!/usr/bin/env python3
"""
Agent Shin — LLM-as-judge triage for external OSS pull requests and issues.
Evaluates a single PR or issue against the contribution rubric and, when the
LLM judge marks it as failing, posts an explanatory comment + closes the
PR/issue. Re-triggers on `reopened` so contributors can iterate back in by
filling in the missing pieces and reopening.
Internal BerriAI contributors (`author_association` in {OWNER, MEMBER,
COLLABORATOR}) and bot accounts are skipped entirely.
Usage:
triage_with_llm.py --repo owner/repo --pr 1234
triage_with_llm.py --repo owner/repo --issue 5678
triage_with_llm.py --repo owner/repo --pr 1234 --close # actually close
triage_with_llm.py --repo owner/repo --pr 1234 --print-prompt # show prompt
Defaults are SAFE: without `--close` the script writes a verdict to stdout (and,
when running in GitHub Actions, to $GITHUB_STEP_SUMMARY) but takes no GitHub
write actions.
Environment:
GH_TOKEN / GITHUB_TOKEN - for `gh` CLI auth (auto-set in Actions)
OPENAI_API_KEY - required when --close is passed
OPENAI_BASE_URL - optional (route to any OpenAI-compatible API)
TRIAGE_MODEL - optional model override (default: gpt-4o-mini)
"""
from __future__ import annotations
import argparse
import json
import os
import re
import subprocess
import sys
import textwrap
from typing import Any
DEFAULT_MODEL = "gpt-4o-mini"
INTERNAL_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"})
# Regexes for picking off "obvious passes" without burning LLM tokens.
LINKED_ISSUE_PATTERN = re.compile(
r"\b(?:fixes|fix|closes|close|resolves|resolve|refs|ref|see|addresses)\s+"
r"(?:#\d+|https?://github\.com/[\w.-]+/[\w.-]+/issues/\d+)",
re.IGNORECASE,
)
HTML_COMMENT_PATTERN = re.compile(r"<!--.*?-->", re.DOTALL)
# ---------------------------------------------------------------------------
# gh helpers
def gh(*args: str) -> str:
"""Run a `gh` CLI command and return stdout. Raises on non-zero exit."""
result = subprocess.run(
["gh", *args],
capture_output=True,
text=True,
check=True,
)
return result.stdout
def fetch_pr(repo: str, number: int) -> dict:
"""Return the full GitHub REST representation of a PR."""
return json.loads(gh("api", f"repos/{repo}/pulls/{number}"))
def fetch_issue(repo: str, number: int) -> dict:
"""Return the full GitHub REST representation of an issue."""
return json.loads(gh("api", f"repos/{repo}/issues/{number}"))
def post_comment(repo: str, number: int, body: str) -> None:
"""Post an issue-style comment (works for both issues and PRs)."""
gh(
"api",
f"repos/{repo}/issues/{number}/comments",
"-X",
"POST",
"-f",
f"body={body}",
)
def close_pr(repo: str, number: int) -> None:
"""Close a pull request (state=closed)."""
gh(
"api",
f"repos/{repo}/pulls/{number}",
"-X",
"PATCH",
"-f",
"state=closed",
)
def close_issue(repo: str, number: int, *, not_planned: bool = True) -> None:
"""Close an issue, marking state_reason=not_planned by default."""
args = [
"api",
f"repos/{repo}/issues/{number}",
"-X",
"PATCH",
"-f",
"state=closed",
]
if not_planned:
args.extend(["-f", "state_reason=not_planned"])
gh(*args)
# ---------------------------------------------------------------------------
# Author classification
def is_internal_contributor(item: dict) -> bool:
"""Return True if the PR/issue author should be exempted from triage."""
association = (item.get("author_association") or "").upper()
if association in INTERNAL_ASSOCIATIONS:
return True
login = ((item.get("user") or {}).get("login") or "").lower()
if login.endswith("[bot]") or login in {"dependabot", "github-actions"}:
return True
return False
# ---------------------------------------------------------------------------
# Prompt construction
def strip_html_comments(text: str) -> str:
"""Remove HTML comments — template placeholder text shouldn't fool the judge."""
return HTML_COMMENT_PATTERN.sub("", text or "")
def has_linked_issue(text: str) -> bool:
"""Heuristic: does this body link to an open issue (Fixes #123 etc.)?"""
return bool(LINKED_ISSUE_PATTERN.search(strip_html_comments(text or "")))
def build_pr_prompt(*, title: str, body: str) -> str:
cleaned_body = strip_html_comments(body or "").strip() or "(empty)"
return textwrap.dedent(
f"""
You are "Agent Shin", the OSS triage bot for the LiteLLM open-source
repository (BerriAI/litellm). Decide whether this external pull request
meets the project's contribution standards.
The PR PASSES triage if it satisfies AT LEAST ONE of:
(A) It links to a related GitHub issue. Acceptable forms:
"Fixes #1234", "Closes #1234", "Resolves #1234",
"Refs https://github.com/BerriAI/litellm/issues/1234". A bare
issue number without a closing keyword counts only if it's
clearly the related issue (not a passing mention).
(B) The PR body contains ALL of:
- A clear problem description (what bug or missing feature this
addresses, beyond the title).
- Expected vs. actual behavior (or, for features, "what's
possible now vs. with this PR").
- Visual QA proof: before/after screenshots, a screen recording,
terminal output, log output, or test output demonstrating the
fix or feature works end-to-end. Saying "I tested it" is NOT
proof.
Bias toward PASS when the PR has structure and context — only FAIL when
the body is empty, copy-paste filler from the template, or genuinely
missing both a linked issue AND the core elements of (B).
Respond with a single JSON object, no prose:
{{
"verdict": "pass" | "fail",
"linked_issue": boolean,
"has_problem_description": boolean,
"has_expected_vs_actual": boolean,
"has_qa_proof": boolean,
"missing": ["plain-english strings naming what is missing"],
"explanation": "1-2 sentence reasoning for the team to skim"
}}
---
PR title: {title}
PR body:
---
{cleaned_body}
---
"""
).strip()
def build_issue_prompt(*, title: str, body: str) -> str:
cleaned_body = strip_html_comments(body or "").strip() or "(empty)"
return textwrap.dedent(
f"""
You are "Agent Shin", the OSS triage bot for the LiteLLM open-source
repository (BerriAI/litellm). Decide whether this GitHub issue meets
the project's reporting standards.
For a BUG REPORT the issue PASSES triage when it contains ALL of:
- A clear reproduction (steps, runnable code snippet, curl command,
or example config the maintainer can paste into their machine).
- Screenshot, terminal output, traceback, or log output as proof of
the bug.
- Expected vs. actual behavior.
For a FEATURE REQUEST the issue PASSES triage when it contains ALL of:
- A clear description of the proposed feature (what should LiteLLM do
that it does not today).
- Motivation / use case with a concrete example (config, API call,
UI flow, or scenario showing what's blocked today).
Bias toward PASS when the issue has structure and context — only FAIL
when the body is empty, copy-paste template placeholder text, or a
one-line "X is broken" with no detail. Asking clarifying questions is
OK content; mark such issues PASS.
Respond with a single JSON object, no prose:
{{
"verdict": "pass" | "fail",
"kind": "bug" | "feature" | "other",
"has_repro": boolean,
"has_proof": boolean,
"has_expected_vs_actual": boolean,
"has_motivation_example": boolean,
"missing": ["plain-english strings naming what is missing"],
"explanation": "1-2 sentence reasoning for the team to skim"
}}
---
Issue title: {title}
Issue body:
---
{cleaned_body}
---
"""
).strip()
# ---------------------------------------------------------------------------
# LLM call + verdict parsing
def call_llm_judge(
prompt: str, *, model: str, api_key: str, base_url: str | None
) -> str:
"""Call an OpenAI-compatible chat completions endpoint. Returns raw text."""
# Import inside the function so unit tests that monkey-patch this never
# need the openai package installed.
from openai import OpenAI
client = (
OpenAI(api_key=api_key, base_url=base_url)
if base_url
else OpenAI(api_key=api_key)
)
response = client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": prompt}],
temperature=0,
response_format={"type": "json_object"},
)
return response.choices[0].message.content or ""
def parse_verdict(raw: str) -> dict:
"""Parse the LLM's JSON response. Tolerates ```json fences and stray text."""
if not raw:
raise ValueError("empty LLM response")
text = raw.strip()
if text.startswith("```"):
text = re.sub(r"^```(?:json)?\s*", "", text)
text = re.sub(r"\s*```$", "", text)
try:
return json.loads(text)
except json.JSONDecodeError:
match = re.search(r"\{.*\}", text, re.DOTALL)
if not match:
raise ValueError(f"could not extract JSON from LLM response: {raw[:200]}")
return json.loads(match.group(0))
# ---------------------------------------------------------------------------
# Comment composition
def _format_missing(missing: list[str]) -> str:
if not missing:
return "- (see explanation below)"
return "\n".join(f"- {m}" for m in missing)
def format_pr_close_comment(verdict: dict) -> str:
missing_lines = _format_missing(verdict.get("missing") or [])
explanation = verdict.get("explanation") or ""
return (
"👋 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this repository.\n"
"\n"
"This PR is being **auto-closed** because it does not yet meet the bar described in our "
"[pull-request template](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md). "
"Specifically, I couldn't find:\n"
"\n"
f"{missing_lines}\n"
"\n"
f"> {explanation}\n"
"\n"
"**This isn't a rejection of the idea.** To bring this PR back:\n"
"\n"
"1. Update the PR description to either:\n"
" - Link a related GitHub issue (e.g. `Fixes #1234`), OR\n"
" - Add a clear **problem description**, **expected vs. actual behavior**, and **visual QA proof** "
"(before/after screenshots, a short screen recording, or terminal/log output).\n"
"2. **Reopen** the PR (or open a fresh one) — I'll re-evaluate automatically.\n"
"\n"
"Internal BerriAI contributors: this rubric doesn't apply to you — ping a maintainer.\n"
"\n"
"_(I'm an LLM, so I'm not infallible. If you think I got this wrong, reopen and ping a maintainer — "
"they'll override me.)_"
)
def format_issue_close_comment(verdict: dict) -> str:
missing_lines = _format_missing(verdict.get("missing") or [])
explanation = verdict.get("explanation") or ""
return (
"👋 Hi, thanks for filing this! I'm **Agent Shin**, the automated triage bot for this repository.\n"
"\n"
"This issue is being **auto-closed** because it doesn't yet have enough detail for a maintainer to act on. "
"Specifically, I couldn't find:\n"
"\n"
f"{missing_lines}\n"
"\n"
f"> {explanation}\n"
"\n"
"**This isn't a \"won't fix\".** To bring this issue back:\n"
"\n"
"1. Edit the issue to add the missing pieces:\n"
" - For **bug reports**: a runnable reproduction (code / curl / config), expected vs. actual behavior, "
"and a screenshot / traceback / log showing the bug.\n"
" - For **feature requests**: a concrete description of what should change, plus a use case and example "
"(config / API call / UI flow).\n"
"2. **Reopen** the issue — I'll re-evaluate automatically.\n"
"\n"
"Internal BerriAI contributors: this rubric doesn't apply to you — ping a maintainer.\n"
"\n"
"_(I'm an LLM, so I'm not infallible. If you think I got this wrong, reopen and ping a maintainer — "
"they'll override me.)_"
)
# ---------------------------------------------------------------------------
# Step-summary helpers
def write_step_summary(content: str) -> None:
"""When running inside GitHub Actions, append to the step summary file."""
path = os.environ.get("GITHUB_STEP_SUMMARY")
if not path:
return
try:
with open(path, "a", encoding="utf-8") as handle:
handle.write(content)
if not content.endswith("\n"):
handle.write("\n")
except OSError as exc:
print(f"warn: failed to write step summary: {exc}", file=sys.stderr)
# ---------------------------------------------------------------------------
# Core orchestration
def triage(
*,
repo: str,
kind: str,
number: int,
close: bool,
model: str,
judge: Any = None,
print_prompt: bool = False,
) -> dict:
"""Triage a single PR or issue. Returns a result dict for logging/tests.
`judge` is an optional callable `(prompt) -> str` for tests / dry-run with
a stub. In production, leave it None and the script uses `call_llm_judge`.
"""
fetcher = {"pr": fetch_pr, "issue": fetch_issue}[kind]
item = fetcher(repo, number)
title = item.get("title") or ""
body = item.get("body") or ""
login = (item.get("user") or {}).get("login") or ""
association = item.get("author_association") or ""
state = item.get("state") or ""
base_result = {
"kind": kind,
"number": number,
"title": title,
"author": login,
"author_association": association,
"state": state,
}
if state != "open":
return {**base_result, "action": "skip-not-open"}
if is_internal_contributor(item):
return {**base_result, "action": "skip-internal-author"}
if kind == "pr":
prompt = build_pr_prompt(title=title, body=body)
# Short-circuit: if body very clearly links a related issue, just pass.
if has_linked_issue(body):
return {
**base_result,
"action": "pass-linked-issue",
"verdict": {
"verdict": "pass",
"linked_issue": True,
"explanation": "Linked-issue regex matched; LLM was not called.",
},
}
else:
prompt = build_issue_prompt(title=title, body=body)
if print_prompt:
return {**base_result, "action": "print-prompt", "prompt": prompt}
if judge is None:
api_key = os.environ.get("OPENAI_API_KEY")
if not api_key:
# No key configured — never take a destructive action. Report skip.
return {
**base_result,
"action": "skip-no-llm-key",
"prompt_preview": prompt[:200],
}
base_url = os.environ.get("OPENAI_BASE_URL") or None
def judge(p: str) -> str:
return call_llm_judge(p, model=model, api_key=api_key, base_url=base_url)
try:
raw = judge(prompt)
verdict = parse_verdict(raw)
except Exception as exc: # noqa: BLE001 - judge errors must never close PRs
return {**base_result, "action": "skip-llm-error", "error": str(exc)}
decision = (verdict.get("verdict") or "").lower()
if decision != "fail":
return {**base_result, "action": "pass-llm", "verdict": verdict}
if not close:
return {**base_result, "action": "would-close", "verdict": verdict}
comment_body = (
format_pr_close_comment(verdict)
if kind == "pr"
else format_issue_close_comment(verdict)
)
post_comment(repo, number, comment_body)
if kind == "pr":
close_pr(repo, number)
else:
close_issue(repo, number)
return {
**base_result,
"action": "closed",
"verdict": verdict,
"comment": comment_body,
}
# ---------------------------------------------------------------------------
# CLI
def render_summary(result: dict) -> str:
"""Render a human-readable summary block (used for stdout + step summary)."""
lines = ["## Agent Shin verdict", ""]
lines.append(
f"- **{result['kind'].upper()} #{result['number']}**: {result.get('title', '')}"
)
lines.append(
f"- **Author**: `{result.get('author', '')}` ({result.get('author_association', '')})"
)
lines.append(f"- **State**: {result.get('state', '')}")
lines.append(f"- **Action**: `{result['action']}`")
verdict = result.get("verdict")
if verdict:
lines.append("")
lines.append("```json")
lines.append(json.dumps(verdict, indent=2))
lines.append("```")
error = result.get("error")
if error:
lines.append("")
lines.append(f"_LLM error: {error}_")
comment = result.get("comment")
if comment:
lines.append("")
lines.append("### Would post comment:")
lines.append("")
lines.append("> " + comment.replace("\n", "\n> "))
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", required=True, help="Repository (owner/repo).")
target = parser.add_mutually_exclusive_group(required=True)
target.add_argument("--pr", type=int, help="Pull request number to triage.")
target.add_argument("--issue", type=int, help="Issue number to triage.")
parser.add_argument(
"--close",
action="store_true",
help="Actually post comment + close on fail (default: dry run).",
)
parser.add_argument(
"--model",
default=os.environ.get("TRIAGE_MODEL", DEFAULT_MODEL),
help=f"OpenAI-compatible model name (default: {DEFAULT_MODEL}).",
)
parser.add_argument(
"--print-prompt",
action="store_true",
help="Print the prompt that would be sent to the judge and exit.",
)
args = parser.parse_args()
kind = "pr" if args.pr is not None else "issue"
number = args.pr if args.pr is not None else args.issue
result = triage(
repo=args.repo,
kind=kind,
number=number,
close=args.close,
model=args.model,
print_prompt=args.print_prompt,
)
if result.get("action") == "print-prompt":
print(result["prompt"])
return 0
summary = render_summary(result)
print(summary)
write_step_summary(summary + "\n")
return 0
if __name__ == "__main__":
sys.exit(main())

75
.github/workflows/triage_issue_with_llm.yml vendored Normal file
View file

@ -0,0 +1,75 @@
name: Agent Shin — Issue triage
# LLM-as-judge triage for external GitHub issues.
#
# DRY-RUN BY DEFAULT. See .github/workflows/triage_pr_with_llm.yml for the
# enablement procedure — same repo variable (`AGENT_SHIN_ENABLED=true`)
# unlocks the PR and issue triage flows together.
on:
issues:
types: [opened, reopened]
workflow_dispatch:
inputs:
issue_number:
description: "Issue number to triage manually."
required: true
close:
description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail."
required: false
default: "false"
type: choice
options:
- "true"
- "false"
permissions:
contents: read
issues: write
jobs:
triage:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir "openai>=1.40.0"
- name: Run Agent Shin
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
DISPATCH_CLOSE: ${{ github.event.inputs.close }}
ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }}
run: |
set -euo pipefail
ARGS=(--repo "${{ github.repository }}" --issue "${ISSUE_NUMBER}")
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" != "false" ]; then
ARGS+=(--close)
echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode."
elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
echo "::notice::Agent Shin is ENABLED but this trigger is dry-run."
else
echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no issues will be closed."
fi
# Automatic `issues` events stay dry-run regardless until the team
# explicitly invokes workflow_dispatch with close=true.
if [ "${GITHUB_EVENT_NAME:-}" = "issues" ]; then
ARGS=("${ARGS[@]/--close/}")
echo "::notice::issues trigger -> forcing dry-run."
fi
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"

89
.github/workflows/triage_pr_with_llm.yml vendored Normal file
View file

@ -0,0 +1,89 @@
name: Agent Shin — PR triage
# LLM-as-judge triage for external pull requests.
#
# DRY-RUN BY DEFAULT. Closures and public comments are gated on the repo
# variable `AGENT_SHIN_ENABLED` being set to the string `"true"`. Until then,
# every run only writes its verdict to the workflow step summary so the team
# can QA the judge's decisions before flipping it on.
#
# To enable for real:
# 1. Add a repo secret `OPENAI_API_KEY` (or compatible).
# 2. Set repo variable `AGENT_SHIN_ENABLED` to `true`
# (Settings > Secrets and variables > Actions > Variables).
#
# We use `pull_request_target` so the workflow has access to repo secrets
# and runs against PRs from forks. We never check out fork code — only read
# PR metadata via `gh api`, so this is safe.
on:
pull_request_target:
types: [opened, reopened]
workflow_dispatch:
inputs:
pr_number:
description: "PR number to triage manually."
required: true
close:
description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail."
required: false
default: "false"
type: choice
options:
- "true"
- "false"
permissions:
contents: read
issues: write
pull-requests: write
jobs:
triage:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir "openai>=1.40.0"
- name: Run Agent Shin
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
DISPATCH_CLOSE: ${{ github.event.inputs.close }}
PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
run: |
set -euo pipefail
ARGS=(--repo "${{ github.repository }}" --pr "${PR_NUMBER}")
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" != "false" ]; then
ARGS+=(--close)
echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode."
elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
echo "::notice::Agent Shin is ENABLED but this trigger is dry-run (workflow_dispatch close=false or scheduled event)."
else
echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no PRs will be closed."
fi
# On the scheduled/automatic pull_request_target trigger we default to
# dry-run regardless, so the team can review verdicts in the step
# summary before any contributor sees a comment. Only the manual
# workflow_dispatch path (with close=true) closes PRs.
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request_target" ]; then
# strip any --close added above
ARGS=("${ARGS[@]/--close/}")
echo "::notice::pull_request_target trigger -> forcing dry-run."
fi
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"

357
tests/test_litellm/test_github_triage_with_llm.py Normal file
View file

@ -0,0 +1,357 @@
"""Unit tests for `.github/scripts/triage_with_llm.py` (Agent Shin)."""
from __future__ import annotations
import importlib.util
import json
import sys
from pathlib import Path
import pytest
SCRIPT_PATH = (
Path(__file__).resolve().parents[2] / ".github" / "scripts" / "triage_with_llm.py"
)
@pytest.fixture(scope="module")
def triage_module():
spec = importlib.util.spec_from_file_location("triage_with_llm", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules["triage_with_llm"] = module
spec.loader.exec_module(module)
return module
class TestIsInternalContributor:
@pytest.mark.parametrize("association", ["OWNER", "MEMBER", "COLLABORATOR"])
def test_should_mark_org_associations_as_internal(self, triage_module, association):
item = {
"author_association": association,
"user": {"login": "krrishdholakia"},
}
assert triage_module.is_internal_contributor(item) is True
@pytest.mark.parametrize(
"association",
["CONTRIBUTOR", "FIRST_TIME_CONTRIBUTOR", "FIRST_TIMER", "NONE", ""],
)
def test_should_mark_outside_associations_as_external(
self, triage_module, association
):
item = {
"author_association": association,
"user": {"login": "random-oss-dev"},
}
assert triage_module.is_internal_contributor(item) is False
@pytest.mark.parametrize(
"login",
["dependabot[bot]", "greptile-apps[bot]", "dependabot", "github-actions"],
)
def test_should_skip_bot_accounts_regardless_of_association(
self, triage_module, login
):
item = {"author_association": "NONE", "user": {"login": login}}
assert triage_module.is_internal_contributor(item) is True
class TestHasLinkedIssue:
@pytest.mark.parametrize(
"body",
[
"Fixes #1234",
"closes #1",
"Resolves #99",
"fix #42 — this addresses the regression",
"Refs https://github.com/BerriAI/litellm/issues/27000",
],
)
def test_should_detect_common_link_phrases(self, triage_module, body):
assert triage_module.has_linked_issue(body) is True
@pytest.mark.parametrize(
"body",
[
"",
"Some change",
"See #1234", # "see" is allowed per regex but we want documented coverage
],
)
def test_should_handle_empty_and_unrelated_bodies(self, triage_module, body):
# "See #1234" is intentionally accepted as a related-issue reference.
# Just make sure empty/unrelated bodies don't crash.
triage_module.has_linked_issue(body)
def test_should_not_detect_when_only_html_comment_template(self, triage_module):
body = "<!-- e.g. Fixes #1234 -->"
assert triage_module.has_linked_issue(body) is False
class TestStripHtmlComments:
def test_should_remove_single_line_comments(self, triage_module):
text = "before <!-- placeholder --> after"
assert "placeholder" not in triage_module.strip_html_comments(text)
def test_should_remove_multiline_comments(self, triage_module):
text = "kept\n<!--\nlots of placeholder text\nFixes #1\n-->\nkept2"
cleaned = triage_module.strip_html_comments(text)
assert "Fixes #1" not in cleaned
assert "kept" in cleaned and "kept2" in cleaned
def test_should_handle_none(self, triage_module):
assert triage_module.strip_html_comments(None) == ""
class TestParseVerdict:
def test_should_parse_plain_json(self, triage_module):
raw = '{"verdict": "pass", "missing": []}'
assert triage_module.parse_verdict(raw)["verdict"] == "pass"
def test_should_strip_markdown_fence(self, triage_module):
raw = '```json\n{"verdict": "fail", "missing": ["foo"]}\n```'
result = triage_module.parse_verdict(raw)
assert result["verdict"] == "fail"
assert result["missing"] == ["foo"]
def test_should_extract_embedded_json_from_prose(self, triage_module):
raw = 'Here you go: {"verdict": "pass", "missing": []}\nThanks.'
assert triage_module.parse_verdict(raw)["verdict"] == "pass"
def test_should_raise_for_unparseable_text(self, triage_module):
with pytest.raises(ValueError):
triage_module.parse_verdict("not even close to json")
def test_should_raise_for_empty(self, triage_module):
with pytest.raises(ValueError):
triage_module.parse_verdict("")
class TestBuildPrompts:
def test_should_include_pr_title_and_body(self, triage_module):
prompt = triage_module.build_pr_prompt(
title="Add foo", body="<!-- comment --> Real body"
)
assert "Add foo" in prompt
assert "Real body" in prompt
assert "comment" not in prompt # HTML comments are stripped
def test_should_show_empty_marker_for_empty_pr_body(self, triage_module):
prompt = triage_module.build_pr_prompt(title="t", body="<!-- nothing -->")
assert "(empty)" in prompt
def test_should_include_issue_title_and_body(self, triage_module):
prompt = triage_module.build_issue_prompt(title="Bug", body="repro here")
assert "Bug" in prompt
assert "repro here" in prompt
class TestTriageOrchestration:
"""End-to-end-ish tests that mock both gh fetchers and the LLM."""
def _make_pr(self, **overrides):
base = {
"number": 1,
"title": "PR title",
"body": "PR body",
"state": "open",
"author_association": "NONE",
"user": {"login": "outside-dev"},
}
base.update(overrides)
return base
def test_should_skip_internal_author(self, triage_module, monkeypatch):
pr = self._make_pr(
author_association="MEMBER", user={"login": "krrishdholakia"}
)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
def boom(*a, **kw):
pytest.fail("LLM should not be called for internal authors")
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=True,
model="m",
judge=boom,
)
assert result["action"] == "skip-internal-author"
def test_should_skip_closed_pr(self, triage_module, monkeypatch):
pr = self._make_pr(state="closed")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=True,
model="m",
judge=lambda p: pytest.fail("should not run on closed PRs"),
)
assert result["action"] == "skip-not-open"
def test_should_short_circuit_on_linked_issue(self, triage_module, monkeypatch):
pr = self._make_pr(body="Fixes #1234\n\nFoo bar")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=True,
model="m",
judge=lambda p: pytest.fail("LLM should not be called"),
)
assert result["action"] == "pass-linked-issue"
assert result["verdict"]["verdict"] == "pass"
def test_should_return_pass_llm_when_judge_passes(self, triage_module, monkeypatch):
pr = self._make_pr(body="Long body, no linked issue.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
captured = {}
def judge(prompt):
captured["prompt"] = prompt
return json.dumps({"verdict": "pass", "missing": [], "explanation": "ok"})
result = triage_module.triage(
repo="o/r", kind="pr", number=1, close=True, model="m", judge=judge
)
assert result["action"] == "pass-llm"
assert "Long body" in captured["prompt"]
def test_should_return_would_close_in_dry_run(self, triage_module, monkeypatch):
pr = self._make_pr(body="just a sentence.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
def fake_post(*a, **kw):
pytest.fail("should not post comments in dry-run")
def fake_close(*a, **kw):
pytest.fail("should not close in dry-run")
monkeypatch.setattr(triage_module, "post_comment", fake_post)
monkeypatch.setattr(triage_module, "close_pr", fake_close)
verdict = {
"verdict": "fail",
"missing": ["problem description", "QA proof"],
"explanation": "Body is one sentence.",
}
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=False,
model="m",
judge=lambda p: json.dumps(verdict),
)
assert result["action"] == "would-close"
assert result["verdict"]["missing"] == ["problem description", "QA proof"]
def test_should_post_comment_and_close_when_close_enabled(
self, triage_module, monkeypatch
):
pr = self._make_pr(body="just a sentence.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
posted = {}
closed = {}
monkeypatch.setattr(
triage_module,
"post_comment",
lambda repo, n, body: posted.update({"repo": repo, "n": n, "body": body}),
)
monkeypatch.setattr(
triage_module,
"close_pr",
lambda repo, n: closed.update({"repo": repo, "n": n}),
)
verdict = {
"verdict": "fail",
"missing": ["QA proof"],
"explanation": "Body too thin.",
}
result = triage_module.triage(
repo="o/r",
kind="pr",
number=42,
close=True,
model="m",
judge=lambda p: json.dumps(verdict),
)
assert result["action"] == "closed"
assert posted["n"] == 42 and closed["n"] == 42
assert "Agent Shin" in posted["body"]
assert "QA proof" in posted["body"]
def test_should_skip_on_llm_error_in_close_mode(self, triage_module, monkeypatch):
pr = self._make_pr(body="something.")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda *a, **kw: pytest.fail("must not comment on LLM error"),
)
monkeypatch.setattr(
triage_module,
"close_pr",
lambda *a, **kw: pytest.fail("must not close on LLM error"),
)
def broken_judge(prompt):
raise RuntimeError("upstream 500")
result = triage_module.triage(
repo="o/r",
kind="pr",
number=1,
close=True,
model="m",
judge=broken_judge,
)
assert result["action"] == "skip-llm-error"
assert "upstream 500" in result["error"]
def test_should_triage_issues_kind(self, triage_module, monkeypatch):
issue = {
"number": 7,
"title": "Bug: X is broken",
"body": "no detail",
"state": "open",
"author_association": "NONE",
"user": {"login": "outside"},
}
monkeypatch.setattr(triage_module, "fetch_issue", lambda repo, n: issue)
closed = {}
posted = {}
monkeypatch.setattr(
triage_module,
"post_comment",
lambda repo, n, body: posted.update(body=body),
)
monkeypatch.setattr(
triage_module, "close_issue", lambda repo, n: closed.update(n=n)
)
verdict = {
"verdict": "fail",
"kind": "bug",
"has_repro": False,
"missing": ["reproduction", "expected vs. actual"],
"explanation": "No repro provided.",
}
result = triage_module.triage(
repo="o/r",
kind="issue",
number=7,
close=True,
model="m",
judge=lambda p: json.dumps(verdict),
)
assert result["action"] == "closed"
assert closed["n"] == 7
assert "reproduction" in posted["body"]