diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py new file mode 100644 index 00000000000..f8ca780ee4c --- /dev/null +++ b/.github/scripts/triage_with_llm.py @@ -0,0 +1,567 @@ +#!/usr/bin/env python3 +""" +Agent Shin — LLM-as-judge triage for external OSS pull requests and issues. + +Evaluates a single PR or issue against the contribution rubric and, when the +LLM judge marks it as failing, posts an explanatory comment + closes the +PR/issue. Re-triggers on `reopened` so contributors can iterate back in by +filling in the missing pieces and reopening. + +Internal BerriAI contributors (`author_association` in {OWNER, MEMBER, +COLLABORATOR}) and bot accounts are skipped entirely. + +Usage: + triage_with_llm.py --repo owner/repo --pr 1234 + triage_with_llm.py --repo owner/repo --issue 5678 + triage_with_llm.py --repo owner/repo --pr 1234 --close # actually close + triage_with_llm.py --repo owner/repo --pr 1234 --print-prompt # show prompt + +Defaults are SAFE: without `--close` the script writes a verdict to stdout (and, +when running in GitHub Actions, to $GITHUB_STEP_SUMMARY) but takes no GitHub +write actions. + +Environment: + GH_TOKEN / GITHUB_TOKEN - for `gh` CLI auth (auto-set in Actions) + OPENAI_API_KEY - required when --close is passed + OPENAI_BASE_URL - optional (route to any OpenAI-compatible API) + TRIAGE_MODEL - optional model override (default: gpt-4o-mini) +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import textwrap +from typing import Any + +DEFAULT_MODEL = "gpt-4o-mini" + +INTERNAL_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"}) + +# Regexes for picking off "obvious passes" without burning LLM tokens. +LINKED_ISSUE_PATTERN = re.compile( + r"\b(?:fixes|fix|closes|close|resolves|resolve|refs|ref|see|addresses)\s+" + r"(?:#\d+|https?://github\.com/[\w.-]+/[\w.-]+/issues/\d+)", + re.IGNORECASE, +) +HTML_COMMENT_PATTERN = re.compile(r"", re.DOTALL) + + +# --------------------------------------------------------------------------- +# gh helpers + + +def gh(*args: str) -> str: + """Run a `gh` CLI command and return stdout. Raises on non-zero exit.""" + result = subprocess.run( + ["gh", *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +def fetch_pr(repo: str, number: int) -> dict: + """Return the full GitHub REST representation of a PR.""" + return json.loads(gh("api", f"repos/{repo}/pulls/{number}")) + + +def fetch_issue(repo: str, number: int) -> dict: + """Return the full GitHub REST representation of an issue.""" + return json.loads(gh("api", f"repos/{repo}/issues/{number}")) + + +def post_comment(repo: str, number: int, body: str) -> None: + """Post an issue-style comment (works for both issues and PRs).""" + gh( + "api", + f"repos/{repo}/issues/{number}/comments", + "-X", + "POST", + "-f", + f"body={body}", + ) + + +def close_pr(repo: str, number: int) -> None: + """Close a pull request (state=closed).""" + gh( + "api", + f"repos/{repo}/pulls/{number}", + "-X", + "PATCH", + "-f", + "state=closed", + ) + + +def close_issue(repo: str, number: int, *, not_planned: bool = True) -> None: + """Close an issue, marking state_reason=not_planned by default.""" + args = [ + "api", + f"repos/{repo}/issues/{number}", + "-X", + "PATCH", + "-f", + "state=closed", + ] + if not_planned: + args.extend(["-f", "state_reason=not_planned"]) + gh(*args) + + +# --------------------------------------------------------------------------- +# Author classification + + +def is_internal_contributor(item: dict) -> bool: + """Return True if the PR/issue author should be exempted from triage.""" + association = (item.get("author_association") or "").upper() + if association in INTERNAL_ASSOCIATIONS: + return True + login = ((item.get("user") or {}).get("login") or "").lower() + if login.endswith("[bot]") or login in {"dependabot", "github-actions"}: + return True + return False + + +# --------------------------------------------------------------------------- +# Prompt construction + + +def strip_html_comments(text: str) -> str: + """Remove HTML comments — template placeholder text shouldn't fool the judge.""" + return HTML_COMMENT_PATTERN.sub("", text or "") + + +def has_linked_issue(text: str) -> bool: + """Heuristic: does this body link to an open issue (Fixes #123 etc.)?""" + return bool(LINKED_ISSUE_PATTERN.search(strip_html_comments(text or ""))) + + +def build_pr_prompt(*, title: str, body: str) -> str: + cleaned_body = strip_html_comments(body or "").strip() or "(empty)" + return textwrap.dedent( + f""" + You are "Agent Shin", the OSS triage bot for the LiteLLM open-source + repository (BerriAI/litellm). Decide whether this external pull request + meets the project's contribution standards. + + The PR PASSES triage if it satisfies AT LEAST ONE of: + + (A) It links to a related GitHub issue. Acceptable forms: + "Fixes #1234", "Closes #1234", "Resolves #1234", + "Refs https://github.com/BerriAI/litellm/issues/1234". A bare + issue number without a closing keyword counts only if it's + clearly the related issue (not a passing mention). + + (B) The PR body contains ALL of: + - A clear problem description (what bug or missing feature this + addresses, beyond the title). + - Expected vs. actual behavior (or, for features, "what's + possible now vs. with this PR"). + - Visual QA proof: before/after screenshots, a screen recording, + terminal output, log output, or test output demonstrating the + fix or feature works end-to-end. Saying "I tested it" is NOT + proof. + + Bias toward PASS when the PR has structure and context — only FAIL when + the body is empty, copy-paste filler from the template, or genuinely + missing both a linked issue AND the core elements of (B). + + Respond with a single JSON object, no prose: + + {{ + "verdict": "pass" | "fail", + "linked_issue": boolean, + "has_problem_description": boolean, + "has_expected_vs_actual": boolean, + "has_qa_proof": boolean, + "missing": ["plain-english strings naming what is missing"], + "explanation": "1-2 sentence reasoning for the team to skim" + }} + + --- + PR title: {title} + + PR body: + --- + {cleaned_body} + --- + """ + ).strip() + + +def build_issue_prompt(*, title: str, body: str) -> str: + cleaned_body = strip_html_comments(body or "").strip() or "(empty)" + return textwrap.dedent( + f""" + You are "Agent Shin", the OSS triage bot for the LiteLLM open-source + repository (BerriAI/litellm). Decide whether this GitHub issue meets + the project's reporting standards. + + For a BUG REPORT the issue PASSES triage when it contains ALL of: + - A clear reproduction (steps, runnable code snippet, curl command, + or example config the maintainer can paste into their machine). + - Screenshot, terminal output, traceback, or log output as proof of + the bug. + - Expected vs. actual behavior. + + For a FEATURE REQUEST the issue PASSES triage when it contains ALL of: + - A clear description of the proposed feature (what should LiteLLM do + that it does not today). + - Motivation / use case with a concrete example (config, API call, + UI flow, or scenario showing what's blocked today). + + Bias toward PASS when the issue has structure and context — only FAIL + when the body is empty, copy-paste template placeholder text, or a + one-line "X is broken" with no detail. Asking clarifying questions is + OK content; mark such issues PASS. + + Respond with a single JSON object, no prose: + + {{ + "verdict": "pass" | "fail", + "kind": "bug" | "feature" | "other", + "has_repro": boolean, + "has_proof": boolean, + "has_expected_vs_actual": boolean, + "has_motivation_example": boolean, + "missing": ["plain-english strings naming what is missing"], + "explanation": "1-2 sentence reasoning for the team to skim" + }} + + --- + Issue title: {title} + + Issue body: + --- + {cleaned_body} + --- + """ + ).strip() + + +# --------------------------------------------------------------------------- +# LLM call + verdict parsing + + +def call_llm_judge( + prompt: str, *, model: str, api_key: str, base_url: str | None +) -> str: + """Call an OpenAI-compatible chat completions endpoint. Returns raw text.""" + # Import inside the function so unit tests that monkey-patch this never + # need the openai package installed. + from openai import OpenAI + + client = ( + OpenAI(api_key=api_key, base_url=base_url) + if base_url + else OpenAI(api_key=api_key) + ) + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=0, + response_format={"type": "json_object"}, + ) + return response.choices[0].message.content or "" + + +def parse_verdict(raw: str) -> dict: + """Parse the LLM's JSON response. Tolerates ```json fences and stray text.""" + if not raw: + raise ValueError("empty LLM response") + text = raw.strip() + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*", "", text) + text = re.sub(r"\s*```$", "", text) + try: + return json.loads(text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", text, re.DOTALL) + if not match: + raise ValueError(f"could not extract JSON from LLM response: {raw[:200]}") + return json.loads(match.group(0)) + + +# --------------------------------------------------------------------------- +# Comment composition + + +def _format_missing(missing: list[str]) -> str: + if not missing: + return "- (see explanation below)" + return "\n".join(f"- {m}" for m in missing) + + +def format_pr_close_comment(verdict: dict) -> str: + missing_lines = _format_missing(verdict.get("missing") or []) + explanation = verdict.get("explanation") or "" + return ( + "👋 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this repository.\n" + "\n" + "This PR is being **auto-closed** because it does not yet meet the bar described in our " + "[pull-request template](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md). " + "Specifically, I couldn't find:\n" + "\n" + f"{missing_lines}\n" + "\n" + f"> {explanation}\n" + "\n" + "**This isn't a rejection of the idea.** To bring this PR back:\n" + "\n" + "1. Update the PR description to either:\n" + " - Link a related GitHub issue (e.g. `Fixes #1234`), OR\n" + " - Add a clear **problem description**, **expected vs. actual behavior**, and **visual QA proof** " + "(before/after screenshots, a short screen recording, or terminal/log output).\n" + "2. **Reopen** the PR (or open a fresh one) — I'll re-evaluate automatically.\n" + "\n" + "Internal BerriAI contributors: this rubric doesn't apply to you — ping a maintainer.\n" + "\n" + "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, reopen and ping a maintainer — " + "they'll override me.)_" + ) + + +def format_issue_close_comment(verdict: dict) -> str: + missing_lines = _format_missing(verdict.get("missing") or []) + explanation = verdict.get("explanation") or "" + return ( + "👋 Hi, thanks for filing this! I'm **Agent Shin**, the automated triage bot for this repository.\n" + "\n" + "This issue is being **auto-closed** because it doesn't yet have enough detail for a maintainer to act on. " + "Specifically, I couldn't find:\n" + "\n" + f"{missing_lines}\n" + "\n" + f"> {explanation}\n" + "\n" + "**This isn't a \"won't fix\".** To bring this issue back:\n" + "\n" + "1. Edit the issue to add the missing pieces:\n" + " - For **bug reports**: a runnable reproduction (code / curl / config), expected vs. actual behavior, " + "and a screenshot / traceback / log showing the bug.\n" + " - For **feature requests**: a concrete description of what should change, plus a use case and example " + "(config / API call / UI flow).\n" + "2. **Reopen** the issue — I'll re-evaluate automatically.\n" + "\n" + "Internal BerriAI contributors: this rubric doesn't apply to you — ping a maintainer.\n" + "\n" + "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, reopen and ping a maintainer — " + "they'll override me.)_" + ) + + +# --------------------------------------------------------------------------- +# Step-summary helpers + + +def write_step_summary(content: str) -> None: + """When running inside GitHub Actions, append to the step summary file.""" + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + try: + with open(path, "a", encoding="utf-8") as handle: + handle.write(content) + if not content.endswith("\n"): + handle.write("\n") + except OSError as exc: + print(f"warn: failed to write step summary: {exc}", file=sys.stderr) + + +# --------------------------------------------------------------------------- +# Core orchestration + + +def triage( + *, + repo: str, + kind: str, + number: int, + close: bool, + model: str, + judge: Any = None, + print_prompt: bool = False, +) -> dict: + """Triage a single PR or issue. Returns a result dict for logging/tests. + + `judge` is an optional callable `(prompt) -> str` for tests / dry-run with + a stub. In production, leave it None and the script uses `call_llm_judge`. + """ + fetcher = {"pr": fetch_pr, "issue": fetch_issue}[kind] + item = fetcher(repo, number) + + title = item.get("title") or "" + body = item.get("body") or "" + login = (item.get("user") or {}).get("login") or "" + association = item.get("author_association") or "" + state = item.get("state") or "" + + base_result = { + "kind": kind, + "number": number, + "title": title, + "author": login, + "author_association": association, + "state": state, + } + + if state != "open": + return {**base_result, "action": "skip-not-open"} + + if is_internal_contributor(item): + return {**base_result, "action": "skip-internal-author"} + + if kind == "pr": + prompt = build_pr_prompt(title=title, body=body) + # Short-circuit: if body very clearly links a related issue, just pass. + if has_linked_issue(body): + return { + **base_result, + "action": "pass-linked-issue", + "verdict": { + "verdict": "pass", + "linked_issue": True, + "explanation": "Linked-issue regex matched; LLM was not called.", + }, + } + else: + prompt = build_issue_prompt(title=title, body=body) + + if print_prompt: + return {**base_result, "action": "print-prompt", "prompt": prompt} + + if judge is None: + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + # No key configured — never take a destructive action. Report skip. + return { + **base_result, + "action": "skip-no-llm-key", + "prompt_preview": prompt[:200], + } + base_url = os.environ.get("OPENAI_BASE_URL") or None + + def judge(p: str) -> str: + return call_llm_judge(p, model=model, api_key=api_key, base_url=base_url) + + try: + raw = judge(prompt) + verdict = parse_verdict(raw) + except Exception as exc: # noqa: BLE001 - judge errors must never close PRs + return {**base_result, "action": "skip-llm-error", "error": str(exc)} + + decision = (verdict.get("verdict") or "").lower() + if decision != "fail": + return {**base_result, "action": "pass-llm", "verdict": verdict} + + if not close: + return {**base_result, "action": "would-close", "verdict": verdict} + + comment_body = ( + format_pr_close_comment(verdict) + if kind == "pr" + else format_issue_close_comment(verdict) + ) + post_comment(repo, number, comment_body) + if kind == "pr": + close_pr(repo, number) + else: + close_issue(repo, number) + + return { + **base_result, + "action": "closed", + "verdict": verdict, + "comment": comment_body, + } + + +# --------------------------------------------------------------------------- +# CLI + + +def render_summary(result: dict) -> str: + """Render a human-readable summary block (used for stdout + step summary).""" + lines = ["## Agent Shin verdict", ""] + lines.append( + f"- **{result['kind'].upper()} #{result['number']}**: {result.get('title', '')}" + ) + lines.append( + f"- **Author**: `{result.get('author', '')}` ({result.get('author_association', '')})" + ) + lines.append(f"- **State**: {result.get('state', '')}") + lines.append(f"- **Action**: `{result['action']}`") + verdict = result.get("verdict") + if verdict: + lines.append("") + lines.append("```json") + lines.append(json.dumps(verdict, indent=2)) + lines.append("```") + error = result.get("error") + if error: + lines.append("") + lines.append(f"_LLM error: {error}_") + comment = result.get("comment") + if comment: + lines.append("") + lines.append("### Would post comment:") + lines.append("") + lines.append("> " + comment.replace("\n", "\n> ")) + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True, help="Repository (owner/repo).") + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--pr", type=int, help="Pull request number to triage.") + target.add_argument("--issue", type=int, help="Issue number to triage.") + parser.add_argument( + "--close", + action="store_true", + help="Actually post comment + close on fail (default: dry run).", + ) + parser.add_argument( + "--model", + default=os.environ.get("TRIAGE_MODEL", DEFAULT_MODEL), + help=f"OpenAI-compatible model name (default: {DEFAULT_MODEL}).", + ) + parser.add_argument( + "--print-prompt", + action="store_true", + help="Print the prompt that would be sent to the judge and exit.", + ) + args = parser.parse_args() + + kind = "pr" if args.pr is not None else "issue" + number = args.pr if args.pr is not None else args.issue + + result = triage( + repo=args.repo, + kind=kind, + number=number, + close=args.close, + model=args.model, + print_prompt=args.print_prompt, + ) + + if result.get("action") == "print-prompt": + print(result["prompt"]) + return 0 + + summary = render_summary(result) + print(summary) + write_step_summary(summary + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/triage_issue_with_llm.yml b/.github/workflows/triage_issue_with_llm.yml new file mode 100644 index 00000000000..c9ddad5d3ea --- /dev/null +++ b/.github/workflows/triage_issue_with_llm.yml @@ -0,0 +1,75 @@ +name: Agent Shin — Issue triage + +# LLM-as-judge triage for external GitHub issues. +# +# DRY-RUN BY DEFAULT. See .github/workflows/triage_pr_with_llm.yml for the +# enablement procedure — same repo variable (`AGENT_SHIN_ENABLED=true`) +# unlocks the PR and issue triage flows together. + +on: + issues: + types: [opened, reopened] + workflow_dispatch: + inputs: + issue_number: + description: "Issue number to triage manually." + required: true + close: + description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + +permissions: + contents: read + issues: write + +jobs: + triage: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir "openai>=1.40.0" + + - name: Run Agent Shin + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + DISPATCH_CLOSE: ${{ github.event.inputs.close }} + ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }} + run: | + set -euo pipefail + ARGS=(--repo "${{ github.repository }}" --issue "${ISSUE_NUMBER}") + if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" != "false" ]; then + ARGS+=(--close) + echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode." + elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then + echo "::notice::Agent Shin is ENABLED but this trigger is dry-run." + else + echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no issues will be closed." + fi + # Automatic `issues` events stay dry-run regardless until the team + # explicitly invokes workflow_dispatch with close=true. + if [ "${GITHUB_EVENT_NAME:-}" = "issues" ]; then + ARGS=("${ARGS[@]/--close/}") + echo "::notice::issues trigger -> forcing dry-run." + fi + python3 .github/scripts/triage_with_llm.py "${ARGS[@]}" diff --git a/.github/workflows/triage_pr_with_llm.yml b/.github/workflows/triage_pr_with_llm.yml new file mode 100644 index 00000000000..f9b6d7fed47 --- /dev/null +++ b/.github/workflows/triage_pr_with_llm.yml @@ -0,0 +1,89 @@ +name: Agent Shin — PR triage + +# LLM-as-judge triage for external pull requests. +# +# DRY-RUN BY DEFAULT. Closures and public comments are gated on the repo +# variable `AGENT_SHIN_ENABLED` being set to the string `"true"`. Until then, +# every run only writes its verdict to the workflow step summary so the team +# can QA the judge's decisions before flipping it on. +# +# To enable for real: +# 1. Add a repo secret `OPENAI_API_KEY` (or compatible). +# 2. Set repo variable `AGENT_SHIN_ENABLED` to `true` +# (Settings > Secrets and variables > Actions > Variables). +# +# We use `pull_request_target` so the workflow has access to repo secrets +# and runs against PRs from forks. We never check out fork code — only read +# PR metadata via `gh api`, so this is safe. + +on: + pull_request_target: + types: [opened, reopened] + workflow_dispatch: + inputs: + pr_number: + description: "PR number to triage manually." + required: true + close: + description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + triage: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir "openai>=1.40.0" + + - name: Run Agent Shin + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + DISPATCH_CLOSE: ${{ github.event.inputs.close }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }} + run: | + set -euo pipefail + ARGS=(--repo "${{ github.repository }}" --pr "${PR_NUMBER}") + if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" != "false" ]; then + ARGS+=(--close) + echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode." + elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then + echo "::notice::Agent Shin is ENABLED but this trigger is dry-run (workflow_dispatch close=false or scheduled event)." + else + echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no PRs will be closed." + fi + # On the scheduled/automatic pull_request_target trigger we default to + # dry-run regardless, so the team can review verdicts in the step + # summary before any contributor sees a comment. Only the manual + # workflow_dispatch path (with close=true) closes PRs. + if [ "${GITHUB_EVENT_NAME:-}" = "pull_request_target" ]; then + # strip any --close added above + ARGS=("${ARGS[@]/--close/}") + echo "::notice::pull_request_target trigger -> forcing dry-run." + fi + python3 .github/scripts/triage_with_llm.py "${ARGS[@]}" diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py new file mode 100644 index 00000000000..de72cc8d7c9 --- /dev/null +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -0,0 +1,357 @@ +"""Unit tests for `.github/scripts/triage_with_llm.py` (Agent Shin).""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[2] / ".github" / "scripts" / "triage_with_llm.py" +) + + +@pytest.fixture(scope="module") +def triage_module(): + spec = importlib.util.spec_from_file_location("triage_with_llm", SCRIPT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules["triage_with_llm"] = module + spec.loader.exec_module(module) + return module + + +class TestIsInternalContributor: + @pytest.mark.parametrize("association", ["OWNER", "MEMBER", "COLLABORATOR"]) + def test_should_mark_org_associations_as_internal(self, triage_module, association): + item = { + "author_association": association, + "user": {"login": "krrishdholakia"}, + } + assert triage_module.is_internal_contributor(item) is True + + @pytest.mark.parametrize( + "association", + ["CONTRIBUTOR", "FIRST_TIME_CONTRIBUTOR", "FIRST_TIMER", "NONE", ""], + ) + def test_should_mark_outside_associations_as_external( + self, triage_module, association + ): + item = { + "author_association": association, + "user": {"login": "random-oss-dev"}, + } + assert triage_module.is_internal_contributor(item) is False + + @pytest.mark.parametrize( + "login", + ["dependabot[bot]", "greptile-apps[bot]", "dependabot", "github-actions"], + ) + def test_should_skip_bot_accounts_regardless_of_association( + self, triage_module, login + ): + item = {"author_association": "NONE", "user": {"login": login}} + assert triage_module.is_internal_contributor(item) is True + + +class TestHasLinkedIssue: + @pytest.mark.parametrize( + "body", + [ + "Fixes #1234", + "closes #1", + "Resolves #99", + "fix #42 — this addresses the regression", + "Refs https://github.com/BerriAI/litellm/issues/27000", + ], + ) + def test_should_detect_common_link_phrases(self, triage_module, body): + assert triage_module.has_linked_issue(body) is True + + @pytest.mark.parametrize( + "body", + [ + "", + "Some change", + "See #1234", # "see" is allowed per regex but we want documented coverage + ], + ) + def test_should_handle_empty_and_unrelated_bodies(self, triage_module, body): + # "See #1234" is intentionally accepted as a related-issue reference. + # Just make sure empty/unrelated bodies don't crash. + triage_module.has_linked_issue(body) + + def test_should_not_detect_when_only_html_comment_template(self, triage_module): + body = "" + assert triage_module.has_linked_issue(body) is False + + +class TestStripHtmlComments: + def test_should_remove_single_line_comments(self, triage_module): + text = "before after" + assert "placeholder" not in triage_module.strip_html_comments(text) + + def test_should_remove_multiline_comments(self, triage_module): + text = "kept\n\nkept2" + cleaned = triage_module.strip_html_comments(text) + assert "Fixes #1" not in cleaned + assert "kept" in cleaned and "kept2" in cleaned + + def test_should_handle_none(self, triage_module): + assert triage_module.strip_html_comments(None) == "" + + +class TestParseVerdict: + def test_should_parse_plain_json(self, triage_module): + raw = '{"verdict": "pass", "missing": []}' + assert triage_module.parse_verdict(raw)["verdict"] == "pass" + + def test_should_strip_markdown_fence(self, triage_module): + raw = '```json\n{"verdict": "fail", "missing": ["foo"]}\n```' + result = triage_module.parse_verdict(raw) + assert result["verdict"] == "fail" + assert result["missing"] == ["foo"] + + def test_should_extract_embedded_json_from_prose(self, triage_module): + raw = 'Here you go: {"verdict": "pass", "missing": []}\nThanks.' + assert triage_module.parse_verdict(raw)["verdict"] == "pass" + + def test_should_raise_for_unparseable_text(self, triage_module): + with pytest.raises(ValueError): + triage_module.parse_verdict("not even close to json") + + def test_should_raise_for_empty(self, triage_module): + with pytest.raises(ValueError): + triage_module.parse_verdict("") + + +class TestBuildPrompts: + def test_should_include_pr_title_and_body(self, triage_module): + prompt = triage_module.build_pr_prompt( + title="Add foo", body=" Real body" + ) + assert "Add foo" in prompt + assert "Real body" in prompt + assert "comment" not in prompt # HTML comments are stripped + + def test_should_show_empty_marker_for_empty_pr_body(self, triage_module): + prompt = triage_module.build_pr_prompt(title="t", body="") + assert "(empty)" in prompt + + def test_should_include_issue_title_and_body(self, triage_module): + prompt = triage_module.build_issue_prompt(title="Bug", body="repro here") + assert "Bug" in prompt + assert "repro here" in prompt + + +class TestTriageOrchestration: + """End-to-end-ish tests that mock both gh fetchers and the LLM.""" + + def _make_pr(self, **overrides): + base = { + "number": 1, + "title": "PR title", + "body": "PR body", + "state": "open", + "author_association": "NONE", + "user": {"login": "outside-dev"}, + } + base.update(overrides) + return base + + def test_should_skip_internal_author(self, triage_module, monkeypatch): + pr = self._make_pr( + author_association="MEMBER", user={"login": "krrishdholakia"} + ) + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + + def boom(*a, **kw): + pytest.fail("LLM should not be called for internal authors") + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=boom, + ) + assert result["action"] == "skip-internal-author" + + def test_should_skip_closed_pr(self, triage_module, monkeypatch): + pr = self._make_pr(state="closed") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("should not run on closed PRs"), + ) + assert result["action"] == "skip-not-open" + + def test_should_short_circuit_on_linked_issue(self, triage_module, monkeypatch): + pr = self._make_pr(body="Fixes #1234\n\nFoo bar") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=lambda p: pytest.fail("LLM should not be called"), + ) + assert result["action"] == "pass-linked-issue" + assert result["verdict"]["verdict"] == "pass" + + def test_should_return_pass_llm_when_judge_passes(self, triage_module, monkeypatch): + pr = self._make_pr(body="Long body, no linked issue.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + captured = {} + + def judge(prompt): + captured["prompt"] = prompt + return json.dumps({"verdict": "pass", "missing": [], "explanation": "ok"}) + + result = triage_module.triage( + repo="o/r", kind="pr", number=1, close=True, model="m", judge=judge + ) + assert result["action"] == "pass-llm" + assert "Long body" in captured["prompt"] + + def test_should_return_would_close_in_dry_run(self, triage_module, monkeypatch): + pr = self._make_pr(body="just a sentence.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + + def fake_post(*a, **kw): + pytest.fail("should not post comments in dry-run") + + def fake_close(*a, **kw): + pytest.fail("should not close in dry-run") + + monkeypatch.setattr(triage_module, "post_comment", fake_post) + monkeypatch.setattr(triage_module, "close_pr", fake_close) + + verdict = { + "verdict": "fail", + "missing": ["problem description", "QA proof"], + "explanation": "Body is one sentence.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=False, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "would-close" + assert result["verdict"]["missing"] == ["problem description", "QA proof"] + + def test_should_post_comment_and_close_when_close_enabled( + self, triage_module, monkeypatch + ): + pr = self._make_pr(body="just a sentence.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + posted = {} + closed = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update({"repo": repo, "n": n, "body": body}), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda repo, n: closed.update({"repo": repo, "n": n}), + ) + + verdict = { + "verdict": "fail", + "missing": ["QA proof"], + "explanation": "Body too thin.", + } + result = triage_module.triage( + repo="o/r", + kind="pr", + number=42, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "closed" + assert posted["n"] == 42 and closed["n"] == 42 + assert "Agent Shin" in posted["body"] + assert "QA proof" in posted["body"] + + def test_should_skip_on_llm_error_in_close_mode(self, triage_module, monkeypatch): + pr = self._make_pr(body="something.") + monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr) + monkeypatch.setattr( + triage_module, + "post_comment", + lambda *a, **kw: pytest.fail("must not comment on LLM error"), + ) + monkeypatch.setattr( + triage_module, + "close_pr", + lambda *a, **kw: pytest.fail("must not close on LLM error"), + ) + + def broken_judge(prompt): + raise RuntimeError("upstream 500") + + result = triage_module.triage( + repo="o/r", + kind="pr", + number=1, + close=True, + model="m", + judge=broken_judge, + ) + assert result["action"] == "skip-llm-error" + assert "upstream 500" in result["error"] + + def test_should_triage_issues_kind(self, triage_module, monkeypatch): + issue = { + "number": 7, + "title": "Bug: X is broken", + "body": "no detail", + "state": "open", + "author_association": "NONE", + "user": {"login": "outside"}, + } + monkeypatch.setattr(triage_module, "fetch_issue", lambda repo, n: issue) + closed = {} + posted = {} + monkeypatch.setattr( + triage_module, + "post_comment", + lambda repo, n, body: posted.update(body=body), + ) + monkeypatch.setattr( + triage_module, "close_issue", lambda repo, n: closed.update(n=n) + ) + + verdict = { + "verdict": "fail", + "kind": "bug", + "has_repro": False, + "missing": ["reproduction", "expected vs. actual"], + "explanation": "No repro provided.", + } + result = triage_module.triage( + repo="o/r", + kind="issue", + number=7, + close=True, + model="m", + judge=lambda p: json.dumps(verdict), + ) + assert result["action"] == "closed" + assert closed["n"] == 7 + assert "reproduction" in posted["body"]