diff --git a/.github/workflows/review_gate.yml b/.github/workflows/review_gate.yml new file mode 100644 index 00000000000..4595f09de7e --- /dev/null +++ b/.github/workflows/review_gate.yml @@ -0,0 +1,133 @@ +name: Agent Shin — review gate + +# Keeps the `ready for review` label in sync with whether an external PR +# currently clears BOTH the LLM rubric AND Greptile's confidence score. +# +# pass -> add `ready for review` + a "passed / all clear" comment +# regress -> remove the label + a "what's missing" comment (PR stays open) +# fail, <24h old -> a one-time "what's missing" notice (grace window) +# fail, >24h old -> close + a comment (reopen via `@agent-shin reconsider`) +# +# DRY-RUN BY DEFAULT. Every side effect (label add/remove, comment, close) is +# gated behind `--close`, which is only added when the repo variable +# `AGENT_SHIN_ENABLED == "true"`. Until then runs only write the verdict to the +# workflow step summary. +# +# Manual single PR: gh workflow run "Agent Shin — review gate" -f pr_number=NNN +# Manual dry-run: gh workflow run "Agent Shin — review gate" -f close=false +# +# This workflow deliberately has NO pull_request/pull_request_target trigger +# (see PR #30784 and the agent-shin bridge repo). Per-PR reconciliation the +# moment a fork PR is opened/updated comes from the BerriAI/agent-shin +# Cloudflare Worker, which dispatches this workflow with the PR number and +# close=true. Only base-repo code runs here, on triggers fork authors cannot +# fire; the daily schedule below re-reconciles everything as a catch-all. + +on: + schedule: + # Daily at 09:30 UTC — re-reconciles labels as Greptile re-reviews land. + - cron: "30 9 * * *" + workflow_dispatch: + inputs: + pr_number: + description: "Single PR to reconcile (omit to sweep all open PRs)." + required: false + close: + description: "If AGENT_SHIN_ENABLED=true, actually act (false = dry run)." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + grace_days: + description: "Hours/24 a failing, un-tagged PR may stay open before close." + required: false + default: "1" + min_greptile_score: + description: "Greptile score below which a PR counts as not passing (1-5)." + required: false + default: "4" + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + review-gate: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt + + - name: Run review gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # workflow_dispatch can only be fired by actors with actions:write + # (maintainers, or the agent-shin bridge app — which itself refuses + # to dispatch unless AGENT_SHIN_ENABLED is "true"), so exposing the + # LLM key here cannot be forced by an external user churning fork + # PRs while the bot is still in dry-run. + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }} + GRACE_DAYS: ${{ github.event.inputs.grace_days || '1' }} + MIN_GREPTILE_SCORE: ${{ github.event.inputs.min_greptile_score || '4' }} + INPUT_PR: ${{ github.event.inputs.pr_number }} + run: | + set -euo pipefail + COMMON=(--review-gate --grace-days "${GRACE_DAYS}" --min-greptile-score "${MIN_GREPTILE_SCORE}") + + # Fail-safe gating, identical philosophy to the Greptile closer: + # - AGENT_SHIN_ENABLED must be the EXACT string "true" to act at all. + # - A manual dispatch can still preview with close=false. + # - The schedule sweep acts once enabled — that is the whole point + # of the gate (re-tag / un-tag automatically). + # - Bridge-dispatched per-PR runs pass close=true explicitly, so + # they act exactly like the old automatic PR-event trigger did. + DO_CLOSE="false" + if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then + echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> dry-run (no labels/comments/closes)." + elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG:-false}" = "true" ]; then + DO_CLOSE="true" + echo "::notice::Dispatched run with close=true -> acting for real." + elif [ "${GITHUB_EVENT_NAME:-}" != "workflow_dispatch" ]; then + DO_CLOSE="true" + echo "::notice::Enabled automatic trigger (${GITHUB_EVENT_NAME:-}) -> acting for real." + else + echo "::notice::Manual dispatch with close=false -> dry-run." + fi + if [ "${DO_CLOSE}" = "true" ]; then + COMMON+=(--close) + fi + + # Single PR (explicit input) vs. sweep over all open PRs. + if [ -n "${INPUT_PR:-}" ]; then + python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${INPUT_PR}" "${COMMON[@]}" + else + echo "::notice::Sweeping all open PRs." + # Match GH_LIST_ALL_LIMIT in agent_shin_shared.py: gh lists newest-first, + # so any cap below the real backlog silently drops the *oldest* PRs — + # exactly the stale ones this daily sweep is meant to reconcile. + mapfile -t NUMBERS < <(gh pr list --repo "${{ github.repository }}" --state open --limit 100000 --json number --jq '.[].number') + for n in "${NUMBERS[@]}"; do + echo "::group::PR #${n}" + python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${n}" "${COMMON[@]}" || echo "::warning::review gate errored on #${n}" + echo "::endgroup::" + done + fi diff --git a/.github/workflows/triage_pr_with_llm.yml b/.github/workflows/triage_pr_with_llm.yml new file mode 100644 index 00000000000..a0b0e960581 --- /dev/null +++ b/.github/workflows/triage_pr_with_llm.yml @@ -0,0 +1,92 @@ +name: Agent Shin — PR triage + +# LLM-as-judge triage for external pull requests. +# +# DRY-RUN BY DEFAULT. Closures and public comments are gated on the repo +# variable `AGENT_SHIN_ENABLED` being set to the string `"true"`. Until then, +# every run only writes its verdict to the workflow step summary so the team +# can QA the judge's decisions before flipping it on. +# +# To enable for real: +# 1. Add a repo secret `OPENAI_API_KEY` (or compatible). +# 2. Set repo variable `AGENT_SHIN_ENABLED` to `true` +# (Settings > Secrets and variables > Actions > Variables). +# +# This workflow deliberately has NO pull_request/pull_request_target trigger +# (see PR #30784 and the agent-shin bridge repo). Instant reaction to fork PR +# events comes from the BerriAI/agent-shin Cloudflare Worker, which receives +# GitHub App webhooks and dispatches this workflow with the PR number. Only +# base-repo code runs here, on a trigger fork authors cannot fire. + +on: + workflow_dispatch: + inputs: + pr_number: + description: "PR number to triage." + required: true + close: + description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail." + required: false + default: "false" + type: choice + options: + - "true" + - "false" + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + triage: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - name: Checkout triage script + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install LLM client + run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt + + - name: Run Agent Shin + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # workflow_dispatch can only be fired by actors with actions:write + # (maintainers, or the agent-shin bridge app — which itself refuses + # to dispatch unless AGENT_SHIN_ENABLED is "true"), so exposing the + # LLM key here cannot be forced by an external user churning fork + # PRs while the bot is still in dry-run. + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }} + TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }} + AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }} + DISPATCH_CLOSE: ${{ github.event.inputs.close }} + PR_NUMBER: ${{ github.event.inputs.pr_number }} + run: | + set -euo pipefail + ARGS=(--repo "${{ github.repository }}" --pr "${PR_NUMBER}") + # Fail-safe gating: only the EXACT string "true" enables the + # destructive --close path. The workflow_dispatch input is a + # `choice` dropdown of "true"/"false" so the UI is constrained, + # but the API (`gh workflow run -f close=...`) accepts any + # string, and a `!= "false"` check would treat "True", "yes", + # "1", "TRUE", typos, and accidental whitespace as enabling + # closure. Mirror the Greptile closer's `= "true"` pattern. + if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" = "true" ]; then + ARGS+=(--close) + echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode." + elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then + echo "::notice::Agent Shin is ENABLED but this run is dry-run (close != 'true'). Bridge-dispatched runs always pass close=false: instant triage never closes, only the review gate's grace-window path does." + else + echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no PRs will be closed." + fi + python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"