litellm/.github/workflows/review_gate.yml

name: Agent Shin — review gate

# Keeps the `ready for review` label in sync with whether an external PR
# currently clears BOTH the LLM rubric AND Greptile's confidence score.
#
#   pass            -> add `ready for review` + a "passed / all clear" comment
#   regress         -> remove the label + a "what's missing" comment (PR stays open)
#   fail, <24h old  -> a one-time "what's missing" notice (grace window)
#   fail, >24h old  -> close + a comment (reopen via `@agent-shin reconsider`)
#
# DRY-RUN BY DEFAULT. Every side effect (label add/remove, comment, close) is
# gated behind `--close`, which is only added when the repo variable
# `AGENT_SHIN_ENABLED == "true"`. Until then runs only write the verdict to the
# workflow step summary.
#
# Manual single PR:   gh workflow run "Agent Shin — review gate" -f pr_number=NNN
# Manual dry-run:     gh workflow run "Agent Shin — review gate" -f close=false
#
# This workflow deliberately has NO pull_request/pull_request_target trigger
# (see PR #30784 and the agent-shin bridge repo). Per-PR reconciliation the
# moment a fork PR is opened/updated comes from the BerriAI/agent-shin
# Cloudflare Worker, which dispatches this workflow with the PR number and
# close=true. Only base-repo code runs here, on triggers fork authors cannot
# fire; the daily schedule below re-reconciles everything as a catch-all.

on:
  schedule:
    # Daily at 09:30 UTC — re-reconciles labels as Greptile re-reviews land.
    - cron: "30 9 * * *"
  workflow_dispatch:
    inputs:
      pr_number:
        description: "Single PR to reconcile (omit to sweep all open PRs)."
        required: false
      close:
        description: "If AGENT_SHIN_ENABLED=true, actually act (false = dry run)."
        required: false
        default: "false"
        type: choice
        options:
          - "true"
          - "false"
      grace_days:
        description: "Days a failing, un-tagged PR may stay open before close."
        required: false
        default: "1"
      min_greptile_score:
        description: "Greptile score below which a PR counts as not passing (1-5)."
        required: false
        default: "4"

permissions:
  contents: read
  issues: write
  pull-requests: write

jobs:
  review-gate:
    if: github.repository == 'BerriAI/litellm'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout triage script
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          sparse-checkout: .github/scripts
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Install LLM client
        run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt

      - name: Run review gate
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # workflow_dispatch can only be fired by actors with actions:write
          # (maintainers, or the agent-shin bridge app — which itself refuses
          # to dispatch unless AGENT_SHIN_ENABLED is "true"), so exposing the
          # LLM key here cannot be forced by an external user churning fork
          # PRs while the bot is still in dry-run.
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
          TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
          AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
          CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }}
          GRACE_DAYS: ${{ github.event.inputs.grace_days || '1' }}
          MIN_GREPTILE_SCORE: ${{ github.event.inputs.min_greptile_score || '4' }}
          INPUT_PR: ${{ github.event.inputs.pr_number }}
        run: |
          set -euo pipefail
          COMMON=(--review-gate --grace-days "${GRACE_DAYS}" --min-greptile-score "${MIN_GREPTILE_SCORE}")

          # Fail-safe gating, identical philosophy to the Greptile closer:
          #   - AGENT_SHIN_ENABLED must be the EXACT string "true" to act at all.
          #   - A manual dispatch can still preview with close=false.
          #   - The schedule sweep acts once enabled — that is the whole point
          #     of the gate (re-tag / un-tag automatically).
          #   - Bridge-dispatched per-PR runs pass close=true explicitly, so
          #     they act exactly like the old automatic PR-event trigger did.
          DO_CLOSE="false"
          if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then
            echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> dry-run (no labels/comments/closes)."
          elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG:-false}" = "true" ]; then
            DO_CLOSE="true"
            echo "::notice::Dispatched run with close=true -> acting for real."
          elif [ "${GITHUB_EVENT_NAME:-}" != "workflow_dispatch" ]; then
            DO_CLOSE="true"
            echo "::notice::Enabled automatic trigger (${GITHUB_EVENT_NAME:-}) -> acting for real."
          else
            echo "::notice::Manual dispatch with close=false -> dry-run."
          fi
          if [ "${DO_CLOSE}" = "true" ]; then
            COMMON+=(--close)
          fi

          # Single PR (explicit input) vs. sweep over all open PRs.
          if [ -n "${INPUT_PR:-}" ]; then
            python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${INPUT_PR}" "${COMMON[@]}"
          else
            echo "::notice::Sweeping all open PRs."
            # Match GH_LIST_ALL_LIMIT in agent_shin_shared.py: gh lists newest-first,
            # so any cap below the real backlog silently drops the *oldest* PRs —
            # exactly the stale ones this daily sweep is meant to reconcile.
            mapfile -t NUMBERS < <(gh pr list --repo "${{ github.repository }}" --state open --limit 100000 --json number --jq '.[].number')
            for n in "${NUMBERS[@]}"; do
              echo "::group::PR #${n}"
              python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${n}" "${COMMON[@]}" || echo "::warning::review gate errored on #${n}"
              echo "::endgroup::"
            done
          fi