mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
ci: restore Agent Shin PR triage and review gate on safe triggers
Reintroduces triage_pr_with_llm.yml and review_gate.yml, removed in #30784 because they ran on pull_request_target. Both now run only on workflow_dispatch (plus the review gate's daily schedule sweep), so fork authors cannot fire them and no privileged context is ever exposed to a fork-controlled event. Instant reaction to fork PR events is provided by the agent-shin GitHub App bridge, which dispatches these workflows with the PR number when AGENT_SHIN_ENABLED is true.
This commit is contained in:
mateo-berri
parent
3e9e52042a
commit
39fefc6c44
2 changed files with 225 additions and 0 deletions
133
.github/workflows/review_gate.yml
vendored
Normal file
133
.github/workflows/review_gate.yml
vendored
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
name: Agent Shin — review gate
|
||||
|
||||
# Keeps the `ready for review` label in sync with whether an external PR
|
||||
# currently clears BOTH the LLM rubric AND Greptile's confidence score.
|
||||
#
|
||||
# pass -> add `ready for review` + a "passed / all clear" comment
|
||||
# regress -> remove the label + a "what's missing" comment (PR stays open)
|
||||
# fail, <24h old -> a one-time "what's missing" notice (grace window)
|
||||
# fail, >24h old -> close + a comment (reopen via `@agent-shin reconsider`)
|
||||
#
|
||||
# DRY-RUN BY DEFAULT. Every side effect (label add/remove, comment, close) is
|
||||
# gated behind `--close`, which is only added when the repo variable
|
||||
# `AGENT_SHIN_ENABLED == "true"`. Until then runs only write the verdict to the
|
||||
# workflow step summary.
|
||||
#
|
||||
# Manual single PR: gh workflow run "Agent Shin — review gate" -f pr_number=NNN
|
||||
# Manual dry-run: gh workflow run "Agent Shin — review gate" -f close=false
|
||||
#
|
||||
# This workflow deliberately has NO pull_request/pull_request_target trigger
|
||||
# (see PR #30784 and the agent-shin bridge repo). Per-PR reconciliation the
|
||||
# moment a fork PR is opened/updated comes from the BerriAI/agent-shin
|
||||
# Cloudflare Worker, which dispatches this workflow with the PR number and
|
||||
# close=true. Only base-repo code runs here, on triggers fork authors cannot
|
||||
# fire; the daily schedule below re-reconciles everything as a catch-all.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Daily at 09:30 UTC — re-reconciles labels as Greptile re-reviews land.
|
||||
- cron: "30 9 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: "Single PR to reconcile (omit to sweep all open PRs)."
|
||||
required: false
|
||||
close:
|
||||
description: "If AGENT_SHIN_ENABLED=true, actually act (false = dry run)."
|
||||
required: false
|
||||
default: "false"
|
||||
type: choice
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
grace_days:
|
||||
description: "Hours/24 a failing, un-tagged PR may stay open before close."
|
||||
required: false
|
||||
default: "1"
|
||||
min_greptile_score:
|
||||
description: "Greptile score below which a PR counts as not passing (1-5)."
|
||||
required: false
|
||||
default: "4"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
review-gate:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout triage script
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install LLM client
|
||||
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
|
||||
|
||||
- name: Run review gate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# workflow_dispatch can only be fired by actors with actions:write
|
||||
# (maintainers, or the agent-shin bridge app — which itself refuses
|
||||
# to dispatch unless AGENT_SHIN_ENABLED is "true"), so exposing the
|
||||
# LLM key here cannot be forced by an external user churning fork
|
||||
# PRs while the bot is still in dry-run.
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
|
||||
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
|
||||
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
|
||||
CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }}
|
||||
GRACE_DAYS: ${{ github.event.inputs.grace_days || '1' }}
|
||||
MIN_GREPTILE_SCORE: ${{ github.event.inputs.min_greptile_score || '4' }}
|
||||
INPUT_PR: ${{ github.event.inputs.pr_number }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
COMMON=(--review-gate --grace-days "${GRACE_DAYS}" --min-greptile-score "${MIN_GREPTILE_SCORE}")
|
||||
|
||||
# Fail-safe gating, identical philosophy to the Greptile closer:
|
||||
# - AGENT_SHIN_ENABLED must be the EXACT string "true" to act at all.
|
||||
# - A manual dispatch can still preview with close=false.
|
||||
# - The schedule sweep acts once enabled — that is the whole point
|
||||
# of the gate (re-tag / un-tag automatically).
|
||||
# - Bridge-dispatched per-PR runs pass close=true explicitly, so
|
||||
# they act exactly like the old automatic PR-event trigger did.
|
||||
DO_CLOSE="false"
|
||||
if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then
|
||||
echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> dry-run (no labels/comments/closes)."
|
||||
elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG:-false}" = "true" ]; then
|
||||
DO_CLOSE="true"
|
||||
echo "::notice::Dispatched run with close=true -> acting for real."
|
||||
elif [ "${GITHUB_EVENT_NAME:-}" != "workflow_dispatch" ]; then
|
||||
DO_CLOSE="true"
|
||||
echo "::notice::Enabled automatic trigger (${GITHUB_EVENT_NAME:-}) -> acting for real."
|
||||
else
|
||||
echo "::notice::Manual dispatch with close=false -> dry-run."
|
||||
fi
|
||||
if [ "${DO_CLOSE}" = "true" ]; then
|
||||
COMMON+=(--close)
|
||||
fi
|
||||
|
||||
# Single PR (explicit input) vs. sweep over all open PRs.
|
||||
if [ -n "${INPUT_PR:-}" ]; then
|
||||
python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${INPUT_PR}" "${COMMON[@]}"
|
||||
else
|
||||
echo "::notice::Sweeping all open PRs."
|
||||
# Match GH_LIST_ALL_LIMIT in agent_shin_shared.py: gh lists newest-first,
|
||||
# so any cap below the real backlog silently drops the *oldest* PRs —
|
||||
# exactly the stale ones this daily sweep is meant to reconcile.
|
||||
mapfile -t NUMBERS < <(gh pr list --repo "${{ github.repository }}" --state open --limit 100000 --json number --jq '.[].number')
|
||||
for n in "${NUMBERS[@]}"; do
|
||||
echo "::group::PR #${n}"
|
||||
python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${n}" "${COMMON[@]}" || echo "::warning::review gate errored on #${n}"
|
||||
echo "::endgroup::"
|
||||
done
|
||||
fi
|
||||
92
.github/workflows/triage_pr_with_llm.yml
vendored
Normal file
92
.github/workflows/triage_pr_with_llm.yml
vendored
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
name: Agent Shin — PR triage
|
||||
|
||||
# LLM-as-judge triage for external pull requests.
|
||||
#
|
||||
# DRY-RUN BY DEFAULT. Closures and public comments are gated on the repo
|
||||
# variable `AGENT_SHIN_ENABLED` being set to the string `"true"`. Until then,
|
||||
# every run only writes its verdict to the workflow step summary so the team
|
||||
# can QA the judge's decisions before flipping it on.
|
||||
#
|
||||
# To enable for real:
|
||||
# 1. Add a repo secret `OPENAI_API_KEY` (or compatible).
|
||||
# 2. Set repo variable `AGENT_SHIN_ENABLED` to `true`
|
||||
# (Settings > Secrets and variables > Actions > Variables).
|
||||
#
|
||||
# This workflow deliberately has NO pull_request/pull_request_target trigger
|
||||
# (see PR #30784 and the agent-shin bridge repo). Instant reaction to fork PR
|
||||
# events comes from the BerriAI/agent-shin Cloudflare Worker, which receives
|
||||
# GitHub App webhooks and dispatches this workflow with the PR number. Only
|
||||
# base-repo code runs here, on a trigger fork authors cannot fire.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: "PR number to triage."
|
||||
required: true
|
||||
close:
|
||||
description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail."
|
||||
required: false
|
||||
default: "false"
|
||||
type: choice
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
triage:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout triage script
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install LLM client
|
||||
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
|
||||
|
||||
- name: Run Agent Shin
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# workflow_dispatch can only be fired by actors with actions:write
|
||||
# (maintainers, or the agent-shin bridge app — which itself refuses
|
||||
# to dispatch unless AGENT_SHIN_ENABLED is "true"), so exposing the
|
||||
# LLM key here cannot be forced by an external user churning fork
|
||||
# PRs while the bot is still in dry-run.
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
|
||||
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
|
||||
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
|
||||
DISPATCH_CLOSE: ${{ github.event.inputs.close }}
|
||||
PR_NUMBER: ${{ github.event.inputs.pr_number }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ARGS=(--repo "${{ github.repository }}" --pr "${PR_NUMBER}")
|
||||
# Fail-safe gating: only the EXACT string "true" enables the
|
||||
# destructive --close path. The workflow_dispatch input is a
|
||||
# `choice` dropdown of "true"/"false" so the UI is constrained,
|
||||
# but the API (`gh workflow run -f close=...`) accepts any
|
||||
# string, and a `!= "false"` check would treat "True", "yes",
|
||||
# "1", "TRUE", typos, and accidental whitespace as enabling
|
||||
# closure. Mirror the Greptile closer's `= "true"` pattern.
|
||||
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" = "true" ]; then
|
||||
ARGS+=(--close)
|
||||
echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode."
|
||||
elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
|
||||
echo "::notice::Agent Shin is ENABLED but this run is dry-run (close != 'true'). Bridge-dispatched runs always pass close=false: instant triage never closes, only the review gate's grace-window path does."
|
||||
else
|
||||
echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no PRs will be closed."
|
||||
fi
|
||||
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"
|
||||
Loading…
Add table
Reference in a new issue