scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

Merge branch 'litellm_internal_staging' into feature/improve-gigachat-provider

Browse source
This commit is contained in:
KnyazSh 2026-06-18 13:29:56 +00:00
commit d185b8fa49
284 changed files with 23082 additions and 2044 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
.circleci/config.yml
View file

@ -133,6 +133,26 @@ commands:
done
echo "record/replay proxy did not become ready" >&2
exit 1
start_fake_openai_endpoint:
description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced."
steps:
- run:
name: Start fake OpenAI endpoint
background: true
command: |
uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190
- run:
name: Wait for fake OpenAI endpoint
command: |
for i in $(seq 1 30); do
if curl -sf http://localhost:8190/health >/dev/null 2>&1; then
echo "fake OpenAI endpoint is up"
exit 0
fi
sleep 1
done
echo "fake OpenAI endpoint did not become ready" >&2
exit 1
setup_litellm_enterprise_pip:
steps:
- run:
@ -168,6 +188,8 @@ jobs:
name: win/default
shell: powershell.exe
working_directory: ~/project
environment:
UV_PYTHON: "3.11"
steps:
- checkout
- run:
@ -200,7 +222,7 @@ jobs:
if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) {
Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`""
}
uv sync --frozen --group dev --python (Get-Command python).Source
uv sync --frozen --group dev --python 3.11
- run:
name: Run Windows-specific test
command: |
@ -594,6 +616,8 @@ jobs:
working_directory: ~/project
resource_class: large
parallelism: 4
environment:
FAKE_OPENAI_API_BASE: http://127.0.0.1:8190
steps:
- checkout
- setup_google_dns
@ -609,6 +633,7 @@ jobs:
paths:
- ~/.cache/uv
key: v1-uv-cache-{{ checksum "uv.lock" }}
- start_fake_openai_endpoint
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@ -1549,6 +1574,7 @@ jobs:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_fake_openai_endpoint
- start_postgres:
db_name: litellm_test
- attach_workspace:
@ -1586,6 +1612,7 @@ jobs:
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DISABLE_SCHEMA_UPDATE="True" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
--name my-app \
--add-host=host.docker.internal:host-gateway \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
@ -1648,6 +1675,7 @@ jobs:
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker tag litellm-docker-database:ci my-app:latest
- start_openai_record_replay_proxy
- start_fake_openai_endpoint
- run:
name: Run Docker container
command: |
@ -1655,6 +1683,7 @@ jobs:
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e USE_PRISMA_MIGRATE=True \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
@ -1817,6 +1846,7 @@ jobs:
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- start_openai_record_replay_proxy
- start_fake_openai_endpoint
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -1830,6 +1860,7 @@ jobs:
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
@ -1889,6 +1920,7 @@ jobs:
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE="bad-license" \
--add-host host.docker.internal:host-gateway \
--name my-app-3 \
@ -1938,6 +1970,7 @@ jobs:
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
- start_redis
- start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
@ -1961,6 +1994,7 @@ jobs:
-e REDIS_PORT=6379 \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
@ -2020,6 +2054,7 @@ jobs:
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
- start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
@ -2039,6 +2074,7 @@ jobs:
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
@ -2060,6 +2096,7 @@ jobs:
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
@ -2112,6 +2149,7 @@ jobs:
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
- start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
@ -2129,6 +2167,7 @@ jobs:
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e STORE_MODEL_IN_DB="True" \
-e LITELLM_MASTER_KEY="sk-1234" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--add-host host.docker.internal:host-gateway \
--name my-app \
@ -2187,6 +2226,7 @@ jobs:
command: |
docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
- start_postgres
- start_fake_openai_endpoint
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -2200,6 +2240,7 @@ jobs:
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \

50
.github/scripts/_agent_shin_actions.py vendored Normal file
View file

@ -0,0 +1,50 @@
"""Dry-run wrapper(s) around Agent Shin GitHub mutations.
The rollout scripts currently need only one mutation wrapped, so this module
exposes a single ``maybe_post_comment`` helper. It takes a ``dry_run: bool``
keyword argument and the body is intentionally trivial:
if dry_run:
print(...) # log what we would do, return
return
real_mutation(...) # otherwise, actually do it
That shape means a dry-run preview differs from the real run in exactly one
line per side effect: the call site. So when you `python3 script.py` locally
without ``--close``, you can be confident the actions printed are the ones the
GitHub Action would have performed (modulo ordering on retry/error paths,
which are deliberately simple). Any further mutation a rollout script needs
should get the same ``maybe_*`` treatment instead of calling the raw
``triage_with_llm`` mutation directly.
Importing from this module pulls in the real mutation from ``triage_with_llm``
— call sites in the rollout scripts should NEVER import ``post_comment``
directly; that would skip the dry-run gate and is the bug class this module
exists to prevent.
"""
from __future__ import annotations
import sys
import textwrap
# Import the module itself rather than the bare names so monkeypatching
# `triage_with_llm.post_comment` (or any of the other mutations) in tests is
# reflected here — `from triage_with_llm import post_comment` would bind the
# original function to a local name and bypass the patch, defeating the whole
# point of these wrappers.
import triage_with_llm
def _log(line: str) -> None:
"""Print a single dry-run line to stdout (one log statement per side effect)."""
print(line, file=sys.stdout, flush=True)
def maybe_post_comment(repo: str, number: int, body: str, *, dry_run: bool) -> None:
"""Post a comment on ``repo#number`` — or, in dry-run, log what we would post."""
if dry_run:
_log(f"[DRY RUN] comment {repo}#{number}:")
_log(textwrap.indent(body, " "))
return
triage_with_llm.post_comment(repo, number, body)

211
.github/scripts/agent_shin_shared.py vendored Normal file
View file

@ -0,0 +1,211 @@
"""Constants and helpers shared by Agent Shin's triage scripts.
Both `triage_with_llm.py` (the LLM-judge entrypoint) and
`close_low_quality_prs.py` (the daily Greptile-score sweep) need to
agree on the same notions of:
* What counts as a Greptile-authored review comment
(``GREPTILE_BOT_LOGINS``) and how to extract a confidence score from
its body (``SCORE_PATTERN`` / :func:`extract_greptile_score`).
* How long the 2-hour grace window is (``GRACE_PERIOD_SECONDS``) and
the HTML marker stamped into a grace-warning comment so the *other*
script can see "Agent Shin already warned" and behave accordingly
(``GRACE_COMMENT_MARKER``).
* Who Agent Shin is on GitHub (``AGENT_SHIN_DEFAULT_BOT_LOGIN``).
* How GitHub-style ISO-8601 timestamps round-trip into timezone-aware
:class:`datetime.datetime` (:func:`parse_iso8601`).
Keeping these in one module means a future change (new Greptile output
format, a longer grace window, a new allowlisted account) is a single edit
instead of two — the original split version had to call out in comments
that the two copies "must stay in sync" precisely because nothing
enforced it.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import re
import subprocess
from typing import Iterable
GREPTILE_BOT_LOGINS = frozenset({"greptile-apps", "greptile-apps[bot]"})
SCORE_PATTERN = re.compile(
r"confidence\s*score\s*[:\-]?\s*(\d+)\s*/\s*5",
re.IGNORECASE,
)
GRACE_COMMENT_MARKER = "<!-- agent-shin:grace-warning -->"
# Hidden HTML marker stamped on every Agent Shin auto-close comment (the LLM
# judge's grace/review-gate close and the daily Greptile sweep's close).
# `was_closed_by_agent_shin` requires this marker — not just the closing actor —
# before `@agent-shin reconsider` may reopen, because the `github-actions[bot]`
# identity is shared with every other workflow in the repo and is not unique to
# Agent Shin. Both close paths must stamp it or the reconsider path silently
# rejects the contributor.
AGENT_SHIN_CLOSE_MARKER = "<!-- agent-shin:closed -->"
# 2 hours between the grace warning and the auto-close. Short enough to
# dogfood the "fix it before it closes" loop in one sitting; bump back up
# (e.g. 86400 for a day) for the public rollout.
GRACE_PERIOD_SECONDS = 7200
AGENT_SHIN_DEFAULT_BOT_LOGIN = "github-actions[bot]"
def _logins(*names: str) -> frozenset[str]:
"""Build a login set normalized for case-insensitive membership checks.
Callers compare via ``login.lower() in <set>``, so the stored values
must be lowercase. Normalizing here lets the literals keep each
account's canonical GitHub casing (e.g. ``SwiftWinds``) for
readability without breaking the lookup.
"""
return frozenset(name.lower() for name in names)
# Dogfood rollout gate. While this set is non-empty, Agent Shin acts ONLY on
# PRs/issues authored by these logins and skips everyone else. For an
# allowlisted author the usual internal/external classification is bypassed, so
# an internal account (e.g. a maintainer's own work login) still gets triaged
# while the bot is being tested on a small set of accounts. Empty the set to
# lift the restriction and restore full triage for the public rollout. Logins
# are compared case-insensitively.
ALLOWLIST_LOGINS = _logins("mateo-berri", "SwiftWinds")
# `gh {pr,issue} list` has no "fetch everything" flag — `--limit` is the only
# control and it defaults to 30. Pass a ceiling far above any realistic open
# backlog (low thousands today) so gh paginates the API until the queue is
# exhausted rather than silently truncating. The bulk sweeps MUST see the whole
# backlog: gh lists newest-first, so a low cap drops the *oldest* PRs/issues —
# exactly the stale ones a low-quality sweep is meant to catch.
GH_LIST_ALL_LIMIT = 100_000
def extract_greptile_score(comments: Iterable[dict]) -> tuple[int, dict] | None:
"""Return (score, comment) for the most recent Greptile-authored comment
that contains a "Confidence Score: X/5". Returns None if no such comment.
"Most recent" is determined by the comment's `updated_at` (falling back to
`created_at`), so re-reviews override earlier passes.
"""
candidates: list[tuple[str, int, dict]] = []
for comment in comments:
user = (comment.get("user") or {}).get("login", "")
if user not in GREPTILE_BOT_LOGINS:
continue
body = comment.get("body") or ""
match = SCORE_PATTERN.search(body)
if not match:
continue
score = int(match.group(1))
timestamp = comment.get("updated_at") or comment.get("created_at") or ""
candidates.append((timestamp, score, comment))
if not candidates:
return None
candidates.sort(key=lambda triple: triple[0])
_, score, comment = candidates[-1]
return score, comment
def parse_iso8601(value: str) -> dt.datetime:
"""Parse a GitHub ISO-8601 timestamp into a timezone-aware datetime."""
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
def gh(*args: str) -> str:
"""Run a `gh` CLI command and return stdout. Raises on non-zero exit.
Shared by both Agent Shin entrypoints so a future change here
(timeout handling, logging, retry on transient failures) only needs
to be made once.
"""
result = subprocess.run(
["gh", *args],
capture_output=True,
text=True,
check=True,
)
return result.stdout
def list_open_items(kind: str, *, repo: str | None, fields: str) -> list[dict]:
"""Return EVERY open PR (``kind="pr"``) or issue (``kind="issue"``) in ``repo``.
Wraps ``gh {pr,issue} list`` with ``--limit GH_LIST_ALL_LIMIT`` so the full
backlog is fetched instead of the default 30 (or any other arbitrary cap).
Both bulk sweeps — the daily Greptile closer and the one-shot rollout
heads-up — rely on this seeing the whole queue, including the oldest items.
``fields`` is the comma-separated ``--json`` field list the caller needs
(e.g. ``"number"`` for the rollout, the full set for the closer).
"""
if kind not in ("pr", "issue"):
raise ValueError(f"kind must be 'pr' or 'issue', got {kind!r}")
repo_args = ["--repo", repo] if repo else []
raw = gh(
kind,
"list",
"--state",
"open",
"--limit",
str(GH_LIST_ALL_LIMIT),
"--json",
fields,
*repo_args,
)
return json.loads(raw)
def seconds_since_latest_marker_comment(
comments: Iterable[dict],
*,
marker: str,
bot_login: str | None = None,
now: dt.datetime | None = None,
) -> float | None:
"""Return seconds since the bot's most recent comment containing ``marker``.
Filters comments by author so a contributor who quotes the HTML
marker (e.g. via GitHub's "Quote reply" feature, which preserves
HTML comments in the raw markdown of the quoted text) is not
mistaken for a bot warning — that would silently reset cooldown
timers and suppress legitimate notifications.
``bot_login`` defaults to the `AGENT_SHIN_BOT_LOGIN` env override or
``AGENT_SHIN_DEFAULT_BOT_LOGIN`` so callers normally don't need to
pass it. ``now`` is injectable for tests / callers (like the daily
sweep) that want every age calculation pinned to one snapshot.
"""
expected_login = (
bot_login
or os.environ.get("AGENT_SHIN_BOT_LOGIN")
or AGENT_SHIN_DEFAULT_BOT_LOGIN
).lower()
latest: dt.datetime | None = None
for comment in comments:
author = ((comment.get("user") or {}).get("login") or "").lower()
if author != expected_login:
continue
body = comment.get("body") or ""
if marker not in body:
continue
created = comment.get("created_at")
if not created:
continue
try:
ts = parse_iso8601(created)
except ValueError:
continue
if latest is None or ts > latest:
latest = ts
if latest is None:
return None
reference = now if now is not None else dt.datetime.now(dt.timezone.utc)
return (reference - latest).total_seconds()

573
.github/scripts/close_low_quality_prs.py vendored Normal file
View file

@ -0,0 +1,573 @@
#!/usr/bin/env python3
"""
Auto-close low-quality pull requests.
Closes open PRs (including drafts, regardless of age) that satisfy ALL of:
1. Have a Greptile (`greptile-apps`) review comment whose latest
"Confidence Score: X/5" is below the configured threshold (default: 4).
2. Are authored by an external OSS contributor (internal BerriAI
contributors are exempt).
3. Do not carry an opt-out label (default: "do not close").
`--min-age-days` is retained as an opt-in safety net for one-off backfill
runs (default: 0). The team's intent is that the count of open PRs equals
the count of PRs internal collaborators need to action on, so neither age
nor draft status acts as a free pass.
For each match, the script posts an explanatory comment and closes the PR.
Because OSS contributors *cannot* reopen a PR closed by the bot/maintainer
(GitHub limitation), the close-comment instructs them to push their fixes
and **open a fresh PR**, or to comment `@agent-shin reconsider` on the
closed PR to have the LLM judge re-evaluate (and reopen on pass).
Requires the `gh` CLI to be authenticated.
Usage examples:
# Dry run (default) - prints what would be closed
python3 close_low_quality_prs.py
# Actually close matching PRs
python3 close_low_quality_prs.py --close
# Restrict to PRs at least N days old (one-off backfill safety net)
python3 close_low_quality_prs.py --min-age-days 7 --min-score 4 --close
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import subprocess
import sys
from typing import Iterable
# Add this script's directory to `sys.path` so the sibling
# `agent_shin_shared` module is importable when the script is invoked
# directly (e.g. `python3 .github/scripts/close_low_quality_prs.py ...`).
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from agent_shin_shared import ( # noqa: E402 -- sys.path adjusted above
AGENT_SHIN_CLOSE_MARKER,
ALLOWLIST_LOGINS,
GRACE_COMMENT_MARKER,
GRACE_PERIOD_SECONDS,
GREPTILE_BOT_LOGINS,
SCORE_PATTERN,
extract_greptile_score,
gh,
list_open_items,
parse_iso8601,
seconds_since_latest_marker_comment,
)
# `GREPTILE_BOT_LOGINS` and `SCORE_PATTERN` (Greptile's GitHub App login
# variants and the "Confidence Score: X/5" regex) are imported from
# `agent_shin_shared` so the LLM judge in `triage_with_llm.py` and this
# daily Greptile sweep read the score through the same set of logins
# and the same regex.
# `author_association` values for internal BerriAI contributors who should be
# exempt from auto-triage.
INTERNAL_AUTHOR_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"})
# Default labels that exempt a PR from auto-close. Defined at module scope (not
# as a mutable argparse default) so that `--optout-label foo` REPLACES the
# defaults instead of appending to them — the argparse `action="append"` +
# `default=[...]` combination silently mutates the shared default list.
DEFAULT_OPTOUT_LABELS = ("do not close", "keep open", "wip")
# `GRACE_COMMENT_MARKER` (HTML marker appended to grace-period warning
# comments — used by either script to recognize that a warning was
# already posted) and `GRACE_PERIOD_SECONDS` (length of the grace
# period between the warning and the actual auto-close, 2 hours) are
# imported from `agent_shin_shared` so the Agent Shin LLM judge and
# this daily Greptile sweep agree on the same marker and duration.
def fetch_open_prs(repo: str | None) -> list[dict]:
"""Fetch all open PRs (number, createdAt, isDraft, labels, author).
Includes drafts: `gh pr list --state open` returns both ready-for-review
and draft PRs by default. This is the desired behavior — drafts are not
a free pass; the internal-collaborator open-PR queue should reflect every
PR that needs human attention regardless of draft status.
"""
fields = "number,title,createdAt,isDraft,labels,author,url"
return list_open_items("pr", repo=repo, fields=fields)
def fetch_pr_author_association(pr_number: int, repo: str | None) -> str:
"""Return the GitHub `author_association` for a PR, uppercase.
Values: OWNER, MEMBER, COLLABORATOR, CONTRIBUTOR, FIRST_TIME_CONTRIBUTOR,
FIRST_TIMER, MANNEQUIN, NONE. Returns "" on lookup failure.
"""
endpoint = (
f"repos/{repo}/pulls/{pr_number}"
if repo
else f"repos/{{owner}}/{{repo}}/pulls/{pr_number}"
)
try:
data = json.loads(gh("api", endpoint))
except subprocess.CalledProcessError:
return ""
return (data.get("author_association") or "").upper()
def is_external_pr_author(pr: dict, repo: str | None) -> bool:
"""Return True if the PR author is an external OSS contributor.
Internal = `OWNER` / `MEMBER` / `COLLABORATOR` association, or a bot login.
"""
login = ((pr.get("author") or {}).get("login") or "").lower()
if login.endswith("[bot]") or login in {"dependabot", "github-actions"}:
return False
association = fetch_pr_author_association(pr["number"], repo)
# Fail-safe: if the API lookup failed (empty string), treat the author as
# internal so we don't auto-close their PR. Auto-close is destructive, so
# an unknown association should never make a PR eligible for closing.
if not association or association in INTERNAL_AUTHOR_ASSOCIATIONS:
return False
return True
def fetch_pr_comments(pr_number: int, repo: str | None) -> list[dict]:
"""Fetch issue-level comments on a PR (where Greptile posts its summary)."""
endpoint = (
f"repos/{repo}/issues/{pr_number}/comments?per_page=100"
if repo
else f"repos/{{owner}}/{{repo}}/issues/{pr_number}/comments?per_page=100"
)
raw = gh("api", "--paginate", endpoint)
comments: list[dict] = []
for line in raw.strip().splitlines():
line = line.strip()
if not line:
continue
try:
parsed = json.loads(line)
except json.JSONDecodeError:
# A malformed line should not blow up the whole sweep. Skip and
# carry on so the remaining PRs in this run still get evaluated.
continue
if isinstance(parsed, list):
comments.extend(parsed)
else:
comments.append(parsed)
return comments
def has_optout_label(pr: dict, optout_labels: set[str]) -> bool:
labels = {label.get("name", "").lower() for label in pr.get("labels", [])}
return bool(labels & {lbl.lower() for lbl in optout_labels})
def seconds_since_last_grace_warning(
comments: Iterable[dict],
*,
bot_login: str | None = None,
now: dt.datetime | None = None,
) -> float | None:
"""Return seconds since the bot's most recent grace-period warning, or
None if no such warning has ever been posted on this PR.
Thin wrapper over
`agent_shin_shared.seconds_since_latest_marker_comment` — the
centralized helper handles the bot-author filter, marker match,
timestamp parsing, and `now` injection. Keeping this wrapper
preserves the closer's "already-fetched comments + injectable now"
interface so callers (and tests) don't need to change.
"""
return seconds_since_latest_marker_comment(
comments,
marker=GRACE_COMMENT_MARKER,
bot_login=bot_login,
now=now,
)
def format_grace_warning_comment(score: int, threshold: int) -> str:
"""Comment posted on the FIRST low-Greptile-score detection — gives
the contributor a 2-hour grace window before the auto-close fires on
the next daily cron run.
Mirrors `format_grace_warning_pr_comment` in
`triage_with_llm.py` in spirit (2-hour grace + escape hatches), but
framed around Greptile's confidence score instead of the LLM judge's
rubric since the close trigger here is the Greptile signal.
"""
return (
"🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this "
"repository.\n"
"\n"
"Heads up: Greptile's most recent review scored this PR "
f"**{score}/5**, below our merge bar of **{threshold}/5**.\n"
"\n"
"If the score isn't lifted in the next **2 hours**, I'll auto-close this PR. That's "
"**not** us saying the change isn't worthwhile. We want the open-PR list to mirror "
"what a maintainer can act on *right now*, so contributors like you don't get lost in "
"a backlog. Take your time; everything below still works after the close.\n"
"\n"
"**During the grace period:** push fixes that address Greptile's feedback, then comment "
"`@greptileai` to request a fresh review. If "
f"the new score is **{threshold}/5 or higher**, the PR stays open and no further "
"action is needed on your side.\n"
"\n"
"**If the PR does get auto-closed in 2 hours, you still have an easy recovery path:**\n"
"\n"
"- Comment `@greptileai` to request a fresh review. **This still works even after "
f"the PR is closed**, and a score of {threshold}/5 or higher is one of the signals "
"that lifts the PR back into the review queue. A low Greptile score isn't a blocker.\n"
"- Comment `@agent-shin reconsider` after pushing fixes; I'll re-run the rubric and "
"reopen the PR if both gates (description rubric + Greptile score) now pass.\n"
"\n"
f"{GRACE_COMMENT_MARKER}"
)
def post_grace_warning(
pr: dict,
score: int,
threshold: int,
repo: str | None,
dry_run: bool,
) -> None:
"""Post the 2-hour grace-period warning comment on `pr`.
The warning carries `GRACE_COMMENT_MARKER` so subsequent runs can
detect that the contributor has already been told about the
pending close. Does NOT close the PR — the close happens on the
next eligible run after `GRACE_PERIOD_SECONDS` elapses (handled
by `close_pr`).
"""
pr_number = pr["number"]
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(
f" [DRY RUN] Would post grace warning to PR #{pr_number} "
f"(greptile={score}/5): {pr['title']}"
)
return
comment_body = format_grace_warning_comment(score, threshold)
gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args)
print(f" Posted grace warning on PR #{pr_number} (greptile={score}/5)")
def format_close_comment(score: int, threshold: int) -> str:
"""Comment posted when a low-Greptile-score PR is auto-closed.
Carries `AGENT_SHIN_CLOSE_MARKER` so the `@agent-shin reconsider` path
(guarded by `was_closed_by_agent_shin`) recognizes this as an Agent Shin
close and is allowed to reopen the PR once it passes again; without the
marker that recovery path the comment advertises silently rejects the
contributor.
"""
score_sentence = (
f"Greptile's most recent review scored this PR **{score}/5**, below "
f"our merge bar of **{threshold}/5**, and the 2-hour grace period since "
"the warning has elapsed.\n\n"
)
return (
f"Closing as part of automated PR triage.\n\n"
f"{score_sentence}"
"We close low-confidence PRs aggressively to keep the review queue "
"manageable for maintainers and contributors alike. **This is not a "
"rejection of the idea.** To bring this back:\n\n"
"1. Push the fixes that address Greptile's feedback (continue using "
"your existing branch is fine).\n"
"2. **Open a new PR** with the updated branch. Greptile will review "
"it again, and if it scores "
f"**{threshold}/5 or higher** a maintainer will take another look.\n\n"
"_Why open a new PR instead of reopening this one?_ GitHub does not "
"let external contributors reopen a PR that was closed by a bot or "
"maintainer, so a fresh PR is the most reliable path forward. If you "
"would prefer this exact PR re-evaluated, comment "
"`@agent-shin reconsider` once you've pushed the fixes; Agent Shin "
"will re-run triage and reopen this PR if it now meets the bar. "
"You can also comment `@greptileai` to request a fresh Greptile "
"review; that works **even after the PR is closed**.\n\n"
"Thanks for contributing to LiteLLM. We know auto-closures can sting; "
"the goal is to keep the project healthy, not to dismiss your work."
f"\n\n{AGENT_SHIN_CLOSE_MARKER}"
)
def close_pr(
pr: dict,
score: int,
threshold: int,
age_days: int,
repo: str | None,
dry_run: bool,
label: str | None,
) -> None:
"""Post the explanatory comment and close the PR."""
pr_number = pr["number"]
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(
f" [DRY RUN] Would close PR #{pr_number} "
f"(age={age_days}d, greptile={score}/5): {pr['title']}"
)
return
comment_body = format_close_comment(score, threshold)
gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args)
if label:
try:
gh("pr", "edit", str(pr_number), "--add-label", label, *repo_args)
except subprocess.CalledProcessError as exc:
stderr = (exc.stderr or "").strip()
print(f" warn: failed to add label '{label}' to #{pr_number}: {stderr}")
gh("pr", "close", str(pr_number), *repo_args)
print(f" Closed PR #{pr_number} (greptile={score}/5, age={age_days}d)")
def evaluate_pr(
pr: dict,
now: dt.datetime,
min_age_days: int,
min_score: int,
repo: str | None,
optout_labels: set[str],
allowlist: frozenset[str] = ALLOWLIST_LOGINS,
) -> tuple[str, int | None, int | None]:
"""Decide what to do with `pr` on this triage run.
Returns (action, score_or_none, age_days_or_none) where action is one of:
"skip-too-young", "skip-optout-label", "skip-not-allowlisted",
"skip-internal", "skip-no-greptile-score", "skip-score-ok",
"warn-grace", "skip-in-grace-period", or "close".
Drafts are NOT skipped — the goal is "open PR count == PRs internal
collaborators need to action on", and a draft that Greptile scored <4/5
is still in that queue. Authors can opt out via the `wip` label (see
`DEFAULT_OPTOUT_LABELS`) if they need to keep a long-lived draft open.
Grace-period semantics: the first time a PR fails the rubric, the
action is `warn-grace` — the caller should post a warning comment but
NOT close the PR. On a subsequent run, if the warning is still less
than `GRACE_PERIOD_SECONDS` old AND the PR still fails, the action is
`skip-in-grace-period`. Once the warning ages out and the rubric is
still failing, the action is `close`.
"""
if has_optout_label(pr, optout_labels):
return ("skip-optout-label", None, None)
created = parse_iso8601(pr["createdAt"])
age_days = (now - created).days
# `min_age_days` defaults to 0 (close as soon as Greptile scores low).
# Set a positive value via --min-age-days for one-off backfill runs that
# want to skip very-young PRs.
if min_age_days > 0 and age_days < min_age_days:
return ("skip-too-young", None, age_days)
# While the allowlist is active it is the sole author gate: only those
# logins are acted on and the external-only restriction is bypassed for
# them. Otherwise auto-close only external OSS contributors — internal
# contributors (BerriAI org members) handle their own backlog.
login = ((pr.get("author") or {}).get("login") or "").lower()
if allowlist:
if login not in allowlist:
return ("skip-not-allowlisted", None, age_days)
elif not is_external_pr_author(pr, repo):
return ("skip-internal", None, age_days)
comments = fetch_pr_comments(pr["number"], repo)
extraction = extract_greptile_score(comments)
if extraction is None:
return ("skip-no-greptile-score", None, age_days)
score, _ = extraction
if score >= min_score:
return ("skip-score-ok", score, age_days)
grace_age = seconds_since_last_grace_warning(comments, now=now)
if grace_age is None:
return ("warn-grace", score, age_days)
if grace_age < GRACE_PERIOD_SECONDS:
return ("skip-in-grace-period", score, age_days)
return ("close", score, age_days)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--repo",
type=str,
default=None,
help="Repository (owner/repo). Auto-detected if omitted.",
)
parser.add_argument(
"--min-age-days",
type=int,
default=0,
help=(
"Minimum age (in days) before a PR is eligible. Default 0 = "
"close as soon as Greptile flags it. Set a positive value for "
"one-off backfill runs that want to spare very-young PRs."
),
)
parser.add_argument(
"--min-score",
type=int,
default=4,
choices=range(1, 6),
help="Greptile score below which a PR is closed (default: 4 -> closes <4/5).",
)
parser.add_argument(
"--optout-label",
action="append",
default=None,
help=(
"Label(s) that exempt a PR from auto-close. Repeat to add more. "
"Case-insensitive. When omitted, defaults to "
f"{list(DEFAULT_OPTOUT_LABELS)!r}; passing this flag REPLACES the "
"defaults (argparse `append` with a mutable default would append "
"instead, which we explicitly avoid)."
),
)
parser.add_argument(
"--close-label",
type=str,
default=None,
help=(
"Optional label to add to PRs that get auto-closed "
"(e.g. 'auto-closed-low-quality'). Must already exist on the repo."
),
)
parser.add_argument(
"--close",
action="store_true",
help="Actually close matching PRs (default is dry-run).",
)
parser.add_argument(
"--limit",
type=int,
default=None,
help="Maximum number of PRs to close in one run (safety net).",
)
args = parser.parse_args()
dry_run = not args.close
if dry_run:
print("=== DRY RUN MODE (pass --close to actually close PRs) ===\n")
print("Fetching open PRs...")
prs = fetch_open_prs(args.repo)
print(f"Found {len(prs)} open PRs.\n")
now = dt.datetime.now(dt.timezone.utc)
optout_labels = set(args.optout_label or DEFAULT_OPTOUT_LABELS)
closed = 0
summary = {
"close": 0,
"warn-grace": 0,
"skip-in-grace-period": 0,
"skip-too-young": 0,
"skip-optout-label": 0,
"skip-not-allowlisted": 0,
"skip-internal": 0,
"skip-no-greptile-score": 0,
"skip-score-ok": 0,
}
# `warned` tracks grace-warning comments posted in this run so the
# `--limit` safety net bounds *all* destructive write actions, not
# just closures. Without this cap, a backlog of PRs failing the
# threshold simultaneously could flood contributors with comments.
warned = 0
for pr in sorted(prs, key=lambda p: p["createdAt"]):
try:
action, score, age_days = evaluate_pr(
pr,
now,
args.min_age_days,
args.min_score,
args.repo,
optout_labels,
)
summary[action] = summary.get(action, 0) + 1
if action == "warn-grace":
assert score is not None
print(
f"#{pr['number']}: \"{pr['title']}\" "
f"(age={age_days}d, greptile={score}/5) -> warn-grace"
)
post_grace_warning(
pr,
score=score,
threshold=args.min_score,
repo=args.repo,
dry_run=dry_run,
)
if not dry_run:
warned += 1
if args.limit is not None and (warned + closed) >= args.limit:
print(
f"\nReached --limit={args.limit} "
f"(closed={closed}, warned={warned}); stopping."
)
break
continue
if action != "close":
continue
assert score is not None and age_days is not None
print(
f"#{pr['number']}: \"{pr['title']}\" "
f"(age={age_days}d, greptile={score}/5) -> close"
)
close_pr(
pr,
score=score,
threshold=args.min_score,
age_days=age_days,
repo=args.repo,
dry_run=dry_run,
label=args.close_label,
)
if not dry_run:
closed += 1
if args.limit is not None and (warned + closed) >= args.limit:
print(
f"\nReached --limit={args.limit} "
f"(closed={closed}, warned={warned}); stopping."
)
break
except Exception as exc: # noqa: BLE001 - per-PR errors don't abort the sweep
summary["error"] = summary.get("error", 0) + 1
print(
f"!! PR #{pr.get('number')}: {exc}",
file=sys.stderr,
)
continue
print("\n=== Summary ===")
for key, value in summary.items():
print(f" {key:28s} {value}")
if dry_run:
print(f"\nTotal would close: {summary['close']}")
else:
print(f"\nTotal closed: {closed}")
print(
f"Total {'would warn (grace)' if dry_run else 'warned (grace)'}: "
f"{summary['warn-grace']}"
)
return 0
if __name__ == "__main__":
sys.exit(main())

282
.github/scripts/triage-requirements.txt vendored Normal file
View file

@ -0,0 +1,282 @@
# Hash-pinned dependency set for the Agent Shin triage scripts.
# Installed in privileged triage workflows, so every package is pinned to an
# exact version with SHA-256 hashes and installed with pip --require-hashes.
#
# Regenerate after bumping openai:
# echo 'openai==<version>' \
# | uv pip compile - --generate-hashes --python-version 3.12 \
# --no-annotate --no-header -o .github/scripts/triage-requirements.txt
annotated-types==0.7.0 \
--hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
--hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
anyio==4.14.0 \
--hash=sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89 \
--hash=sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9
certifi==2026.6.17 \
--hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \
--hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db
distro==1.9.0 \
--hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \
--hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2
h11==0.16.0 \
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
httpcore==1.0.9 \
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
httpx==0.28.1 \
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
idna==3.18 \
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
jiter==0.15.0 \
--hash=sha256:01a8222cf05ab1128e239421156c207949808acaaea2bdfd33130ae666786e86 \
--hash=sha256:032396229564bca02440396bd327710719f724f5e7b7e9f7a8eb3faa4a2c2281 \
--hash=sha256:04b400bbf8c9efb03d9bdd976475c919c1d85593b04b9fff7ae234065daf87ae \
--hash=sha256:05906b93d72f03339e6bb7cf8dc10ebda64a0266126eed6beba79e20abcf5fd4 \
--hash=sha256:066f8f33f18b2419cd8213b2436fa7fbc9c499f315971cfa3ce1f9820c001b1b \
--hash=sha256:0ab068bce62a45aa3e7367eceaffb5dde60b7eb853be8dece45132e3d0ff4879 \
--hash=sha256:0be6f5ad41a809f303f416d17cec92a7a725902fb9b4f3de3d19362ac0ef8554 \
--hash=sha256:0e90a1c315a0226ec822d973817967f9223b7701546c8c2a7913e7ab0926294d \
--hash=sha256:0f862193b8696249d22ec433e85fd2ab0ad9596bc3e45e6c0bc55e8aeba97be2 \
--hash=sha256:1303d4d68a9b051ea90502402063ecf3807da00ad2affa19ca1ae3b90b3c5f67 \
--hash=sha256:144f8e72cb53dab146347b91cceac01f5481237f2b93b4a339a1ee8f8878b67c \
--hash=sha256:182226cbc930c9fab81bc2e41a4da672f89539906dadb05e75670ac07b94f71f \
--hash=sha256:1c11465f97e2abf45a014b83b730222f8f1c5335e802c7055a67d50de6f1f4e3 \
--hash=sha256:1c15024a3d892223b18f597c86d59387249dc396590844ce6b9f6131d1093bae \
--hash=sha256:1d54fb5b31dea401a41af3f8a7d2512e9b6a6a005491e6166c7e4ffab9639a9c \
--hash=sha256:25ffbe229aa8cd98c28879d8aa1a6e34ae77992ab984a65fba800859dab16269 \
--hash=sha256:2a77aadd57cac1682e4401a72724d2796d89a4ba129b1a5812aa94ee480826eb \
--hash=sha256:2ae901f3a55bfafdde31d289590fa25e3245735a2b1e8c7cc15871710a002871 \
--hash=sha256:2b0074e2f56eb2dacca1689760fd2852a068f85a0547a157b82cb4cafeb6768b \
--hash=sha256:2c8aea7781d2a372227871de4e1a1332aa96f5a89fd76c5e835dafdbad102887 \
--hash=sha256:2c9cb907439d20bd0c7d7565ca01ee52234203208433749bae5b516907526928 \
--hash=sha256:2fb6a5d26af81fc0f00f9360a891e05cf755e149bba391c4d563adc54812973d \
--hash=sha256:2fd73e3da91a0a722d67165e849ce2cdc10de0e0d48738c142be8c6c5f310f4c \
--hash=sha256:30ce1a5d16b5641dc935d50ef775af6a0871e3d14ab05d6fc54dff371b78e558 \
--hash=sha256:30ce785d2adb8e32c3f7741442370a74834ec4c01f3c48f0750227a0b4ef27d6 \
--hash=sha256:30f2218e6a9e5c18bc10fe6d41ac189c442c88eacf11bad9f28ef95a9bef00e6 \
--hash=sha256:351a341c2105aa430b7047e30f1bf7975f6313b00165d3fc07be2edaf741f279 \
--hash=sha256:37a10c377ce3a4a85f4a67f28b7afe093154cde77eaf248a72e856aa08b4d865 \
--hash=sha256:392b8ab019e5502d08aff85c6272209c24bc2cbe706ea82a56368f524236614a \
--hash=sha256:3e4540b8e74e4268811ac05db226a6a128ff572e7e0ce3f1163b693cadb184cd \
--hash=sha256:40b2c7e92c44a84d748d21706c68dc6ff8161d80b59c99d774721a0d2317d7c7 \
--hash=sha256:411fa4dfa5a7ae3d11491027ffb9beadec3996010a986862db70d91abba1c750 \
--hash=sha256:4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76 \
--hash=sha256:42bfb257930800cf43e7c62c832402c704ab60797c992faf88d20e903eac8f32 \
--hash=sha256:4363818355dbc70ae1a8e9eaba9de350d93ede4ff6992b8f8eb8cbb6e5122d42 \
--hash=sha256:4ab395feec8d249ec4044e228e98a7033f043426a265df439dc3698823f0a4e4 \
--hash=sha256:50164d7610c00e7cd913a873fce30b6beeebf4b37e53983e33f22de4c900f6b8 \
--hash=sha256:50e51156192722a9c58db112837d3f8ef96fb3c5ecc14e95f409134b08b158ec \
--hash=sha256:510c8b3c17a0ed9ac69850c0438dada3c9b82d9c4d589fcb62002a5a9cf3a866 \
--hash=sha256:5157de9f76eb4bc5ea74a1219366a25f945ad305641d74e04f59c54087091aa9 \
--hash=sha256:54d5d6090cdc1b7c9e780dfb04949a990adb1e301a2fc0bbcee7de4638d33f9a \
--hash=sha256:553fcac2ef2cb990877f9fc0833b8b629a3e6a5670b6b5fd58219b41a653ddc4 \
--hash=sha256:5607e6013ed7e6b0ec9661e467b7ffde0aa7ab36833a04850f26fcf88ed4845b \
--hash=sha256:5d6a60072b44c3c2b797a7ddcbcbbf2b34ea3cfd4721580fbfd2a09d9d9b84ba \
--hash=sha256:5f30bae8bc1c2d613e28e5af3e8cceb09b742f1c8a8a5f839fb67afaffc03b61 \
--hash=sha256:62ebd14e47e9aed9df4472afcb2663668ce4d74891cd54f86bf6e44029d6dc89 \
--hash=sha256:631f13a3d04e97d4e083993b10f4b99530e3a10d953e2eb5e196b7dc7f812ce0 \
--hash=sha256:6550fa135c7deb8ead6af49ed7ff648532ea8334a1447fe34a36315ef79c5c29 \
--hash=sha256:66b1880df2d01e206e8339769d1c7c1753bcb653efd6289e203f6f24ebada0c0 \
--hash=sha256:6eac374c5c975709b69c10f09afd199df74150172156ad10c8d4fd785b7da995 \
--hash=sha256:71683c38c825452999b5717fcae07ea708e8c93003e808be4319c1b02e3d176e \
--hash=sha256:7553333dd0930c104a5a0db8df72bf7219fe663d731383b576bb6ed6351c984d \
--hash=sha256:75e8a04e91432dde9f1838373cf93d23726c79d3e908d319acf0e796f85592e7 \
--hash=sha256:773b6eb282ce11ee19f05f6b2d4404fa308e5bbd353b0b80a0262caad6db2cd7 \
--hash=sha256:774f93f65031856bf14ad9f59bdcab8b8cad501e5ceabd51ba3525f76937a25b \
--hash=sha256:7c468136b8bd6bb18c8786e4236a1fa27362f24cb23450ba0cb204ab379b8e6f \
--hash=sha256:7ce8902f939970048b233087082e7bb829db29375811c7ad50687b8624c6fd08 \
--hash=sha256:7d3d6683288c11cbab50e865f2e2f13950179aa45410e30b2cfbd3fb7b0177bf \
--hash=sha256:7f6163c0f10b055245f814dcc59f4818da60dfe72f3e72ab89fc24b6bd5e9c52 \
--hash=sha256:8020c99ec13a7db2b6f96cbe82ef4721c88b426a4892f27478044af0284615ef \
--hash=sha256:813dfbb17d65328bf86e5f0905dd277ba2265d3ca20556e86c0c7035b7182e5a \
--hash=sha256:860a74063284a2ae9bfedd694f299cc2c68e2696c5f3d440cc9d18bb81b9dd04 \
--hash=sha256:8c9004af7c8d67cce7f1aae1026fb55607f4aa600710d08ede3a3ce4aeefe7e0 \
--hash=sha256:8d2c0c44d569ce0f2850f5c926f8caeb5f245fbc84475aeb36efccc2103e6dbd \
--hash=sha256:8f7e9bc0f1135039b22ee6eab588d42df1ce55842b30740a352885eb267bd941 \
--hash=sha256:90c5db5527c221249a876160663ab891ace358c17f7b9c93ec1478b7f0550e5c \
--hash=sha256:9100ddbec09741cc66feb0fc6773f8bdbd0e3c345689368f260082ff85dcc0cd \
--hash=sha256:913d02d29c9606643418d9ccfc3b72492ab25a6bf7889934e09a3490f8d3438b \
--hash=sha256:980c256edb05b78a111b99c4de3b1d32e31634b867fd1fc2cf726e7b7bba9854 \
--hash=sha256:9f924585cdacf631cd382b657966847bb537bf9ed0a6f9b991da5f05a631480f \
--hash=sha256:a254e10b593624d230c365b6d616b22ca0ad65e63a16e6631c2b3466022e6ba8 \
--hash=sha256:a2a438005b6f22d0273413484d6094d7c2c5d10ec1b3a3bf128e0d1d3ba53258 \
--hash=sha256:a97261f1fccb8e50ecd2890a96e46efdc3f57c80a197324c6777827231eca712 \
--hash=sha256:ab596fa3837e91e7e6a31b5f639988bfc6a35d1f915ac3932d946062219d588f \
--hash=sha256:abbf258599526ad0326fe51e252e24f2bd6f24f1852681b4b78feda3808f1d18 \
--hash=sha256:ac0d9ddea4350974be7a221fc25895f251a8fee748c889bdced2141c0fec1a49 \
--hash=sha256:acf4ee4d1fc55917239fe72972fb292dd773055d05eb040d36f4326e02cc2c0e \
--hash=sha256:ae1b0d82ac2d987f9ea512b1c9adfcc71a28de3dea3a6039b54d76cffda9901e \
--hash=sha256:b15741f501469009ae0ae90b7147958a664a7dede40aa7ff174a8a4645f546d0 \
--hash=sha256:b15d3ec9b0449c40e85319bdb4caa8b77ab526e74f5532ed94bec15e2f66822c \
--hash=sha256:b3b3b775e33d3bfaec9899edc526ae97b0da0bf9d071a46124ba419149a414f8 \
--hash=sha256:b6c0ffae686c39bf3737be60793783267628783ea42545632c10b291105aee45 \
--hash=sha256:c210f8b35dc6f30aafd4b4365ca89b9d1189f21ab49b8e68fa6322a847aef138 \
--hash=sha256:c2f6bb8b5216ab9e7873bc08b5d7bef2b8abbb578a3069bf1cd14a45d71d771d \
--hash=sha256:c60e71b6d10cfc284c9bf36bd885e8d44c46f688ce50aa91b5edd90181dea687 \
--hash=sha256:c6694a173ecabc12eb60efbc0b474464ead1951ff65cd8b1e72100715c64512b \
--hash=sha256:c77496cb10bd7549690fbbab3e5ec05857b83e49276f4a9423a766ddd2afcd4c \
--hash=sha256:c84c1b7be454b0c16f8499b4ebfbfd82ea5cca6527cceefcbbc06a7557b5ed2e \
--hash=sha256:cc0bc345cf2df9d1c00ac443f50d543c1ccfa8b0422cb85b1ab70d681c0b255b \
--hash=sha256:ceb8fc27d38793f9c97149be8302720c5b22e5c195a37bf2c45dc36c4600a512 \
--hash=sha256:cf4bd113a69c0a740e27cb962ce10630c36d2b8f59d759a651b955ee9d18a823 \
--hash=sha256:d1aa62e277fc1cbd80e6deacae6f4d983b41b3d7728e0645c5d741a6149bba45 \
--hash=sha256:d1e7b1776f0797956c509e123d0952d10d293a9492dea9f288ab9570ec01d1a5 \
--hash=sha256:d636d5095155afd364247f65070fab7beda13498d7ff4de331046e704ab9657f \
--hash=sha256:d726e3ceeb337191324b49de298142f27c3ad10886341555d1d5315b5f252c6a \
--hash=sha256:d72d8af5c1013656a8870c866660627d1a75bc185814ee022c8533caa1de88ae \
--hash=sha256:d8d2955167274e15d79a7a020afdd9b39c990eb80b2d89fca695d92dcfdd38ec \
--hash=sha256:d92a5cd21fdb083931d546c207aa29633787c5dc5b02daab2d32b843f88a2c53 \
--hash=sha256:e58585a58209d72691ce2d62a9147445f5a87beb0bde97fde284c96ae392a3d1 \
--hash=sha256:e7196e56f1cd69af1dbb07dff02dcfb260a50b45a82d409d92a06fedb32473b5 \
--hash=sha256:eda3071db3346334beae1360b46da4606da57bf3528c167b3c38533afaf9f2c5 \
--hash=sha256:edebcf7d1f601199084bb6e844d7dc67e03e04f6ac786b0332d616635c4ff7a4 \
--hash=sha256:ef1fd24d9413f6209e00d3d5a453e67acfe004a25cc6c8e8484faed4311ab9e8 \
--hash=sha256:f0b271b462769543716f92d3a4f90527df6ef5ed05ee95ec4137f513e21e1b77 \
--hash=sha256:f18f85e4218d1b40f000f42a92239a7a61a902cd42c65e6c360dbd17dcb20894 \
--hash=sha256:f1e1754960f38ec40613a07e5e372df67acb3b890fb383b6fb3de3e49ddbf3c7 \
--hash=sha256:f2143ab06181d2b029eedcb6af3cebe95f11bbac62441781860f98ee9330a6a6 \
--hash=sha256:f3d37768fce7f88dd2a8c6091f2325dea27d30d30d5c6e7a1c0f0af77723b708 \
--hash=sha256:fa248c9eb220197d363f688818dac2fd4b2f0cd7d843ca7105d652034823427d
openai==2.33.0 \
--hash=sha256:03ac37d70e8c9e3a8124214e3afa785e2cbc12e627fbd98177a086ef2fd87ad5 \
--hash=sha256:f850c435e2a4685bba3295bd54912dd26315d9c1b7733068186134d6e0599f9a
pydantic==2.13.4 \
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
pydantic-core==2.46.4 \
--hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \
--hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \
--hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \
--hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \
--hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \
--hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \
--hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \
--hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \
--hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \
--hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \
--hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \
--hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \
--hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \
--hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \
--hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \
--hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \
--hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \
--hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \
--hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \
--hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \
--hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \
--hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \
--hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \
--hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \
--hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \
--hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \
--hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \
--hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \
--hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \
--hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \
--hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \
--hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \
--hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \
--hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \
--hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \
--hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \
--hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \
--hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \
--hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \
--hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \
--hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \
--hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \
--hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \
--hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \
--hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \
--hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \
--hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \
--hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \
--hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \
--hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \
--hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \
--hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \
--hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \
--hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \
--hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \
--hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \
--hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \
--hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \
--hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \
--hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \
--hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \
--hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \
--hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \
--hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \
--hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \
--hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \
--hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \
--hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \
--hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \
--hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \
--hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \
--hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \
--hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \
--hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \
--hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \
--hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \
--hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \
--hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \
--hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \
--hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \
--hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \
--hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \
--hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \
--hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \
--hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \
--hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \
--hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \
--hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \
--hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \
--hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \
--hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \
--hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \
--hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \
--hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \
--hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \
--hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \
--hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \
--hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \
--hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \
--hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \
--hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \
--hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \
--hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \
--hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \
--hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \
--hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \
--hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \
--hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \
--hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \
--hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \
--hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \
--hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \
--hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \
--hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \
--hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \
--hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \
--hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \
--hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \
--hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \
--hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae
sniffio==1.3.1 \
--hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
--hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
tqdm==4.68.3 \
--hash=sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482 \
--hash=sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03
typing-extensions==4.15.0 \
--hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \
--hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548
typing-inspection==0.4.2 \
--hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \
--hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464

557
.github/scripts/triage_rollout_heads_up.py vendored Normal file
View file

@ -0,0 +1,557 @@
#!/usr/bin/env python3
"""One-shot 7-day heads-up sweep for the Agent Shin rollout.
Posts a friendly "the OSS triage bot kicks in next Monday" comment on every
open external PR/issue that currently *would* fail the new rubric — i.e.,
every PR/issue Agent Shin would close once the rollout completes. The point
is to give contributors a full week to fix their description before the bot
ever takes a destructive action, so nobody is surprised by an auto-close.
The script is designed to run **exactly once** at rollout, fired by a manual
``workflow_dispatch`` (``dry_run=false``) on the heads-up workflow. Re-runs
are safe: every comment is stamped with the hidden ``HEADS_UP_MARKER`` and
PRs/issues that already carry the marker are skipped.
Dry-run vs. real run
--------------------
Defaults to dry-run. Passing ``--close`` flips into real mode. Every GitHub
mutation goes through ``_agent_shin_actions``, which has a one-line
``if dry_run: log else: do_it`` per call, so the only difference between a
dry-run preview and the real run is the call site that actually hits the
GitHub API.
Local preview::
python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm
Real run (the manual rollout dispatch uses this)::
python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm --close
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import sys
from pathlib import Path
from typing import Any
# Make the sibling triage_with_llm + _agent_shin_actions importable when this
# script is invoked directly (the GitHub workflow does `python3 .github/scripts/...`).
_SCRIPTS_DIR = Path(__file__).resolve().parent
if str(_SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(_SCRIPTS_DIR))
from _agent_shin_actions import maybe_post_comment # noqa: E402
from agent_shin_shared import ( # noqa: E402
AGENT_SHIN_DEFAULT_BOT_LOGIN,
ALLOWLIST_LOGINS,
list_open_items,
)
from triage_with_llm import ( # noqa: E402
DEFAULT_MODEL,
call_llm_judge,
fetch_issue,
fetch_pr,
gh,
is_internal_contributor,
review_gate,
triage,
)
# Hidden marker so re-runs skip PRs/issues we've already notified. Distinct from
# the within-grace / ready / regressed markers so it can't be confused with the
# steady-state lifecycle comments.
HEADS_UP_MARKER = "<!-- agent-shin:rollout-heads-up -->"
# Placeholder until the litellm-docs PR ships. The rollout blog post explains
# the new rubric, the 7-day grace, and how to recover after an auto-close.
# TODO(docs): replace with the canonical URL once the litellm-docs PR merges.
ROLLOUT_BLOG_URL = "https://docs.litellm.ai/docs/agent_shin_triage_rollout"
# Default cutoff is one week from "now". Computed at runtime so the wording
# stays correct even if the rollout is merged later than planned. The user can
# override with --close-on YYYY-MM-DD when running the script manually.
DEFAULT_GRACE_DAYS = 7
# The daily auto-close sweeps (close_low_quality_prs.yml at 09:00 UTC and
# review_gate.yml at 09:30 UTC) are what actually close a still-failing item,
# so the deadline we promise contributors has to name that wall-clock moment.
ACTIVATION_TIME_UTC = "09:00 UTC"
def _format_cutoff(cutoff: dt.date) -> str:
"""Human-readable, timezone-explicit cutoff, e.g. ``Monday, June 1, 2026
(09:00 UTC)`` — the moment a still-failing PR/issue gets closed."""
return (
f"{cutoff.strftime('%A, %B')} {cutoff.day}, {cutoff.year} "
f"({ACTIVATION_TIME_UTC})"
)
def _rubric_section_pr() -> str:
return (
"**Going forward, every external PR needs ONE of:**\n"
"\n"
"- A linked GitHub issue using a closing keyword: "
"`Fixes #1234`, `Closes #1234`, or `Resolves #1234`, OR\n"
"- All three of: a clear **problem description**, **expected vs. "
"actual behavior**, and **end-to-end QA proof** (at least one of a "
"short screen recording / video, before/after screenshots, or the "
"exact commands you ran with their real output; mocked or stubbed "
"runs don't count).\n"
"\n"
"PRs also need a **Greptile confidence score of 4/5 or higher** before "
"the bot will tag them `ready for review`. You can `@greptileai` to "
"request a fresh review at any time, including after the PR is closed."
)
def _rubric_section_issue() -> str:
return (
"**Going forward, every external issue needs:**\n"
"\n"
"- For **bug reports**: end-to-end evidence of the bug (at least one "
"of a screen recording / video, a screenshot, or the exact commands "
"you ran with their real output / traceback) plus expected vs. actual "
"behavior. Written steps with no run output don't count, and mocked "
"or stubbed runs don't count.\n"
"- For **feature requests**: a clear description of the proposed "
"feature plus a use case + concrete example (config, API call, UI "
"flow, or scenario showing what's blocked today)."
)
def _description_only_note(kind: str) -> str:
noun = "PR" if kind == "pr" else "issue"
return (
f"⚠️ **The requirements must live in the {noun} *description*, not in "
"comments.** Some PRs/issues collect 100+ comments from humans and "
"bots; reading the entire thread on every triage run would balloon "
"GitHub API usage (we'd start getting 429'd) and blow out the LLM "
"judge's context. The bot only reads the description, so anything "
"you add as a comment will be invisible to it."
)
def _missing_section(verdict: dict, greptile_score: int | None) -> str:
"""Bullet list of what's currently missing on this PR/issue.
Combines the LLM judge's `missing` list (rubric items) with a Greptile
shortfall (for PRs) so the contributor sees one list of things to fix.
"""
missing = list(verdict.get("missing") or [])
if greptile_score is not None and greptile_score < 4:
missing.insert(
0,
f"Greptile's most recent review scored this PR {greptile_score}/5 "
"(below the 4/5 bar Agent Shin will require).",
)
if not missing:
return (
"_The bot couldn't articulate a specific missing piece; see the "
"rubric link above and double-check the description includes all "
"of it before the rollout._"
)
bullets = "\n".join(f"- {m}" for m in missing)
return f"**What this one is currently missing:**\n\n{bullets}"
def _recovery_section(kind: str) -> str:
if kind == "pr":
return (
"**If the bot closes this PR after the rollout:** update the "
"description with the missing pieces, then either open a fresh "
"PR or comment `@agent-shin reconsider` on the closed PR. If "
"Greptile re-scores you at 4/5 or higher I'll reopen and tag "
"the PR `ready for review`. (`@greptileai` works on closed PRs "
"too; a fresh review is one of the signals that lifts you back "
"into the queue.) This is **not** us losing interest in your "
"change; far from it. We just need open PRs to be a list of "
"things a maintainer can act on, so we can get to yours faster."
)
return (
"**If the bot closes this issue after the rollout:** edit the issue "
"description to add the missing pieces, then comment `@agent-shin "
"reconsider` on the closed issue. I'll re-evaluate and, if the rubric "
"is met, reopen it. (GitHub doesn't let external authors reopen an "
"issue a maintainer or bot closed, so the comment is the reliable "
"path.) This is **not** us saying the bug isn't real or the request "
"isn't useful; it's so the remaining open issues are a list of things "
"a maintainer can act on."
)
def format_heads_up_comment(
*, kind: str, verdict: dict, greptile_score: int | None, cutoff: dt.date
) -> str:
"""Compose the friendly 7-day heads-up comment posted on a failing PR/issue."""
noun = "PR" if kind == "pr" else "issue"
rubric = _rubric_section_pr() if kind == "pr" else _rubric_section_issue()
cutoff_str = _format_cutoff(cutoff)
explanation = (verdict.get("explanation") or "").strip()
explanation_block = (
f"> _(The judge's note for this one: {explanation})_\n\n" if explanation else ""
)
return (
"🚅 **Heads-up: we're turning on the OSS triage bot in "
f"{DEFAULT_GRACE_DAYS} days, on {cutoff_str}.**\n"
"\n"
"We're rolling out **Agent Shin**, an LLM-as-judge triage bot for "
f"external {noun}s. Once it's live, the bot reads each open "
f"{noun}'s description, scores it against a small rubric, and "
f"auto-closes any {noun} that's missing the basics, with a single "
f"comment explaining what's missing and how to recover. Full "
f"context: [Agent Shin rollout blog post]({ROLLOUT_BLOG_URL}).\n"
"\n"
f"{rubric}\n"
"\n"
f"{_description_only_note(kind)}\n"
"\n"
f"{_missing_section(verdict, greptile_score)}\n"
"\n"
f"{explanation_block}"
"**Timeline (you have a week):**\n"
"\n"
f"- We turn the bot on in {DEFAULT_GRACE_DAYS} days, on "
f"**{cutoff_str}**. You have until then to update this {noun}'s "
"description with the missing pieces above.\n"
f"- If this {noun} still fails the rubric at **{cutoff_str}**, "
"we'll close it.\n"
f"- From then on the bot runs daily, and every {noun} that fails "
"the rubric gets a **2-hour lifetime**: one warning comment, then "
"auto-close 2 hours later.\n"
"\n"
f"{_recovery_section(kind)}\n"
"\n"
f"{HEADS_UP_MARKER}"
)
def _list_open_numbers(repo: str, kind: str) -> list[int]:
"""Return every open PR or issue number in ``repo``.
Delegates to ``list_open_items`` so the full backlog is fetched (no cap)
and the `gh {pr,issue} list` invocation stays in one shared place. ``gh
issue list`` would include PRs, but ``list_open_items`` uses the dedicated
command per kind, so the two never mix.
"""
return [
item["number"] for item in list_open_items(kind, repo=repo, fields="number")
]
def _has_heads_up_marker(item: dict) -> bool:
"""Cheap fast-path: check the PR/issue body itself for the marker.
The marker is appended to the *comment* we post, not the body, so this
will only fire if the body literally contains the marker text. We still
do the comment-marker check separately below; this body check just lets
us short-circuit for PRs/issues that quote the marker for any reason.
"""
body = item.get("body") or ""
return HEADS_UP_MARKER in body
def _comments_have_marker(repo: str, number: int) -> bool:
"""True if the bot already posted a comment carrying the marker.
Used for idempotency: a re-run skips items the previous run notified.
Filters by author (matching the sibling marker-checks in
``triage_with_llm._has_marker`` and
``agent_shin_shared.seconds_since_latest_marker_comment``) so a
contributor who quotes the heads-up via GitHub's "Quote reply" — which
preserves HTML comments in the raw markdown — can't trick the
idempotency check into silently skipping a real heads-up.
Comments live on the unified issues endpoint regardless of whether the
item is a PR or an issue, so no ``kind`` argument is required here.
"""
expected_login = (
os.environ.get("AGENT_SHIN_BOT_LOGIN") or AGENT_SHIN_DEFAULT_BOT_LOGIN
).lower()
raw = gh(
"api",
"--paginate",
f"repos/{repo}/issues/{number}/comments?per_page=100",
)
for line in raw.splitlines():
line = line.strip()
if not line:
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
comments = payload if isinstance(payload, list) else [payload]
for comment in comments:
author = ((comment.get("user") or {}).get("login") or "").lower()
if author != expected_login:
continue
if HEADS_UP_MARKER in (comment.get("body") or ""):
return True
return False
def _evaluate_pr(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
"""Run the future PR rubric (review_gate) in dry-run and return the result."""
return review_gate(
repo=repo,
number=number,
close=False, # we only want the verdict, never act here
model=model,
judge=judge,
)
def _evaluate_issue(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
"""Run the future issue rubric (triage kind='issue') in dry-run."""
return triage(
repo=repo,
kind="issue",
number=number,
close=False,
model=model,
judge=judge,
)
def _would_be_closed(kind: str, result: dict) -> bool:
"""True if the future triage would auto-close this PR/issue based on the
rubric (regardless of grace-period gating).
For PRs we trust ``review_gate``'s ``passing`` field — it combines the LLM
verdict and the Greptile score. For issues we read the LLM verdict
directly. Both fields are ``None``/missing on skip paths
(skip-internal-author, skip-llm-error, etc.) where the future bot would
NOT close the item — those return False.
"""
if kind == "pr":
passing = result.get("passing")
if passing is None:
return False # skipped — nothing for the heads-up to warn about
return passing is False
verdict = result.get("verdict") or {}
return (verdict.get("verdict") or "").lower() == "fail"
def _process_one(
*,
repo: str,
kind: str,
number: int,
model: str,
cutoff: dt.date,
dry_run: bool,
judge: Any = None,
skip_marker_check: bool = False,
allowlist: frozenset[str] = ALLOWLIST_LOGINS,
) -> dict:
"""Evaluate one PR/issue and post a heads-up if it would be auto-closed.
Returns a per-item dict for the summary table.
"""
base = {"kind": kind, "number": number}
fetcher = fetch_pr if kind == "pr" else fetch_issue
item = fetcher(repo, number)
if (item.get("state") or "") != "open":
return {**base, "action": "skip-not-open"}
if allowlist:
login = (item.get("user") or {}).get("login") or ""
if login.lower() not in allowlist:
return {**base, "action": "skip-not-allowlisted"}
elif is_internal_contributor(item):
return {**base, "action": "skip-internal-author"}
if not skip_marker_check and _has_heads_up_marker(item):
return {**base, "action": "skip-already-marked-in-body"}
if not skip_marker_check and _comments_have_marker(repo, number):
return {**base, "action": "skip-already-notified"}
if kind == "pr":
result = _evaluate_pr(repo=repo, number=number, model=model, judge=judge)
else:
result = _evaluate_issue(repo=repo, number=number, model=model, judge=judge)
if not _would_be_closed(kind, result):
return {**base, "action": "skip-passing", "evaluator": result.get("action")}
verdict = result.get("verdict") or {}
greptile_score = result.get("greptile_score") if kind == "pr" else None
comment = format_heads_up_comment(
kind=kind, verdict=verdict, greptile_score=greptile_score, cutoff=cutoff
)
maybe_post_comment(repo, number, comment, dry_run=dry_run)
return {
**base,
"action": "heads-up-posted" if not dry_run else "would-post-heads-up",
"verdict": (verdict.get("verdict") or "").lower(),
"greptile_score": greptile_score,
}
def _print_summary(results: list[dict]) -> None:
"""Tally per-action counts so a dry-run preview tells you at a glance how
many comments the real run would post."""
counts: dict[str, int] = {}
for r in results:
counts[r["action"]] = counts.get(r["action"], 0) + 1
print("\n=== rollout heads-up summary ===")
for action in sorted(counts):
print(f" {action:35s} {counts[action]}")
print(f" total {len(results)}")
def run(
*,
repo: str,
close: bool,
cutoff: dt.date,
model: str,
kinds: tuple[str, ...] = ("pr", "issue"),
judge: Any = None,
only_numbers: dict[str, list[int]] | None = None,
skip_marker_check: bool = False,
) -> list[dict]:
"""Sweep ``repo`` and post heads-up comments. Returns the per-item results."""
dry_run = not close
if dry_run:
print(
f"[DRY RUN] sweeping {repo}; --close not passed, no comments will be posted."
)
else:
print(f"[REAL RUN] sweeping {repo}; comments WILL be posted.")
print(f"Cutoff date in comment body: {cutoff.isoformat()}")
results: list[dict] = []
for kind in kinds:
if only_numbers and kind in only_numbers:
numbers = list(only_numbers[kind])
else:
numbers = _list_open_numbers(repo, kind)
print(f"\n--- {kind}s: {len(numbers)} open ---")
for n in numbers:
try:
result = _process_one(
repo=repo,
kind=kind,
number=n,
model=model,
cutoff=cutoff,
dry_run=dry_run,
judge=judge,
skip_marker_check=skip_marker_check,
)
except (
Exception
) as exc: # noqa: BLE001 - per-item errors don't abort the sweep
result = {
"kind": kind,
"number": n,
"action": "error",
"error": str(exc),
}
print(f"!! {kind}#{n}: {exc}", file=sys.stderr)
print(f" {kind}#{n}: {result['action']}")
results.append(result)
_print_summary(results)
return results
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", required=True, help="owner/repo")
parser.add_argument(
"--close",
action="store_true",
help=(
"Actually post comments. Without this flag the script is in "
"dry-run mode and only logs what it would do."
),
)
parser.add_argument(
"--close-on",
type=dt.date.fromisoformat,
default=None,
help=(
"Cutoff date shown in the heads-up comment as the rollout date "
f"(default: today + {DEFAULT_GRACE_DAYS} days)."
),
)
parser.add_argument(
"--model",
default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL,
help=f"Model for the rubric LLM judge (default: {DEFAULT_MODEL}).",
)
parser.add_argument(
"--kind",
choices=("pr", "issue", "both"),
default="both",
help="Restrict the sweep to PRs or issues only (default: both).",
)
parser.add_argument(
"--only-pr",
type=int,
action="append",
default=[],
help="Limit the PR sweep to these PR numbers (repeat for several).",
)
parser.add_argument(
"--only-issue",
type=int,
action="append",
default=[],
help="Limit the issue sweep to these issue numbers (repeat for several).",
)
parser.add_argument(
"--ignore-existing-marker",
action="store_true",
help=(
"Re-post on PRs/issues that already carry the heads-up marker. "
"Useful for testing the comment wording on a known PR."
),
)
args = parser.parse_args()
cutoff = args.close_on or (
dt.datetime.now(dt.timezone.utc).date() + dt.timedelta(days=DEFAULT_GRACE_DAYS)
)
kinds: tuple[str, ...]
if args.kind == "pr":
kinds = ("pr",)
elif args.kind == "issue":
kinds = ("issue",)
else:
kinds = ("pr", "issue")
only: dict[str, list[int]] = {}
if args.only_pr:
only["pr"] = args.only_pr
if args.only_issue:
only["issue"] = args.only_issue
# The script must NOT hit the LLM in dry-run if no key is set — we still
# want a useful preview that says "skip-no-llm-key" for items that would
# have been judged. Production runs require OPENAI_API_KEY.
if args.close and not os.environ.get("OPENAI_API_KEY"):
parser.error("OPENAI_API_KEY must be set for --close (real-run) mode.")
run(
repo=args.repo,
close=args.close,
cutoff=cutoff,
model=args.model,
kinds=kinds,
only_numbers=only or None,
skip_marker_check=args.ignore_existing_marker,
)
return 0
if __name__ == "__main__":
sys.exit(main())

1778
.github/scripts/triage_with_llm.py vendored Normal file
View file

File diff suppressed because it is too large Load diff

92
.github/workflows/close_low_quality_prs.yml vendored Normal file
View file

@ -0,0 +1,92 @@
name: Close Low-Quality PRs
# Auto-close any open PR (including drafts, regardless of age) authored by an
# external OSS contributor that Greptile reviewed with a confidence score
# below 4/5. Closures are explained in a comment that tells the contributor
# to push fixes and open a fresh PR (since OSS authors cannot reopen a PR
# closed by a bot/maintainer) or comment `@agent-shin reconsider` to have
# Agent Shin re-evaluate.
#
# Manual one-off run:
# gh workflow run "Close Low-Quality PRs" -f close=true
#
# Dry-run preview (no PRs are touched):
# gh workflow run "Close Low-Quality PRs" -f close=false
on:
schedule:
# Daily at 09:00 UTC. Pairs well with the stale-issue workflow at midnight.
- cron: "0 9 * * *"
workflow_dispatch:
inputs:
close:
description: "Actually close matching PRs (false = dry run)."
required: false
default: "false"
type: choice
options:
- "true"
- "false"
min_age_days:
description: "Minimum PR age in days (default 0 = no age filter)."
required: false
default: "0"
min_score:
description: "Greptile score below which a PR is closed (1-5)."
required: false
default: "4"
limit:
description: "Maximum number of PRs to close in a single run."
required: false
default: "25"
permissions:
contents: read
pull-requests: write
issues: write
jobs:
close-low-quality-prs:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Run low-quality PR closer
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Scheduled runs are ALWAYS dry-run, even when AGENT_SHIN_ENABLED is
# "true", so the team can QA the closer's verdicts in step summaries
# before any contributor sees a PR closed. Real closures only happen
# on manual workflow_dispatch with close=true (and the variable set).
CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
MIN_AGE_DAYS: ${{ github.event.inputs.min_age_days || '0' }}
MIN_SCORE: ${{ github.event.inputs.min_score || '4' }}
LIMIT: ${{ github.event.inputs.limit || '25' }}
run: |
set -euo pipefail
ARGS=(
--repo "${{ github.repository }}"
--min-age-days "${MIN_AGE_DAYS}"
--min-score "${MIN_SCORE}"
--limit "${LIMIT}"
)
if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then
echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> forcing dry-run regardless of close input."
elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG}" = "true" ]; then
ARGS+=(--close)
echo "::notice::Running in close-on-fail mode."
else
echo "::notice::AGENT_SHIN_ENABLED is true but this trigger is dry-run (scheduled event or close=false)."
fi
python3 .github/scripts/close_low_quality_prs.py "${ARGS[@]}"

131
.github/workflows/review_gate.yml vendored Normal file
View file

@ -0,0 +1,131 @@
name: Agent Shin — review gate
# Keeps the `ready for review` label in sync with whether an external PR
# currently clears BOTH the LLM rubric AND Greptile's confidence score.
#
# pass -> add `ready for review` + a "passed / all clear" comment
# regress -> remove the label + a "what's missing" comment (PR stays open)
# fail, <24h old -> a one-time "what's missing" notice (grace window)
# fail, >24h old -> close + a comment (reopen via `@agent-shin reconsider`)
#
# DRY-RUN BY DEFAULT. Every side effect (label add/remove, comment, close) is
# gated behind `--close`, which is only added when the repo variable
# `AGENT_SHIN_ENABLED == "true"`. Until then runs only write the verdict to the
# workflow step summary.
#
# Manual single PR: gh workflow run "Agent Shin — review gate" -f pr_number=NNN
# Manual dry-run: gh workflow run "Agent Shin — review gate" -f close=false
#
# We use `pull_request_target` so the workflow can read repo secrets and run
# against fork PRs. Fork code is never checked out — only PR metadata is read
# via `gh api`.
on:
pull_request_target:
types: [opened, reopened, synchronize, ready_for_review]
schedule:
# Daily at 09:30 UTC — re-reconciles labels as Greptile re-reviews land.
- cron: "30 9 * * *"
workflow_dispatch:
inputs:
pr_number:
description: "Single PR to reconcile (omit to sweep all open PRs)."
required: false
close:
description: "If AGENT_SHIN_ENABLED=true, actually act (false = dry run)."
required: false
default: "false"
type: choice
options:
- "true"
- "false"
grace_days:
description: "Hours/24 a failing, un-tagged PR may stay open before close."
required: false
default: "1"
min_greptile_score:
description: "Greptile score below which a PR counts as not passing (1-5)."
required: false
default: "4"
permissions:
contents: read
issues: write
pull-requests: write
jobs:
review-gate:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run review gate
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Mirror the triage workflow: only expose the LLM key when the bot is
# enabled or a collaborator triggers it manually, so an external user
# can't force paid LLM calls by churning a fork PR while the bot is
# still in dry-run.
OPENAI_API_KEY: ${{ (vars.AGENT_SHIN_ENABLED == 'true' || github.event_name == 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }}
GRACE_DAYS: ${{ github.event.inputs.grace_days || '1' }}
MIN_GREPTILE_SCORE: ${{ github.event.inputs.min_greptile_score || '4' }}
EVENT_PR: ${{ github.event.pull_request.number }}
INPUT_PR: ${{ github.event.inputs.pr_number }}
run: |
set -euo pipefail
COMMON=(--review-gate --grace-days "${GRACE_DAYS}" --min-greptile-score "${MIN_GREPTILE_SCORE}")
# Fail-safe gating, identical philosophy to the Greptile closer:
# - AGENT_SHIN_ENABLED must be the EXACT string "true" to act at all.
# - A manual dispatch can still preview with close=false.
# - Automatic triggers (PR events, schedule) act once enabled — that
# is the whole point of the gate (re-tag / un-tag automatically).
DO_CLOSE="false"
if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then
echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> dry-run (no labels/comments/closes)."
elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG:-false}" = "true" ]; then
DO_CLOSE="true"
echo "::notice::Manual run -> acting for real."
elif [ "${GITHUB_EVENT_NAME:-}" != "workflow_dispatch" ]; then
DO_CLOSE="true"
echo "::notice::Enabled automatic trigger (${GITHUB_EVENT_NAME:-}) -> acting for real."
else
echo "::notice::Manual dispatch with close=false -> dry-run."
fi
if [ "${DO_CLOSE}" = "true" ]; then
COMMON+=(--close)
fi
# Single PR (PR event or explicit input) vs. sweep over all open PRs.
TARGET_PR="${EVENT_PR:-${INPUT_PR:-}}"
if [ -n "${TARGET_PR}" ]; then
python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${TARGET_PR}" "${COMMON[@]}"
else
echo "::notice::Sweeping all open PRs."
# Match GH_LIST_ALL_LIMIT in agent_shin_shared.py: gh lists newest-first,
# so any cap below the real backlog silently drops the *oldest* PRs —
# exactly the stale ones this daily sweep is meant to reconcile.
mapfile -t NUMBERS < <(gh pr list --repo "${{ github.repository }}" --state open --limit 100000 --json number --jq '.[].number')
for n in "${NUMBERS[@]}"; do
echo "::group::PR #${n}"
python3 .github/scripts/triage_with_llm.py --repo "${{ github.repository }}" --pr "${n}" "${COMMON[@]}" || echo "::warning::review gate errored on #${n}"
echo "::endgroup::"
done
fi

56
.github/workflows/test-linting.yml vendored
View file

@ -77,18 +77,19 @@ jobs:
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
- name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$BASE_SHA"
- name: Print OpenAI version
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Run MyPy type checking
run: |
cd litellm
(uv run --no-sync mypy . || true) | uv run --no-sync python ../scripts/type_check_gate.py --tool mypy
- name: Run basedpyright type checking
run: |
(uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --tool basedpyright
(uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py
- name: Check for circular imports
run: |
@ -100,22 +101,20 @@ jobs:
run: |
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
any-discipline:
# Separate job: the first run cold-builds litellm's type cache (~2 min, ~3 GB),
# so keep it off the main lint job's time budget. Subsequent runs reuse the
# cached .mypy_cache_any and only re-type-check the changed files.
# Intentionally NON-GATING. This job turns red when a *-budget.json ceiling is
# raised (or a rule/budget is dropped) so a loosening is obvious in review, but it
# must be kept OUT of the branch-protection required-checks list so a justified
# bump can still be merged by a human who has seen and accepted the red.
budget-ratchet:
runs-on: ubuntu-latest
timeout-minutes: 10
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
# Check out the PR head, not the default refs/pull/N/merge: the merge ref
# folds in newer base commits, which the diff-based gates (ruff delta,
# Any-discipline) would otherwise blame on this branch.
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
clean: true
persist-credentials: false
- name: Set up Python
@ -123,32 +122,11 @@ jobs:
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Install dependencies
run: |
uv sync --frozen
# Keyed on deps + mypy config (which fix the type cache's validity), not on
# source content, so changed files always differ from the restored cache.
# The gate also defensively invalidates each target's cache entry, so
# correctness never depends on cache freshness -- this is purely for speed.
- name: Restore Any-gate type cache
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .mypy_cache_any
key: any-mypy-cache-${{ runner.os }}-py3.12-${{ hashFiles('uv.lock', 'litellm/mypy.ini') }}
restore-keys: |
any-mypy-cache-${{ runner.os }}-py3.12-
- name: Check Any discipline on changed lines
- name: Ratchet check (budgets may only decrease; non-gating)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
uv run --no-sync python scripts/check_any_discipline.py --changed --base "$BASE_SHA"
python scripts/budget_ratchet_check.py --base "$BASE_SHA"
secret-scan:
runs-on: ubuntu-latest

96
.github/workflows/triage_issue_with_llm.yml vendored Normal file
View file

@ -0,0 +1,96 @@
name: Agent Shin — Issue triage
# LLM-as-judge triage for external GitHub issues.
#
# DRY-RUN BY DEFAULT. See .github/workflows/triage_pr_with_llm.yml for the
# enablement procedure — same repo variable (`AGENT_SHIN_ENABLED=true`)
# unlocks the PR and issue triage flows together.
on:
issues:
types: [opened, reopened]
workflow_dispatch:
inputs:
issue_number:
description: "Issue number to triage manually."
required: true
close:
description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail."
required: false
default: "false"
type: choice
options:
- "true"
- "false"
permissions:
contents: read
issues: write
jobs:
triage:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run Agent Shin
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Only expose the LLM key when the bot is enabled or a collaborator
# triggers it manually, so an external user can't force paid LLM
# calls by churning issues while the bot is still in dry-run.
# The Python script calls the LLM whenever this var is set
# (regardless of `--close`); stripping `--close` doesn't suppress
# the API call, only the destructive side effects.
OPENAI_API_KEY: ${{ (vars.AGENT_SHIN_ENABLED == 'true' || github.event_name == 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
DISPATCH_CLOSE: ${{ github.event.inputs.close }}
ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }}
run: |
set -euo pipefail
ARGS=(--repo "${{ github.repository }}" --issue "${ISSUE_NUMBER}")
# Fail-safe gating: only the EXACT string "true" enables the
# destructive --close path. The workflow_dispatch input is a
# `choice` dropdown of "true"/"false" so the UI is constrained,
# but the API (`gh workflow run -f close=...`) accepts any
# string, and a `!= "false"` check would treat "True", "yes",
# "1", "TRUE", typos, and accidental whitespace as enabling
# closure. Mirror the Greptile closer's `= "true"` pattern.
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" = "true" ]; then
ARGS+=(--close)
echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode."
elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
echo "::notice::Agent Shin is ENABLED but this trigger is dry-run (workflow_dispatch close != 'true')."
else
echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no issues will be closed."
fi
# Automatic `issues` events stay dry-run regardless until the team
# explicitly invokes workflow_dispatch with close=true.
if [ "${GITHUB_EVENT_NAME:-}" = "issues" ]; then
# filter out --close rather than substituting to "" (which would
# leave an empty positional arg that argparse rejects)
FILTERED=()
for arg in "${ARGS[@]}"; do
if [ "${arg}" != "--close" ]; then
FILTERED+=("${arg}")
fi
done
ARGS=("${FILTERED[@]}")
echo "::notice::issues trigger -> forcing dry-run."
fi
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"

110
.github/workflows/triage_pr_with_llm.yml vendored Normal file
View file

@ -0,0 +1,110 @@
name: Agent Shin — PR triage
# LLM-as-judge triage for external pull requests.
#
# DRY-RUN BY DEFAULT. Closures and public comments are gated on the repo
# variable `AGENT_SHIN_ENABLED` being set to the string `"true"`. Until then,
# every run only writes its verdict to the workflow step summary so the team
# can QA the judge's decisions before flipping it on.
#
# To enable for real:
# 1. Add a repo secret `OPENAI_API_KEY` (or compatible).
# 2. Set repo variable `AGENT_SHIN_ENABLED` to `true`
# (Settings > Secrets and variables > Actions > Variables).
#
# We use `pull_request_target` so the workflow has access to repo secrets
# and runs against PRs from forks. We never check out fork code — only read
# PR metadata via `gh api`, so this is safe.
on:
pull_request_target:
types: [opened, reopened]
workflow_dispatch:
inputs:
pr_number:
description: "PR number to triage manually."
required: true
close:
description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail."
required: false
default: "false"
type: choice
options:
- "true"
- "false"
permissions:
contents: read
issues: write
pull-requests: write
jobs:
triage:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage script
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run Agent Shin
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Only expose the LLM key when the bot is enabled or a collaborator
# triggers it manually, so an external user can't force paid LLM
# calls by churning a fork PR while the bot is still in dry-run.
# The Python script calls the LLM whenever this var is set
# (regardless of `--close`); stripping `--close` doesn't suppress
# the API call, only the destructive side effects.
OPENAI_API_KEY: ${{ (vars.AGENT_SHIN_ENABLED == 'true' || github.event_name == 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
DISPATCH_CLOSE: ${{ github.event.inputs.close }}
PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
run: |
set -euo pipefail
ARGS=(--repo "${{ github.repository }}" --pr "${PR_NUMBER}")
# Fail-safe gating: only the EXACT string "true" enables the
# destructive --close path. The workflow_dispatch input is a
# `choice` dropdown of "true"/"false" so the UI is constrained,
# but the API (`gh workflow run -f close=...`) accepts any
# string, and a `!= "false"` check would treat "True", "yes",
# "1", "TRUE", typos, and accidental whitespace as enabling
# closure. Mirror the Greptile closer's `= "true"` pattern.
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" = "true" ]; then
ARGS+=(--close)
echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode."
elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
echo "::notice::Agent Shin is ENABLED but this trigger is dry-run (workflow_dispatch close != 'true' or scheduled event)."
else
echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no PRs will be closed."
fi
# On the scheduled/automatic pull_request_target trigger we default to
# dry-run regardless, so the team can review verdicts in the step
# summary before any contributor sees a comment. Only the manual
# workflow_dispatch path (with close=true) closes PRs.
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request_target" ]; then
# strip any --close added above (filter out, don't substitute
# to empty string — that would leave a stray "" positional arg
# that argparse rejects)
FILTERED=()
for arg in "${ARGS[@]}"; do
if [ "${arg}" != "--close" ]; then
FILTERED+=("${arg}")
fi
done
ARGS=("${FILTERED[@]}")
echo "::notice::pull_request_target trigger -> forcing dry-run."
fi
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"

172
.github/workflows/triage_reconsider.yml vendored Normal file
View file

@ -0,0 +1,172 @@
name: Agent Shin — reconsider
# Comment-trigger workflow: when the PR/issue author (or an internal
# collaborator) comments `@agent-shin reconsider` on a CLOSED PR/issue,
# Agent Shin re-runs LLM-judge triage on the current title+body and:
#
# - on PASS: posts a "re-evaluated and reopened" comment + reopens.
# - on FAIL: posts a "still missing X" comment and leaves it closed,
# so the contributor can iterate again.
#
# This exists because GitHub does NOT let an external (non-write-access)
# OSS contributor reopen a PR/issue closed by a bot or maintainer. Without
# this comment trigger, a contributor whose PR Agent Shin auto-closed
# would have no path back into the review queue except opening a fresh PR
# (which loses the original PR's history). The bot, on the other hand,
# has write access via GH_TOKEN and can reopen on their behalf.
#
# DRY-RUN BY DEFAULT — gated on `vars.AGENT_SHIN_ENABLED == 'true'` just
# like the other Agent Shin workflows. The workflow also gates on the
# commenter being either the PR/issue author or an internal collaborator
# (OWNER/MEMBER/COLLABORATOR) so random commenters cannot DOS the LLM
# judge or force a reopen.
on:
issue_comment:
types: [created]
permissions:
contents: read
issues: write
pull-requests: write
jobs:
reconsider:
if: |
github.repository == 'BerriAI/litellm'
&& contains(github.event.comment.body, '@agent-shin reconsider')
runs-on: ubuntu-latest
steps:
- name: Authorize commenter
# Only the PR/issue author OR an internal collaborator may trigger
# a reconsider. Outside random commenters could otherwise spam the
# phrase to burn LLM budget or, if a fail-open bug were ever
# introduced, force a reopen on someone else's behalf.
#
# We expose the authorization decision as a step output and gate
# every subsequent (potentially destructive) step on it. A `run:`
# step with `exit 0` would NOT stop the job — only `if:` gating
# on a known-true output is safe here.
id: auth
env:
COMMENTER: ${{ github.event.comment.user.login }}
AUTHOR: ${{ github.event.issue.user.login }}
ASSOCIATION: ${{ github.event.comment.author_association }}
run: |
set -euo pipefail
if [ "${COMMENTER}" = "${AUTHOR}" ]; then
echo "::notice::Authorized: commenter is the PR/issue author."
echo "authorized=true" >> "$GITHUB_OUTPUT"
exit 0
fi
case "${ASSOCIATION}" in
OWNER|MEMBER|COLLABORATOR)
echo "::notice::Authorized: commenter is an internal collaborator (${ASSOCIATION})."
echo "authorized=true" >> "$GITHUB_OUTPUT"
;;
*)
echo "::notice::Commenter '${COMMENTER}' (${ASSOCIATION}) is not authorized to trigger reconsider; skipping subsequent steps."
echo "authorized=false" >> "$GITHUB_OUTPUT"
;;
esac
- name: React 👀 to acknowledge the reconsider
# Add an eyes reaction to the triggering comment the moment we accept
# it, so the contributor gets instant feedback that the bot saw their
# `@agent-shin reconsider` before the slower triage steps run. Gated on
# AGENT_SHIN_ENABLED so dry-run leaves no visible trace. Best-effort:
# a reactions API hiccup must never fail the actual reconsider.
if: steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMENT_ID: ${{ github.event.comment.id }}
run: |
set -euo pipefail
gh api --method POST \
-H "Accept: application/vnd.github+json" \
"repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \
-f content=eyes \
|| echo "::warning::failed to add 👀 reaction (non-fatal)"
- name: Checkout triage script
if: steps.auth.outputs.authorized == 'true'
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
if: steps.auth.outputs.authorized == 'true'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
if: steps.auth.outputs.authorized == 'true'
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run Agent Shin reconsider
if: steps.auth.outputs.authorized == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Only expose the LLM key when the bot is enabled, so a PR/issue
# author can't force paid LLM calls by spamming `@agent-shin
# reconsider` while the bot is still in dry-run. The Python script
# calls the LLM whenever this var is set (regardless of `--close`);
# stripping `--close` doesn't suppress the API call, only the
# destructive side effects. Mirror the gating used by every other
# Agent Shin workflow (triage_pr_with_llm.yml, review_gate.yml, ...).
OPENAI_API_KEY: ${{ vars.AGENT_SHIN_ENABLED == 'true' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
# `issue_comment` events fire for both issues and PR comments.
# `issue.pull_request` is set iff this is a PR comment, so we use
# its presence to decide whether to invoke `--pr N` or `--issue N`.
IS_PR: ${{ github.event.issue.pull_request != null }}
NUMBER: ${{ github.event.issue.number }}
run: |
set -euo pipefail
if [ "${IS_PR}" = "true" ]; then
ARGS=(--repo "${{ github.repository }}" --pr "${NUMBER}" --reconsider)
else
ARGS=(--repo "${{ github.repository }}" --issue "${NUMBER}" --reconsider)
fi
# Reconsider's destructive actions (post comment + reopen) are
# gated on `--close`, mirroring the regular triage workflows.
# When AGENT_SHIN_ENABLED is not the EXACT string "true", we
# still run the script so its verdict + would-X action lands in
# the step summary for QA — but without `--close`, the script
# returns `would-reopen` / `would-reconsider-still-failing`
# instead of touching GitHub state.
#
# Use the positive `= "true"` gate (not `!= "true" -> exit`) so
# the workflow guardrails in
# tests/test_litellm/test_github_triage_workflows.py see the
# canonical fail-safe enable pattern. Unknown values like
# "True", "yes", "1", or typos fall through to the dry-run
# branch, which is the safe default.
if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
ARGS+=(--close)
echo "::notice::Agent Shin reconsider ENABLED — running real triage (close=true)."
else
echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> reconsider stays in dry-run (no comment, no reopen)."
fi
python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"
- name: React 👍 when the reconsider finishes
# Once the reconsider run has completed successfully, add a thumbs-up so
# the contributor sees the bot is done (the 👀 stays, signalling
# seen -> handled). `success()` keeps this from firing if the run
# errored, and the AGENT_SHIN_ENABLED gate keeps dry-run inert.
if: success() && steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMENT_ID: ${{ github.event.comment.id }}
run: |
set -euo pipefail
gh api --method POST \
-H "Accept: application/vnd.github+json" \
"repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \
-f content=+1 \
|| echo "::warning::failed to add 👍 reaction (non-fatal)"

92
.github/workflows/triage_rollout_heads_up.yml vendored Normal file
View file

@ -0,0 +1,92 @@
name: Agent Shin — rollout heads-up (one-shot)
# Fires the 7-day heads-up comment on every open external PR/issue that the
# new triage bot would auto-close. The real sweep is a deliberate one-shot:
# trigger it at rollout via a manual `workflow_dispatch` with `dry_run=false`.
# The script is idempotent (skips items that already carry the
# `<!-- agent-shin:rollout-heads-up -->` marker), so a re-run is harmless.
#
# The automatic push trigger runs DRY-RUN only, so merging the script to
# `litellm_internal_staging` never posts a comment; it just confirms the
# workflow is wired up. Posting real comments requires the manual dispatch,
# which is also the only trigger that exposes `OPENAI_API_KEY`. The heads-up
# is intentionally NOT gated on `AGENT_SHIN_ENABLED`: it has to warn
# contributors while that flag is still off, ahead of the flip that turns on
# auto-closing.
#
# The workflow is a thin shell over `.github/scripts/triage_rollout_heads_up.py`.
# Dry-run vs. real run differ in EXACTLY one CLI flag (`--close`), added only
# on a manual dispatch with `dry_run=false`.
on:
push:
branches:
- litellm_internal_staging
paths:
# The presence of this script on staging IS the rollout merge marker.
# Editing the file later would re-fire the workflow; that's safe because
# the script skips PRs/issues that already have the heads-up marker.
- ".github/scripts/triage_rollout_heads_up.py"
workflow_dispatch:
inputs:
dry_run:
description: "Dry run (true = preview only, false = actually post comments)."
required: false
default: "true"
type: choice
options:
- "true"
- "false"
permissions:
contents: read
issues: write
pull-requests: write
jobs:
heads-up:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage scripts
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run heads-up sweep
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Only the manual dispatch (the real-run trigger) needs the LLM key.
# The automatic push trigger runs dry-run and never posts, so it gets
# no key. Mirrors the sibling triage workflows, which expose the key
# only on an enabled/dispatched run rather than unconditionally.
OPENAI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
# The real run is a deliberate manual dispatch with dry_run=false.
# Use the EXACT "false" comparison so any unexpected input value
# fail-closes to dry-run (mirrors the AGENT_SHIN_ENABLED pattern in
# the sibling workflows). The automatic push trigger always stays
# dry-run, so merging the script never posts.
DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }}
run: |
set -euo pipefail
ARGS=(--repo "${{ github.repository }}")
if [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${DRY_RUN_INPUT:-true}" = "false" ]; then
ARGS+=(--close)
echo "::notice::Manual rollout dispatch with dry_run=false -> heads-up comments WILL be posted."
elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ]; then
echo "::notice::Manual dispatch in dry-run mode -> previewing only, no comments will be posted."
else
echo "::notice::Automatic push trigger -> dry-run preview only. Fire the real rollout sweep with a manual workflow_dispatch (dry_run=false)."
fi
python3 .github/scripts/triage_rollout_heads_up.py "${ARGS[@]}"

2
.gitignore vendored
View file

@ -74,8 +74,6 @@ tests/local_testing/log.txt
.codegpt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
**/.mypy_cache/
**/.mypy_cache_any/
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
tests/llm_translation/test_vertex_key.json

6
CLAUDE.md
View file

@ -36,11 +36,11 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Run tests, format your code, and lint your code before each commit
When you fix violations gated by `ruff-strict-budget.json`, `mypy-code-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom
When you fix violations gated by `ruff-strict-budget.json` or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and bringing it closer to the max, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
The Any-discipline gate (`make lint-any`, also a CI job) fails when a line you changed under `litellm/` holds a value typed `Any`, including the `X | Any`. Ideally `# any-ok: <reason>` is never used; treat it as a last resort for a genuine typed/untyped boundary that Pydantic truly can't model
If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it)

9
CONTRIBUTING.md
View file

@ -154,8 +154,7 @@ Individual linting commands:
```bash
make format-check # Check Black formatting
make lint-ruff # Run Ruff linting
make lint-mypy # Run MyPy type checking
make lint-any # Fail on Any-typed values on changed lines
make lint-basedpyright # Run basedpyright type checking
make check-circular-imports # Check for circular imports
make check-import-safety # Check import safety
```
@ -217,7 +216,7 @@ LiteLLM follows the [Google Python Style Guide](https://google.github.io/stylegu
Our automated quality checks include:
- **Black** for consistent code formatting
- **Ruff** for linting and code quality
- **MyPy** for static type checking
- **basedpyright** for static type checking
- **Circular import detection**
- **Import safety validation**
@ -231,7 +230,7 @@ If `make lint` fails:
1. **Formatting issues**: Run `make format` to auto-fix
2. **Ruff issues**: Check the output and fix manually
3. **MyPy issues**: Add proper type hints
3. **basedpyright issues**: Add proper type hints
4. **Circular imports**: Refactor import dependencies
5. **Import safety**: Fix any unprotected imports
@ -246,7 +245,7 @@ If `make test-unit` fails:
### 3. Common Development Tips
- **Use type hints**: MyPy requires proper type annotations
- **Use type hints**: basedpyright requires proper type annotations
- **Write descriptive commit messages**: Help reviewers understand your changes
- **Keep PRs focused**: One feature/fix per PR
- **Test edge cases**: Don't just test the happy path

32
Makefile
View file

@ -5,8 +5,8 @@
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev format \
lint-mypy lint-mypy-budget-update lint-basedpyright lint-basedpyright-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-any \
lint-basedpyright lint-basedpyright-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety
@ -22,17 +22,14 @@ help:
@echo " make install-hooks - Install git hooks (Conventional Commits + Branches)"
@echo " make format - Apply Black code formatting"
@echo " make format-check - Check Black code formatting (matches CI)"
@echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
@echo " make lint - Run all linting (Ruff, basedpyright, Black check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-mypy - Run MyPy (disallow_untyped_defs), gated by per-rule error counts"
@echo " make lint-mypy-budget-update - Re-capture the MyPy per-rule budget (ratchet)"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
@echo " make lint-black - Check Black formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
@echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
@echo " make lint-budget-update - Re-capture all three ratchet budgets (ruff + mypy + basedpyright)"
@echo " make lint-any - Fail if changed lines under litellm/ hold an Any-typed value"
@echo " make lint-budget-update - Re-capture all ratchet budgets (ruff + basedpyright)"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -126,17 +123,11 @@ lint-ruff-FULL-dev: install-dev
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
else echo "No changed .py files to check."; fi
lint-mypy: install-dev
cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy
lint-mypy-budget-update: install-dev
cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy --update
lint-basedpyright: install-dev
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py
lint-basedpyright-budget-update: install-dev
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright --update
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
lint-black: format-check
@ -146,11 +137,8 @@ lint-ruff-budget: install-dev
lint-ruff-budget-update: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --update
# Ratchet all three budgets in one shot (ruff strict + mypy + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-mypy-budget-update lint-basedpyright-budget-update
lint-any: install-dev
$(UV_RUN) python scripts/check_any_discipline.py --changed
# Ratchet all budgets in one shot (ruff strict + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-basedpyright-budget-update
check-circular-imports: install-dev
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
@ -159,10 +147,10 @@ check-import-safety: install-dev
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting (matches test-linting.yml workflow)
lint: format-check lint-ruff lint-mypy lint-basedpyright check-circular-imports check-import-safety lint-ruff-budget lint-any
lint: format-check lint-ruff lint-basedpyright check-circular-imports check-import-safety lint-ruff-budget
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed lint-mypy lint-any check-circular-imports check-import-safety
lint-dev: lint-format-changed check-circular-imports check-import-safety
# Testing targets
test: install-test-deps

18
basedpyright-code-budget.json
View file

@ -1,11 +1,11 @@
{
"reportAny": {
"baseline": 24954,
"slack": 10
"slack": 2500
},
"reportArgumentType": {
"baseline": 1863,
"slack": 3
"slack": 180
},
"reportAssignmentType": {
"baseline": 220,
@ -33,7 +33,7 @@
},
"reportExplicitAny": {
"baseline": 6931,
"slack": 10
"slack": 700
},
"reportFunctionMemberAccess": {
"baseline": 7,
@ -77,7 +77,7 @@
},
"reportMissingTypeArgument": {
"baseline": 10612,
"slack": 10
"slack": 1000
},
"reportMissingTypeStubs": {
"baseline": 27,
@ -113,7 +113,7 @@
},
"reportPrivateUsage": {
"baseline": 1625,
"slack": 10
"slack": 160
},
"reportRedeclaration": {
"baseline": 8,
@ -133,7 +133,7 @@
},
"reportUnknownArgumentType": {
"baseline": 30603,
"slack": 10
"slack": 3000
},
"reportUnknownLambdaType": {
"baseline": 76,
@ -141,15 +141,15 @@
},
"reportUnknownMemberType": {
"baseline": 27322,
"slack": 10
"slack": 2500
},
"reportUnknownParameterType": {
"baseline": 13636,
"slack": 10
"slack": 1000
},
"reportUnknownVariableType": {
"baseline": 21776,
"slack": 10
"slack": 2000
},
"reportUnnecessaryCast": {
"baseline": 118,

8
db_scripts/create_views.py
View file

@ -15,7 +15,7 @@ db = Prisma(
)
async def check_view_exists(): # noqa: PLR0915
async def check_view_exists():
"""
Checks if the LiteLLM_VerificationTokenView and MonthlyGlobalSpend exists in the user's db.
@ -34,8 +34,7 @@ async def check_view_exists(): # noqa: PLR0915
print("LiteLLM_VerificationTokenView Exists!") # noqa
except Exception:
# If an error occurs, the view does not exist, so create it
await db.execute_raw(
"""
await db.execute_raw("""
CREATE VIEW "LiteLLM_VerificationTokenView" AS
SELECT
v.*,
@ -45,8 +44,7 @@ async def check_view_exists(): # noqa: PLR0915
t.rpm_limit AS team_rpm_limit
FROM "LiteLLM_VerificationToken" v
LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id;
"""
)
""")
print("LiteLLM_VerificationTokenView Created!") # noqa

2
docker/build_from_pip/litellm_config.yaml
View file

@ -3,7 +3,7 @@ model_list:
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
api_base: os.environ/FAKE_OPENAI_API_BASE
general_settings:
alerting: ["slack"]

2
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -412,7 +412,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
)
async def async_pre_call_hook( # noqa: PLR0915
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,

2
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -483,7 +483,7 @@ async def new_project(
response_model=LiteLLM_ProjectTable,
)
@management_endpoint_wrapper
async def update_project( # noqa: PLR0915
async def update_project(
data: UpdateProjectRequest,
http_request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),

22
litellm/__init__.py
View file

@ -213,6 +213,15 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
# When True (default — preserves historical behavior), the Router appends
# internal config names (model_group, fallback model groups, deployment
# timeouts, fallback failure details) onto exception messages and surfaces
# them to clients via ProxyException.message. Set to False if you do NOT
# want the proxy's internal model_name / fallback wiring visible to clients.
# Deprecation: planned to flip to False (redact by default) in a future
# major release; opt in early with `litellm.expose_router_debug_in_errors
# = False`.
expose_router_debug_in_errors: bool = True
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
@ -235,6 +244,17 @@ modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))
use_chat_completions_url_for_anthropic_messages: bool = bool(
os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API
# When True, strip the OpenAI-flavored `usage.total_tokens` field that
# LiteLLM injects into non-streaming /v1/messages responses, bringing the
# wire response into line with the Anthropic spec (matches the streaming
# SSE path, which already omits total_tokens). Default False to preserve
# backward compatibility for clients that read the LiteLLM-shaped
# `usage.total_tokens` today. Planned to flip to True in a future major
# release; opt in early via Python:
# `litellm.strip_anthropic_total_tokens = True`
# Or via `litellm_settings.strip_anthropic_total_tokens: true` in
# config.yaml.
strip_anthropic_total_tokens: bool = False
route_all_chat_openai_to_responses: bool = (
os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true"
) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge
@ -413,7 +433,7 @@ anthropic_beta_headers_url: str = os.getenv(
"LITELLM_ANTHROPIC_BETA_HEADERS_URL",
"https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json",
)
suppress_debug_info = False
suppress_debug_info: bool = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
s3_audit_callback_params: Optional[Dict] = None

2
litellm/_redis.py
View file

@ -311,7 +311,7 @@ def get_redis_url_from_environment():
return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
def _get_redis_client_logic(**env_overrides): # noqa: PLR0915
def _get_redis_client_logic(**env_overrides):
"""
Common functionality across sync + async redis client implementations
"""

2
litellm/a2a_protocol/main.py
View file

@ -436,7 +436,7 @@ def _build_streaming_logging_obj(
return logging_obj
async def asend_message_streaming( # noqa: PLR0915
async def asend_message_streaming(
a2a_client: Optional["A2AClientType"] = None,
request: Optional["SendStreamingMessageRequest"] = None,
api_base: Optional[str] = None,

2
litellm/batches/main.py
View file

@ -157,7 +157,7 @@ async def acreate_batch(
@client
def create_batch( # noqa: PLR0915
def create_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
input_file_id: str,

2
litellm/caching/caching_handler.py
View file

@ -394,7 +394,7 @@ class LLMCachingHandler:
return cr["model"]
return None
def _process_async_embedding_cached_response( # noqa: PLR0915
def _process_async_embedding_cached_response(
self,
final_embedding_cached_response: Optional[EmbeddingResponse],
cached_result: List[Optional[CachedEmbedding]],

2
litellm/caching/qdrant_semantic_cache.py
View file

@ -28,7 +28,7 @@ from .base_cache import BaseCache
class QdrantSemanticCache(BaseCache):
CACHE_KEY_FIELD_NAME = "litellm_cache_key"
def __init__( # noqa: PLR0915
def __init__(
self,
qdrant_api_base=None,
qdrant_api_key=None,

2
litellm/caching/redis_cache.py
View file

@ -369,6 +369,8 @@ class RedisCache(BaseCache):
"""
Make sure each key starts with the given namespace
"""
if key is None:
return key # type: ignore[return-value]
if self.namespace is not None and not key.startswith(self.namespace):
key = self.namespace + ":" + key

2
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -1211,7 +1211,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
return self.chunk_parser(json.loads(str_line))
@staticmethod
def translate_responses_chunk_to_openai_stream( # noqa: PLR0915
def translate_responses_chunk_to_openai_stream(
parsed_chunk: Union[dict, BaseModel],
) -> "ModelResponseStream":
"""

2
litellm/constants.py
View file

@ -510,6 +510,8 @@ DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE = os.getenv(
"DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield"
)
LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED = 499
EMAIL_BUDGET_ALERT_TTL = int(
os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)
) # 24 hours in seconds

32
litellm/cost_calculator.py
View file

@ -94,6 +94,7 @@ from litellm.types.utils import (
LlmProviders,
LlmProvidersSet,
ModelInfo,
ServiceTier,
StandardBuiltInToolsParams,
TranscriptionUsageDurationObject,
TranscriptionUsageTokensObject,
@ -288,7 +289,7 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def cost_per_token( # noqa: PLR0915
def cost_per_token(
model: str = "",
prompt_tokens: int = 0,
completion_tokens: int = 0,
@ -614,7 +615,9 @@ def cost_per_token( # noqa: PLR0915
service_tier=service_tier,
)
elif custom_llm_provider == "anthropic":
return anthropic_cost_per_token(model=model, usage=usage_block)
return anthropic_cost_per_token(
model=model, usage=usage_block, service_tier=service_tier
)
elif custom_llm_provider == "bedrock":
return bedrock_cost_per_token(
model=model, usage=usage_block, service_tier=service_tier
@ -885,6 +888,23 @@ def _map_traffic_type_to_service_tier(traffic_type: Optional[str]) -> Optional[s
return service_tier
def _normalize_service_tier(service_tier: object) -> str | None:
"""
Reduce a service_tier value to a concrete billable tier string or None.
"auto" is a routing preference and any non-string value is not a billable
tier, so both defer to standard pricing (or to the tier the provider reports
on the response usage) instead of crashing the downstream cost-key lookup,
which calls service_tier.lower()
"""
if (
not isinstance(service_tier, str)
or service_tier.lower() == ServiceTier.AUTO.value
):
return None
return service_tier
def _get_usage_object(
completion_response: Any,
) -> Optional[Usage]:
@ -1136,7 +1156,7 @@ def _store_cost_breakdown_in_logging_obj(
pass
def completion_cost( # noqa: PLR0915
def completion_cost(
completion_response=None,
model: Optional[str] = None,
prompt="",
@ -1224,6 +1244,8 @@ def completion_cost( # noqa: PLR0915
if service_tier is None and optional_params is not None:
service_tier = optional_params.get("service_tier")
service_tier = _normalize_service_tier(service_tier)
# Extract service_tier from completion_response if not provided
if service_tier is None and completion_response is not None:
if isinstance(completion_response, BaseModel):
@ -1231,6 +1253,8 @@ def completion_cost( # noqa: PLR0915
elif isinstance(completion_response, dict):
service_tier = completion_response.get("service_tier")
service_tier = _normalize_service_tier(service_tier)
# Extract service_tier from usage object if not provided
if service_tier is None and cost_per_token_usage_object is not None:
if isinstance(cost_per_token_usage_object, BaseModel):
@ -1240,6 +1264,8 @@ def completion_cost( # noqa: PLR0915
elif isinstance(cost_per_token_usage_object, dict):
service_tier = cost_per_token_usage_object.get("service_tier")
service_tier = _normalize_service_tier(service_tier)
selected_model = _select_model_name_for_cost_calc(
model=model,
completion_response=completion_response,

4
litellm/images/main.py
View file

@ -195,7 +195,7 @@ def image_generation(
@client
def image_generation( # noqa: PLR0915
def image_generation(
prompt: str,
model: Optional[str] = None,
n: Optional[int] = None,
@ -738,7 +738,7 @@ def image_variation(
@client
def image_edit( # noqa: PLR0915
def image_edit(
image: Optional[Union[FileTypes, List[FileTypes]]] = None,
prompt: Optional[str] = None,
model: Optional[str] = None,

4
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -351,7 +351,7 @@ class SlackAlerting(CustomBatchLogger):
except Exception:
return 0
async def send_daily_reports(self, router) -> bool: # noqa: PLR0915
async def send_daily_reports(self, router) -> bool:
"""
Send a daily report on:
- Top 5 deployments with most failed requests
@ -1373,7 +1373,7 @@ Model Info:
return False
async def send_alert( # noqa: PLR0915
async def send_alert(
self,
message: str,
level: Literal["Low", "Medium", "High"],

8
litellm/integrations/braintrust_logging.py
View file

@ -133,9 +133,7 @@ class BraintrustLogger(CustomLogger):
self.default_project_id = project_dict["id"]
def log_success_event( # noqa: PLR0915
self, kwargs, response_obj, start_time, end_time
):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
try:
litellm_call_id = kwargs.get("litellm_call_id")
@ -271,9 +269,7 @@ class BraintrustLogger(CustomLogger):
except Exception as e:
raise e # don't use verbose_logger.exception, if exception is raised
async def async_log_success_event( # noqa: PLR0915
self, kwargs, response_obj, start_time, end_time
):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
try:
litellm_call_id = kwargs.get("litellm_call_id")

2
litellm/integrations/langfuse/langfuse.py
View file

@ -549,7 +549,7 @@ class LangFuseLogger:
)
)
def _log_langfuse_v2( # noqa: PLR0915
def _log_langfuse_v2(
self,
user_id: Optional[str],
metadata: dict,

2
litellm/integrations/mock_client_factory.py
View file

@ -107,7 +107,7 @@ def _is_url_match(url, matchers: List[str]) -> bool:
return False
def create_mock_client_factory(config: MockClientConfig): # noqa: PLR0915
def create_mock_client_factory(config: MockClientConfig):
"""
Factory function that creates mock client functions based on configuration.

4
litellm/integrations/opentelemetry.py
View file

@ -2198,9 +2198,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
return kv_pairs
def set_attributes( # noqa: PLR0915
self, span: Span, kwargs, response_obj: Optional[Any]
):
def set_attributes(self, span: Span, kwargs, response_obj: Optional[Any]):
try:
if self.callback_name == "langtrace":
from litellm.integrations.langtrace import LangtraceAttributes

14
litellm/integrations/otel/model/config.py
View file

@ -184,6 +184,20 @@ class OpenTelemetryV2Config(BaseSettings):
),
)
@field_validator("capture_message_content", mode="before")
@classmethod
def _normalize_capture_message_content(cls, value: object) -> object:
"""Fold the capture mode to its canonical lower_snake_case form.
V1 read this env var case-insensitively, so operators set the
UPPER_SNAKE_CASE form (e.g. ``SPAN_AND_EVENT``). The canonical values
here are lower_snake_case; normalizing at the boundary keeps both
spellings working and lets every downstream comparison stay exact.
"""
if isinstance(value, str):
return value.lower()
return value
@field_validator(
"baggage_promoted_keys",
"baggage_metadata_keys",

4
litellm/integrations/prometheus.py
View file

@ -75,7 +75,7 @@ class PrometheusLogger(CustomLogger):
return cb
return None
def __init__( # noqa: PLR0915
def __init__(
self,
**kwargs,
):
@ -2255,7 +2255,7 @@ class PrometheusLogger(CustomLogger):
or _litellm_params_metadata.get("user_agent"),
}
def set_llm_deployment_failure_metrics(self, request_kwargs: dict): # noqa: PLR0915
def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
"""
Sets Failure metrics when an LLM API call fails

35
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -170,6 +170,16 @@ def get_error_message(error_obj) -> Optional[str]:
####### EXCEPTION MAPPING ################
def _get_body_error_code(error_str: str) -> int | None:
"""Return error.code from a JSON error body, or None if not parseable."""
try:
body = json.loads(error_str)
code = body.get("error", {}).get("code")
return int(code) if code is not None else None
except Exception:
return None
def _get_response_headers(original_exception: Exception) -> Optional[httpx.Headers]:
"""
Extract and return the response headers from an exception, if present.
@ -234,7 +244,7 @@ def extract_and_raise_litellm_exception(
)
def exception_type( # type: ignore # noqa: PLR0915
def exception_type( # type: ignore
model,
original_exception,
custom_llm_provider,
@ -1415,6 +1425,29 @@ def exception_type( # type: ignore # noqa: PLR0915
),
),
)
elif (
isinstance(getattr(original_exception, "status_code", None), int)
and 500 <= original_exception.status_code < 600
and _get_body_error_code(error_str) == 429
):
# upstream gateway wraps a 429 inside a 5xx envelope
# e.g. HTTP 500/503 with {"error":{"code":429,...}}.
# Scoped to 5xx so HTTP 400/401 with body code:429
# still maps to BadRequestError / AuthenticationError.
exception_mapping_worked = True
raise RateLimitError(
message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=429,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
),
)
elif (
"500 Internal Server Error" in error_str
or "The model is overloaded." in error_str

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -154,7 +154,7 @@ def handle_anthropic_text_model_custom_llm_provider(
return model, custom_llm_provider
def get_llm_provider( # noqa: PLR0915
def get_llm_provider(
model: str,
custom_llm_provider: Optional[str] = None,
api_base: Optional[str] = None,
@ -573,7 +573,7 @@ def get_llm_provider( # noqa: PLR0915
)
def _get_openai_compatible_provider_info( # noqa: PLR0915
def _get_openai_compatible_provider_info(
model: str,
api_base: Optional[str],
api_key: Optional[str],

2
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -5,7 +5,7 @@ from litellm.exceptions import BadRequestError
from litellm.types.utils import LlmProviders, LlmProvidersSet
def get_supported_openai_params( # noqa: PLR0915
def get_supported_openai_params(
model: str,
custom_llm_provider: Optional[str] = None,
request_type: Literal[

78
litellm/litellm_core_utils/litellm_logging.py
View file

@ -986,7 +986,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._get_masked_api_base(additional_args.get("api_base", ""))
)
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
def pre_call(self, input, api_key, model=None, additional_args={}):
# Log the exact input to the LLM API
litellm.error_logs["PRE_CALL"] = locals()
try:
@ -2124,7 +2124,7 @@ class Logging(LiteLLMLoggingBaseClass):
await self.async_success_handler(result=complete_streaming_response)
return
def success_handler( # noqa: PLR0915
def success_handler(
self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs
):
verbose_logger.debug(
@ -2589,7 +2589,7 @@ class Logging(LiteLLMLoggingBaseClass):
),
)
async def async_success_handler( # noqa: PLR0915
async def async_success_handler(
self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs
):
"""
@ -3041,7 +3041,7 @@ class Logging(LiteLLMLoggingBaseClass):
kwargs=self.model_call_details,
) # type: ignore
def failure_handler( # noqa: PLR0915
def failure_handler(
self, exception, traceback_exception, start_time=None, end_time=None
):
verbose_logger.debug(
@ -3758,7 +3758,7 @@ def _get_masked_values(
}
def set_callbacks(callback_list, function_id=None): # noqa: PLR0915
def set_callbacks(callback_list, function_id=None):
"""
Globally sets the callback client
"""
@ -3859,7 +3859,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915
return None
def _init_custom_logger_compatible_class( # noqa: PLR0915
def _init_custom_logger_compatible_class(
logging_integration: _custom_logger_compatible_callbacks_literal,
internal_usage_cache: Optional[DualCache],
llm_router: Optional[
@ -4616,7 +4616,7 @@ def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list) -> None:
)
def get_custom_logger_compatible_class( # noqa: PLR0915
def get_custom_logger_compatible_class(
logging_integration: _custom_logger_compatible_callbacks_literal,
) -> Optional[CustomLogger]:
try:
@ -5421,19 +5421,20 @@ class StandardLoggingPayloadSetup:
tb_lines[:MAXIMUM_TRACEBACK_LINES_TO_LOG]
) # Limit to first 100 lines
# Prefer the `.message` attribute (set by ProxyException and every
# litellm.exceptions.* class) over str(exc); ProxyException does not
# call super().__init__() nor define __str__, so str() on it returns
# an empty string, which used to silently strip the human-readable
# message from spend_logs.metadata.error_information.
# Use isinstance, not truthiness: an explicit empty string on
# `.message` is a deliberate value and must not be replaced by
# `str(exc)`.
explicit_message = getattr(original_exception, "message", None)
error_message = (
explicit_message
if isinstance(explicit_message, str) and explicit_message
else str(original_exception)
)
if isinstance(explicit_message, str):
error_message = explicit_message
else:
error_message = str(original_exception) if original_exception else ""
# Duck-typed read so bare-Exception subclasses like
# `litellm.BudgetExceededError` can participate without joining the
# RateLimitError hierarchy (which would break `except BudgetExceededError`).
# Validated against the enum value sets so a third-party exception that
# happens to declare a `.category` or `.rate_limit_type` string attribute
# can't leak garbage into the payload or Prometheus label cardinality.
rate_limit_category = validate_rate_limit_category(
getattr(original_exception, "category", None)
)
@ -5446,11 +5447,42 @@ class StandardLoggingPayloadSetup:
error_class=error_class,
llm_provider=_llm_provider_in_exception,
traceback=traceback_info,
error_message=error_message if original_exception else "",
error_message=error_message,
error_rate_limit_category=rate_limit_category,
error_rate_limit_type=rate_limit_type,
)
@staticmethod
def get_error_information_for_logging_payload(
metadata: dict,
original_exception: Exception | None,
error_str: str | None,
) -> tuple[StandardLoggingPayloadErrorInformation, str | None]:
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,
)
if not metadata.get("client_disconnected"):
return error_information, error_str
client_disconnect_error = metadata.get("error_information")
if isinstance(client_disconnect_error, dict):
error_information = cast(
StandardLoggingPayloadErrorInformation,
client_disconnect_error,
)
else:
error_information = cast(
StandardLoggingPayloadErrorInformation,
{
"error_code": "499",
"error_message": "Client disconnected the request",
"error_class": "ClientDisconnected",
},
)
if not error_str:
error_str = "Client disconnected the request"
return error_information, error_str
@staticmethod
def get_response_time(
start_time_float: float,
@ -5778,8 +5810,12 @@ def get_standard_logging_object_payload(
api_base=litellm_params.get("api_base"),
)
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,
error_information, error_str = (
StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata=metadata,
original_exception=original_exception,
error_str=error_str,
)
)
## get final response object ##

83
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -191,6 +191,11 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st
return base_key
def _parse_above_token_threshold(key: str) -> float:
threshold_str = key.split("_above_")[1].split("_tokens")[0]
return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1)
def _get_token_base_cost(
model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None
) -> Tuple[float, float, float, float, float]:
@ -256,15 +261,13 @@ def _get_token_base_cost(
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
threshold: Optional[float] = None
for key in sorted(threshold_keys, reverse=True):
for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True):
value = model_info.get(key)
if value is not None:
try:
# Handle both formats: _above_128k_tokens and _above_128_tokens
threshold_str = key.split("_above_")[1].split("_tokens")[0]
threshold = float(threshold_str.replace("k", "")) * (
1000 if "k" in threshold_str else 1
)
threshold = _parse_above_token_threshold(key)
if usage.prompt_tokens > threshold:
# Prefer a service_tier-specific above-threshold key when available,
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
@ -303,40 +306,54 @@ def _get_token_base_cost(
# Apply tiered pricing to cache costs
cache_creation_tiered_key = (
f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
_get_service_tier_cost_key(
f"cache_creation_input_token_cost_above_{threshold_str}_tokens",
service_tier,
)
if service_tier
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
)
cache_creation_1hr_tiered_key = (
_get_service_tier_cost_key(
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
service_tier,
)
if service_tier
else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
)
cache_creation_1hr_tiered_key = f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
cache_read_tiered_key = (
f"cache_read_input_token_cost_above_{threshold_str}_tokens"
_get_service_tier_cost_key(
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
service_tier,
)
if service_tier
else f"cache_read_input_token_cost_above_{threshold_str}_tokens"
)
if cache_creation_tiered_key in model_info:
cache_creation_cost = cast(
float,
_get_cost_per_unit(
model_info,
cache_creation_tiered_key,
cache_creation_cost,
),
)
cache_creation_cost = cast(
float,
_get_cost_per_unit(
model_info,
cache_creation_tiered_key,
cache_creation_cost,
),
)
if cache_creation_1hr_tiered_key in model_info:
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(
model_info,
cache_creation_1hr_tiered_key,
cache_creation_cost_above_1hr,
),
)
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(
model_info,
cache_creation_1hr_tiered_key,
cache_creation_cost_above_1hr,
),
)
if cache_read_tiered_key in model_info:
cache_read_cost = cast(
float,
_get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
),
)
cache_read_cost = cast(
float,
_get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
),
)
break
except (IndexError, ValueError):
@ -683,7 +700,7 @@ def _get_regional_uplift_multiplier(
return 1.0
def generic_cost_per_token( # noqa: PLR0915
def generic_cost_per_token(
model: str,
usage: Usage,
custom_llm_provider: str,

2
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -471,7 +471,7 @@ def _should_convert_tool_call_to_json_mode(
return False
def convert_to_model_response_object( # noqa: PLR0915
def convert_to_model_response_object(
response_object: Optional[dict] = None,
model_response_object: Optional[
Union[

12
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1475,7 +1475,7 @@ def convert_to_gemini_tool_call_invoke(
)
def convert_to_gemini_tool_call_result( # noqa: PLR0915
def convert_to_gemini_tool_call_result(
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
last_message_with_tool_calls: Optional[dict],
model: Optional[str] = None,
@ -2227,7 +2227,7 @@ def _sanitize_empty_text_content(
return message
def _add_missing_tool_results( # noqa: PLR0915
def _add_missing_tool_results(
current_message: AllMessageValues,
messages: List[AllMessageValues],
current_index: int,
@ -2484,7 +2484,7 @@ def sanitize_messages_for_tool_calling(
return sanitized_messages
def anthropic_messages_pt( # noqa: PLR0915
def anthropic_messages_pt(
messages: List[AllMessageValues],
model: str,
llm_provider: str,
@ -3278,7 +3278,7 @@ def convert_to_cohere_tool_invoke(tool_calls: list) -> List[ToolCallObject]:
return cohere_tool_invoke
def cohere_messages_pt_v2( # noqa: PLR0915
def cohere_messages_pt_v2(
messages: List,
model: str,
llm_provider: str,
@ -4703,7 +4703,7 @@ class BedrockConverseMessagesProcessor:
return messages
@staticmethod
async def _bedrock_converse_messages_pt_async( # noqa: PLR0915
async def _bedrock_converse_messages_pt_async(
messages: List,
model: str,
llm_provider: str,
@ -5133,7 +5133,7 @@ class BedrockConverseMessagesProcessor:
return assistant_parts
def _bedrock_converse_messages_pt( # noqa: PLR0915
def _bedrock_converse_messages_pt(
messages: List,
model: str,
llm_provider: str,

2
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1198,7 +1198,7 @@ class RealTimeStreaming:
item["content"] = new_content
return item
async def client_ack_messages(self): # noqa: PLR0915
async def client_ack_messages(self):
try:
while True:
message = await self.websocket.receive_text()

2
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -209,7 +209,7 @@ class ChunkProcessor:
)
return response
def get_combined_tool_content( # noqa: PLR0915
def get_combined_tool_content(
self, tool_call_chunks: List[Dict[str, Any]]
) -> List[ChatCompletionMessageToolCall]:
tool_calls_list: List[ChatCompletionMessageToolCall] = []

8
litellm/litellm_core_utils/streaming_handler.py
View file

@ -967,7 +967,7 @@ class CustomStreamWrapper:
delta, model_response.choices[0].delta, attribute
)
def return_processed_chunk_logic( # noqa: PLR0915, C901
def return_processed_chunk_logic( # noqa: C901
self,
completion_obj: Dict[str, Any],
model_response: ModelResponseStream,
@ -1145,7 +1145,7 @@ class CustomStreamWrapper:
del model_response.choices[0].delta.reasoning_content
return
def chunk_creator(self, chunk: Any): # type: ignore # noqa: PLR0915
def chunk_creator(self, chunk: Any): # type: ignore
if hasattr(chunk, "id"):
self.response_id = chunk.id
model_response = self.model_response_creator()
@ -1887,7 +1887,7 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason = "tool_calls"
return model_response
def __next__(self) -> "ModelResponseStream": # noqa: PLR0915
def __next__(self) -> "ModelResponseStream":
cache_hit = False
if (
self.custom_llm_provider is not None
@ -2077,7 +2077,7 @@ class CustomStreamWrapper:
return self.completion_stream
async def __anext__(self) -> "ModelResponseStream": # noqa: PLR0915
async def __anext__(self) -> "ModelResponseStream":
cache_hit = False
if (
self.custom_llm_provider is not None

13
litellm/litellm_core_utils/token_counter.py
View file

@ -744,6 +744,17 @@ def _count_content_list(
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
elif c["type"] == "tool_reference":
# Anthropic tool-search reference block: a lightweight pointer to
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
# The full tool definition is counted via the `tools` param, so we
# only count the referenced name here. Without this branch,
# token_counter raises on tool-search traffic; on the streaming
# anthropic_messages path that nulls response_cost and causes the
# proxy to drop the SpendLogs row entirely (silent cost undercount).
tool_name = str(c.get("tool_name") or "")
if tool_name:
num_tokens += count_function(tool_name)
else:
content_type = (
c.get("type", type(c).__name__)
@ -752,7 +763,7 @@ def _count_content_list(
)
raise ValueError(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking)."
f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)."
)
return num_tokens
except Exception as e:

2
litellm/llms/anthropic/chat/handler.py
View file

@ -772,7 +772,7 @@ class ModelResponseIterator:
)
return results
def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
try:
type_chunk = chunk.get("type", "") or ""

9
litellm/llms/anthropic/chat/transformation.py
View file

@ -605,7 +605,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return _tool_choice
def _map_tool_helper( # noqa: PLR0915
def _map_tool_helper(
self,
tool: ChatCompletionToolParam,
) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]:
@ -1399,7 +1399,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return None
def map_openai_params( # noqa: PLR0915
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
@ -2213,6 +2213,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
inference_geo: Optional[str] = None
if "inference_geo" in _usage and _usage["inference_geo"] is not None:
inference_geo = _usage["inference_geo"]
service_tier = cast(
str | None,
_usage.get("service_tier"),
)
iterations: Optional[List[Any]] = _usage.get("iterations")
if iterations:
@ -2324,6 +2328,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
),
inference_geo=inference_geo,
speed=speed,
service_tier=service_tier,
)
return usage

23
litellm/llms/anthropic/cost_calculation.py
View file

@ -18,7 +18,9 @@ if TYPE_CHECKING:
import litellm
def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float:
def _compute_cache_only_cost(
model_info: "ModelInfo", usage: "Usage", service_tier: str | None = None
) -> float:
"""
Return only the cache-related portion of the prompt cost (cache read + cache write).
@ -36,7 +38,9 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float:
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(model_info=model_info, usage=usage)
) = _get_token_base_cost(
model_info=model_info, usage=usage, service_tier=service_tier
)
cache_cost = float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
@ -56,19 +60,26 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float:
return cache_cost
def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
def cost_per_token(
model: str, usage: "Usage", service_tier: str | None = None
) -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
Input:
- model: str, the model name without provider prefix
- usage: LiteLLM Usage block, containing anthropic caching information
- service_tier: the service tier the request was served at (e.g. "priority"),
read from the Anthropic response usage and used to select tier-specific pricing
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
prompt_cost, completion_cost = generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="anthropic"
model=model,
usage=usage,
custom_llm_provider="anthropic",
service_tier=service_tier,
)
# Apply provider_specific_entry multipliers for geo/speed routing
@ -89,7 +100,9 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
multiplier *= provider_specific_entry.get("fast", 1.0)
if multiplier != 1.0:
cache_cost = _compute_cache_only_cost(model_info=model_info, usage=usage)
cache_cost = _compute_cache_only_cost(
model_info=model_info, usage=usage, service_tier=service_tier
)
prompt_cost = (prompt_cost - cache_cost) * multiplier + cache_cost
completion_cost *= multiplier
except Exception:

4
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -372,7 +372,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
cache_read_input_tokens=0,
)
def __next__(self): # noqa: PLR0915
def __next__(self):
from .transformation import LiteLLMAnthropicMessagesAdapter
try:
@ -618,7 +618,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
)
raise StopIteration
async def __anext__(self): # noqa: PLR0915
async def __anext__(self):
from .transformation import LiteLLMAnthropicMessagesAdapter
try:

25
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -332,7 +332,14 @@ class LiteLLMAnthropicMessagesAdapter:
if isinstance(source, dict)
else getattr(source, "cache_control", None)
)
if cache_control and model and self.is_anthropic_claude_model(model):
if (
cache_control
and model
and (
self.is_anthropic_claude_model(model)
or self.is_bedrock_arn_model(model)
)
):
# TypedDict objects support dict operations at runtime
# Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
if isinstance(target, dict):
@ -376,7 +383,7 @@ class LiteLLMAnthropicMessagesAdapter:
isinstance(tool_type, str) and tool_type.startswith("web_search")
) or tool_name == "web_search"
def translate_anthropic_messages_to_openai( # noqa: PLR0915
def translate_anthropic_messages_to_openai(
self,
messages: List[
Union[
@ -752,6 +759,20 @@ class LiteLLMAnthropicMessagesAdapter:
model_lower = model.lower()
return "anthropic" in model_lower or "claude" in model_lower
@staticmethod
def is_bedrock_arn_model(model: str) -> bool:
"""
Check if the model string is a Bedrock ARN, such as an Application
Inference Profile (e.g. arn:aws:bedrock:us-east-1:123:application-inference-profile/id).
These ARNs contain neither "anthropic" nor "claude", so is_anthropic_claude_model
cannot identify them even though, on the /v1/messages endpoint, they point at Claude.
Match ":bedrock:" in the ARN service field so another service's ARN that merely names
bedrock in a resource (arn:aws:sagemaker:.../my-bedrock-endpoint) is not matched.
"""
model_lower = model.lower()
return "arn:" in model_lower and ":bedrock:" in model_lower
@staticmethod
def translate_thinking_for_model(
thinking: Dict[str, Any],

4
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -97,7 +97,7 @@ def _read_summary_max_tokens_setting() -> int:
return COMPACT_SUMMARY_MAX_TOKENS
async def _check_summary_model_access( # noqa: PLR0915
async def _check_summary_model_access(
user_api_key_auth: Any,
summary_model: str,
llm_router: Any,
@ -970,7 +970,7 @@ def apply_client_compaction_block_history(
)
async def apply_compact_20260112( # noqa: PLR0915
async def apply_compact_20260112(
*,
model: str,
messages: List[Dict[str, Any]],

2
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -66,7 +66,7 @@ class AnthropicResponsesStreamWrapper:
self._current_block_index += 1
return self._current_block_index
def _process_event(self, event: Any) -> None: # noqa: PLR0915
def _process_event(self, event: Any) -> None:
"""Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
event_type = getattr(event, "type", None)
if event_type is None and isinstance(event, dict):

2
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -51,7 +51,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
return source.get("url")
return None
def translate_messages_to_responses_input( # noqa: PLR0915
def translate_messages_to_responses_input(
self,
messages: List[
Union[

51
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -218,8 +218,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
- Qualifier goes as query parameter
- Only the payload goes in the request body
Payload shape:
- ``prompt`` is always present and contains the text-only flatten of the
last message's content (existing behavior).
- ``content`` is added ONLY when the ``forward_multimodal_content`` litellm
param is truthy AND the last message's ``content`` is a list containing a
non-text block (e.g. ``image_url``, ``file``, ``input_audio``). The list is
forwarded verbatim so the agent's ``@app.entrypoint`` handler can parse the
OpenAI-shaped multimodal blocks. This is opt-in because an AgentCore agent
must be explicitly written to read ``payload["content"]``; by default the
payload stays byte-identical to the legacy ``{"prompt": "..."}`` shape.
Returns:
dict: Payload dict containing the prompt
dict: Payload dict containing the prompt and (optionally) the OpenAI
content list.
"""
verbose_logger.debug(
f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}"
@ -231,6 +243,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
# Create the payload - this is what goes in the body (raw JSON)
payload: dict = {"prompt": prompt}
# Opt-in: when forward_multimodal_content is set, forward the OpenAI content
# list verbatim under "content" so an attachment-aware agent can read the raw
# blocks (image_url, file, etc.). Default off keeps the payload byte-identical
# to the legacy {"prompt": "..."} shape for agents that only read the prompt.
if self._should_forward_multimodal_content(optional_params, litellm_params):
last_content = messages[-1].get("content")
if isinstance(last_content, list) and any(
isinstance(block, dict) and block.get("type") not in (None, "text")
for block in last_content
):
# Copy so the payload never aliases messages[-1]["content"]; shallow,
# not deep, to avoid cloning large base64 media on the request path.
payload["content"] = list(last_content)
# Get or generate session ID - this goes in the header
runtime_session_id = self._get_runtime_session_id(optional_params)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = runtime_session_id
@ -246,6 +272,29 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
verbose_logger.debug(f"PAYLOAD: {payload}")
return payload
@staticmethod
def _should_forward_multimodal_content(
optional_params: dict, litellm_params: dict
) -> bool:
"""Whether to forward raw OpenAI content blocks under ``payload["content"]``.
Opt-in via the ``forward_multimodal_content`` litellm param (default ``False``)
because AgentCore agents must be explicitly written to read the field. The
value may arrive as a bool or a config/env string ("true", "1", ...). Checks
``optional_params`` first (where other AgentCore params land), then
``litellm_params``.
"""
for source in (optional_params, litellm_params):
if not isinstance(source, dict):
continue
value = source.get("forward_multimodal_content")
if value is None:
continue
if isinstance(value, str):
return value.strip().lower() in ("1", "true", "yes", "on")
return bool(value)
return False
def _extract_sse_json(self, line: str) -> Optional[Dict]:
"""Extract and parse JSON from an SSE data line."""
if not line.startswith("data:"):

2
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -248,7 +248,7 @@ class BedrockConverseLLM(BaseAWSLLM):
encoding=encoding,
)
def completion( # noqa: PLR0915
def completion(
self,
model: str,
messages: list,

2
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -2189,7 +2189,7 @@ class AmazonConverseConfig(BaseConfig):
real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices]
return real_tools if real_tools else None
def _transform_response( # noqa: PLR0915
def _transform_response(
self,
model: str,
response: httpx.Response,

4
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -473,7 +473,7 @@ class BedrockLLM(BaseAWSLLM):
prompt += f"{message['content']}"
return prompt, chat_history # type: ignore
def process_response( # noqa: PLR0915
def process_response(
self,
model: str,
response: httpx.Response,
@ -765,7 +765,7 @@ class BedrockLLM(BaseAWSLLM):
return model_response
def completion( # noqa: PLR0915
def completion(
self,
model: str,
messages: list,

2
litellm/llms/bedrock/embed/embedding.py
View file

@ -388,7 +388,7 @@ class BedrockEmbedding(BaseAWSLLM):
batch_data=batch_data,
)
def embeddings( # noqa: PLR0915
def embeddings(
self,
model: str,
input: List[str],

2
litellm/llms/bedrock/image_edit/stability_transformation.py
View file

@ -149,7 +149,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
return mapped_params
def transform_image_edit_request( # noqa: PLR0915
def transform_image_edit_request(
self,
model: str,
prompt: Optional[str],

2
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -226,7 +226,7 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
return _is_converse_endpoint(endpoint)
@staticmethod
async def de_anonymize_event_stream( # noqa: PLR0915
async def de_anonymize_event_stream(
body_bytes: bytes,
proxy_logging_obj: "ProxyLogging",
user_api_key_dict: "UserAPIKeyAuth",

20
litellm/llms/bedrock_mantle/chat/transformation.py
View file

@ -4,8 +4,10 @@ Amazon Bedrock Mantle - OpenAI-compatible inference engine in Amazon Bedrock.
API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html
Base URL: https://bedrock-mantle.{region}.api.aws/v1
Auth: AWS Bedrock API key as Bearer token (set via BEDROCK_MANTLE_API_KEY env var)
or region-aware key via BEDROCK_MANTLE_{REGION}_API_KEY.
Auth: Bearer token (litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the
standard AWS_BEARER_TOKEN_BEDROCK) when present; otherwise AWS SigV4
(service "bedrock") over the standard credential chain. See
BedrockMantleAuthMixin in common_utils.
"""
from typing import Iterator, AsyncIterator, Any, List, Optional, Tuple, Union
@ -13,20 +15,26 @@ from typing import Iterator, AsyncIterator, Any, List, Optional, Tuple, Union
import litellm
from litellm._logging import verbose_logger
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock_mantle.common_utils import (
BEDROCK_MANTLE_DEFAULT_REGION,
BedrockMantleAuthMixin,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from ...openai_like.chat.transformation import OpenAILikeChatConfig
BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1"
class BedrockMantleChatConfig(OpenAILikeChatConfig):
class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
"""
Transformation config for Amazon Bedrock Mantle OpenAI-compatible API.
"""
def __init__(self, aws_signer: BaseAWSLLM | None = None):
super().__init__()
self._aws_signer = aws_signer or BaseAWSLLM()
@property
def custom_llm_provider(self) -> Optional[str]:
return "bedrock_mantle"
@ -54,7 +62,7 @@ class BedrockMantleChatConfig(OpenAILikeChatConfig):
or get_secret_str("BEDROCK_MANTLE_API_BASE")
or f"https://bedrock-mantle.{region}.api.aws/v1"
)
dynamic_api_key = api_key or get_secret_str("BEDROCK_MANTLE_API_KEY")
dynamic_api_key = self._resolve_bearer_token(api_key)
return api_base, dynamic_api_key
def validate_environment(

115
litellm/llms/bedrock_mantle/common_utils.py Normal file
View file

@ -0,0 +1,115 @@
"""
Shared auth and region resolution for the Amazon Bedrock Mantle backends.
Mantle authenticates with a Bearer token when one is available
(litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the standard
AWS_BEARER_TOKEN_BEDROCK); otherwise it falls back to AWS SigV4 (service
"bedrock") over the standard credential chain (IAM role / access key / profile /
web identity). The Chat Completions and Responses backends share this behaviour
through BedrockMantleAuthMixin so the two paths can never drift apart.
"""
import re
from typing import Tuple
from botocore.exceptions import (
CredentialRetrievalError,
NoCredentialsError,
PartialCredentialsError,
ProfileNotFound,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.secret_managers.main import get_secret_str
BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1"
# Standard Mantle host: https://bedrock-mantle.<region>.api.aws (group 1 = region).
MANTLE_HOST_RE = re.compile(
r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE
)
class BedrockMantleAuthMixin:
_aws_signer: BaseAWSLLM
@staticmethod
def _resolve_bearer_token(api_key: str | None) -> str | None:
return (
api_key
or get_secret_str("BEDROCK_MANTLE_API_KEY")
or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
)
@staticmethod
def _resolve_region(params: dict) -> str:
region = params.get("aws_region_name")
if region:
BaseAWSLLM._validate_aws_region_name(region)
return region
base = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
if base:
match = MANTLE_HOST_RE.match(base.rstrip("/"))
if match:
return match.group(1)
return (
get_secret_str("BEDROCK_MANTLE_REGION")
or get_secret_str("AWS_REGION_NAME")
or get_secret_str("AWS_REGION")
or BEDROCK_MANTLE_DEFAULT_REGION
)
def sign_request(
self,
headers: dict,
optional_params: dict,
request_data: dict,
api_base: str,
api_key: str | None = None,
model: str | None = None,
stream: bool | None = None,
fake_stream: bool | None = None,
) -> Tuple[dict, bytes | None]:
bearer = self._resolve_bearer_token(api_key)
if not bearer:
# SigV4 path. Pin the credential-scope region to the region of the actual
# signing URL so the SigV4 scope and the URL host can never disagree, even
# when a stale api_base and aws_region_name point at different regions.
# Fall back to _resolve_region only for custom proxy hosts that do not
# match the standard Mantle URL pattern. Also drop any caller Authorization
# so _sign_request's restore-original-Authorization step cannot override
# the SigV4 header.
host_match = MANTLE_HOST_RE.match(api_base.rstrip("/"))
optional_params = {
**optional_params,
"aws_region_name": (
host_match.group(1)
if host_match
else self._resolve_region({**optional_params, "api_base": api_base})
),
}
headers = {k: v for k, v in headers.items() if k.lower() != "authorization"}
try:
return self._aws_signer._sign_request(
service_name="bedrock",
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
api_key=bearer,
model=model,
stream=stream,
fake_stream=fake_stream,
)
except (
NoCredentialsError,
PartialCredentialsError,
ProfileNotFound,
CredentialRetrievalError,
) as e:
raise ValueError(
"Bedrock Mantle auth failed: no Bearer token and no usable AWS "
"credentials. Set BEDROCK_MANTLE_API_KEY (or AWS_BEARER_TOKEN_BEDROCK) "
"or pass api_key for Bearer auth, or provide AWS credentials "
"(IAM role / access key / profile / web identity) for SigV4."
) from e

108
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -15,26 +15,20 @@ role / access key / profile / web identity), signed via the shared
BaseAWSLLM._sign_request after the request body is finalized.
"""
import re
from typing import Any, Dict, List, Optional, Tuple
from botocore.exceptions import (
CredentialRetrievalError,
NoCredentialsError,
PartialCredentialsError,
ProfileNotFound,
)
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock_mantle.common_utils import (
MANTLE_HOST_RE,
BedrockMantleAuthMixin,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1"
# Checked longest/most-specific first so a full endpoint URL collapses to host
# in one pass and the appended path never doubles.
_BASE_SUFFIXES_TO_STRIP = (
@ -45,18 +39,13 @@ _BASE_SUFFIXES_TO_STRIP = (
"/v1",
)
# Standard Mantle host: https://bedrock-mantle.<region>.api.aws (group 1 = region).
_MANTLE_HOST_RE = re.compile(
r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE
)
# Per Bedrock Mantle Responses API validation errors.
_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES = frozenset(
{"function", "mcp", "custom", "namespace", "tool_search"}
)
class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
def __init__(
self,
aws_signer: Optional[BaseAWSLLM] = None,
@ -70,24 +59,6 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK_MANTLE
@staticmethod
def _resolve_region(params: dict) -> str:
region = params.get("aws_region_name")
if region:
BaseAWSLLM._validate_aws_region_name(region)
return region
base = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
if base:
match = _MANTLE_HOST_RE.match(base.rstrip("/"))
if match:
return match.group(1)
return (
get_secret_str("BEDROCK_MANTLE_REGION")
or get_secret_str("AWS_REGION_NAME")
or get_secret_str("AWS_REGION")
or BEDROCK_MANTLE_DEFAULT_REGION
)
def get_complete_url(
self,
api_base: Optional[str],
@ -107,7 +78,7 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
# For the standard Mantle host (including the default-region base that
# responses/main.py auto-injects into litellm_params.api_base), pin to the
# single resolved region so aws_region_name wins; preserve custom proxy hosts.
if _MANTLE_HOST_RE.match(base):
if MANTLE_HOST_RE.match(base):
base = f"https://bedrock-mantle.{region}.api.aws"
path = "/openai/v1/responses" if self.use_openai_path else "/v1/responses"
return f"{base}{path}"
@ -116,13 +87,9 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()
api_key = (
litellm_params.api_key
or get_secret_str("BEDROCK_MANTLE_API_KEY")
or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
)
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
bearer = self._resolve_bearer_token(litellm_params.api_key)
if bearer:
headers["Authorization"] = f"Bearer {bearer}"
if litellm_params.aws_bedrock_project_id:
headers["OpenAI-Project"] = litellm_params.aws_bedrock_project_id
return headers
@ -182,58 +149,3 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
params.pop("tools", None)
return params
def sign_request(
self,
headers: dict,
optional_params: dict,
request_data: dict,
api_base: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
stream: Optional[bool] = None,
fake_stream: Optional[bool] = None,
) -> Tuple[dict, Optional[bytes]]:
bearer = (
api_key
or get_secret_str("BEDROCK_MANTLE_API_KEY")
or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
)
if not bearer:
# SigV4 path. Pin the credential-scope region to the region of the actual
# signing URL (api_base, already region-resolved by get_complete_url) so the
# SigV4 scope and the URL host can never disagree. Resolve from api_base first,
# then fall back to the regular precedence. Also drop any caller Authorization
# so _sign_request's restore-original-Authorization step cannot override the
# SigV4 header.
optional_params = {
**optional_params,
"aws_region_name": self._resolve_region(
{**optional_params, "api_base": api_base}
),
}
headers = {k: v for k, v in headers.items() if k.lower() != "authorization"}
try:
return self._aws_signer._sign_request(
service_name="bedrock",
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
api_key=bearer,
model=model,
stream=stream,
fake_stream=fake_stream,
)
except (
NoCredentialsError,
PartialCredentialsError,
ProfileNotFound,
CredentialRetrievalError,
) as e:
raise ValueError(
"Bedrock Mantle auth failed: no Bearer token and no usable AWS "
"credentials. Set BEDROCK_MANTLE_API_KEY (or AWS_BEARER_TOKEN_BEDROCK) "
"or pass api_key for Bearer auth, or provide AWS credentials "
"(IAM role / access key / profile / web identity) for SigV4."
) from e

10
litellm/llms/custom_httpx/aiohttp_transport.py
View file

@ -116,6 +116,16 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
# For other exceptions, use the normal mapping
with map_aiohttp_exceptions():
raise
finally:
# Release the aiohttp connection when iteration ends for any
# reason (read timeout, cancellation from a client disconnect,
# GeneratorExit). Without this, abnormally terminated streams
# permanently hold a slot in the TCPConnector pool; once the
# pool is exhausted every request to that host times out (408)
# until the proxy is restarted, even after the backend recovers.
# On a fully-read response the connection was already released
# at EOF and close() is a no-op.
self._aiohttp_response.close()
async def aclose(self) -> None:
with map_aiohttp_exceptions():

2
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -5676,7 +5676,7 @@ class BaseLLMHTTPHandler:
)
raise
async def async_responses_websocket( # noqa: PLR0915
async def async_responses_websocket(
self,
model: str,
websocket: Any,

5
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -392,7 +392,10 @@ class FireworksAIConfig(OpenAIGPTConfig):
headers: dict,
) -> dict:
if not model.startswith("accounts/") and "#" not in model:
model = f"accounts/fireworks/models/{model}"
if model.endswith("-fast"):
model = f"accounts/fireworks/routers/{model}"
else:
model = f"accounts/fireworks/models/{model}"
messages = self._transform_messages_helper(
messages=messages, model=model, litellm_params=litellm_params
)

2
litellm/llms/gemini/realtime/transformation.py
View file

@ -1378,7 +1378,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
raise ValueError(f"Unknown openai event: {key}, value: {value}")
return openai_event
def transform_realtime_response( # noqa: PLR0915
def transform_realtime_response(
self,
message: Union[str, bytes],
model: str,

101
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -2,6 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
import json
from typing import (
Any,
Coroutine,
@ -22,7 +23,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import _parse_mime_type
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantToolCall,
ChatCompletionFileObject,
ChatCompletionToolCallFunctionChunk,
ChatCompletionVideoObject,
ChatCompletionVideoUrlObject,
)
@ -101,26 +104,18 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
) -> dict:
_tools = non_default_params.pop("tools", None)
if _tools is not None:
# remove 'additionalProperties' from tools
_tools = _remove_additional_properties(_tools)
# remove 'strict' from tools
_tools = _remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)
if _tools is not None:
non_default_params["tools"] = _tools
# Handle thinking parameter - convert Anthropic-style to OpenAI-style reasoning_effort
# vLLM is OpenAI-compatible, so it understands reasoning_effort, not thinking
# Reference: https://github.com/BerriAI/litellm/issues/19761
thinking = non_default_params.pop("thinking", None)
if thinking is not None and isinstance(thinking, dict):
if thinking.get("type") == "enabled":
# Only convert if reasoning_effort not already set
if "reasoning_effort" not in non_default_params:
budget_tokens = thinking.get("budget_tokens", 0)
# Map budget_tokens to reasoning_effort level
# Same logic as Anthropic adapter (translate_anthropic_thinking_to_reasoning_effort)
if budget_tokens >= 10000:
non_default_params["reasoning_effort"] = "high"
elif budget_tokens >= 5000:
@ -137,20 +132,13 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
api_base = api_base or get_secret_str("HOSTED_VLLM_API_BASE") # type: ignore
api_base = api_base or get_secret_str("HOSTED_VLLM_API_BASE")
dynamic_api_key = (
api_key or get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key"
) # vllm does not require an api key
)
return api_base, dynamic_api_key
def _is_video_file(self, content_item: ChatCompletionFileObject) -> bool:
"""
Check if the file is a video
- format: video/<extension>
- file_data: base64 encoded video data
- file_id: infer mp4 from extension
"""
file = content_item.get("file", {})
format = file.get("format")
file_data = file.get("file_data")
@ -205,29 +193,69 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
"""
Support translating:
- video files from file_id or file_data to video_url
- thinking_blocks on assistant messages to content blocks
- thinking_blocks on assistant messages are removed, and content lists
are converted to strings for vLLM compatibility
"""
for message in messages:
if message["role"] == "assistant":
thinking_blocks = message.pop("thinking_blocks", None) # type: ignore
if thinking_blocks:
new_content: list = [
(
{
"type": block["type"],
"thinking": block.get("thinking", ""),
message.pop("thinking_blocks", None)
existing_content = message.get("content")
if isinstance(existing_content, list):
text_parts = []
tool_calls: list[ChatCompletionAssistantToolCall] = []
content_blocks: list[object] = []
has_structured_content = False
for c in existing_content:
if isinstance(c, dict) and c.get("type") == "text":
text_parts.append(c.get("text", ""))
content_blocks.append(c)
elif isinstance(c, dict) and c.get("type") == "tool_use":
tool_input = c.get("input", {})
tool_calls.append(
ChatCompletionAssistantToolCall(
id=c.get("id"),
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=c.get("name"),
arguments=(
tool_input
if isinstance(
tool_input,
str,
)
else json.dumps(tool_input)
),
),
)
)
else:
content_blocks.append(c)
has_structured_content = True
if tool_calls:
existing_tool_calls = message.get("tool_calls")
if isinstance(existing_tool_calls, list):
existing_tool_call_ids = {
tool_call.get("id")
for tool_call in existing_tool_calls
if isinstance(tool_call, dict)
and tool_call.get("id") is not None
}
if block.get("type") == "thinking"
else {"type": block["type"], "data": block.get("data", "")}
)
for block in thinking_blocks
]
existing_content = message.get("content")
if isinstance(existing_content, str):
new_content.append({"type": "text", "text": existing_content})
elif isinstance(existing_content, list):
new_content.extend(existing_content)
message["content"] = new_content # type: ignore
new_tool_calls = [
tool_call
for tool_call in tool_calls
if tool_call.get("id") not in existing_tool_call_ids
]
if new_tool_calls:
message["tool_calls"] = (
existing_tool_calls + new_tool_calls
)
else:
message["tool_calls"] = tool_calls
content_str = "\n".join(text_parts)
new_content = (
content_blocks if has_structured_content else content_str
)
message["content"] = new_content # type: ignore[typeddict-item]
elif message["role"] == "user":
message_content = message.get("content")
if message_content and isinstance(message_content, list):
@ -243,6 +271,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
message_content[idx] = self._convert_file_to_video_url(
content_item
)
if is_async:
return super()._transform_messages(
messages, model, is_async=cast(Literal[True], True)

2
litellm/llms/huggingface/embedding/transformation.py
View file

@ -404,7 +404,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
)
return completion_response
def convert_to_model_response_object( # noqa: PLR0915
def convert_to_model_response_object(
self,
completion_response: Union[List[Dict[str, Any]], Dict[str, Any]],
model_response: ModelResponse,

65
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -18,6 +18,9 @@ from typing import (
overload,
)
import os
from urllib.parse import urlparse
import httpx
import litellm
@ -426,6 +429,32 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
)
return messages, tools
def _should_preserve_cache_control_for_endpoint(
self,
custom_llm_provider: str | None,
api_base: str | None,
) -> bool:
"""
The generic `openai` provider also reaches OpenAI-compatible endpoints
(a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom
api_base. Those can understand cache_control, so it must survive there.
Real OpenAI cannot, so it is still stripped for an openai.com host.
"""
if custom_llm_provider != "openai":
return False
resolved_api_base = (
api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
)
if not resolved_api_base:
return False
hostname = urlparse(resolved_api_base).hostname
if hostname is None:
return False
return hostname != "openai.com" and not hostname.endswith(".openai.com")
def transform_request(
self,
model: str,
@ -441,11 +470,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
dict: The transformed request. Sent as the body of the API call.
"""
messages = self._transform_messages(messages=messages, model=model)
messages, tools = self.remove_cache_control_flag_from_messages_and_tools(
model=model, messages=messages, tools=optional_params.get("tools", [])
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
if not self._should_preserve_cache_control_for_endpoint(
litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
):
messages, tools = self.remove_cache_control_flag_from_messages_and_tools(
model=model, messages=messages, tools=optional_params.get("tools", [])
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
optional_params.pop("max_retries", None)
@ -466,16 +498,19 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
transformed_messages = await self._transform_messages(
messages=messages, model=model, is_async=True
)
(
transformed_messages,
tools,
) = self.remove_cache_control_flag_from_messages_and_tools(
model=model,
messages=transformed_messages,
tools=optional_params.get("tools", []),
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
if not self._should_preserve_cache_control_for_endpoint(
litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
):
(
transformed_messages,
tools,
) = self.remove_cache_control_flag_from_messages_and_tools(
model=model,
messages=transformed_messages,
tools=optional_params.get("tools", []),
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
if self.__class__._is_base_class:
return {
"model": model,

2
litellm/llms/openai/openai.py
View file

@ -608,7 +608,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
return streaming_response
def completion( # type: ignore # noqa: PLR0915
def completion( # type: ignore
self,
model_response: ModelResponse,
timeout: Union[float, httpx.Timeout],

14
litellm/llms/openai/transcriptions/whisper_transformation.py
View file

@ -1,3 +1,4 @@
import json
from typing import List, Optional, Union
from httpx import Headers, Response
@ -107,9 +108,7 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
"""
data = {"model": model, "file": audio_file, **optional_params}
if "response_format" not in data or (
data["response_format"] == "text" or data["response_format"] == "json"
):
if "response_format" not in data:
data["response_format"] = (
"verbose_json" # ensures 'duration' is received - used for cost calculation
)
@ -133,10 +132,11 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
) -> TranscriptionResponse:
try:
raw_response_json = raw_response.json()
except Exception as e:
raise ValueError(
f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}"
)
except json.JSONDecodeError:
content_type = raw_response.headers.get("content-type", "").lower()
if "application/json" in content_type:
raise
return TranscriptionResponse(text=raw_response.text)
if any(
key in raw_response_json

6
litellm/llms/openrouter/chat/transformation.py
View file

@ -50,11 +50,15 @@ class OpenrouterConfig(OpenAIGPTConfig):
def map_openai_params(
self,
non_default_params: dict,
non_default_params: dict[str, object],
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
# OpenRouter expects "xhigh" instead of "max" for reasoning_effort.
if non_default_params.get("reasoning_effort") == "max":
non_default_params = {**non_default_params, "reasoning_effort": "xhigh"}
mapped_openai_params = super().map_openai_params(
non_default_params, optional_params, model, drop_params
)

17
litellm/llms/perplexity/cost_calculator.py
View file

@ -58,11 +58,8 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
## CALCULATE OUTPUT COST
output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token"))
completion_cost: float = (usage.completion_tokens or 0) * output_cost_per_token
## ADD REASONING TOKENS COST (if present)
reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0
# Also check completion_tokens_details if reasoning_tokens is not directly available
if (
reasoning_tokens == 0
and hasattr(usage, "completion_tokens_details")
@ -73,9 +70,19 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
)
reasoning_cost_value = model_info.get("output_cost_per_reasoning_token")
# `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage
# convention (codified for the central path in PR #18607). When a reasoning rate is
# configured we subtract before the output-rate multiplication so the reasoning
# tokens are not billed twice.
if reasoning_tokens > 0 and reasoning_cost_value is not None:
reasoning_cost_per_token = _safe_float_cast(reasoning_cost_value)
completion_cost += reasoning_tokens * reasoning_cost_per_token
non_reasoning_completion_tokens = max(
0, (usage.completion_tokens or 0) - reasoning_tokens
)
completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token
completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value)
else:
completion_cost = (usage.completion_tokens or 0) * output_cost_per_token
## ADD SEARCH QUERIES COST (if present)
num_search_queries = 0

2
litellm/llms/predibase/chat/transformation.py
View file

@ -129,7 +129,7 @@ class PredibaseConfig(BaseConfig):
optional_params["response_format"] = value
return optional_params
def transform_response( # noqa: PLR0915
def transform_response(
self,
model: str,
raw_response: Response,

4
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -678,7 +678,7 @@ def check_if_part_exists_in_parts(
return False
def _gemini_convert_messages_with_history( # noqa: PLR0915
def _gemini_convert_messages_with_history(
messages: List[AllMessageValues],
model: Optional[str] = None,
litellm_params: Optional[dict] = None,
@ -1176,7 +1176,7 @@ def _rewrite_google_maps_response_format(data: RequestBody) -> None:
_rewrite_mime_type_to_response_format(generation_config)
def _transform_request_body( # noqa: PLR0915
def _transform_request_body(
messages: List[AllMessageValues],
model: str,
optional_params: dict,

52
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -614,9 +614,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return googleSearch, googleSearchRetrieval, enterpriseWebSearch, urlContext
def _map_function( # noqa: PLR0915
self, value: List[dict], optional_params: dict
) -> List[Tools]:
def _map_function(self, value: List[dict], optional_params: dict) -> List[Tools]:
"""
Map OpenAI-style tools/functions to Vertex AI format.
@ -1173,7 +1171,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
optional_params["include_server_side_tool_invocations"] = True
return
def map_openai_params( # noqa: PLR0915
def map_openai_params(
self,
non_default_params: Dict,
optional_params: Dict,
@ -1904,7 +1902,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return False
@staticmethod
def _calculate_usage( # noqa: PLR0915
def _calculate_usage(
completion_response: Union[
GenerateContentResponseBody, BidiGenerateContentServerMessage
],
@ -2380,7 +2378,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return annotations
@staticmethod
def _process_candidates( # noqa: PLR0915
def _process_candidates(
_candidates: List[Candidates],
model_response: Union[ModelResponse, "ModelResponseStream"],
standard_optional_params: dict,
@ -2846,6 +2844,7 @@ async def make_call(
sync_stream=False,
logging_obj=logging_obj,
response_headers=response.headers,
response=response,
)
# LOGGING
logging_obj.post_call(
@ -2889,6 +2888,7 @@ def make_sync_call(
sync_stream=True,
logging_obj=logging_obj,
response_headers=response.headers,
response=response,
)
# LOGGING
@ -3350,12 +3350,14 @@ class ModelResponseIterator:
sync_stream: bool,
logging_obj: LoggingClass,
response_headers: Optional[Dict[str, str]] = None,
response: httpx.Response | None = None,
):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
check_is_function_call,
)
self.streaming_response = streaming_response
self.response = response
self.chunk_type: Literal["valid_json", "accumulated_json"] = "valid_json"
self.accumulated_json = ""
self.sent_first_chunk = False
@ -3657,3 +3659,41 @@ class ModelResponseIterator:
raise StopAsyncIteration
except ValueError as e:
raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}")
async def aclose(self) -> None:
iterator = getattr(
self,
"async_response_iterator",
self.streaming_response,
)
if iterator is not None and hasattr(iterator, "aclose"):
try:
await iterator.aclose()
except Exception as e: # noqa: BLE001
verbose_logger.debug(
"ModelResponseIterator.aclose: error closing iterator: %s", e
)
if self.response is not None:
try:
await self.response.aclose()
except Exception as e: # noqa: BLE001
verbose_logger.debug(
"ModelResponseIterator.aclose: error closing response: %s", e
)
def close(self) -> None:
iterator = getattr(self, "response_iterator", self.streaming_response)
if iterator is not None and hasattr(iterator, "close"):
try:
iterator.close()
except Exception as e: # noqa: BLE001
verbose_logger.debug(
"ModelResponseIterator.close: error closing iterator: %s", e
)
if self.response is not None:
try:
self.response.close()
except Exception as e: # noqa: BLE001
verbose_logger.debug(
"ModelResponseIterator.close: error closing response: %s", e
)

2
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py
View file

@ -124,7 +124,7 @@ class GoogleBatchEmbeddings(VertexLLM):
return resolved_files
def batch_embeddings( # noqa: PLR0915
def batch_embeddings(
self,
model: str,
input: GeminiEmbeddingInput,

4
litellm/llms/vertex_ai/vertex_ai_non_gemini.py
View file

@ -77,7 +77,7 @@ def _set_client_in_cache(client_cache_key: str, vertex_llm_model: Any):
)
def completion( # noqa: PLR0915
def completion(
model: str,
messages: list,
model_response: ModelResponse,
@ -485,7 +485,7 @@ def completion( # noqa: PLR0915
)
async def async_completion( # noqa: PLR0915
async def async_completion(
llm_model,
mode: str,
prompt: str,

29
litellm/main.py
View file

@ -1086,7 +1086,7 @@ def _build_custom_pricing_entry(
@tracer.wrap()
@client
def completion( # type: ignore # noqa: PLR0915
def completion( # type: ignore
model: str,
# Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create
messages: List = [],
@ -4881,7 +4881,7 @@ def embedding(
@client
def embedding( # noqa: PLR0915
def embedding(
model,
input=[],
# Optional params
@ -6128,7 +6128,7 @@ async def atext_completion(
@client
def text_completion( # noqa: PLR0915
def text_completion(
prompt: Union[
str, List[Union[str, List[Union[str, List[int]]]]]
], # Required: The prompt(s) to generate completions for.
@ -6667,7 +6667,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
@client
def transcription( # noqa: PLR0915
def transcription(
model: str,
file: FileTypes,
## OPTIONAL OPENAI PARAMS ##
@ -6974,7 +6974,7 @@ async def aspeech(*args, **kwargs) -> HttpxBinaryResponseContent:
@client
def speech( # noqa: PLR0915
def speech(
model: str,
input: str,
voice: Optional[Union[str, dict]] = None,
@ -7437,22 +7437,7 @@ def speech( # noqa: PLR0915
async def ahealth_check(
model_params: dict,
mode: Optional[
Literal[
"chat",
"completion",
"embedding",
"audio_speech",
"audio_transcription",
"image_generation",
"video_generation",
"batch",
"rerank",
"realtime",
"responses",
"ocr",
]
] = "chat",
mode: str | None = "chat",
prompt: Optional[str] = None,
input: Optional[List] = None,
):
@ -7665,7 +7650,7 @@ def stream_chunk_builder_text_completion(
return TextCompletionResponse(**response)
def stream_chunk_builder( # noqa: PLR0915
def stream_chunk_builder(
chunks: list,
messages: Optional[list] = None,
start_time=None,

649
litellm/model_prices_and_context_window_backup.json
View file

@ -2528,6 +2528,100 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"azure_ai/gpt-5.5": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.5",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.5",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
@ -10068,6 +10162,8 @@
},
"claude-sonnet-4-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
@ -10097,6 +10193,8 @@
},
"claude-sonnet-4-5-20250929": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
@ -10127,6 +10225,7 @@
},
"claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "anthropic",
@ -10155,6 +10254,8 @@
},
"claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
@ -10811,13 +10912,13 @@
"supports_tool_choice": true
},
"command-r7b-12-2024": {
"input_cost_per_token": 1.5e-07,
"input_cost_per_token": 3.75e-08,
"litellm_provider": "cohere_chat",
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 3.75e-08,
"output_cost_per_token": 1.5e-07,
"source": "https://docs.cohere.com/v2/docs/command-r7b",
"supports_function_calling": true,
"supports_tool_choice": true
@ -14511,6 +14612,38 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": {
"cache_read_input_token_cost": 2.8e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": {
"cache_read_input_token_cost": 1.45e-07,
"input_cost_per_token": 1.74e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.48e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/firefunction-v2": {
"input_cost_per_token": 9e-07,
"litellm_provider": "fireworks_ai",
@ -14586,43 +14719,64 @@
"input_cost_per_token": 1.4e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 202800,
"max_output_tokens": 202800,
"max_tokens": 202800,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://fireworks.ai/models/fireworks/glm-5p1",
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/glm-5p2": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/gpt-oss-120b": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://fireworks.ai/pricing",
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/gpt-oss-20b": {
"input_cost_per_token": 5e-08,
"cache_read_input_token_cost": 3.5e-08,
"input_cost_per_token": 7e-08,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 2e-07,
"source": "https://fireworks.ai/pricing",
"output_cost_per_token": 3e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": {
"input_cost_per_token": 6e-07,
@ -14678,6 +14832,38 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/kimi-k2p6": {
"cache_read_input_token_cost": 1.6e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/accounts/fireworks/models/kimi-k2p7-code": {
"cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": {
"input_cost_per_token": 3e-06,
"litellm_provider": "fireworks_ai",
@ -14795,6 +14981,38 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/minimax-m2p7": {
"cache_read_input_token_cost": 6e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 196608,
"max_output_tokens": 196608,
"max_tokens": 196608,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/minimax-m3": {
"cache_read_input_token_cost": 6e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 512000,
"max_output_tokens": 512000,
"max_tokens": 512000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
"litellm_provider": "fireworks_ai",
@ -14847,6 +15065,38 @@
"supports_response_schema": true,
"supports_tool_choice": false
},
"fireworks_ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 2.8e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/deepseek-v4-pro": {
"cache_read_input_token_cost": 1.45e-07,
"input_cost_per_token": 1.74e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.48e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/glm-4p7": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 6e-07,
@ -14867,15 +15117,80 @@
"input_cost_per_token": 1.4e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 202800,
"max_output_tokens": 202800,
"max_tokens": 202800,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://fireworks.ai/models/fireworks/glm-5p1",
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/glm-5p1-fast": {
"cache_read_input_token_cost": 5.2e-07,
"input_cost_per_token": 2.8e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 202800,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8.8e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/glm-5p2": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/gpt-oss-120b": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/gpt-oss-20b": {
"cache_read_input_token_cost": 3.5e-08,
"input_cost_per_token": 7e-08,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 3e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/kimi-k2p5": {
"cache_read_input_token_cost": 1e-07,
@ -14891,6 +15206,70 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/kimi-k2p6": {
"cache_read_input_token_cost": 1.6e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/kimi-k2p6-fast": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 8e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/kimi-k2p7-code": {
"cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/kimi-k2p7-code-fast": {
"cache_read_input_token_cost": 3.8e-07,
"input_cost_per_token": 1.9e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 8e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/minimax-m2p1": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
@ -14905,6 +15284,54 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/minimax-m2p7": {
"cache_read_input_token_cost": 6e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 196608,
"max_output_tokens": 196608,
"max_tokens": 196608,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/minimax-m3": {
"cache_read_input_token_cost": 6e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 512000,
"max_output_tokens": 512000,
"max_tokens": 512000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 4e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.6e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/nomic-ai/nomic-embed-text-v1": {
"input_cost_per_token": 8e-09,
"litellm_provider": "fireworks_ai-embedding-models",
@ -25103,6 +25530,21 @@
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-medium-3-5": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-small": {
"input_cost_per_token": 1e-07,
"litellm_provider": "mistral",
@ -39351,6 +39793,22 @@
"litellm_provider": "fireworks_ai",
"mode": "chat"
},
"fireworks_ai/accounts/fireworks/models/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 4e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.6e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/accounts/fireworks/models/qwq-32b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -39513,6 +39971,54 @@
"litellm_provider": "fireworks_ai",
"mode": "chat"
},
"fireworks_ai/accounts/fireworks/routers/glm-5p1-fast": {
"cache_read_input_token_cost": 5.2e-07,
"input_cost_per_token": 2.8e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 202800,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8.8e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 8e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast": {
"cache_read_input_token_cost": 3.8e-07,
"input_cost_per_token": 1.9e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 8e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"novita/deepseek/deepseek-v3.2": {
"litellm_provider": "novita",
"mode": "chat",
@ -42456,4 +42962,105 @@
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
}
}
,
"deepseek-v4-flash": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 2.8e-09,
"input_cost_per_token": 1.4e-07,
"input_cost_per_token_cache_hit": 2.8e-09,
"litellm_provider": "deepseek",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"source": "https://api-docs.deepseek.com/quick_start/pricing",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
},
"deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 3.625e-09,
"input_cost_per_token": 4.35e-07,
"input_cost_per_token_cache_hit": 3.625e-09,
"litellm_provider": "deepseek",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 8.7e-07,
"source": "https://api-docs.deepseek.com/quick_start/pricing",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
},
"deepseek/deepseek-v4-flash": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 2.8e-09,
"input_cost_per_token": 1.4e-07,
"input_cost_per_token_cache_hit": 2.8e-09,
"litellm_provider": "deepseek",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"source": "https://api-docs.deepseek.com/quick_start/pricing",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
},
"deepseek/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 3.625e-09,
"input_cost_per_token": 4.35e-07,
"input_cost_per_token_cache_hit": 3.625e-09,
"litellm_provider": "deepseek",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 8.7e-07,
"source": "https://api-docs.deepseek.com/quick_start/pricing",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
}
}

22
litellm/mypy.ini
View file

@ -1,22 +0,0 @@
[mypy]
warn_return_any = True
ignore_missing_imports = True
disallow_untyped_defs = True
mypy_path = litellm/stubs
namespace_packages = True
disable_error_code =
annotation-unchecked,
import-untyped
[mypy-litellm.*]
ignore_missing_imports = False
[mypy-google.*]
ignore_missing_imports = True
[mypy-cryptography.hazmat.bindings._rust.x509]
ignore_errors = True
[mypy-fastuuid.*]
ignore_missing_imports = True
ignore_errors = True

2
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -125,7 +125,7 @@ class MCPRequestHandler:
LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME = SpecialHeaders.mcp_access_groups.value
@staticmethod
async def process_mcp_request( # noqa: PLR0915
async def process_mcp_request(
scope: Scope,
) -> Tuple[
UserAPIKeyAuth,

9
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -1421,8 +1421,11 @@ class MCPServerManager:
"No allowed MCP Servers found for user api key auth."
)
return list(combined_servers)
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers: {str(e)}.")
except Exception: # noqa: BLE001
verbose_logger.exception(
"Failed to get allowed MCP servers; team-level object_permission "
"grants may be dropped. Falling back to global servers only."
)
return allow_all_server_ids
async def resolve_toolset_tool_permissions(
@ -3355,7 +3358,7 @@ class MCPServerManager:
)
)
async def _call_regular_mcp_tool( # noqa: PLR0915
async def _call_regular_mcp_tool(
self,
mcp_server: MCPServer,
original_tool_name: str,

2
litellm/proxy/_experimental/mcp_server/sampling_handler.py
View file

@ -661,7 +661,7 @@ def _convert_openai_response_to_mcp_result(
)
async def _check_model_access( # noqa: PLR0915
async def _check_model_access(
model: str, user_api_key_auth: Any
) -> Optional["ErrorData"]:
"""Enforce model-permission checks for MCP sampling requests.

28
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -617,7 +617,7 @@ if MCP_AVAILABLE:
active_mcp_session_var.reset(_session_reset_token)
@server.call_tool()
async def mcp_server_tool_call( # noqa: PLR0915
async def mcp_server_tool_call(
name: str, arguments: Dict[str, Any] | None
) -> CallToolResult:
"""
@ -1036,7 +1036,14 @@ if MCP_AVAILABLE:
allowed_mcp_servers: List[MCPServer],
) -> List[MCPServer]:
"""
Get the filtered MCP servers from the MCP server names
Get the filtered MCP servers from the MCP server names.
Fails closed when ``mcp_servers`` is explicitly provided (path- or
header-derived) but none of the names resolve to a server alias or
access group the caller can access. The previous behavior returned
the full ``allowed_mcp_servers`` set, which silently widened scope
when a client targeted ``/mcp/<unknown>/`` and made URL/header
namespacing appear to work when it did not.
"""
filtered_server: dict[str, MCPServer] = {}
@ -1076,6 +1083,17 @@ if MCP_AVAILABLE:
if filtered_server:
return list(filtered_server.values())
if mcp_servers is not None:
# Caller asked for a specific scope but nothing resolved. Fail
# closed so URL/header namespacing cannot silently fall back to
# the caller's full allowed-server set.
verbose_logger.debug(
"MCP scope filter resolved to no servers for requested names %s; "
"returning empty list (fail-closed).",
mcp_servers,
)
return []
return allowed_mcp_servers
def _tool_name_matches(tool_name: str, filter_list: List[str]) -> bool:
@ -1591,7 +1609,7 @@ if MCP_AVAILABLE:
_mcp_gateway_initialize_instructions.reset(instructions_token)
_mcp_gateway_server_name.reset(server_name_token)
async def _get_tools_from_mcp_servers( # noqa: PLR0915
async def _get_tools_from_mcp_servers(
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
@ -2435,7 +2453,7 @@ if MCP_AVAILABLE:
},
)
async def execute_mcp_tool( # noqa: PLR0915
async def execute_mcp_tool(
name: str,
arguments: Dict[str, Any],
allowed_mcp_servers: List[MCPServer],
@ -3642,7 +3660,7 @@ if MCP_AVAILABLE:
detail="Forbidden",
)
async def handle_streamable_http_mcp( # noqa: PLR0915
async def handle_streamable_http_mcp(
scope: Scope, receive: Receive, send: Send
) -> None:
"""Handle MCP requests through StreamableHTTP."""

15
litellm/proxy/_experimental/mcp_server/utils.py
View file

@ -23,9 +23,20 @@ import os
from urllib.parse import quote
# Constants
LITELLM_MCP_SERVER_NAME = "litellm-mcp-server"
#
# NOTE: The environment-backed values below are read once, when this module is
# first imported, and cached for the lifetime of the process. Changing the
# corresponding environment variables after import has no effect unless the
# module is reloaded (e.g. ``importlib.reload``). Tests that override these
# variables must reload this module — see
# ``tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py``.
LITELLM_MCP_SERVER_NAME = os.environ.get(
"LITELLM_MCP_SERVER_NAME", "litellm-mcp-server"
)
LITELLM_MCP_SERVER_VERSION = "1.0.0"
LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM"
LITELLM_MCP_SERVER_DESCRIPTION = os.environ.get(
"LITELLM_MCP_SERVER_DESCRIPTION", "MCP Server for LiteLLM"
)
MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-")
MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}"

14
litellm/proxy/_types.py
View file

@ -361,6 +361,16 @@ class LiteLLMRoutes(enum.Enum):
"/realtime?{model}",
"/v1/realtime?{model}",
"/openai/v1/realtime?{model}",
# realtime (GA WebRTC HTTP routes)
"/realtime/client_secrets",
"/v1/realtime/client_secrets",
"/openai/v1/realtime/client_secrets",
"/realtime/calls",
"/v1/realtime/calls",
"/openai/v1/realtime/calls",
"/realtime/transcription_sessions",
"/v1/realtime/transcription_sessions",
"/openai/v1/realtime/transcription_sessions",
# responses API
"/responses",
"/v1/responses",
@ -2153,6 +2163,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
master_key: Optional[str] = Field(
None, description="require a key for all calls to proxy"
)
allow_cli_sso_verification_uri_complete: bool | None = Field(
None,
description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine",
)
database_url: Optional[str] = Field(
None,
description="connect to a postgres db - needed for generating temporary keys + tracking spend / key",

2
litellm/proxy/agent_endpoints/a2a_endpoints.py
View file

@ -509,7 +509,7 @@ async def get_agent_card(
tags=["[beta] A2A Agents"],
dependencies=[Depends(user_api_key_auth)],
)
async def invoke_agent_a2a( # noqa: PLR0915
async def invoke_agent_a2a(
agent_id: str,
request: Request,
fastapi_response: Response,

47
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -5,6 +5,7 @@ Unified /v1/messages endpoint - (Anthropic Spec)
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi.responses import JSONResponse
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping
from litellm.integrations.custom_guardrail import ModifyResponseException
@ -23,6 +24,40 @@ from litellm.types.utils import TokenCountResponse
router = APIRouter()
def _strip_total_tokens_from_anthropic_response(response: Any) -> None:
"""Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM
injects into Anthropic /v1/messages responses.
The Anthropic /v1/messages spec only defines:
input_tokens, output_tokens, cache_creation_input_tokens,
cache_read_input_tokens, cache_creation.{ephemeral_5m,ephemeral_1h}
The streaming SSE path (message_delta.usage) already does not include
total_tokens; this brings the non-streaming path into the same shape.
Handles both shapes returned by `base_process_llm_request`:
- plain `dict` (most common — `AnthropicMessagesResponse` is a TypedDict
and is `dict` at runtime)
- Pydantic model whose `usage` attribute is dict-shaped (e.g. a
BaseModel that holds raw Anthropic usage as a `dict[str, int]`)
Streaming results (StreamingResponse, AsyncIterator, etc.) and Pydantic
models with strongly-typed Usage sub-models are left untouched —
those paths either have separate serialization handling or impose
type constraints the helper does not try to subvert.
"""
if response is None:
return
if isinstance(response, dict):
usage = response.get("usage")
if isinstance(usage, dict) and "total_tokens" in usage:
usage.pop("total_tokens", None)
return
# Pydantic-model fallback: only mutate if `usage` is a dict.
usage = getattr(response, "usage", None)
if isinstance(usage, dict) and "total_tokens" in usage:
usage.pop("total_tokens", None)
@router.post(
"/v1/messages",
tags=["[beta] Anthropic `/v1/messages`"],
@ -72,6 +107,18 @@ async def anthropic_response(
user_api_base=user_api_base,
version=version,
)
# Optionally strip the non-Anthropic `usage.total_tokens` field
# LiteLLM adds internally. Anthropic's official /v1/messages spec
# only defines input_tokens / output_tokens / cache_*_input_tokens;
# total_tokens is an OpenAI convention. Default off
# (`litellm.strip_anthropic_total_tokens = False`) to preserve
# backward compatibility for clients that currently read it; set
# to True to align the wire response with the spec (and with the
# streaming SSE path, which already omits total_tokens).
# spend_logs / Prometheus still compute total internally — this
# only affects the wire response.
if litellm.strip_anthropic_total_tokens:
_strip_total_tokens_from_anthropic_response(result)
return result
except ModifyResponseException as e:
# Guardrail flagged content in passthrough mode - return 200 with violation message

2
litellm/proxy/auth/auth_checks.py
View file

@ -519,7 +519,7 @@ MODEL_DISCOVERY_ROUTES = frozenset(
)
async def common_checks( # noqa: PLR0915
async def common_checks(
request_body: dict,
team_object: Optional[LiteLLM_TeamTable],
user_object: Optional[LiteLLM_UserTable],

54
litellm/proxy/auth/auth_utils.py
View file

@ -2,6 +2,7 @@ import os
import re
import sys
from functools import lru_cache
from logging import Logger
from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Tuple, Union
from fastapi import HTTPException, Request, status
@ -995,6 +996,45 @@ def get_project_model_tpm_limit(
return None
def custom_auth_common_checks_warning(
*,
custom_auth_configured: bool,
run_common_checks: bool,
) -> str | None:
if not custom_auth_configured or run_common_checks:
return None
return (
"custom_auth is configured but 'custom_auth_run_common_checks' is not set. "
"Problem: budgets, model-access allowlists, and per-model rate limits configured "
"on your DB team/project records will NOT be enforced for custom-auth requests "
"(rate limits set directly on the returned UserAPIKeyAuth still apply). "
"Fix: set 'general_settings.custom_auth_run_common_checks: true'. "
"Docs: https://docs.litellm.ai/docs/proxy/custom_auth"
)
_custom_auth_common_checks_warning_emitted = False
def warn_once_if_custom_auth_skips_common_checks(
*,
custom_auth_configured: bool,
run_common_checks: bool,
logger: Logger = verbose_proxy_logger,
) -> None:
global _custom_auth_common_checks_warning_emitted
if _custom_auth_common_checks_warning_emitted:
return
message = custom_auth_common_checks_warning(
custom_auth_configured=custom_auth_configured,
run_common_checks=run_common_checks,
)
if message is None:
return
logger.warning(message)
_custom_auth_common_checks_warning_emitted = True
def is_pass_through_provider_route(route: str) -> bool:
PROVIDER_SPECIFIC_PASS_THROUGH_ROUTES = [
"vertex-ai",
@ -1227,6 +1267,14 @@ _MODEL_ROUTING_BODY_TARGET_MODEL_ROUTE_MARKERS = (
"/vector_stores",
)
_MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS = ("/evals",)
# Realtime WebRTC routes carry the effective model inside the nested
# ``session.model`` field (see realtime_endpoints.endpoints), so the model the
# request will actually use is not present at the top level. Extract it here so
# can_key_call_model() validates the real target model.
_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS = (
"/realtime/client_secrets",
"/realtime/calls",
)
_MODEL_ROUTING_ID_FIELDS = (
"file_id",
"input_file_id",
@ -1409,6 +1457,12 @@ def _extract_model_candidates_from_request(
_append_model_candidates(candidates, body_model)
if uses_body_target_model_sources or not body_model:
_append_model_candidates(candidates, request_data.get("target_model_names"))
if _route_matches_any_marker(
route=route, markers=_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS
):
session = request_data.get("session")
if isinstance(session, dict):
_append_model_candidates(candidates, session.get("model"))
if uses_completion_model_sources and isinstance(
request_data.get("completion"), dict
):

2
litellm/proxy/auth/handle_jwt.py
View file

@ -1954,7 +1954,7 @@ class JWTAuthManager:
return None, None, None
@staticmethod
async def auth_builder( # noqa: PLR0915
async def auth_builder(
api_key: str,
jwt_handler: JWTHandler,
request_data: dict,

Some files were not shown because too many files have changed in this diff Show more