diff --git a/.circleci/config.yml b/.circleci/config.yml
index dbeb412506f..abcdbf45187 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -133,6 +133,26 @@ commands:
done
echo "record/replay proxy did not become ready" >&2
exit 1
+ start_fake_openai_endpoint:
+ description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced."
+ steps:
+ - run:
+ name: Start fake OpenAI endpoint
+ background: true
+ command: |
+ uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190
+ - run:
+ name: Wait for fake OpenAI endpoint
+ command: |
+ for i in $(seq 1 30); do
+ if curl -sf http://localhost:8190/health >/dev/null 2>&1; then
+ echo "fake OpenAI endpoint is up"
+ exit 0
+ fi
+ sleep 1
+ done
+ echo "fake OpenAI endpoint did not become ready" >&2
+ exit 1
setup_litellm_enterprise_pip:
steps:
- run:
@@ -168,6 +188,8 @@ jobs:
name: win/default
shell: powershell.exe
working_directory: ~/project
+ environment:
+ UV_PYTHON: "3.11"
steps:
- checkout
- run:
@@ -200,7 +222,7 @@ jobs:
if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) {
Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`""
}
- uv sync --frozen --group dev --python (Get-Command python).Source
+ uv sync --frozen --group dev --python 3.11
- run:
name: Run Windows-specific test
command: |
@@ -594,6 +616,8 @@ jobs:
working_directory: ~/project
resource_class: large
parallelism: 4
+ environment:
+ FAKE_OPENAI_API_BASE: http://127.0.0.1:8190
steps:
- checkout
- setup_google_dns
@@ -609,6 +633,7 @@ jobs:
paths:
- ~/.cache/uv
key: v1-uv-cache-{{ checksum "uv.lock" }}
+ - start_fake_openai_endpoint
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -1549,6 +1574,7 @@ jobs:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
+ - start_fake_openai_endpoint
- start_postgres:
db_name: litellm_test
- attach_workspace:
@@ -1586,6 +1612,7 @@ jobs:
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DISABLE_SCHEMA_UPDATE="True" \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
--name my-app \
--add-host=host.docker.internal:host-gateway \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
@@ -1648,6 +1675,7 @@ jobs:
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker tag litellm-docker-database:ci my-app:latest
- start_openai_record_replay_proxy
+ - start_fake_openai_endpoint
- run:
name: Run Docker container
command: |
@@ -1655,6 +1683,7 @@ jobs:
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e USE_PRISMA_MIGRATE=True \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
@@ -1817,6 +1846,7 @@ jobs:
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- start_openai_record_replay_proxy
+ - start_fake_openai_endpoint
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@@ -1830,6 +1860,7 @@ jobs:
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
@@ -1889,6 +1920,7 @@ jobs:
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE="bad-license" \
--add-host host.docker.internal:host-gateway \
--name my-app-3 \
@@ -1938,6 +1970,7 @@ jobs:
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
- start_redis
+ - start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
@@ -1961,6 +1994,7 @@ jobs:
-e REDIS_PORT=6379 \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
@@ -2020,6 +2054,7 @@ jobs:
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
+ - start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
@@ -2039,6 +2074,7 @@ jobs:
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
@@ -2060,6 +2096,7 @@ jobs:
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
@@ -2112,6 +2149,7 @@ jobs:
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
+ - start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
@@ -2129,6 +2167,7 @@ jobs:
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e STORE_MODEL_IN_DB="True" \
-e LITELLM_MASTER_KEY="sk-1234" \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--add-host host.docker.internal:host-gateway \
--name my-app \
@@ -2187,6 +2226,7 @@ jobs:
command: |
docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
- start_postgres
+ - start_fake_openai_endpoint
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@@ -2200,6 +2240,7 @@ jobs:
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
+ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
diff --git a/.github/deploy-on-aws.png b/.github/deploy-on-aws.png
new file mode 100644
index 00000000000..06d41f2a5e0
Binary files /dev/null and b/.github/deploy-on-aws.png differ
diff --git a/.github/deploy-on-gcp.png b/.github/deploy-on-gcp.png
new file mode 100644
index 00000000000..e831a8c2e4e
Binary files /dev/null and b/.github/deploy-on-gcp.png differ
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 99f79c0b272..9658baeb89a 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -4,7 +4,7 @@
## Linear ticket
-
+
## Pre-Submission checklist
diff --git a/.github/scripts/_agent_shin_actions.py b/.github/scripts/_agent_shin_actions.py
new file mode 100644
index 00000000000..b3d1ff055b3
--- /dev/null
+++ b/.github/scripts/_agent_shin_actions.py
@@ -0,0 +1,50 @@
+"""Dry-run wrapper(s) around Agent Shin GitHub mutations.
+
+The rollout scripts currently need only one mutation wrapped, so this module
+exposes a single ``maybe_post_comment`` helper. It takes a ``dry_run: bool``
+keyword argument and the body is intentionally trivial:
+
+ if dry_run:
+ print(...) # log what we would do, return
+ return
+ real_mutation(...) # otherwise, actually do it
+
+That shape means a dry-run preview differs from the real run in exactly one
+line per side effect: the call site. So when you `python3 script.py` locally
+without ``--close``, you can be confident the actions printed are the ones the
+GitHub Action would have performed (modulo ordering on retry/error paths,
+which are deliberately simple). Any further mutation a rollout script needs
+should get the same ``maybe_*`` treatment instead of calling the raw
+``triage_with_llm`` mutation directly.
+
+Importing from this module pulls in the real mutation from ``triage_with_llm``
+— call sites in the rollout scripts should NEVER import ``post_comment``
+directly; that would skip the dry-run gate and is the bug class this module
+exists to prevent.
+"""
+
+from __future__ import annotations
+
+import sys
+import textwrap
+
+# Import the module itself rather than the bare names so monkeypatching
+# `triage_with_llm.post_comment` (or any of the other mutations) in tests is
+# reflected here — `from triage_with_llm import post_comment` would bind the
+# original function to a local name and bypass the patch, defeating the whole
+# point of these wrappers.
+import triage_with_llm
+
+
+def _log(line: str) -> None:
+ """Print a single dry-run line to stdout (one log statement per side effect)."""
+ print(line, file=sys.stdout, flush=True)
+
+
+def maybe_post_comment(repo: str, number: int, body: str, *, dry_run: bool) -> None:
+ """Post a comment on ``repo#number`` — or, in dry-run, log what we would post."""
+ if dry_run:
+ _log(f"[DRY RUN] comment {repo}#{number}:")
+ _log(textwrap.indent(body, " "))
+ return
+ triage_with_llm.post_comment(repo, number, body)
diff --git a/.github/scripts/agent_shin_shared.py b/.github/scripts/agent_shin_shared.py
new file mode 100644
index 00000000000..8f3dc3c2322
--- /dev/null
+++ b/.github/scripts/agent_shin_shared.py
@@ -0,0 +1,211 @@
+"""Constants and helpers shared by Agent Shin's triage scripts.
+
+Both `triage_with_llm.py` (the LLM-judge entrypoint) and
+`close_low_quality_prs.py` (the daily Greptile-score sweep) need to
+agree on the same notions of:
+
+ * What counts as a Greptile-authored review comment
+ (``GREPTILE_BOT_LOGINS``) and how to extract a confidence score from
+ its body (``SCORE_PATTERN`` / :func:`extract_greptile_score`).
+ * How long the 2-hour grace window is (``GRACE_PERIOD_SECONDS``) and
+ the HTML marker stamped into a grace-warning comment so the *other*
+ script can see "Agent Shin already warned" and behave accordingly
+ (``GRACE_COMMENT_MARKER``).
+ * Who Agent Shin is on GitHub (``AGENT_SHIN_DEFAULT_BOT_LOGIN``).
+ * How GitHub-style ISO-8601 timestamps round-trip into timezone-aware
+ :class:`datetime.datetime` (:func:`parse_iso8601`).
+
+Keeping these in one module means a future change (new Greptile output
+format, a longer grace window, a new allowlisted account) is a single edit
+instead of two — the original split version had to call out in comments
+that the two copies "must stay in sync" precisely because nothing
+enforced it.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import os
+import re
+import subprocess
+from typing import Iterable
+
+GREPTILE_BOT_LOGINS = frozenset({"greptile-apps", "greptile-apps[bot]"})
+
+SCORE_PATTERN = re.compile(
+ r"confidence\s*score\s*[:\-]?\s*(\d+)\s*/\s*5",
+ re.IGNORECASE,
+)
+
+GRACE_COMMENT_MARKER = ""
+
+# Hidden HTML marker stamped on every Agent Shin auto-close comment (the LLM
+# judge's grace/review-gate close and the daily Greptile sweep's close).
+# `was_closed_by_agent_shin` requires this marker — not just the closing actor —
+# before `@agent-shin reconsider` may reopen, because the `github-actions[bot]`
+# identity is shared with every other workflow in the repo and is not unique to
+# Agent Shin. Both close paths must stamp it or the reconsider path silently
+# rejects the contributor.
+AGENT_SHIN_CLOSE_MARKER = ""
+
+# 2 hours between the grace warning and the auto-close. Short enough to
+# dogfood the "fix it before it closes" loop in one sitting; bump back up
+# (e.g. 86400 for a day) for the public rollout.
+GRACE_PERIOD_SECONDS = 7200
+
+AGENT_SHIN_DEFAULT_BOT_LOGIN = "github-actions[bot]"
+
+
+def _logins(*names: str) -> frozenset[str]:
+ """Build a login set normalized for case-insensitive membership checks.
+
+ Callers compare via ``login.lower() in ``, so the stored values
+ must be lowercase. Normalizing here lets the literals keep each
+ account's canonical GitHub casing (e.g. ``SwiftWinds``) for
+ readability without breaking the lookup.
+ """
+ return frozenset(name.lower() for name in names)
+
+
+# Dogfood rollout gate. While this set is non-empty, Agent Shin acts ONLY on
+# PRs/issues authored by these logins and skips everyone else. For an
+# allowlisted author the usual internal/external classification is bypassed, so
+# an internal account (e.g. a maintainer's own work login) still gets triaged
+# while the bot is being tested on a small set of accounts. Empty the set to
+# lift the restriction and restore full triage for the public rollout. Logins
+# are compared case-insensitively.
+ALLOWLIST_LOGINS = _logins("mateo-berri", "SwiftWinds")
+
+# `gh {pr,issue} list` has no "fetch everything" flag — `--limit` is the only
+# control and it defaults to 30. Pass a ceiling far above any realistic open
+# backlog (low thousands today) so gh paginates the API until the queue is
+# exhausted rather than silently truncating. The bulk sweeps MUST see the whole
+# backlog: gh lists newest-first, so a low cap drops the *oldest* PRs/issues —
+# exactly the stale ones a low-quality sweep is meant to catch.
+GH_LIST_ALL_LIMIT = 100_000
+
+
+def extract_greptile_score(comments: Iterable[dict]) -> tuple[int, dict] | None:
+ """Return (score, comment) for the most recent Greptile-authored comment
+ that contains a "Confidence Score: X/5". Returns None if no such comment.
+
+ "Most recent" is determined by the comment's `updated_at` (falling back to
+ `created_at`), so re-reviews override earlier passes.
+ """
+ candidates: list[tuple[str, int, dict]] = []
+ for comment in comments:
+ user = (comment.get("user") or {}).get("login", "")
+ if user not in GREPTILE_BOT_LOGINS:
+ continue
+ body = comment.get("body") or ""
+ match = SCORE_PATTERN.search(body)
+ if not match:
+ continue
+ score = int(match.group(1))
+ timestamp = comment.get("updated_at") or comment.get("created_at") or ""
+ candidates.append((timestamp, score, comment))
+
+ if not candidates:
+ return None
+
+ candidates.sort(key=lambda triple: triple[0])
+ _, score, comment = candidates[-1]
+ return score, comment
+
+
+def parse_iso8601(value: str) -> dt.datetime:
+ """Parse a GitHub ISO-8601 timestamp into a timezone-aware datetime."""
+ return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+
+
+def gh(*args: str) -> str:
+ """Run a `gh` CLI command and return stdout. Raises on non-zero exit.
+
+ Shared by both Agent Shin entrypoints so a future change here
+ (timeout handling, logging, retry on transient failures) only needs
+ to be made once.
+ """
+ result = subprocess.run(
+ ["gh", *args],
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ return result.stdout
+
+
+def list_open_items(kind: str, *, repo: str | None, fields: str) -> list[dict]:
+ """Return EVERY open PR (``kind="pr"``) or issue (``kind="issue"``) in ``repo``.
+
+ Wraps ``gh {pr,issue} list`` with ``--limit GH_LIST_ALL_LIMIT`` so the full
+ backlog is fetched instead of the default 30 (or any other arbitrary cap).
+ Both bulk sweeps — the daily Greptile closer and the one-shot rollout
+ heads-up — rely on this seeing the whole queue, including the oldest items.
+
+ ``fields`` is the comma-separated ``--json`` field list the caller needs
+ (e.g. ``"number"`` for the rollout, the full set for the closer).
+ """
+ if kind not in ("pr", "issue"):
+ raise ValueError(f"kind must be 'pr' or 'issue', got {kind!r}")
+ repo_args = ["--repo", repo] if repo else []
+ raw = gh(
+ kind,
+ "list",
+ "--state",
+ "open",
+ "--limit",
+ str(GH_LIST_ALL_LIMIT),
+ "--json",
+ fields,
+ *repo_args,
+ )
+ return json.loads(raw)
+
+
+def seconds_since_latest_marker_comment(
+ comments: Iterable[dict],
+ *,
+ marker: str,
+ bot_login: str | None = None,
+ now: dt.datetime | None = None,
+) -> float | None:
+ """Return seconds since the bot's most recent comment containing ``marker``.
+
+ Filters comments by author so a contributor who quotes the HTML
+ marker (e.g. via GitHub's "Quote reply" feature, which preserves
+ HTML comments in the raw markdown of the quoted text) is not
+ mistaken for a bot warning — that would silently reset cooldown
+ timers and suppress legitimate notifications.
+
+ ``bot_login`` defaults to the `AGENT_SHIN_BOT_LOGIN` env override or
+ ``AGENT_SHIN_DEFAULT_BOT_LOGIN`` so callers normally don't need to
+ pass it. ``now`` is injectable for tests / callers (like the daily
+ sweep) that want every age calculation pinned to one snapshot.
+ """
+ expected_login = (
+ bot_login
+ or os.environ.get("AGENT_SHIN_BOT_LOGIN")
+ or AGENT_SHIN_DEFAULT_BOT_LOGIN
+ ).lower()
+ latest: dt.datetime | None = None
+ for comment in comments:
+ author = ((comment.get("user") or {}).get("login") or "").lower()
+ if author != expected_login:
+ continue
+ body = comment.get("body") or ""
+ if marker not in body:
+ continue
+ created = comment.get("created_at")
+ if not created:
+ continue
+ try:
+ ts = parse_iso8601(created)
+ except ValueError:
+ continue
+ if latest is None or ts > latest:
+ latest = ts
+ if latest is None:
+ return None
+ reference = now if now is not None else dt.datetime.now(dt.timezone.utc)
+ return (reference - latest).total_seconds()
diff --git a/.github/scripts/close_low_quality_prs.py b/.github/scripts/close_low_quality_prs.py
new file mode 100644
index 00000000000..7b9bbb579e3
--- /dev/null
+++ b/.github/scripts/close_low_quality_prs.py
@@ -0,0 +1,573 @@
+#!/usr/bin/env python3
+"""
+Auto-close low-quality pull requests.
+
+Closes open PRs (including drafts, regardless of age) that satisfy ALL of:
+ 1. Have a Greptile (`greptile-apps`) review comment whose latest
+ "Confidence Score: X/5" is below the configured threshold (default: 4).
+ 2. Are authored by an external OSS contributor (internal BerriAI
+ contributors are exempt).
+ 3. Do not carry an opt-out label (default: "do not close").
+
+`--min-age-days` is retained as an opt-in safety net for one-off backfill
+runs (default: 0). The team's intent is that the count of open PRs equals
+the count of PRs internal collaborators need to action on, so neither age
+nor draft status acts as a free pass.
+
+For each match, the script posts an explanatory comment and closes the PR.
+Because OSS contributors *cannot* reopen a PR closed by the bot/maintainer
+(GitHub limitation), the close-comment instructs them to push their fixes
+and **open a fresh PR**, or to comment `@agent-shin reconsider` on the
+closed PR to have the LLM judge re-evaluate (and reopen on pass).
+
+Requires the `gh` CLI to be authenticated.
+
+Usage examples:
+ # Dry run (default) - prints what would be closed
+ python3 close_low_quality_prs.py
+
+ # Actually close matching PRs
+ python3 close_low_quality_prs.py --close
+
+ # Restrict to PRs at least N days old (one-off backfill safety net)
+ python3 close_low_quality_prs.py --min-age-days 7 --min-score 4 --close
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import subprocess
+import sys
+from typing import Iterable
+
+# Add this script's directory to `sys.path` so the sibling
+# `agent_shin_shared` module is importable when the script is invoked
+# directly (e.g. `python3 .github/scripts/close_low_quality_prs.py ...`).
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from agent_shin_shared import ( # noqa: E402 -- sys.path adjusted above
+ AGENT_SHIN_CLOSE_MARKER,
+ ALLOWLIST_LOGINS,
+ GRACE_COMMENT_MARKER,
+ GRACE_PERIOD_SECONDS,
+ GREPTILE_BOT_LOGINS,
+ SCORE_PATTERN,
+ extract_greptile_score,
+ gh,
+ list_open_items,
+ parse_iso8601,
+ seconds_since_latest_marker_comment,
+)
+
+# `GREPTILE_BOT_LOGINS` and `SCORE_PATTERN` (Greptile's GitHub App login
+# variants and the "Confidence Score: X/5" regex) are imported from
+# `agent_shin_shared` so the LLM judge in `triage_with_llm.py` and this
+# daily Greptile sweep read the score through the same set of logins
+# and the same regex.
+
+# `author_association` values for internal BerriAI contributors who should be
+# exempt from auto-triage.
+INTERNAL_AUTHOR_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"})
+
+# Default labels that exempt a PR from auto-close. Defined at module scope (not
+# as a mutable argparse default) so that `--optout-label foo` REPLACES the
+# defaults instead of appending to them — the argparse `action="append"` +
+# `default=[...]` combination silently mutates the shared default list.
+DEFAULT_OPTOUT_LABELS = ("do not close", "keep open", "wip")
+
+# `GRACE_COMMENT_MARKER` (HTML marker appended to grace-period warning
+# comments — used by either script to recognize that a warning was
+# already posted) and `GRACE_PERIOD_SECONDS` (length of the grace
+# period between the warning and the actual auto-close, 2 hours) are
+# imported from `agent_shin_shared` so the Agent Shin LLM judge and
+# this daily Greptile sweep agree on the same marker and duration.
+
+
+def fetch_open_prs(repo: str | None) -> list[dict]:
+ """Fetch all open PRs (number, createdAt, isDraft, labels, author).
+
+ Includes drafts: `gh pr list --state open` returns both ready-for-review
+ and draft PRs by default. This is the desired behavior — drafts are not
+ a free pass; the internal-collaborator open-PR queue should reflect every
+ PR that needs human attention regardless of draft status.
+ """
+ fields = "number,title,createdAt,isDraft,labels,author,url"
+ return list_open_items("pr", repo=repo, fields=fields)
+
+
+def fetch_pr_author_association(pr_number: int, repo: str | None) -> str:
+ """Return the GitHub `author_association` for a PR, uppercase.
+
+ Values: OWNER, MEMBER, COLLABORATOR, CONTRIBUTOR, FIRST_TIME_CONTRIBUTOR,
+ FIRST_TIMER, MANNEQUIN, NONE. Returns "" on lookup failure.
+ """
+ endpoint = (
+ f"repos/{repo}/pulls/{pr_number}"
+ if repo
+ else f"repos/{{owner}}/{{repo}}/pulls/{pr_number}"
+ )
+ try:
+ data = json.loads(gh("api", endpoint))
+ except subprocess.CalledProcessError:
+ return ""
+ return (data.get("author_association") or "").upper()
+
+
+def is_external_pr_author(pr: dict, repo: str | None) -> bool:
+ """Return True if the PR author is an external OSS contributor.
+
+ Internal = `OWNER` / `MEMBER` / `COLLABORATOR` association, or a bot login.
+ """
+ login = ((pr.get("author") or {}).get("login") or "").lower()
+ if login.endswith("[bot]") or login in {"dependabot", "github-actions"}:
+ return False
+ association = fetch_pr_author_association(pr["number"], repo)
+ # Fail-safe: if the API lookup failed (empty string), treat the author as
+ # internal so we don't auto-close their PR. Auto-close is destructive, so
+ # an unknown association should never make a PR eligible for closing.
+ if not association or association in INTERNAL_AUTHOR_ASSOCIATIONS:
+ return False
+ return True
+
+
+def fetch_pr_comments(pr_number: int, repo: str | None) -> list[dict]:
+ """Fetch issue-level comments on a PR (where Greptile posts its summary)."""
+ endpoint = (
+ f"repos/{repo}/issues/{pr_number}/comments?per_page=100"
+ if repo
+ else f"repos/{{owner}}/{{repo}}/issues/{pr_number}/comments?per_page=100"
+ )
+ raw = gh("api", "--paginate", endpoint)
+ comments: list[dict] = []
+ for line in raw.strip().splitlines():
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ parsed = json.loads(line)
+ except json.JSONDecodeError:
+ # A malformed line should not blow up the whole sweep. Skip and
+ # carry on so the remaining PRs in this run still get evaluated.
+ continue
+ if isinstance(parsed, list):
+ comments.extend(parsed)
+ else:
+ comments.append(parsed)
+ return comments
+
+
+def has_optout_label(pr: dict, optout_labels: set[str]) -> bool:
+ labels = {label.get("name", "").lower() for label in pr.get("labels", [])}
+ return bool(labels & {lbl.lower() for lbl in optout_labels})
+
+
+def seconds_since_last_grace_warning(
+ comments: Iterable[dict],
+ *,
+ bot_login: str | None = None,
+ now: dt.datetime | None = None,
+) -> float | None:
+ """Return seconds since the bot's most recent grace-period warning, or
+ None if no such warning has ever been posted on this PR.
+
+ Thin wrapper over
+ `agent_shin_shared.seconds_since_latest_marker_comment` — the
+ centralized helper handles the bot-author filter, marker match,
+ timestamp parsing, and `now` injection. Keeping this wrapper
+ preserves the closer's "already-fetched comments + injectable now"
+ interface so callers (and tests) don't need to change.
+ """
+ return seconds_since_latest_marker_comment(
+ comments,
+ marker=GRACE_COMMENT_MARKER,
+ bot_login=bot_login,
+ now=now,
+ )
+
+
+def format_grace_warning_comment(score: int, threshold: int) -> str:
+ """Comment posted on the FIRST low-Greptile-score detection — gives
+ the contributor a 2-hour grace window before the auto-close fires on
+ the next daily cron run.
+
+ Mirrors `format_grace_warning_pr_comment` in
+ `triage_with_llm.py` in spirit (2-hour grace + escape hatches), but
+ framed around Greptile's confidence score instead of the LLM judge's
+ rubric since the close trigger here is the Greptile signal.
+ """
+ return (
+ "🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this "
+ "repository.\n"
+ "\n"
+ "Heads up: Greptile's most recent review scored this PR "
+ f"**{score}/5**, below our merge bar of **{threshold}/5**.\n"
+ "\n"
+ "If the score isn't lifted in the next **2 hours**, I'll auto-close this PR. That's "
+ "**not** us saying the change isn't worthwhile. We want the open-PR list to mirror "
+ "what a maintainer can act on *right now*, so contributors like you don't get lost in "
+ "a backlog. Take your time; everything below still works after the close.\n"
+ "\n"
+ "**During the grace period:** push fixes that address Greptile's feedback, then comment "
+ "`@greptileai` to request a fresh review. If "
+ f"the new score is **{threshold}/5 or higher**, the PR stays open and no further "
+ "action is needed on your side.\n"
+ "\n"
+ "**If the PR does get auto-closed in 2 hours, you still have an easy recovery path:**\n"
+ "\n"
+ "- Comment `@greptileai` to request a fresh review. **This still works even after "
+ f"the PR is closed**, and a score of {threshold}/5 or higher is one of the signals "
+ "that lifts the PR back into the review queue. A low Greptile score isn't a blocker.\n"
+ "- Comment `@agent-shin reconsider` after pushing fixes; I'll re-run the rubric and "
+ "reopen the PR if both gates (description rubric + Greptile score) now pass.\n"
+ "\n"
+ f"{GRACE_COMMENT_MARKER}"
+ )
+
+
+def post_grace_warning(
+ pr: dict,
+ score: int,
+ threshold: int,
+ repo: str | None,
+ dry_run: bool,
+) -> None:
+ """Post the 2-hour grace-period warning comment on `pr`.
+
+ The warning carries `GRACE_COMMENT_MARKER` so subsequent runs can
+ detect that the contributor has already been told about the
+ pending close. Does NOT close the PR — the close happens on the
+ next eligible run after `GRACE_PERIOD_SECONDS` elapses (handled
+ by `close_pr`).
+ """
+ pr_number = pr["number"]
+ repo_args = ["--repo", repo] if repo else []
+
+ if dry_run:
+ print(
+ f" [DRY RUN] Would post grace warning to PR #{pr_number} "
+ f"(greptile={score}/5): {pr['title']}"
+ )
+ return
+
+ comment_body = format_grace_warning_comment(score, threshold)
+ gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args)
+ print(f" Posted grace warning on PR #{pr_number} (greptile={score}/5)")
+
+
+def format_close_comment(score: int, threshold: int) -> str:
+ """Comment posted when a low-Greptile-score PR is auto-closed.
+
+ Carries `AGENT_SHIN_CLOSE_MARKER` so the `@agent-shin reconsider` path
+ (guarded by `was_closed_by_agent_shin`) recognizes this as an Agent Shin
+ close and is allowed to reopen the PR once it passes again; without the
+ marker that recovery path the comment advertises silently rejects the
+ contributor.
+ """
+ score_sentence = (
+ f"Greptile's most recent review scored this PR **{score}/5**, below "
+ f"our merge bar of **{threshold}/5**, and the 2-hour grace period since "
+ "the warning has elapsed.\n\n"
+ )
+ return (
+ f"Closing as part of automated PR triage.\n\n"
+ f"{score_sentence}"
+ "We close low-confidence PRs aggressively to keep the review queue "
+ "manageable for maintainers and contributors alike. **This is not a "
+ "rejection of the idea.** To bring this back:\n\n"
+ "1. Push the fixes that address Greptile's feedback (continue using "
+ "your existing branch is fine).\n"
+ "2. **Open a new PR** with the updated branch. Greptile will review "
+ "it again, and if it scores "
+ f"**{threshold}/5 or higher** a maintainer will take another look.\n\n"
+ "_Why open a new PR instead of reopening this one?_ GitHub does not "
+ "let external contributors reopen a PR that was closed by a bot or "
+ "maintainer, so a fresh PR is the most reliable path forward. If you "
+ "would prefer this exact PR re-evaluated, comment "
+ "`@agent-shin reconsider` once you've pushed the fixes; Agent Shin "
+ "will re-run triage and reopen this PR if it now meets the bar. "
+ "You can also comment `@greptileai` to request a fresh Greptile "
+ "review; that works **even after the PR is closed**.\n\n"
+ "Thanks for contributing to LiteLLM. We know auto-closures can sting; "
+ "the goal is to keep the project healthy, not to dismiss your work."
+ f"\n\n{AGENT_SHIN_CLOSE_MARKER}"
+ )
+
+
+def close_pr(
+ pr: dict,
+ score: int,
+ threshold: int,
+ age_days: int,
+ repo: str | None,
+ dry_run: bool,
+ label: str | None,
+) -> None:
+ """Post the explanatory comment and close the PR."""
+ pr_number = pr["number"]
+ repo_args = ["--repo", repo] if repo else []
+
+ if dry_run:
+ print(
+ f" [DRY RUN] Would close PR #{pr_number} "
+ f"(age={age_days}d, greptile={score}/5): {pr['title']}"
+ )
+ return
+
+ comment_body = format_close_comment(score, threshold)
+ gh("pr", "comment", str(pr_number), "--body", comment_body, *repo_args)
+
+ if label:
+ try:
+ gh("pr", "edit", str(pr_number), "--add-label", label, *repo_args)
+ except subprocess.CalledProcessError as exc:
+ stderr = (exc.stderr or "").strip()
+ print(f" warn: failed to add label '{label}' to #{pr_number}: {stderr}")
+
+ gh("pr", "close", str(pr_number), *repo_args)
+ print(f" Closed PR #{pr_number} (greptile={score}/5, age={age_days}d)")
+
+
+def evaluate_pr(
+ pr: dict,
+ now: dt.datetime,
+ min_age_days: int,
+ min_score: int,
+ repo: str | None,
+ optout_labels: set[str],
+ allowlist: frozenset[str] = ALLOWLIST_LOGINS,
+) -> tuple[str, int | None, int | None]:
+ """Decide what to do with `pr` on this triage run.
+
+ Returns (action, score_or_none, age_days_or_none) where action is one of:
+ "skip-too-young", "skip-optout-label", "skip-not-allowlisted",
+ "skip-internal", "skip-no-greptile-score", "skip-score-ok",
+ "warn-grace", "skip-in-grace-period", or "close".
+
+ Drafts are NOT skipped — the goal is "open PR count == PRs internal
+ collaborators need to action on", and a draft that Greptile scored <4/5
+ is still in that queue. Authors can opt out via the `wip` label (see
+ `DEFAULT_OPTOUT_LABELS`) if they need to keep a long-lived draft open.
+
+ Grace-period semantics: the first time a PR fails the rubric, the
+ action is `warn-grace` — the caller should post a warning comment but
+ NOT close the PR. On a subsequent run, if the warning is still less
+ than `GRACE_PERIOD_SECONDS` old AND the PR still fails, the action is
+ `skip-in-grace-period`. Once the warning ages out and the rubric is
+ still failing, the action is `close`.
+ """
+ if has_optout_label(pr, optout_labels):
+ return ("skip-optout-label", None, None)
+
+ created = parse_iso8601(pr["createdAt"])
+ age_days = (now - created).days
+ # `min_age_days` defaults to 0 (close as soon as Greptile scores low).
+ # Set a positive value via --min-age-days for one-off backfill runs that
+ # want to skip very-young PRs.
+ if min_age_days > 0 and age_days < min_age_days:
+ return ("skip-too-young", None, age_days)
+
+ # While the allowlist is active it is the sole author gate: only those
+ # logins are acted on and the external-only restriction is bypassed for
+ # them. Otherwise auto-close only external OSS contributors — internal
+ # contributors (BerriAI org members) handle their own backlog.
+ login = ((pr.get("author") or {}).get("login") or "").lower()
+ if allowlist:
+ if login not in allowlist:
+ return ("skip-not-allowlisted", None, age_days)
+ elif not is_external_pr_author(pr, repo):
+ return ("skip-internal", None, age_days)
+
+ comments = fetch_pr_comments(pr["number"], repo)
+ extraction = extract_greptile_score(comments)
+ if extraction is None:
+ return ("skip-no-greptile-score", None, age_days)
+
+ score, _ = extraction
+ if score >= min_score:
+ return ("skip-score-ok", score, age_days)
+
+ grace_age = seconds_since_last_grace_warning(comments, now=now)
+ if grace_age is None:
+ return ("warn-grace", score, age_days)
+ if grace_age < GRACE_PERIOD_SECONDS:
+ return ("skip-in-grace-period", score, age_days)
+
+ return ("close", score, age_days)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--repo",
+ type=str,
+ default=None,
+ help="Repository (owner/repo). Auto-detected if omitted.",
+ )
+ parser.add_argument(
+ "--min-age-days",
+ type=int,
+ default=0,
+ help=(
+ "Minimum age (in days) before a PR is eligible. Default 0 = "
+ "close as soon as Greptile flags it. Set a positive value for "
+ "one-off backfill runs that want to spare very-young PRs."
+ ),
+ )
+ parser.add_argument(
+ "--min-score",
+ type=int,
+ default=4,
+ choices=range(1, 6),
+ help="Greptile score below which a PR is closed (default: 4 -> closes <4/5).",
+ )
+ parser.add_argument(
+ "--optout-label",
+ action="append",
+ default=None,
+ help=(
+ "Label(s) that exempt a PR from auto-close. Repeat to add more. "
+ "Case-insensitive. When omitted, defaults to "
+ f"{list(DEFAULT_OPTOUT_LABELS)!r}; passing this flag REPLACES the "
+ "defaults (argparse `append` with a mutable default would append "
+ "instead, which we explicitly avoid)."
+ ),
+ )
+ parser.add_argument(
+ "--close-label",
+ type=str,
+ default=None,
+ help=(
+ "Optional label to add to PRs that get auto-closed "
+ "(e.g. 'auto-closed-low-quality'). Must already exist on the repo."
+ ),
+ )
+ parser.add_argument(
+ "--close",
+ action="store_true",
+ help="Actually close matching PRs (default is dry-run).",
+ )
+ parser.add_argument(
+ "--limit",
+ type=int,
+ default=None,
+ help="Maximum number of PRs to close in one run (safety net).",
+ )
+ args = parser.parse_args()
+
+ dry_run = not args.close
+ if dry_run:
+ print("=== DRY RUN MODE (pass --close to actually close PRs) ===\n")
+
+ print("Fetching open PRs...")
+ prs = fetch_open_prs(args.repo)
+ print(f"Found {len(prs)} open PRs.\n")
+
+ now = dt.datetime.now(dt.timezone.utc)
+ optout_labels = set(args.optout_label or DEFAULT_OPTOUT_LABELS)
+
+ closed = 0
+ summary = {
+ "close": 0,
+ "warn-grace": 0,
+ "skip-in-grace-period": 0,
+ "skip-too-young": 0,
+ "skip-optout-label": 0,
+ "skip-not-allowlisted": 0,
+ "skip-internal": 0,
+ "skip-no-greptile-score": 0,
+ "skip-score-ok": 0,
+ }
+
+ # `warned` tracks grace-warning comments posted in this run so the
+ # `--limit` safety net bounds *all* destructive write actions, not
+ # just closures. Without this cap, a backlog of PRs failing the
+ # threshold simultaneously could flood contributors with comments.
+ warned = 0
+ for pr in sorted(prs, key=lambda p: p["createdAt"]):
+ try:
+ action, score, age_days = evaluate_pr(
+ pr,
+ now,
+ args.min_age_days,
+ args.min_score,
+ args.repo,
+ optout_labels,
+ )
+ summary[action] = summary.get(action, 0) + 1
+
+ if action == "warn-grace":
+ assert score is not None
+ print(
+ f"#{pr['number']}: \"{pr['title']}\" "
+ f"(age={age_days}d, greptile={score}/5) -> warn-grace"
+ )
+ post_grace_warning(
+ pr,
+ score=score,
+ threshold=args.min_score,
+ repo=args.repo,
+ dry_run=dry_run,
+ )
+ if not dry_run:
+ warned += 1
+ if args.limit is not None and (warned + closed) >= args.limit:
+ print(
+ f"\nReached --limit={args.limit} "
+ f"(closed={closed}, warned={warned}); stopping."
+ )
+ break
+ continue
+
+ if action != "close":
+ continue
+
+ assert score is not None and age_days is not None
+ print(
+ f"#{pr['number']}: \"{pr['title']}\" "
+ f"(age={age_days}d, greptile={score}/5) -> close"
+ )
+ close_pr(
+ pr,
+ score=score,
+ threshold=args.min_score,
+ age_days=age_days,
+ repo=args.repo,
+ dry_run=dry_run,
+ label=args.close_label,
+ )
+
+ if not dry_run:
+ closed += 1
+ if args.limit is not None and (warned + closed) >= args.limit:
+ print(
+ f"\nReached --limit={args.limit} "
+ f"(closed={closed}, warned={warned}); stopping."
+ )
+ break
+ except Exception as exc: # noqa: BLE001 - per-PR errors don't abort the sweep
+ summary["error"] = summary.get("error", 0) + 1
+ print(
+ f"!! PR #{pr.get('number')}: {exc}",
+ file=sys.stderr,
+ )
+ continue
+
+ print("\n=== Summary ===")
+ for key, value in summary.items():
+ print(f" {key:28s} {value}")
+ if dry_run:
+ print(f"\nTotal would close: {summary['close']}")
+ else:
+ print(f"\nTotal closed: {closed}")
+ print(
+ f"Total {'would warn (grace)' if dry_run else 'warned (grace)'}: "
+ f"{summary['warn-grace']}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/triage-requirements.txt b/.github/scripts/triage-requirements.txt
new file mode 100644
index 00000000000..a18f05fbb95
--- /dev/null
+++ b/.github/scripts/triage-requirements.txt
@@ -0,0 +1,282 @@
+# Hash-pinned dependency set for the Agent Shin triage scripts.
+# Installed in privileged triage workflows, so every package is pinned to an
+# exact version with SHA-256 hashes and installed with pip --require-hashes.
+#
+# Regenerate after bumping openai:
+# echo 'openai==' \
+# | uv pip compile - --generate-hashes --python-version 3.12 \
+# --no-annotate --no-header -o .github/scripts/triage-requirements.txt
+
+annotated-types==0.7.0 \
+ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
+ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
+anyio==4.14.0 \
+ --hash=sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89 \
+ --hash=sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9
+certifi==2026.6.17 \
+ --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \
+ --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db
+distro==1.9.0 \
+ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \
+ --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2
+h11==0.16.0 \
+ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
+ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
+httpcore==1.0.9 \
+ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
+ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
+httpx==0.28.1 \
+ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
+ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
+idna==3.18 \
+ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
+ --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
+jiter==0.15.0 \
+ --hash=sha256:01a8222cf05ab1128e239421156c207949808acaaea2bdfd33130ae666786e86 \
+ --hash=sha256:032396229564bca02440396bd327710719f724f5e7b7e9f7a8eb3faa4a2c2281 \
+ --hash=sha256:04b400bbf8c9efb03d9bdd976475c919c1d85593b04b9fff7ae234065daf87ae \
+ --hash=sha256:05906b93d72f03339e6bb7cf8dc10ebda64a0266126eed6beba79e20abcf5fd4 \
+ --hash=sha256:066f8f33f18b2419cd8213b2436fa7fbc9c499f315971cfa3ce1f9820c001b1b \
+ --hash=sha256:0ab068bce62a45aa3e7367eceaffb5dde60b7eb853be8dece45132e3d0ff4879 \
+ --hash=sha256:0be6f5ad41a809f303f416d17cec92a7a725902fb9b4f3de3d19362ac0ef8554 \
+ --hash=sha256:0e90a1c315a0226ec822d973817967f9223b7701546c8c2a7913e7ab0926294d \
+ --hash=sha256:0f862193b8696249d22ec433e85fd2ab0ad9596bc3e45e6c0bc55e8aeba97be2 \
+ --hash=sha256:1303d4d68a9b051ea90502402063ecf3807da00ad2affa19ca1ae3b90b3c5f67 \
+ --hash=sha256:144f8e72cb53dab146347b91cceac01f5481237f2b93b4a339a1ee8f8878b67c \
+ --hash=sha256:182226cbc930c9fab81bc2e41a4da672f89539906dadb05e75670ac07b94f71f \
+ --hash=sha256:1c11465f97e2abf45a014b83b730222f8f1c5335e802c7055a67d50de6f1f4e3 \
+ --hash=sha256:1c15024a3d892223b18f597c86d59387249dc396590844ce6b9f6131d1093bae \
+ --hash=sha256:1d54fb5b31dea401a41af3f8a7d2512e9b6a6a005491e6166c7e4ffab9639a9c \
+ --hash=sha256:25ffbe229aa8cd98c28879d8aa1a6e34ae77992ab984a65fba800859dab16269 \
+ --hash=sha256:2a77aadd57cac1682e4401a72724d2796d89a4ba129b1a5812aa94ee480826eb \
+ --hash=sha256:2ae901f3a55bfafdde31d289590fa25e3245735a2b1e8c7cc15871710a002871 \
+ --hash=sha256:2b0074e2f56eb2dacca1689760fd2852a068f85a0547a157b82cb4cafeb6768b \
+ --hash=sha256:2c8aea7781d2a372227871de4e1a1332aa96f5a89fd76c5e835dafdbad102887 \
+ --hash=sha256:2c9cb907439d20bd0c7d7565ca01ee52234203208433749bae5b516907526928 \
+ --hash=sha256:2fb6a5d26af81fc0f00f9360a891e05cf755e149bba391c4d563adc54812973d \
+ --hash=sha256:2fd73e3da91a0a722d67165e849ce2cdc10de0e0d48738c142be8c6c5f310f4c \
+ --hash=sha256:30ce1a5d16b5641dc935d50ef775af6a0871e3d14ab05d6fc54dff371b78e558 \
+ --hash=sha256:30ce785d2adb8e32c3f7741442370a74834ec4c01f3c48f0750227a0b4ef27d6 \
+ --hash=sha256:30f2218e6a9e5c18bc10fe6d41ac189c442c88eacf11bad9f28ef95a9bef00e6 \
+ --hash=sha256:351a341c2105aa430b7047e30f1bf7975f6313b00165d3fc07be2edaf741f279 \
+ --hash=sha256:37a10c377ce3a4a85f4a67f28b7afe093154cde77eaf248a72e856aa08b4d865 \
+ --hash=sha256:392b8ab019e5502d08aff85c6272209c24bc2cbe706ea82a56368f524236614a \
+ --hash=sha256:3e4540b8e74e4268811ac05db226a6a128ff572e7e0ce3f1163b693cadb184cd \
+ --hash=sha256:40b2c7e92c44a84d748d21706c68dc6ff8161d80b59c99d774721a0d2317d7c7 \
+ --hash=sha256:411fa4dfa5a7ae3d11491027ffb9beadec3996010a986862db70d91abba1c750 \
+ --hash=sha256:4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76 \
+ --hash=sha256:42bfb257930800cf43e7c62c832402c704ab60797c992faf88d20e903eac8f32 \
+ --hash=sha256:4363818355dbc70ae1a8e9eaba9de350d93ede4ff6992b8f8eb8cbb6e5122d42 \
+ --hash=sha256:4ab395feec8d249ec4044e228e98a7033f043426a265df439dc3698823f0a4e4 \
+ --hash=sha256:50164d7610c00e7cd913a873fce30b6beeebf4b37e53983e33f22de4c900f6b8 \
+ --hash=sha256:50e51156192722a9c58db112837d3f8ef96fb3c5ecc14e95f409134b08b158ec \
+ --hash=sha256:510c8b3c17a0ed9ac69850c0438dada3c9b82d9c4d589fcb62002a5a9cf3a866 \
+ --hash=sha256:5157de9f76eb4bc5ea74a1219366a25f945ad305641d74e04f59c54087091aa9 \
+ --hash=sha256:54d5d6090cdc1b7c9e780dfb04949a990adb1e301a2fc0bbcee7de4638d33f9a \
+ --hash=sha256:553fcac2ef2cb990877f9fc0833b8b629a3e6a5670b6b5fd58219b41a653ddc4 \
+ --hash=sha256:5607e6013ed7e6b0ec9661e467b7ffde0aa7ab36833a04850f26fcf88ed4845b \
+ --hash=sha256:5d6a60072b44c3c2b797a7ddcbcbbf2b34ea3cfd4721580fbfd2a09d9d9b84ba \
+ --hash=sha256:5f30bae8bc1c2d613e28e5af3e8cceb09b742f1c8a8a5f839fb67afaffc03b61 \
+ --hash=sha256:62ebd14e47e9aed9df4472afcb2663668ce4d74891cd54f86bf6e44029d6dc89 \
+ --hash=sha256:631f13a3d04e97d4e083993b10f4b99530e3a10d953e2eb5e196b7dc7f812ce0 \
+ --hash=sha256:6550fa135c7deb8ead6af49ed7ff648532ea8334a1447fe34a36315ef79c5c29 \
+ --hash=sha256:66b1880df2d01e206e8339769d1c7c1753bcb653efd6289e203f6f24ebada0c0 \
+ --hash=sha256:6eac374c5c975709b69c10f09afd199df74150172156ad10c8d4fd785b7da995 \
+ --hash=sha256:71683c38c825452999b5717fcae07ea708e8c93003e808be4319c1b02e3d176e \
+ --hash=sha256:7553333dd0930c104a5a0db8df72bf7219fe663d731383b576bb6ed6351c984d \
+ --hash=sha256:75e8a04e91432dde9f1838373cf93d23726c79d3e908d319acf0e796f85592e7 \
+ --hash=sha256:773b6eb282ce11ee19f05f6b2d4404fa308e5bbd353b0b80a0262caad6db2cd7 \
+ --hash=sha256:774f93f65031856bf14ad9f59bdcab8b8cad501e5ceabd51ba3525f76937a25b \
+ --hash=sha256:7c468136b8bd6bb18c8786e4236a1fa27362f24cb23450ba0cb204ab379b8e6f \
+ --hash=sha256:7ce8902f939970048b233087082e7bb829db29375811c7ad50687b8624c6fd08 \
+ --hash=sha256:7d3d6683288c11cbab50e865f2e2f13950179aa45410e30b2cfbd3fb7b0177bf \
+ --hash=sha256:7f6163c0f10b055245f814dcc59f4818da60dfe72f3e72ab89fc24b6bd5e9c52 \
+ --hash=sha256:8020c99ec13a7db2b6f96cbe82ef4721c88b426a4892f27478044af0284615ef \
+ --hash=sha256:813dfbb17d65328bf86e5f0905dd277ba2265d3ca20556e86c0c7035b7182e5a \
+ --hash=sha256:860a74063284a2ae9bfedd694f299cc2c68e2696c5f3d440cc9d18bb81b9dd04 \
+ --hash=sha256:8c9004af7c8d67cce7f1aae1026fb55607f4aa600710d08ede3a3ce4aeefe7e0 \
+ --hash=sha256:8d2c0c44d569ce0f2850f5c926f8caeb5f245fbc84475aeb36efccc2103e6dbd \
+ --hash=sha256:8f7e9bc0f1135039b22ee6eab588d42df1ce55842b30740a352885eb267bd941 \
+ --hash=sha256:90c5db5527c221249a876160663ab891ace358c17f7b9c93ec1478b7f0550e5c \
+ --hash=sha256:9100ddbec09741cc66feb0fc6773f8bdbd0e3c345689368f260082ff85dcc0cd \
+ --hash=sha256:913d02d29c9606643418d9ccfc3b72492ab25a6bf7889934e09a3490f8d3438b \
+ --hash=sha256:980c256edb05b78a111b99c4de3b1d32e31634b867fd1fc2cf726e7b7bba9854 \
+ --hash=sha256:9f924585cdacf631cd382b657966847bb537bf9ed0a6f9b991da5f05a631480f \
+ --hash=sha256:a254e10b593624d230c365b6d616b22ca0ad65e63a16e6631c2b3466022e6ba8 \
+ --hash=sha256:a2a438005b6f22d0273413484d6094d7c2c5d10ec1b3a3bf128e0d1d3ba53258 \
+ --hash=sha256:a97261f1fccb8e50ecd2890a96e46efdc3f57c80a197324c6777827231eca712 \
+ --hash=sha256:ab596fa3837e91e7e6a31b5f639988bfc6a35d1f915ac3932d946062219d588f \
+ --hash=sha256:abbf258599526ad0326fe51e252e24f2bd6f24f1852681b4b78feda3808f1d18 \
+ --hash=sha256:ac0d9ddea4350974be7a221fc25895f251a8fee748c889bdced2141c0fec1a49 \
+ --hash=sha256:acf4ee4d1fc55917239fe72972fb292dd773055d05eb040d36f4326e02cc2c0e \
+ --hash=sha256:ae1b0d82ac2d987f9ea512b1c9adfcc71a28de3dea3a6039b54d76cffda9901e \
+ --hash=sha256:b15741f501469009ae0ae90b7147958a664a7dede40aa7ff174a8a4645f546d0 \
+ --hash=sha256:b15d3ec9b0449c40e85319bdb4caa8b77ab526e74f5532ed94bec15e2f66822c \
+ --hash=sha256:b3b3b775e33d3bfaec9899edc526ae97b0da0bf9d071a46124ba419149a414f8 \
+ --hash=sha256:b6c0ffae686c39bf3737be60793783267628783ea42545632c10b291105aee45 \
+ --hash=sha256:c210f8b35dc6f30aafd4b4365ca89b9d1189f21ab49b8e68fa6322a847aef138 \
+ --hash=sha256:c2f6bb8b5216ab9e7873bc08b5d7bef2b8abbb578a3069bf1cd14a45d71d771d \
+ --hash=sha256:c60e71b6d10cfc284c9bf36bd885e8d44c46f688ce50aa91b5edd90181dea687 \
+ --hash=sha256:c6694a173ecabc12eb60efbc0b474464ead1951ff65cd8b1e72100715c64512b \
+ --hash=sha256:c77496cb10bd7549690fbbab3e5ec05857b83e49276f4a9423a766ddd2afcd4c \
+ --hash=sha256:c84c1b7be454b0c16f8499b4ebfbfd82ea5cca6527cceefcbbc06a7557b5ed2e \
+ --hash=sha256:cc0bc345cf2df9d1c00ac443f50d543c1ccfa8b0422cb85b1ab70d681c0b255b \
+ --hash=sha256:ceb8fc27d38793f9c97149be8302720c5b22e5c195a37bf2c45dc36c4600a512 \
+ --hash=sha256:cf4bd113a69c0a740e27cb962ce10630c36d2b8f59d759a651b955ee9d18a823 \
+ --hash=sha256:d1aa62e277fc1cbd80e6deacae6f4d983b41b3d7728e0645c5d741a6149bba45 \
+ --hash=sha256:d1e7b1776f0797956c509e123d0952d10d293a9492dea9f288ab9570ec01d1a5 \
+ --hash=sha256:d636d5095155afd364247f65070fab7beda13498d7ff4de331046e704ab9657f \
+ --hash=sha256:d726e3ceeb337191324b49de298142f27c3ad10886341555d1d5315b5f252c6a \
+ --hash=sha256:d72d8af5c1013656a8870c866660627d1a75bc185814ee022c8533caa1de88ae \
+ --hash=sha256:d8d2955167274e15d79a7a020afdd9b39c990eb80b2d89fca695d92dcfdd38ec \
+ --hash=sha256:d92a5cd21fdb083931d546c207aa29633787c5dc5b02daab2d32b843f88a2c53 \
+ --hash=sha256:e58585a58209d72691ce2d62a9147445f5a87beb0bde97fde284c96ae392a3d1 \
+ --hash=sha256:e7196e56f1cd69af1dbb07dff02dcfb260a50b45a82d409d92a06fedb32473b5 \
+ --hash=sha256:eda3071db3346334beae1360b46da4606da57bf3528c167b3c38533afaf9f2c5 \
+ --hash=sha256:edebcf7d1f601199084bb6e844d7dc67e03e04f6ac786b0332d616635c4ff7a4 \
+ --hash=sha256:ef1fd24d9413f6209e00d3d5a453e67acfe004a25cc6c8e8484faed4311ab9e8 \
+ --hash=sha256:f0b271b462769543716f92d3a4f90527df6ef5ed05ee95ec4137f513e21e1b77 \
+ --hash=sha256:f18f85e4218d1b40f000f42a92239a7a61a902cd42c65e6c360dbd17dcb20894 \
+ --hash=sha256:f1e1754960f38ec40613a07e5e372df67acb3b890fb383b6fb3de3e49ddbf3c7 \
+ --hash=sha256:f2143ab06181d2b029eedcb6af3cebe95f11bbac62441781860f98ee9330a6a6 \
+ --hash=sha256:f3d37768fce7f88dd2a8c6091f2325dea27d30d30d5c6e7a1c0f0af77723b708 \
+ --hash=sha256:fa248c9eb220197d363f688818dac2fd4b2f0cd7d843ca7105d652034823427d
+openai==2.33.0 \
+ --hash=sha256:03ac37d70e8c9e3a8124214e3afa785e2cbc12e627fbd98177a086ef2fd87ad5 \
+ --hash=sha256:f850c435e2a4685bba3295bd54912dd26315d9c1b7733068186134d6e0599f9a
+pydantic==2.13.4 \
+ --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
+ --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
+pydantic-core==2.46.4 \
+ --hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \
+ --hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \
+ --hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \
+ --hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \
+ --hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \
+ --hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \
+ --hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \
+ --hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \
+ --hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \
+ --hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \
+ --hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \
+ --hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \
+ --hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \
+ --hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \
+ --hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \
+ --hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \
+ --hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \
+ --hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \
+ --hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \
+ --hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \
+ --hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \
+ --hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \
+ --hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \
+ --hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \
+ --hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \
+ --hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \
+ --hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \
+ --hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \
+ --hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \
+ --hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \
+ --hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \
+ --hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \
+ --hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \
+ --hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \
+ --hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \
+ --hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \
+ --hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \
+ --hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \
+ --hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \
+ --hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \
+ --hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \
+ --hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \
+ --hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \
+ --hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \
+ --hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \
+ --hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \
+ --hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \
+ --hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \
+ --hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \
+ --hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \
+ --hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \
+ --hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \
+ --hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \
+ --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \
+ --hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \
+ --hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \
+ --hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \
+ --hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \
+ --hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \
+ --hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \
+ --hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \
+ --hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \
+ --hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \
+ --hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \
+ --hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \
+ --hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \
+ --hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \
+ --hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \
+ --hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \
+ --hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \
+ --hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \
+ --hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \
+ --hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \
+ --hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \
+ --hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \
+ --hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \
+ --hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \
+ --hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \
+ --hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \
+ --hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \
+ --hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \
+ --hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \
+ --hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \
+ --hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \
+ --hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \
+ --hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \
+ --hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \
+ --hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \
+ --hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \
+ --hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \
+ --hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \
+ --hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \
+ --hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \
+ --hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \
+ --hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \
+ --hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \
+ --hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \
+ --hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \
+ --hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \
+ --hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \
+ --hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \
+ --hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \
+ --hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \
+ --hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \
+ --hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \
+ --hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \
+ --hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \
+ --hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \
+ --hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \
+ --hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \
+ --hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \
+ --hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \
+ --hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \
+ --hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \
+ --hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \
+ --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \
+ --hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \
+ --hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \
+ --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \
+ --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae
+sniffio==1.3.1 \
+ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
+ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
+tqdm==4.68.3 \
+ --hash=sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482 \
+ --hash=sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03
+typing-extensions==4.15.0 \
+ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \
+ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548
+typing-inspection==0.4.2 \
+ --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \
+ --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464
diff --git a/.github/scripts/triage_rollout_heads_up.py b/.github/scripts/triage_rollout_heads_up.py
new file mode 100644
index 00000000000..a5dedb1c9e7
--- /dev/null
+++ b/.github/scripts/triage_rollout_heads_up.py
@@ -0,0 +1,557 @@
+#!/usr/bin/env python3
+"""One-shot 7-day heads-up sweep for the Agent Shin rollout.
+
+Posts a friendly "the OSS triage bot kicks in next Monday" comment on every
+open external PR/issue that currently *would* fail the new rubric — i.e.,
+every PR/issue Agent Shin would close once the rollout completes. The point
+is to give contributors a full week to fix their description before the bot
+ever takes a destructive action, so nobody is surprised by an auto-close.
+
+The script is designed to run **exactly once** at rollout, fired by a manual
+``workflow_dispatch`` (``dry_run=false``) on the heads-up workflow. Re-runs
+are safe: every comment is stamped with the hidden ``HEADS_UP_MARKER`` and
+PRs/issues that already carry the marker are skipped.
+
+Dry-run vs. real run
+--------------------
+Defaults to dry-run. Passing ``--close`` flips into real mode. Every GitHub
+mutation goes through ``_agent_shin_actions``, which has a one-line
+``if dry_run: log else: do_it`` per call, so the only difference between a
+dry-run preview and the real run is the call site that actually hits the
+GitHub API.
+
+Local preview::
+
+ python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm
+
+Real run (the manual rollout dispatch uses this)::
+
+ python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm --close
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+# Make the sibling triage_with_llm + _agent_shin_actions importable when this
+# script is invoked directly (the GitHub workflow does `python3 .github/scripts/...`).
+_SCRIPTS_DIR = Path(__file__).resolve().parent
+if str(_SCRIPTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_SCRIPTS_DIR))
+
+from _agent_shin_actions import maybe_post_comment # noqa: E402
+from agent_shin_shared import ( # noqa: E402
+ AGENT_SHIN_DEFAULT_BOT_LOGIN,
+ ALLOWLIST_LOGINS,
+ list_open_items,
+)
+from triage_with_llm import ( # noqa: E402
+ DEFAULT_MODEL,
+ call_llm_judge,
+ fetch_issue,
+ fetch_pr,
+ gh,
+ is_internal_contributor,
+ review_gate,
+ triage,
+)
+
+# Hidden marker so re-runs skip PRs/issues we've already notified. Distinct from
+# the within-grace / ready / regressed markers so it can't be confused with the
+# steady-state lifecycle comments.
+HEADS_UP_MARKER = ""
+
+# Placeholder until the litellm-docs PR ships. The rollout blog post explains
+# the new rubric, the 7-day grace, and how to recover after an auto-close.
+# TODO(docs): replace with the canonical URL once the litellm-docs PR merges.
+ROLLOUT_BLOG_URL = "https://docs.litellm.ai/docs/agent_shin_triage_rollout"
+
+# Default cutoff is one week from "now". Computed at runtime so the wording
+# stays correct even if the rollout is merged later than planned. The user can
+# override with --close-on YYYY-MM-DD when running the script manually.
+DEFAULT_GRACE_DAYS = 7
+
+# The daily auto-close sweeps (close_low_quality_prs.yml at 09:00 UTC and
+# review_gate.yml at 09:30 UTC) are what actually close a still-failing item,
+# so the deadline we promise contributors has to name that wall-clock moment.
+ACTIVATION_TIME_UTC = "09:00 UTC"
+
+
+def _format_cutoff(cutoff: dt.date) -> str:
+ """Human-readable, timezone-explicit cutoff, e.g. ``Monday, June 1, 2026
+ (09:00 UTC)`` — the moment a still-failing PR/issue gets closed."""
+ return (
+ f"{cutoff.strftime('%A, %B')} {cutoff.day}, {cutoff.year} "
+ f"({ACTIVATION_TIME_UTC})"
+ )
+
+
+def _rubric_section_pr() -> str:
+ return (
+ "**Going forward, every external PR needs ONE of:**\n"
+ "\n"
+ "- A linked GitHub issue using a closing keyword: "
+ "`Fixes #1234`, `Closes #1234`, or `Resolves #1234`, OR\n"
+ "- All three of: a clear **problem description**, **expected vs. "
+ "actual behavior**, and **end-to-end QA proof** (at least one of a "
+ "short screen recording / video, before/after screenshots, or the "
+ "exact commands you ran with their real output; mocked or stubbed "
+ "runs don't count).\n"
+ "\n"
+ "PRs also need a **Greptile confidence score of 4/5 or higher** before "
+ "the bot will tag them `ready for review`. You can `@greptileai` to "
+ "request a fresh review at any time, including after the PR is closed."
+ )
+
+
+def _rubric_section_issue() -> str:
+ return (
+ "**Going forward, every external issue needs:**\n"
+ "\n"
+ "- For **bug reports**: end-to-end evidence of the bug (at least one "
+ "of a screen recording / video, a screenshot, or the exact commands "
+ "you ran with their real output / traceback) plus expected vs. actual "
+ "behavior. Written steps with no run output don't count, and mocked "
+ "or stubbed runs don't count.\n"
+ "- For **feature requests**: a clear description of the proposed "
+ "feature plus a use case + concrete example (config, API call, UI "
+ "flow, or scenario showing what's blocked today)."
+ )
+
+
+def _description_only_note(kind: str) -> str:
+ noun = "PR" if kind == "pr" else "issue"
+ return (
+ f"⚠️ **The requirements must live in the {noun} *description*, not in "
+ "comments.** Some PRs/issues collect 100+ comments from humans and "
+ "bots; reading the entire thread on every triage run would balloon "
+ "GitHub API usage (we'd start getting 429'd) and blow out the LLM "
+ "judge's context. The bot only reads the description, so anything "
+ "you add as a comment will be invisible to it."
+ )
+
+
+def _missing_section(verdict: dict, greptile_score: int | None) -> str:
+ """Bullet list of what's currently missing on this PR/issue.
+
+ Combines the LLM judge's `missing` list (rubric items) with a Greptile
+ shortfall (for PRs) so the contributor sees one list of things to fix.
+ """
+ missing = list(verdict.get("missing") or [])
+ if greptile_score is not None and greptile_score < 4:
+ missing.insert(
+ 0,
+ f"Greptile's most recent review scored this PR {greptile_score}/5 "
+ "(below the 4/5 bar Agent Shin will require).",
+ )
+ if not missing:
+ return (
+ "_The bot couldn't articulate a specific missing piece; see the "
+ "rubric link above and double-check the description includes all "
+ "of it before the rollout._"
+ )
+ bullets = "\n".join(f"- {m}" for m in missing)
+ return f"**What this one is currently missing:**\n\n{bullets}"
+
+
+def _recovery_section(kind: str) -> str:
+ if kind == "pr":
+ return (
+ "**If the bot closes this PR after the rollout:** update the "
+ "description with the missing pieces, then either open a fresh "
+ "PR or comment `@agent-shin reconsider` on the closed PR. If "
+ "Greptile re-scores you at 4/5 or higher I'll reopen and tag "
+ "the PR `ready for review`. (`@greptileai` works on closed PRs "
+ "too; a fresh review is one of the signals that lifts you back "
+ "into the queue.) This is **not** us losing interest in your "
+ "change; far from it. We just need open PRs to be a list of "
+ "things a maintainer can act on, so we can get to yours faster."
+ )
+ return (
+ "**If the bot closes this issue after the rollout:** edit the issue "
+ "description to add the missing pieces, then comment `@agent-shin "
+ "reconsider` on the closed issue. I'll re-evaluate and, if the rubric "
+ "is met, reopen it. (GitHub doesn't let external authors reopen an "
+ "issue a maintainer or bot closed, so the comment is the reliable "
+ "path.) This is **not** us saying the bug isn't real or the request "
+ "isn't useful; it's so the remaining open issues are a list of things "
+ "a maintainer can act on."
+ )
+
+
+def format_heads_up_comment(
+ *, kind: str, verdict: dict, greptile_score: int | None, cutoff: dt.date
+) -> str:
+ """Compose the friendly 7-day heads-up comment posted on a failing PR/issue."""
+ noun = "PR" if kind == "pr" else "issue"
+ rubric = _rubric_section_pr() if kind == "pr" else _rubric_section_issue()
+ cutoff_str = _format_cutoff(cutoff)
+ explanation = (verdict.get("explanation") or "").strip()
+ explanation_block = (
+ f"> _(The judge's note for this one: {explanation})_\n\n" if explanation else ""
+ )
+
+ return (
+ "🚅 **Heads-up: we're turning on the OSS triage bot in "
+ f"{DEFAULT_GRACE_DAYS} days, on {cutoff_str}.**\n"
+ "\n"
+ "We're rolling out **Agent Shin**, an LLM-as-judge triage bot for "
+ f"external {noun}s. Once it's live, the bot reads each open "
+ f"{noun}'s description, scores it against a small rubric, and "
+ f"auto-closes any {noun} that's missing the basics, with a single "
+ f"comment explaining what's missing and how to recover. Full "
+ f"context: [Agent Shin rollout blog post]({ROLLOUT_BLOG_URL}).\n"
+ "\n"
+ f"{rubric}\n"
+ "\n"
+ f"{_description_only_note(kind)}\n"
+ "\n"
+ f"{_missing_section(verdict, greptile_score)}\n"
+ "\n"
+ f"{explanation_block}"
+ "**Timeline (you have a week):**\n"
+ "\n"
+ f"- We turn the bot on in {DEFAULT_GRACE_DAYS} days, on "
+ f"**{cutoff_str}**. You have until then to update this {noun}'s "
+ "description with the missing pieces above.\n"
+ f"- If this {noun} still fails the rubric at **{cutoff_str}**, "
+ "we'll close it.\n"
+ f"- From then on the bot runs daily, and every {noun} that fails "
+ "the rubric gets a **2-hour lifetime**: one warning comment, then "
+ "auto-close 2 hours later.\n"
+ "\n"
+ f"{_recovery_section(kind)}\n"
+ "\n"
+ f"{HEADS_UP_MARKER}"
+ )
+
+
+def _list_open_numbers(repo: str, kind: str) -> list[int]:
+ """Return every open PR or issue number in ``repo``.
+
+ Delegates to ``list_open_items`` so the full backlog is fetched (no cap)
+ and the `gh {pr,issue} list` invocation stays in one shared place. ``gh
+ issue list`` would include PRs, but ``list_open_items`` uses the dedicated
+ command per kind, so the two never mix.
+ """
+ return [
+ item["number"] for item in list_open_items(kind, repo=repo, fields="number")
+ ]
+
+
+def _has_heads_up_marker(item: dict) -> bool:
+ """Cheap fast-path: check the PR/issue body itself for the marker.
+
+ The marker is appended to the *comment* we post, not the body, so this
+ will only fire if the body literally contains the marker text. We still
+ do the comment-marker check separately below; this body check just lets
+ us short-circuit for PRs/issues that quote the marker for any reason.
+ """
+ body = item.get("body") or ""
+ return HEADS_UP_MARKER in body
+
+
+def _comments_have_marker(repo: str, number: int) -> bool:
+ """True if the bot already posted a comment carrying the marker.
+
+ Used for idempotency: a re-run skips items the previous run notified.
+ Filters by author (matching the sibling marker-checks in
+ ``triage_with_llm._has_marker`` and
+ ``agent_shin_shared.seconds_since_latest_marker_comment``) so a
+ contributor who quotes the heads-up via GitHub's "Quote reply" — which
+ preserves HTML comments in the raw markdown — can't trick the
+ idempotency check into silently skipping a real heads-up.
+
+ Comments live on the unified issues endpoint regardless of whether the
+ item is a PR or an issue, so no ``kind`` argument is required here.
+ """
+ expected_login = (
+ os.environ.get("AGENT_SHIN_BOT_LOGIN") or AGENT_SHIN_DEFAULT_BOT_LOGIN
+ ).lower()
+ raw = gh(
+ "api",
+ "--paginate",
+ f"repos/{repo}/issues/{number}/comments?per_page=100",
+ )
+ for line in raw.splitlines():
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ payload = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ comments = payload if isinstance(payload, list) else [payload]
+ for comment in comments:
+ author = ((comment.get("user") or {}).get("login") or "").lower()
+ if author != expected_login:
+ continue
+ if HEADS_UP_MARKER in (comment.get("body") or ""):
+ return True
+ return False
+
+
+def _evaluate_pr(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
+ """Run the future PR rubric (review_gate) in dry-run and return the result."""
+ return review_gate(
+ repo=repo,
+ number=number,
+ close=False, # we only want the verdict, never act here
+ model=model,
+ judge=judge,
+ )
+
+
+def _evaluate_issue(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
+ """Run the future issue rubric (triage kind='issue') in dry-run."""
+ return triage(
+ repo=repo,
+ kind="issue",
+ number=number,
+ close=False,
+ model=model,
+ judge=judge,
+ )
+
+
+def _would_be_closed(kind: str, result: dict) -> bool:
+ """True if the future triage would auto-close this PR/issue based on the
+ rubric (regardless of grace-period gating).
+
+ For PRs we trust ``review_gate``'s ``passing`` field — it combines the LLM
+ verdict and the Greptile score. For issues we read the LLM verdict
+ directly. Both fields are ``None``/missing on skip paths
+ (skip-internal-author, skip-llm-error, etc.) where the future bot would
+ NOT close the item — those return False.
+ """
+ if kind == "pr":
+ passing = result.get("passing")
+ if passing is None:
+ return False # skipped — nothing for the heads-up to warn about
+ return passing is False
+ verdict = result.get("verdict") or {}
+ return (verdict.get("verdict") or "").lower() == "fail"
+
+
+def _process_one(
+ *,
+ repo: str,
+ kind: str,
+ number: int,
+ model: str,
+ cutoff: dt.date,
+ dry_run: bool,
+ judge: Any = None,
+ skip_marker_check: bool = False,
+ allowlist: frozenset[str] = ALLOWLIST_LOGINS,
+) -> dict:
+ """Evaluate one PR/issue and post a heads-up if it would be auto-closed.
+
+ Returns a per-item dict for the summary table.
+ """
+ base = {"kind": kind, "number": number}
+ fetcher = fetch_pr if kind == "pr" else fetch_issue
+ item = fetcher(repo, number)
+
+ if (item.get("state") or "") != "open":
+ return {**base, "action": "skip-not-open"}
+ if allowlist:
+ login = (item.get("user") or {}).get("login") or ""
+ if login.lower() not in allowlist:
+ return {**base, "action": "skip-not-allowlisted"}
+ elif is_internal_contributor(item):
+ return {**base, "action": "skip-internal-author"}
+ if not skip_marker_check and _has_heads_up_marker(item):
+ return {**base, "action": "skip-already-marked-in-body"}
+ if not skip_marker_check and _comments_have_marker(repo, number):
+ return {**base, "action": "skip-already-notified"}
+
+ if kind == "pr":
+ result = _evaluate_pr(repo=repo, number=number, model=model, judge=judge)
+ else:
+ result = _evaluate_issue(repo=repo, number=number, model=model, judge=judge)
+
+ if not _would_be_closed(kind, result):
+ return {**base, "action": "skip-passing", "evaluator": result.get("action")}
+
+ verdict = result.get("verdict") or {}
+ greptile_score = result.get("greptile_score") if kind == "pr" else None
+ comment = format_heads_up_comment(
+ kind=kind, verdict=verdict, greptile_score=greptile_score, cutoff=cutoff
+ )
+ maybe_post_comment(repo, number, comment, dry_run=dry_run)
+ return {
+ **base,
+ "action": "heads-up-posted" if not dry_run else "would-post-heads-up",
+ "verdict": (verdict.get("verdict") or "").lower(),
+ "greptile_score": greptile_score,
+ }
+
+
+def _print_summary(results: list[dict]) -> None:
+ """Tally per-action counts so a dry-run preview tells you at a glance how
+ many comments the real run would post."""
+ counts: dict[str, int] = {}
+ for r in results:
+ counts[r["action"]] = counts.get(r["action"], 0) + 1
+ print("\n=== rollout heads-up summary ===")
+ for action in sorted(counts):
+ print(f" {action:35s} {counts[action]}")
+ print(f" total {len(results)}")
+
+
+def run(
+ *,
+ repo: str,
+ close: bool,
+ cutoff: dt.date,
+ model: str,
+ kinds: tuple[str, ...] = ("pr", "issue"),
+ judge: Any = None,
+ only_numbers: dict[str, list[int]] | None = None,
+ skip_marker_check: bool = False,
+) -> list[dict]:
+ """Sweep ``repo`` and post heads-up comments. Returns the per-item results."""
+ dry_run = not close
+ if dry_run:
+ print(
+ f"[DRY RUN] sweeping {repo}; --close not passed, no comments will be posted."
+ )
+ else:
+ print(f"[REAL RUN] sweeping {repo}; comments WILL be posted.")
+ print(f"Cutoff date in comment body: {cutoff.isoformat()}")
+
+ results: list[dict] = []
+ for kind in kinds:
+ if only_numbers and kind in only_numbers:
+ numbers = list(only_numbers[kind])
+ else:
+ numbers = _list_open_numbers(repo, kind)
+ print(f"\n--- {kind}s: {len(numbers)} open ---")
+ for n in numbers:
+ try:
+ result = _process_one(
+ repo=repo,
+ kind=kind,
+ number=n,
+ model=model,
+ cutoff=cutoff,
+ dry_run=dry_run,
+ judge=judge,
+ skip_marker_check=skip_marker_check,
+ )
+ except (
+ Exception
+ ) as exc: # noqa: BLE001 - per-item errors don't abort the sweep
+ result = {
+ "kind": kind,
+ "number": n,
+ "action": "error",
+ "error": str(exc),
+ }
+ print(f"!! {kind}#{n}: {exc}", file=sys.stderr)
+ print(f" {kind}#{n}: {result['action']}")
+ results.append(result)
+ _print_summary(results)
+ return results
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--repo", required=True, help="owner/repo")
+ parser.add_argument(
+ "--close",
+ action="store_true",
+ help=(
+ "Actually post comments. Without this flag the script is in "
+ "dry-run mode and only logs what it would do."
+ ),
+ )
+ parser.add_argument(
+ "--close-on",
+ type=dt.date.fromisoformat,
+ default=None,
+ help=(
+ "Cutoff date shown in the heads-up comment as the rollout date "
+ f"(default: today + {DEFAULT_GRACE_DAYS} days)."
+ ),
+ )
+ parser.add_argument(
+ "--model",
+ default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL,
+ help=f"Model for the rubric LLM judge (default: {DEFAULT_MODEL}).",
+ )
+ parser.add_argument(
+ "--kind",
+ choices=("pr", "issue", "both"),
+ default="both",
+ help="Restrict the sweep to PRs or issues only (default: both).",
+ )
+ parser.add_argument(
+ "--only-pr",
+ type=int,
+ action="append",
+ default=[],
+ help="Limit the PR sweep to these PR numbers (repeat for several).",
+ )
+ parser.add_argument(
+ "--only-issue",
+ type=int,
+ action="append",
+ default=[],
+ help="Limit the issue sweep to these issue numbers (repeat for several).",
+ )
+ parser.add_argument(
+ "--ignore-existing-marker",
+ action="store_true",
+ help=(
+ "Re-post on PRs/issues that already carry the heads-up marker. "
+ "Useful for testing the comment wording on a known PR."
+ ),
+ )
+ args = parser.parse_args()
+
+ cutoff = args.close_on or (
+ dt.datetime.now(dt.timezone.utc).date() + dt.timedelta(days=DEFAULT_GRACE_DAYS)
+ )
+
+ kinds: tuple[str, ...]
+ if args.kind == "pr":
+ kinds = ("pr",)
+ elif args.kind == "issue":
+ kinds = ("issue",)
+ else:
+ kinds = ("pr", "issue")
+
+ only: dict[str, list[int]] = {}
+ if args.only_pr:
+ only["pr"] = args.only_pr
+ if args.only_issue:
+ only["issue"] = args.only_issue
+
+ # The script must NOT hit the LLM in dry-run if no key is set — we still
+ # want a useful preview that says "skip-no-llm-key" for items that would
+ # have been judged. Production runs require OPENAI_API_KEY.
+ if args.close and not os.environ.get("OPENAI_API_KEY"):
+ parser.error("OPENAI_API_KEY must be set for --close (real-run) mode.")
+
+ run(
+ repo=args.repo,
+ close=args.close,
+ cutoff=cutoff,
+ model=args.model,
+ kinds=kinds,
+ only_numbers=only or None,
+ skip_marker_check=args.ignore_existing_marker,
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py
new file mode 100644
index 00000000000..d2536058e01
--- /dev/null
+++ b/.github/scripts/triage_with_llm.py
@@ -0,0 +1,1778 @@
+#!/usr/bin/env python3
+"""
+Agent Shin — LLM-as-judge triage for external OSS pull requests and issues.
+
+Evaluates a single PR or issue against the contribution rubric and, when the
+LLM judge marks it as failing, posts an explanatory comment + closes the
+PR/issue. Re-triggers on `reopened` so contributors can iterate back in by
+filling in the missing pieces and reopening.
+
+Internal BerriAI contributors (`author_association` in {OWNER, MEMBER,
+COLLABORATOR}) and bot accounts are skipped entirely.
+
+Usage:
+ triage_with_llm.py --repo owner/repo --pr 1234
+ triage_with_llm.py --repo owner/repo --issue 5678
+ triage_with_llm.py --repo owner/repo --pr 1234 --close # actually close
+ triage_with_llm.py --repo owner/repo --pr 1234 --print-prompt # show prompt
+
+Defaults are SAFE: without `--close` the script writes a verdict to stdout (and,
+when running in GitHub Actions, to $GITHUB_STEP_SUMMARY) but takes no GitHub
+write actions.
+
+Environment:
+ GH_TOKEN / GITHUB_TOKEN - for `gh` CLI auth (auto-set in Actions)
+ OPENAI_API_KEY - required when --close is passed
+ OPENAI_BASE_URL - optional (route to any OpenAI-compatible API)
+ TRIAGE_MODEL - optional model override (default: gpt-5.4-mini)
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import re
+import subprocess
+import sys
+import textwrap
+import urllib.parse
+from typing import Any, Iterable
+
+# Add this script's directory to `sys.path` so the sibling
+# `agent_shin_shared` module is importable when the script is invoked
+# directly (e.g. `python3 .github/scripts/triage_with_llm.py ...`) and
+# also when the tests load this script via
+# `importlib.util.spec_from_file_location`.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from agent_shin_shared import ( # noqa: E402 -- sys.path adjusted above
+ AGENT_SHIN_CLOSE_MARKER,
+ AGENT_SHIN_DEFAULT_BOT_LOGIN,
+ ALLOWLIST_LOGINS,
+ GRACE_COMMENT_MARKER,
+ GRACE_PERIOD_SECONDS,
+ GREPTILE_BOT_LOGINS,
+ SCORE_PATTERN,
+ extract_greptile_score,
+ gh,
+ parse_iso8601,
+ seconds_since_latest_marker_comment,
+)
+
+DEFAULT_MODEL = "gpt-5.4-mini"
+
+INTERNAL_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"})
+
+# `AGENT_SHIN_DEFAULT_BOT_LOGIN` is imported from `agent_shin_shared`.
+# When the workflow uses the default `secrets.GITHUB_TOKEN`, the
+# closure / reopen event's `actor.login` is `github-actions[bot]`. The
+# env override `AGENT_SHIN_BOT_LOGIN` exists for local debugging and for
+# repos that wire Agent Shin to a PAT.
+
+# HTML marker appended to every reconsider verdict comment. We grep for this
+# on subsequent reconsider triggers to enforce a short cooldown so that
+# repeated `@agent-shin reconsider` comments don't burn CI/LLM budget.
+# Using a unique HTML comment keeps the marker invisible to humans while
+# being trivially greppable from a comments-list API response.
+RECONSIDER_COMMENT_MARKER = ""
+
+# Minimum gap between two reconsider verdicts on the same PR/issue. Set to
+# 10 minutes — long enough that a contributor can't trivially spam the
+# trigger, short enough that a genuine "I just pushed a fix and reupdated
+# the body" iteration loop isn't punished.
+RECONSIDER_RATE_LIMIT_SECONDS = 600
+
+# `GRACE_COMMENT_MARKER` (HTML marker on the grace-period warning comment
+# posted on the first low-quality detection — used on subsequent triage
+# runs to detect that a warning was already posted and measure how long
+# ago it was posted) and `GRACE_PERIOD_SECONDS` (length of the grace
+# period between the warning and the actual auto-close, 2 hours) are
+# imported from `agent_shin_shared` so the daily Greptile sweep and the
+# LLM judge agree on the same marker and duration.
+
+# --- Review-gate ("ready for review" label lifecycle) configuration ----------
+# The review gate keeps a single label in sync with whether a PR currently
+# clears BOTH quality bars: the LLM rubric (clear problem + expected/actual +
+# QA proof, or a linked issue) AND Greptile's most recent confidence score.
+READY_FOR_REVIEW_LABEL = "ready for review"
+DEFAULT_GRACE_DAYS = 1 # 24h before an un-passing, un-tagged PR is auto-closed
+DEFAULT_MIN_GREPTILE_SCORE = 4 # Greptile < 4/5 counts as "not passing"
+
+# Hidden HTML-comment markers stamped into review-gate comments. They never
+# render in the GitHub UI but let the gate detect its own prior actions so it
+# (a) posts the within-grace "what's missing" notice at most once and (b) can
+# tell a first-time pass ("ready for review") from a recovery after a
+# regression ("all clear again").
+READY_MARKER = ""
+REGRESSED_MARKER = ""
+WITHIN_GRACE_MARKER = ""
+
+# `GREPTILE_BOT_LOGINS` (Greptile's GitHub App login variants —
+# `greptile-apps[bot]` in REST API comments, `greptile-apps` in
+# `gh pr view --json` output) and `SCORE_PATTERN` (regex matching lines
+# like `Confidence Score: 3/5`) are imported from `agent_shin_shared`
+# so the daily sweep and the review gate read the score through the
+# same set of logins / patterns.
+
+# `AGENT_SHIN_CLOSE_MARKER` is imported from `agent_shin_shared` so this LLM
+# judge and the daily Greptile sweep stamp the same marker on their close
+# comments — `was_closed_by_agent_shin` keys the reconsider reopen path off it.
+
+# Model families that require `reasoning_effort` to be set, and that reject
+# `temperature != 1` unless `reasoning_effort` is "none". For these models we
+# pass `reasoning_effort="none"` so a `temperature=0` deterministic judgment
+# is still accepted. See litellm/llms/openai/chat/gpt_5_transformation.py for
+# the full set of constraints LiteLLM applies to these models.
+GPT5_FAMILY_PREFIX = "gpt-5"
+
+# Regexes for picking off "obvious passes" without burning LLM tokens.
+#
+# Keep this list to GitHub's documented PR-closing keywords only
+# (https://docs.github.com/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).
+# Casual mentions like "see #1234" or "ref #1234" are intentionally NOT
+# auto-passed — they should fall through to the LLM judge, which has the
+# stricter rubric "a bare issue number without a closing keyword counts only
+# if it's clearly the related issue (not a passing mention)".
+LINKED_ISSUE_PATTERN = re.compile(
+ r"\b(?:fixes|fix|fixed|closes|close|closed|resolves|resolve|resolved)\s+"
+ r"(?:#\d+|https?://github\.com/[\w.-]+/[\w.-]+/issues/\d+)",
+ re.IGNORECASE,
+)
+HTML_COMMENT_PATTERN = re.compile(r"", re.DOTALL)
+
+
+# ---------------------------------------------------------------------------
+# gh helpers
+#
+# `gh` is imported from `agent_shin_shared` so a future change (timeout,
+# logging, retry) only needs to be made once.
+
+
+def fetch_pr(repo: str, number: int) -> dict:
+ """Return the full GitHub REST representation of a PR."""
+ return json.loads(gh("api", f"repos/{repo}/pulls/{number}"))
+
+
+def fetch_issue(repo: str, number: int) -> dict:
+ """Return the full GitHub REST representation of an issue."""
+ return json.loads(gh("api", f"repos/{repo}/issues/{number}"))
+
+
+def post_comment(repo: str, number: int, body: str) -> None:
+ """Post an issue-style comment (works for both issues and PRs)."""
+ gh(
+ "api",
+ f"repos/{repo}/issues/{number}/comments",
+ "-X",
+ "POST",
+ "-f",
+ f"body={body}",
+ )
+
+
+def close_pr(repo: str, number: int) -> None:
+ """Close a pull request (state=closed)."""
+ gh(
+ "api",
+ f"repos/{repo}/pulls/{number}",
+ "-X",
+ "PATCH",
+ "-f",
+ "state=closed",
+ )
+
+
+def reopen_pr(repo: str, number: int) -> None:
+ """Reopen a previously-closed pull request (state=open).
+
+ Used by the `@agent-shin reconsider` comment-trigger flow: the bot has
+ write access via GH_TOKEN, so it can reopen on the contributor's behalf
+ even though GitHub doesn't let the OSS author do it themselves.
+ """
+ gh(
+ "api",
+ f"repos/{repo}/pulls/{number}",
+ "-X",
+ "PATCH",
+ "-f",
+ "state=open",
+ )
+
+
+def close_issue(repo: str, number: int, *, not_planned: bool = True) -> None:
+ """Close an issue, marking state_reason=not_planned by default."""
+ args = [
+ "api",
+ f"repos/{repo}/issues/{number}",
+ "-X",
+ "PATCH",
+ "-f",
+ "state=closed",
+ ]
+ if not_planned:
+ args.extend(["-f", "state_reason=not_planned"])
+ gh(*args)
+
+
+def reopen_issue(repo: str, number: int) -> None:
+ """Reopen a previously-closed issue (state=open, state_reason=reopened)."""
+ gh(
+ "api",
+ f"repos/{repo}/issues/{number}",
+ "-X",
+ "PATCH",
+ "-f",
+ "state=open",
+ "-f",
+ "state_reason=reopened",
+ )
+
+
+def add_label(repo: str, number: int, label: str) -> None:
+ """Add a label to a PR/issue (GitHub creates the label if it's missing)."""
+ gh(
+ "api",
+ f"repos/{repo}/issues/{number}/labels",
+ "-X",
+ "POST",
+ "-f",
+ f"labels[]={label}",
+ )
+
+
+def remove_label(repo: str, number: int, label: str) -> None:
+ """Remove a label from a PR/issue. A missing label (404) is not an error."""
+ encoded = urllib.parse.quote(label, safe="")
+ try:
+ gh(
+ "api",
+ f"repos/{repo}/issues/{number}/labels/{encoded}",
+ "-X",
+ "DELETE",
+ )
+ except subprocess.CalledProcessError as exc:
+ stderr = (exc.stderr or "").lower()
+ if "404" in stderr or "not found" in stderr:
+ return
+ raise
+
+
+def _iter_paginated_json(*api_args: str) -> Any:
+ """Yield JSON objects from `gh api --paginate ... -q '.[]'`.
+
+ `gh api --paginate` on a JSON-array endpoint concatenates pages into
+ one stream; `-q '.[]'` flattens that stream into newline-delimited
+ objects (jq-style). This keeps memory bounded for chatty endpoints
+ like issue events/comments on long-lived PRs.
+ """
+ raw = gh("api", "--paginate", *api_args, "-q", ".[]")
+ for line in raw.splitlines():
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ yield json.loads(line)
+ except json.JSONDecodeError:
+ # A malformed line should not blow up the whole guard. Skip and
+ # carry on — at worst the guard fail-closes (returns False /
+ # None) and the caller treats it as "unknown".
+ continue
+
+
+def fetch_last_close_event(
+ repo: str, number: int
+) -> tuple[str | None, dt.datetime | None]:
+ """Return the actor login and timestamp of the most recent `closed` event.
+
+ Either field may be None: actor when the events API returns nothing
+ (unusual for a closed item, but possible on transient errors), and
+ timestamp when the event lacks `created_at` or the value can't be
+ parsed. `was_closed_by_agent_shin` fail-closes on either.
+ """
+ actor: str | None = None
+ closed_at: dt.datetime | None = None
+ for event in _iter_paginated_json(f"repos/{repo}/issues/{number}/events"):
+ if event.get("event") != "closed":
+ continue
+ actor = (event.get("actor") or {}).get("login")
+ created = event.get("created_at")
+ if not created:
+ closed_at = None
+ continue
+ try:
+ closed_at = parse_iso8601(created)
+ except ValueError:
+ closed_at = None
+ return actor, closed_at
+
+
+# How much older than the latest `closed` event the Agent Shin marker
+# comment is allowed to be while still counting as "this close was Agent
+# Shin's". Agent Shin posts the close comment immediately before closing,
+# so the marker timestamp is normally at most a few seconds before the
+# close event; the buffer just absorbs clock skew between the comments
+# API and the events API.
+AGENT_SHIN_CLOSE_MARKER_SKEW_SECONDS = 300
+
+
+def was_closed_by_agent_shin(
+ repo: str, number: int, *, bot_login: str | None = None
+) -> bool:
+ """Return True iff Agent Shin itself most-recently closed this PR/issue.
+
+ This is the guard that stops `@agent-shin reconsider` from reopening an
+ item Agent Shin did not close — a maintainer closing for non-rubric
+ reasons (security, duplicate, design rejection), or a different workflow
+ (stale/duplicate sweeps) closing under the shared `github-actions[bot]`
+ identity. Three independent signals must all hold, because that identity
+ is not unique to Agent Shin and a marker comment from a prior
+ closed/reopened cycle would otherwise vouch for an unrelated close:
+
+ 1. The most recent `closed` event's actor is the bot identity.
+ 2. Agent Shin left one of its auto-close comments, detected via
+ `AGENT_SHIN_CLOSE_MARKER`. The actor check alone can't tell an
+ Agent Shin close from any other `github-actions[bot]` close.
+ 3. That marker comment was posted at (or just before) the latest
+ close event, not on a previous close in an
+ Agent-Shin-close -> reconsider-reopen -> other-bot-reclose cycle.
+
+ The check is intentionally fail-closed: any uncertainty about who closed
+ the item is treated as "not Agent Shin" so the destructive reopen path
+ stays gated.
+ """
+ expected = (
+ bot_login
+ or os.environ.get("AGENT_SHIN_BOT_LOGIN")
+ or AGENT_SHIN_DEFAULT_BOT_LOGIN
+ ).lower()
+ actor, closed_at = fetch_last_close_event(repo, number)
+ if not actor or actor.lower() != expected or closed_at is None:
+ return False
+ marker_seconds = seconds_since_last_agent_shin_close(
+ repo, number, bot_login=bot_login
+ )
+ if marker_seconds is None:
+ return False
+ close_age_seconds = (dt.datetime.now(dt.timezone.utc) - closed_at).total_seconds()
+ return marker_seconds <= close_age_seconds + AGENT_SHIN_CLOSE_MARKER_SKEW_SECONDS
+
+
+def _seconds_since_latest_marker_comment(
+ repo: str,
+ number: int,
+ *,
+ marker: str,
+ bot_login: str | None = None,
+) -> float | None:
+ """Return seconds since the bot's most recent comment with ``marker``.
+
+ Fetches comments via `_iter_paginated_json` and delegates the
+ iteration / author-filter / timestamp logic to
+ `agent_shin_shared.seconds_since_latest_marker_comment` so the daily
+ Greptile sweep and the LLM judge use one source of truth for the
+ "bot already posted X" detection. The wall-clock `now` is resolved
+ against this module's `dt` so tests that freeze time via
+ `monkeypatch.setattr(triage_module, "dt", ...)` still apply.
+ """
+ return seconds_since_latest_marker_comment(
+ _iter_paginated_json(f"repos/{repo}/issues/{number}/comments"),
+ marker=marker,
+ bot_login=bot_login,
+ now=dt.datetime.now(dt.timezone.utc),
+ )
+
+
+def seconds_since_last_reconsider_verdict(
+ repo: str, number: int, *, bot_login: str | None = None
+) -> float | None:
+ """Return seconds since the bot's most recent reconsider verdict comment.
+
+ Detects comments by matching the HTML marker `RECONSIDER_COMMENT_MARKER`
+ appended by `format_reopen_comment` and
+ `format_reconsider_still_failing_comment`. Returns None when the bot
+ has never posted a reconsider verdict on this PR/issue (or when the
+ only matching comments are missing a `created_at` timestamp, which
+ shouldn't happen on a real GitHub response).
+ """
+ return _seconds_since_latest_marker_comment(
+ repo, number, marker=RECONSIDER_COMMENT_MARKER, bot_login=bot_login
+ )
+
+
+def seconds_since_last_grace_warning(
+ repo: str, number: int, *, bot_login: str | None = None
+) -> float | None:
+ """Return seconds since the bot's most recent grace-period warning.
+
+ Detects warning comments by matching the HTML marker
+ `GRACE_COMMENT_MARKER` appended by `format_grace_warning_pr_comment`
+ and `format_grace_warning_issue_comment`. Returns None when no
+ grace warning has ever been posted on this PR/issue — that's the
+ "first low-quality detection" signal that drives the warning path.
+ """
+ return _seconds_since_latest_marker_comment(
+ repo, number, marker=GRACE_COMMENT_MARKER, bot_login=bot_login
+ )
+
+
+def seconds_since_last_agent_shin_close(
+ repo: str, number: int, *, bot_login: str | None = None
+) -> float | None:
+ """Return seconds since Agent Shin's most recent auto-close comment.
+
+ Detects close comments by matching `AGENT_SHIN_CLOSE_MARKER` (stamped by
+ `format_pr_close_comment` / `format_issue_close_comment`). Returns None
+ when Agent Shin has never closed this PR/issue — the signal
+ `was_closed_by_agent_shin` uses to keep the reconsider reopen path gated
+ against closures performed by other workflows sharing the bot identity.
+ """
+ return _seconds_since_latest_marker_comment(
+ repo, number, marker=AGENT_SHIN_CLOSE_MARKER, bot_login=bot_login
+ )
+
+
+# ---------------------------------------------------------------------------
+# Author classification
+
+
+def is_internal_contributor(item: dict) -> bool:
+ """Return True if the PR/issue author should be exempted from triage.
+
+ Fail-safe: if `author_association` is missing or empty (which should never
+ happen on a successful GitHub REST response but is possible on schema
+ changes or partial responses), treat the author as INTERNAL so the
+ destructive close path never fires on an unknown contributor. This matches
+ the sibling `is_external_pr_author` in `close_low_quality_prs.py`.
+ """
+ login = ((item.get("user") or {}).get("login") or "").lower()
+ if login.endswith("[bot]") or login in {"dependabot", "github-actions"}:
+ return True
+ association = (item.get("author_association") or "").upper()
+ if not association or association in INTERNAL_ASSOCIATIONS:
+ return True
+ return False
+
+
+# ---------------------------------------------------------------------------
+# Greptile score + age helpers (`extract_greptile_score`, `parse_iso8601`)
+# live in `agent_shin_shared` — they're imported at the top of this module
+# so both `triage_with_llm.py` and `close_low_quality_prs.py` share a
+# single source of truth for the Confidence-Score regex and ISO-8601
+# parsing.
+
+
+# ---------------------------------------------------------------------------
+# Prompt construction
+
+
+def strip_html_comments(text: str) -> str:
+ """Remove HTML comments — template placeholder text shouldn't fool the judge."""
+ return HTML_COMMENT_PATTERN.sub("", text or "")
+
+
+def has_linked_issue(text: str) -> bool:
+ """Heuristic: does this body link to an open issue (Fixes #123 etc.)?"""
+ return bool(LINKED_ISSUE_PATTERN.search(strip_html_comments(text or "")))
+
+
+def build_pr_prompt(*, title: str, body: str) -> str:
+ cleaned_body = strip_html_comments(body or "").strip() or "(empty)"
+ # Dedent the static template *before* interpolating dynamic fields so that
+ # multi-line bodies (whose 2nd+ lines start at column 0) don't defeat the
+ # common-indent computation in textwrap.dedent.
+ template = textwrap.dedent("""
+ You are "Agent Shin", the OSS triage bot for the LiteLLM open-source
+ repository (BerriAI/litellm). Decide whether this external pull request
+ meets the project's contribution standards.
+
+ A PR PASSES triage only if BOTH (1) AND (2) are satisfied. A linked
+ issue alone is NOT enough — it covers context, not proof.
+
+ (1) CONTEXT — the PR provides AT LEAST ONE of:
+ (a) A link to a related GitHub issue. Acceptable forms:
+ "Fixes #1234", "Closes #1234", "Resolves #1234",
+ "Refs https://github.com/BerriAI/litellm/issues/1234". A
+ bare "#1234" without a closing keyword counts only if it
+ is clearly the related issue (not a passing mention).
+ (b) A clear problem description in the body (what bug or
+ missing feature this addresses, beyond the title) AND
+ expected vs. actual behavior (or, for features, "what's
+ possible now vs. with this PR").
+
+ (2) END-TO-END QA PROOF: the PR body contains AT LEAST ONE of:
+ (a) A screen recording / video showing the behavior before
+ and after the change (the bug reproducing, then the fix
+ working). For a brand-new feature with no meaningful
+ "before", a recording of it working end-to-end is fine.
+ (b) A screenshot (or before/after screenshots) showing the
+ fix or feature working.
+ (c) Specific commands that were actually run (curl, python,
+ a CLI invocation, etc.) PAIRED WITH their real
+ output, demonstrating the change works end-to-end against
+ the real system. Commands whose external dependencies
+ (LLM provider, DB, network) are mocked or stubbed do NOT
+ satisfy (2c); they are not end-to-end.
+
+ `has_qa_proof` must be set to `true` only when (2a), (2b),
+ or a non-mocked (2c) is actually present in the body. If the
+ only "proof" is mocked tests, `has_qa_proof` is `false` and
+ the verdict is "fail".
+
+ The following do NOT count as QA proof:
+ - Generic claims like "I tested it", "works locally", "all
+ tests pass", or a checked "I added tests" checkbox with no
+ output shown.
+ - A description of what tests exist or were added, without
+ their actual output in the PR body.
+ - `pytest` (or any test runner) executed against the
+ repository's own unit tests. Those mock the LLM provider,
+ DB, and network, so they are NOT end-to-end and never
+ satisfy (2), no matter how much passing output is pasted.
+ - A linked issue. The linked issue is context (1a), never
+ proof (2).
+
+ FAIL the PR if EITHER (1) or (2) is missing. Do not bias toward PASS:
+ if QA proof is absent, the verdict is "fail" even when the rest of
+ the PR is well-written.
+
+ Respond with a single JSON object, no prose:
+
+ {{
+ "verdict": "pass" | "fail",
+ "linked_issue": boolean,
+ "has_problem_description": boolean,
+ "has_expected_vs_actual": boolean,
+ "has_qa_proof": boolean,
+ "qa_proof_type": "video" | "screenshot" | "commands_with_output" | "none",
+ "missing": ["plain-english strings naming what is missing"],
+ "explanation": "1-2 sentence reasoning for the team to skim"
+ }}
+
+ ---
+ PR title: {title}
+
+ PR body:
+ ---
+ {cleaned_body}
+ ---
+ """).strip()
+ return template.format(title=title, cleaned_body=cleaned_body)
+
+
+def build_issue_prompt(*, title: str, body: str) -> str:
+ cleaned_body = strip_html_comments(body or "").strip() or "(empty)"
+ # Dedent the static template *before* interpolating dynamic fields so that
+ # multi-line bodies (whose 2nd+ lines start at column 0) don't defeat the
+ # common-indent computation in textwrap.dedent.
+ template = textwrap.dedent("""
+ You are "Agent Shin", the OSS triage bot for the LiteLLM open-source
+ repository (BerriAI/litellm). Decide whether this GitHub issue meets
+ the project's reporting standards.
+
+ For a BUG REPORT the issue PASSES triage only when it contains BOTH:
+ (1) END-TO-END EVIDENCE OF THE BUG (the "before"; set
+ `has_repro=true` only when this is present): AT LEAST ONE of:
+ (a) A screen recording / video of the bug happening.
+ (b) A screenshot of the bug.
+ (c) The exact command(s) actually run (curl, python, a CLI
+ invocation, etc.) PAIRED WITH their real output, traceback,
+ or logs showing the failure against the real system.
+ Commands whose external dependencies (LLM provider, DB,
+ network) are mocked or stubbed do NOT count.
+ Prose-only "steps to reproduce" with no run output, video, or
+ screenshot do NOT satisfy (1).
+ (2) Expected vs. actual behavior (`has_expected_vs_actual`).
+
+ FAIL the bug report if either (1) or (2) is missing. Do not bias
+ toward PASS: if the bug isn't demonstrated end-to-end, the verdict is
+ "fail" even when the report is well-written.
+
+ For a FEATURE REQUEST the issue PASSES triage only when it contains
+ ALL of:
+ - A clear description of the proposed feature (what should LiteLLM do
+ that it does not today).
+ - Motivation / use case with a concrete example (config, API call,
+ UI flow, or scenario showing what's blocked today).
+
+ For an issue that is neither a bug report nor a feature request (a
+ question, support request, or discussion), PASS as long as it has a
+ clear, specific ask and is not empty or template placeholder text.
+
+ Respond with a single JSON object, no prose:
+
+ {{
+ "verdict": "pass" | "fail",
+ "kind": "bug" | "feature" | "other",
+ "has_repro": boolean,
+ "has_expected_vs_actual": boolean,
+ "has_motivation_example": boolean,
+ "missing": ["plain-english strings naming what is missing"],
+ "explanation": "1-2 sentence reasoning for the team to skim"
+ }}
+
+ ---
+ Issue title: {title}
+
+ Issue body:
+ ---
+ {cleaned_body}
+ ---
+ """).strip()
+ return template.format(title=title, cleaned_body=cleaned_body)
+
+
+# ---------------------------------------------------------------------------
+# LLM call + verdict parsing
+
+
+def call_llm_judge(
+ prompt: str, *, model: str, api_key: str, base_url: str | None
+) -> str:
+ """Call an OpenAI-compatible chat completions endpoint. Returns raw text."""
+ # Import inside the function so unit tests that monkey-patch this never
+ # need the openai package installed.
+ from openai import OpenAI
+
+ client = (
+ OpenAI(api_key=api_key, base_url=base_url)
+ if base_url
+ else OpenAI(api_key=api_key)
+ )
+ kwargs: dict[str, Any] = {
+ "model": model,
+ "messages": [{"role": "user", "content": prompt}],
+ "temperature": 0,
+ "response_format": {"type": "json_object"},
+ }
+ # gpt-5.x reasoning models reject `temperature != 1` unless
+ # `reasoning_effort` is explicitly "none". Set it via `extra_body` so this
+ # works across openai SDK versions regardless of whether the SDK natively
+ # types `reasoning_effort` as a top-level chat-completions param yet.
+ if model.lower().startswith(GPT5_FAMILY_PREFIX):
+ kwargs["extra_body"] = {"reasoning_effort": "none"}
+ response = client.chat.completions.create(**kwargs)
+ return response.choices[0].message.content or ""
+
+
+def parse_verdict(raw: str) -> dict:
+ """Parse the LLM's JSON response. Tolerates ```json fences and stray text."""
+ if not raw:
+ raise ValueError("empty LLM response")
+ text = raw.strip()
+ if text.startswith("```"):
+ text = re.sub(r"^```(?:json)?\s*", "", text)
+ text = re.sub(r"\s*```$", "", text)
+ try:
+ return json.loads(text)
+ except json.JSONDecodeError:
+ match = re.search(r"\{.*\}", text, re.DOTALL)
+ if not match:
+ raise ValueError(f"could not extract JSON from LLM response: {raw[:200]}")
+ return json.loads(match.group(0))
+
+
+# ---------------------------------------------------------------------------
+# Comment composition
+
+
+def _format_missing(missing: list[str]) -> str:
+ if not missing:
+ return "- (see explanation below)"
+ return "\n".join(f"- {m}" for m in missing)
+
+
+# Rubric items the judge can mark present. The first element of each tuple is
+# the verdict-JSON boolean field, the second is the human-readable label we
+# render in the "what you got right" section of close / grace-warning comments.
+_PR_PRESENT_LABELS: tuple[tuple[str, str], ...] = (
+ ("linked_issue", "Linked a related GitHub issue"),
+ ("has_problem_description", "Clear problem description"),
+ ("has_expected_vs_actual", "Expected vs. actual behavior"),
+ ("has_qa_proof", "End-to-end QA proof"),
+)
+
+# Issue rubric labels grouped by `kind`. The judge sets `kind` to one of
+# {"bug", "feature", "other"}; when "other" we render both groups so we don't
+# silently drop a present-flag the judge actually set to True.
+_ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = (
+ (
+ "has_repro",
+ "End-to-end evidence of the bug (video, screenshot, or command + real output)",
+ ),
+ ("has_expected_vs_actual", "Expected vs. actual behavior"),
+)
+_ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = (
+ ("has_motivation_example", "Motivation and concrete example"),
+)
+
+
+def _format_present_for_pr(verdict: dict) -> list[str]:
+ """Human-readable rubric items the judge confirmed are present on a PR.
+
+ Drives the "what you got right" section in close / grace-warning comments.
+ The user gave explicit feedback: contributors should see what they nailed
+ *before* the list of gaps, so the comment doesn't read as pure rejection.
+ """
+ return [label for field, label in _PR_PRESENT_LABELS if verdict.get(field)]
+
+
+def _format_present_for_issue(verdict: dict) -> list[str]:
+ """Human-readable rubric items the judge confirmed are present on an issue.
+
+ Branches on the judge's `kind` field. For `"other"` (or missing kind) we
+ render the union so a present-flag isn't dropped just because the judge
+ couldn't classify the issue cleanly.
+ """
+ kind = (verdict.get("kind") or "").lower()
+ groups: list[tuple[tuple[str, str], ...]] = []
+ if kind in ("bug", "other", ""):
+ groups.append(_ISSUE_BUG_LABELS)
+ if kind in ("feature", "other", ""):
+ groups.append(_ISSUE_FEATURE_LABELS)
+ out: list[str] = []
+ for group in groups:
+ for field, label in group:
+ if verdict.get(field) and label not in out:
+ out.append(label)
+ return out
+
+
+def _format_present_block(items: list[str]) -> str:
+ """Render the optional "what you got right" block. Empty string when the
+ judge didn't confirm anything as present — better to omit the section
+ entirely than to show "What you got right: (nothing)".
+ """
+ if not items:
+ return ""
+ bullets = "\n".join(f"- ✅ {item}" for item in items)
+ return f"**What you got right:**\n\n{bullets}\n\n"
+
+
+def format_pr_close_comment(verdict: dict) -> str:
+ missing_lines = _format_missing(verdict.get("missing") or [])
+ present_block = _format_present_block(_format_present_for_pr(verdict))
+ explanation = verdict.get("explanation") or ""
+ return (
+ "🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this "
+ "repository. "
+ "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n"
+ "\n"
+ "I read the description against our "
+ "[contribution rubric](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md). "
+ "Here's how it lined up:\n"
+ "\n"
+ f"{present_block}"
+ "**What's still missing:**\n"
+ "\n"
+ f"{missing_lines}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ "**Closing this PR isn't a rejection of the change.** We want the open-PR list to "
+ "mirror what a maintainer can act on *right now*, so contributors don't get lost in a "
+ 'backlog. A closed PR is a soft "park this for later"; your work is still here, '
+ "the diff is still here, and getting it reopened is one comment away. Take your time.\n"
+ "\n"
+ "**To bring this PR back:**\n"
+ "\n"
+ "- Update the description with the missing pieces, then comment `@agent-shin reconsider` "
+ "on this PR. I'll re-evaluate and reopen if it now passes.\n"
+ "- Or **Open a new PR** with the same fix and the updated description. GitHub doesn't "
+ "always let external contributors reopen a bot-closed PR, so a fresh PR is the most "
+ "reliable path back into the review queue.\n"
+ "- If Greptile's most recent score on this PR was below 4/5, comment `@greptileai` to "
+ "request a fresh review; that **still works even after the PR is closed**, and a "
+ "stronger score is one of the signals that lifts the PR back into the queue. A low "
+ "Greptile score isn't a blocker.\n"
+ "\n"
+ '**What "end-to-end QA proof" means**, since it\'s the most common gap: at least one '
+ "of a short before/after screen recording / video (the bug reproducing, then the fix "
+ "working; for a brand-new feature, a recording of it working end-to-end), a screenshot "
+ "(or before/after screenshots) of it working, or the exact commands you ran paired "
+ "with their **real output** against the real system. Running `pytest` on the repo's "
+ "unit tests doesn't count; those mock the LLM provider, DB, and network, so they "
+ "aren't end-to-end. Output from a real, no-mocks integration run is what we look "
+ "for. A linked issue alone isn't enough either: it covers context, not proof. See "
+ "[the full rubric](https://docs.litellm.ai/blog/agent-shin-triage#the-rubric-for-pull-requests).\n"
+ "\n"
+ "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n"
+ "\n"
+ "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, comment "
+ "`@agent-shin reconsider` or ping a maintainer; they'll override me.)_"
+ f"\n\n{AGENT_SHIN_CLOSE_MARKER}"
+ )
+
+
+def format_issue_close_comment(verdict: dict) -> str:
+ missing_lines = _format_missing(verdict.get("missing") or [])
+ present_block = _format_present_block(_format_present_for_issue(verdict))
+ explanation = verdict.get("explanation") or ""
+ return (
+ "🚅 Hi, thanks for filing this! I'm **Agent Shin**, the automated triage bot for this "
+ "repository. "
+ "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n"
+ "\n"
+ "I read the issue against our reporting checklist. Here's how it lined up:\n"
+ "\n"
+ f"{present_block}"
+ "**What's still missing:**\n"
+ "\n"
+ f"{missing_lines}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ "**Closing this isn't us saying the bug isn't real or the request isn't useful.** We "
+ "want the open-issue list to mirror what a maintainer can act on *right now*, so "
+ "reports like yours don't get buried in a backlog. A closed issue is a soft \"park "
+ 'this for later"; your report is still here, and getting it reopened is one comment '
+ "away. Take your time.\n"
+ "\n"
+ "**To bring this issue back:**\n"
+ "\n"
+ "1. Edit the issue description to add the missing pieces:\n"
+ " - For **bug reports**: end-to-end evidence of the bug (a screen recording / "
+ "video, a screenshot, or the exact commands you ran with their real output / "
+ "traceback) plus expected vs. actual behavior. Written steps with no run output, "
+ "video, or screenshot don't count, and mocked or stubbed runs don't count.\n"
+ " - For **feature requests**: a concrete description of what should change, plus a "
+ "use case and example (config / API call / UI flow).\n"
+ "2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it "
+ "now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer "
+ "or bot closed, so the comment-based reconsider is the reliable path.)\n"
+ "\n"
+ "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n"
+ "\n"
+ "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, comment "
+ "`@agent-shin reconsider` or ping a maintainer; they'll override me.)_"
+ f"\n\n{AGENT_SHIN_CLOSE_MARKER}"
+ )
+
+
+def format_grace_warning_pr_comment(verdict: dict) -> str:
+ """Comment posted on the FIRST low-quality detection — gives the
+ contributor a 2-hour grace window to fix the PR before the next
+ triage run actually closes it.
+
+ This is the "before-close" warning. On the second triage run, if the
+ grace marker is older than `GRACE_PERIOD_SECONDS` AND the PR still
+ fails the rubric, the close path runs (which posts
+ `format_pr_close_comment` and closes the PR).
+ """
+ missing_lines = _format_missing(verdict.get("missing") or [])
+ present_block = _format_present_block(_format_present_for_pr(verdict))
+ explanation = verdict.get("explanation") or ""
+ return (
+ "🚅 Hi, thanks for the PR! I'm **Agent Shin**, the automated triage bot for this "
+ "repository. "
+ "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n"
+ "\n"
+ "I read the description against our "
+ "[contribution rubric](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md). "
+ "Here's how it lined up:\n"
+ "\n"
+ f"{present_block}"
+ "**What's still missing:**\n"
+ "\n"
+ f"{missing_lines}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ "If the description isn't updated in the next **2 hours**, I'll auto-close this PR. "
+ "That's **not** us saying we don't care about the change; we want the open-PR list to "
+ "mirror what a maintainer can act on *right now*, so contributors don't get lost in a "
+ 'backlog. A closed PR is a soft "park this for later," not a rejection. Take your '
+ "time; everything below still works after the close.\n"
+ "\n"
+ "**During the grace period:** just update the PR description with the missing pieces. "
+ "No need to ping me; I'll re-check on the next sweep and skip the auto-close if it "
+ "now passes. See "
+ "[what counts as QA proof](https://docs.litellm.ai/blog/agent-shin-triage#the-rubric-for-pull-requests) "
+ "for the full rubric (a linked issue alone isn't enough; it covers context, not proof).\n"
+ "\n"
+ "**If the PR does get auto-closed in 2 hours, you still have easy recovery paths:**\n"
+ "\n"
+ "- Comment `@agent-shin reconsider` after updating the description. I'll re-evaluate "
+ "and reopen the PR if it now passes.\n"
+ "- Comment `@greptileai` to request a fresh Greptile review; that **still works even "
+ "after the PR is closed**, and a stronger score is one of the signals that lifts the "
+ "PR back into the queue. So a low Greptile score isn't a blocker either.\n"
+ "\n"
+ "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n"
+ "\n"
+ "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, ping a "
+ "maintainer; they'll override me.)_\n"
+ "\n"
+ f"{GRACE_COMMENT_MARKER}"
+ )
+
+
+def format_grace_warning_issue_comment(verdict: dict) -> str:
+ """Issue analogue of `format_grace_warning_pr_comment`."""
+ missing_lines = _format_missing(verdict.get("missing") or [])
+ present_block = _format_present_block(_format_present_for_issue(verdict))
+ explanation = verdict.get("explanation") or ""
+ return (
+ "🚅 Hi, thanks for filing this! I'm **Agent Shin**, the automated triage bot for this "
+ "repository. "
+ "[What's this and why am I getting it?](https://docs.litellm.ai/blog/agent-shin-triage)\n"
+ "\n"
+ "I read the issue against our reporting checklist. Here's how it lined up:\n"
+ "\n"
+ f"{present_block}"
+ "**What's still missing:**\n"
+ "\n"
+ f"{missing_lines}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ "If the issue isn't updated in the next **2 hours**, I'll auto-close it. That's **not** us "
+ "saying the bug isn't real or the request isn't useful; we want the open-issue list "
+ "to mirror what a maintainer can act on *right now*, so reports like yours don't get "
+ 'buried in a backlog. A closed issue is a soft "park this for later," not a '
+ "rejection. Take your time; reopening is one comment away.\n"
+ "\n"
+ "**During the grace period:** just edit the issue description with the missing "
+ "pieces. No need to ping me; I'll re-check on the next sweep and skip the auto-close "
+ "if it now passes.\n"
+ "\n"
+ "Missing pieces, depending on what this is:\n"
+ "\n"
+ "- For **bug reports**: end-to-end evidence of the bug (a screen recording / video, a "
+ "screenshot, or the exact commands you ran with their real output / traceback) plus "
+ "expected vs. actual behavior. Written steps with no run output don't count, and "
+ "mocked or stubbed runs don't count.\n"
+ "- For **feature requests**: a concrete description of what should change, plus a use "
+ "case and example (config / API call / UI flow).\n"
+ "\n"
+ "**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` "
+ "and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n"
+ "\n"
+ "Internal BerriAI contributors: this rubric doesn't apply to you; ping a maintainer.\n"
+ "\n"
+ "_(I'm an LLM, so I'm not infallible. If you think I got this wrong, ping a "
+ "maintainer; they'll override me.)_\n"
+ "\n"
+ f"{GRACE_COMMENT_MARKER}"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Step-summary helpers
+
+
+def write_step_summary(content: str) -> None:
+ """When running inside GitHub Actions, append to the step summary file."""
+ path = os.environ.get("GITHUB_STEP_SUMMARY")
+ if not path:
+ return
+ try:
+ with open(path, "a", encoding="utf-8") as handle:
+ handle.write(content)
+ if not content.endswith("\n"):
+ handle.write("\n")
+ except OSError as exc:
+ print(f"warn: failed to write step summary: {exc}", file=sys.stderr)
+
+
+# ---------------------------------------------------------------------------
+# Core orchestration
+
+
+def format_reopen_comment(kind: str) -> str:
+ """Comment posted when Agent Shin reopens after a successful reconsider."""
+ noun = "PR" if kind == "pr" else "issue"
+ # The trailing HTML marker is used by `seconds_since_last_reconsider_verdict`
+ # to enforce a cooldown between repeated `@agent-shin reconsider` triggers.
+ # Keep the marker on its own line so it doesn't disturb the rendered text.
+ return (
+ f"♻️ **Re-evaluated and reopened.** Thanks for updating the {noun}!\n"
+ "\n"
+ "Agent Shin re-ran triage on the latest description and it now meets "
+ "the bar. A maintainer will take another look soon; please don't "
+ f"close this {noun} again unless asked to.\n"
+ "\n"
+ "_(If a maintainer ends up closing this for non-rubric reasons, that "
+ "decision stands; comment `@agent-shin reconsider` again only if you "
+ "have substantively new information.)_\n"
+ "\n"
+ f"{RECONSIDER_COMMENT_MARKER}"
+ )
+
+
+def format_reconsider_still_failing_comment(kind: str, verdict: dict) -> str:
+ """Comment posted when reconsider re-runs triage but the verdict is still fail."""
+ missing_lines = _format_missing(verdict.get("missing") or [])
+ explanation = verdict.get("explanation") or ""
+ noun = "PR" if kind == "pr" else "issue"
+ # The trailing HTML marker is used by `seconds_since_last_reconsider_verdict`
+ # to enforce a cooldown between repeated `@agent-shin reconsider` triggers.
+ return (
+ f"⏸️ **Re-evaluated; this {noun} still doesn't meet the rubric.**\n"
+ "\n"
+ "Agent Shin re-ran triage on the current description but is still "
+ "missing:\n"
+ "\n"
+ f"{missing_lines}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ "Update the description with the missing pieces and comment "
+ "`@agent-shin reconsider` again, or ping a maintainer if you think "
+ "I got this wrong.\n"
+ "\n"
+ "_(I'm an LLM and I'm not infallible.)_\n"
+ "\n"
+ f"{RECONSIDER_COMMENT_MARKER}"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Review gate — "ready for review" label lifecycle
+
+_UNSET = object()
+
+
+def _combine_missing(
+ verdict: dict, greptile_score: int | None, min_score: int
+) -> list[str]:
+ """Merge the LLM rubric's `missing` list with a Greptile-score shortfall."""
+ missing = list(verdict.get("missing") or [])
+ if greptile_score is not None and greptile_score < min_score:
+ missing.insert(
+ 0,
+ f"Greptile's most recent review scored this PR {greptile_score}/5 "
+ f"(below the {min_score}/5 bar)",
+ )
+ return missing or ["(see explanation below)"]
+
+
+def _has_marker(
+ comments: Iterable[dict], marker: str, *, bot_login: str | None = None
+) -> bool:
+ """Return True iff the bot itself posted a comment containing ``marker``.
+
+ Filters by author so a contributor who quotes the marker (e.g. via
+ GitHub's "Quote reply" feature, which preserves HTML comments in
+ raw markdown) is not mistaken for a bot action — that would
+ silently suppress notifications or change which "recovered" wording
+ is selected. Matches the author-filter pattern used by the sibling
+ `_seconds_since_latest_marker_comment` helper.
+ """
+ expected_login = (
+ bot_login
+ or os.environ.get("AGENT_SHIN_BOT_LOGIN")
+ or AGENT_SHIN_DEFAULT_BOT_LOGIN
+ ).lower()
+ for comment in comments:
+ author = ((comment.get("user") or {}).get("login") or "").lower()
+ if author != expected_login:
+ continue
+ if marker in (comment.get("body") or ""):
+ return True
+ return False
+
+
+def format_ready_for_review_comment(
+ verdict: dict,
+ greptile_score: int | None,
+ min_greptile_score: int = DEFAULT_MIN_GREPTILE_SCORE,
+) -> str:
+ """Posted the first time a PR clears the bar (label added)."""
+ score_line = (
+ f" Greptile scored it **{greptile_score}/5**."
+ if greptile_score is not None
+ else ""
+ )
+ explanation = verdict.get("explanation") or ""
+ return (
+ "✅ **Triage passed, tagging `ready for review`.**\n"
+ "\n"
+ "Agent Shin checked this PR against the "
+ "[contribution rubric](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md) "
+ "and it clears the bar (a linked issue, or a clear problem description "
+ f"+ expected vs. actual + QA proof).{score_line}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ "A maintainer will take it from here. If a later re-check finds the PR "
+ f"has regressed (Greptile drops below {min_greptile_score}/5, "
+ "the QA proof is removed, etc.) I'll pull the tag and comment with "
+ "what's missing; fix it and the tag comes back automatically.\n"
+ f"{READY_MARKER}"
+ )
+
+
+def format_all_clear_comment(verdict: dict, greptile_score: int | None) -> str:
+ """Posted when a PR recovers after a regression (label re-added)."""
+ score_line = (
+ f" Greptile is back to **{greptile_score}/5**."
+ if greptile_score is not None
+ else ""
+ )
+ explanation = verdict.get("explanation") or ""
+ return (
+ "✅ **All clear again, re-adding `ready for review`.**\n"
+ "\n"
+ "Thanks for addressing the earlier feedback. On re-check this PR meets "
+ f"the contribution bar once more.{score_line}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ "A maintainer will take another look.\n"
+ f"{READY_MARKER}"
+ )
+
+
+def format_regression_comment(
+ missing: list[str], explanation: str, grace_days: int
+) -> str:
+ """Posted when a previously-tagged PR regresses (label removed, PR stays open).
+
+ Discloses the same ``grace_days`` deadline the state machine enforces:
+ once that window elapses with the PR still failing, the close path fires.
+ Hiding the deadline behind a bare "stays open" would surprise contributors
+ with an auto-close they were never warned about.
+ """
+ window = "24 hours" if grace_days == 1 else f"{grace_days} days"
+ return (
+ "⚠️ **Removing the `ready for review` tag.**\n"
+ "\n"
+ "On a re-check this PR no longer meets the contribution bar. What's "
+ "missing now:\n"
+ "\n"
+ f"{_format_missing(missing)}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ f"The PR stays open for ~{window}; address the points above and Agent "
+ 'Shin will post an "all clear" comment and re-add the tag '
+ "automatically. If the points still aren't addressed after that "
+ "window, the PR is auto-closed; that's not a rejection, and you can "
+ "comment `@agent-shin reconsider` to have it re-evaluated and reopened "
+ "once it passes.\n"
+ f"{REGRESSED_MARKER}"
+ )
+
+
+def format_within_grace_comment(
+ missing: list[str], explanation: str, grace_days: int
+) -> str:
+ """Posted once while a failing PR is still inside its grace window."""
+ window = "24 hours" if grace_days == 1 else f"{grace_days} days"
+ return (
+ "🚅 Hi, thanks for the PR! This is **Agent Shin**, the automated triage "
+ "bot. This PR doesn't quite meet the contribution bar yet:\n"
+ "\n"
+ f"{_format_missing(missing)}\n"
+ "\n"
+ f"> {explanation}\n"
+ "\n"
+ f"You have ~{window} from when this PR was opened to add the missing "
+ "pieces; just update the description and I'll re-check on the next "
+ "sweep. Once it passes I'll tag it `ready for review`. If it does get "
+ "auto-closed, that's not a rejection; comment `@agent-shin reconsider` "
+ "and I'll re-evaluate and reopen if it now passes.\n"
+ f"{WITHIN_GRACE_MARKER}"
+ )
+
+
+def review_gate(
+ *,
+ repo: str,
+ number: int,
+ close: bool,
+ model: str,
+ judge: Any = None,
+ greptile_score: Any = _UNSET,
+ comments: Any = _UNSET,
+ now: dt.datetime | None = None,
+ grace_days: int = DEFAULT_GRACE_DAYS,
+ min_greptile_score: int = DEFAULT_MIN_GREPTILE_SCORE,
+ label: str = READY_FOR_REVIEW_LABEL,
+ allowlist: frozenset[str] = ALLOWLIST_LOGINS,
+) -> dict:
+ """Reconcile the `ready for review` label with a PR's current quality.
+
+ A PR is *passing* when it clears BOTH gates: the LLM rubric (linked issue,
+ or problem description + expected/actual + QA proof) AND Greptile's most
+ recent confidence score (>= ``min_greptile_score``; absence of a score is
+ not held against the PR). The gate then drives a small state machine, using
+ the label itself as the persisted state so comments fire only on
+ transitions (never on every scheduled run):
+
+ passing, untagged -> add label + "ready for review" / "all clear"
+ passing, tagged -> noop-passing
+ not passing, tagged -> remove label + regression comment (stays open)
+ not passing, untagged, old -> close + comment (past the grace window)
+ not passing, untagged, new -> one-time "what's missing" notice (within grace)
+
+ ``close`` gates every destructive side effect: with ``close=False`` the
+ function returns a ``would-*`` preview and touches nothing, mirroring the
+ dry-run contract of :func:`triage`. ``judge``/``greptile_score``/
+ ``comments``/``now`` are injectable for tests; in production they are
+ resolved from the OpenAI judge, the PR's Greptile comment, the live comment
+ list, and the wall clock respectively.
+ """
+ item = fetch_pr(repo, number)
+
+ title = item.get("title") or ""
+ body = item.get("body") or ""
+ login = (item.get("user") or {}).get("login") or ""
+ association = item.get("author_association") or ""
+ state = item.get("state") or ""
+ # GitHub label names are case-insensitive; compare lowercased so a repo
+ # that already has e.g. "Ready for Review" is recognized as the same
+ # label as our READY_FOR_REVIEW_LABEL constant ("ready for review").
+ labels_now = {(lbl.get("name") or "").lower() for lbl in (item.get("labels") or [])}
+ label_key = label.lower()
+ created_raw = item.get("created_at") or ""
+
+ base_result = {
+ "kind": "pr",
+ "number": number,
+ "title": title,
+ "author": login,
+ "author_association": association,
+ "state": state,
+ "labeled": label_key in labels_now,
+ "review_gate": True,
+ }
+
+ if state != "open":
+ return {**base_result, "action": "skip-not-open"}
+
+ if allowlist:
+ if login.lower() not in allowlist:
+ return {**base_result, "action": "skip-not-allowlisted"}
+ elif is_internal_contributor(item):
+ return {**base_result, "action": "skip-internal-author"}
+
+ # Resolve the comment list once — used for both the Greptile score and the
+ # marker-based dedup below.
+ if comments is _UNSET:
+ comments = list(_iter_paginated_json(f"repos/{repo}/issues/{number}/comments"))
+
+ # --- rubric verdict: linked-issue short-circuit, else the LLM judge -------
+ if has_linked_issue(body):
+ verdict = {
+ "verdict": "pass",
+ "linked_issue": True,
+ "missing": [],
+ "explanation": "Linked-issue regex matched; LLM was not called.",
+ }
+ rubric_pass = True
+ else:
+ prompt = build_pr_prompt(title=title, body=body)
+ if judge is None:
+ api_key = os.environ.get("OPENAI_API_KEY")
+ if not api_key:
+ return {**base_result, "action": "skip-no-llm-key"}
+ base_url = os.environ.get("OPENAI_BASE_URL") or None
+
+ def judge(p: str) -> str:
+ return call_llm_judge(
+ p, model=model, api_key=api_key, base_url=base_url
+ )
+
+ try:
+ verdict = parse_verdict(judge(prompt))
+ except Exception as exc: # noqa: BLE001 - judge errors must never act
+ return {**base_result, "action": "skip-llm-error", "error": str(exc)}
+ rubric_pass = (verdict.get("verdict") or "").lower() == "pass"
+
+ # --- Greptile score -------------------------------------------------------
+ if greptile_score is _UNSET:
+ extraction = extract_greptile_score(comments)
+ greptile_score = extraction[0] if extraction else None
+ greptile_ok = greptile_score is None or greptile_score >= min_greptile_score
+ passing = rubric_pass and greptile_ok
+
+ # --- age ------------------------------------------------------------------
+ age_days = None
+ if created_raw:
+ reference = now or dt.datetime.now(dt.timezone.utc)
+ age_days = (reference - parse_iso8601(created_raw)).days
+
+ label_present = label_key in labels_now
+ explanation = verdict.get("explanation") or ""
+ # When the rubric short-circuited to pass (linked-issue regex) but
+ # Greptile dragged the PR below the bar, the synthetic verdict's
+ # explanation ("LLM was not called") would mislead a contributor reading
+ # the regression / close comment. Surface the real reason instead.
+ if rubric_pass and not greptile_ok:
+ explanation = (
+ f"Greptile's most recent review scored this PR "
+ f"{greptile_score}/5 (below the {min_greptile_score}/5 bar)."
+ )
+ verdict = {**verdict, "explanation": explanation}
+ base_result = {
+ **base_result,
+ "verdict": verdict,
+ "greptile_score": greptile_score,
+ "passing": passing,
+ "age_days": age_days,
+ }
+
+ if passing:
+ if label_present:
+ return {**base_result, "action": "noop-passing"}
+ recovered = _has_marker(comments, REGRESSED_MARKER)
+ comment = (
+ format_all_clear_comment(verdict, greptile_score)
+ if recovered
+ else format_ready_for_review_comment(
+ verdict, greptile_score, min_greptile_score
+ )
+ )
+ if not close:
+ return {**base_result, "action": "would-label-ready", "comment": comment}
+ post_comment(repo, number, comment)
+ add_label(repo, number, label)
+ return {**base_result, "action": "labeled-ready", "comment": comment}
+
+ missing = _combine_missing(verdict, greptile_score, min_greptile_score)
+
+ if label_present:
+ comment = format_regression_comment(missing, explanation, grace_days)
+ if not close:
+ return {**base_result, "action": "would-remove-label", "comment": comment}
+ remove_label(repo, number, label)
+ post_comment(repo, number, comment)
+ return {**base_result, "action": "label-removed-regressed", "comment": comment}
+
+ # Not passing and not tagged. If the PR was previously tagged and then
+ # regressed (we removed the label and posted REGRESSED_MARKER), honor the
+ # "PR stays open — fix it and the tag comes back" promise from
+ # `format_regression_comment` and skip the close path. Without this guard,
+ # any PR older than `grace_days` would be closed on the next evaluation,
+ # giving the contributor no realistic window to address the regression.
+ #
+ # The promise has a deliberate expiration: once `grace_days` have elapsed
+ # since the regression notice, fall through to the close path so a PR that
+ # was abandoned post-regression doesn't sit open forever.
+ if _has_marker(comments, REGRESSED_MARKER):
+ reference = now or dt.datetime.now(dt.timezone.utc)
+ seconds_since_regression = seconds_since_latest_marker_comment(
+ comments, marker=REGRESSED_MARKER, now=reference
+ )
+ grace_seconds = grace_days * 86400
+ if seconds_since_regression is None or seconds_since_regression < grace_seconds:
+ return {**base_result, "action": "regressed-already-notified"}
+
+ # Not passing and not tagged: close if past the grace window, else notify once.
+ if age_days is not None and age_days >= grace_days:
+ comment = format_pr_close_comment({**verdict, "missing": missing})
+ if not close:
+ return {**base_result, "action": "would-close", "comment": comment}
+ post_comment(repo, number, comment)
+ close_pr(repo, number)
+ return {**base_result, "action": "closed", "comment": comment}
+
+ if _has_marker(comments, WITHIN_GRACE_MARKER):
+ return {**base_result, "action": "within-grace-already-notified"}
+ comment = format_within_grace_comment(missing, explanation, grace_days)
+ if not close:
+ return {
+ **base_result,
+ "action": "would-notify-within-grace",
+ "comment": comment,
+ }
+ post_comment(repo, number, comment)
+ return {**base_result, "action": "within-grace-notified", "comment": comment}
+
+
+def triage(
+ *,
+ repo: str,
+ kind: str,
+ number: int,
+ close: bool,
+ model: str,
+ judge: Any = None,
+ print_prompt: bool = False,
+ reconsider: bool = False,
+ allowlist: frozenset[str] = ALLOWLIST_LOGINS,
+) -> dict:
+ """Triage a single PR or issue. Returns a result dict for logging/tests.
+
+ `judge` is an optional callable `(prompt) -> str` for tests / dry-run with
+ a stub. In production, leave it None and the script uses `call_llm_judge`.
+
+ When `reconsider=True`, the closed-state guard is skipped and a
+ fail-but-no-comment is replaced with a "still failing" comment + leave
+ closed; a pass triggers `reopen_pr`/`reopen_issue` plus a reopen comment.
+ Reconsider mode is intended for the `@agent-shin reconsider` comment
+ trigger. Like regular triage, `close=False` keeps reconsider in dry-run
+ (returns `would-reopen` / `would-reconsider-still-failing` so a local
+ operator can preview without write side effects); the workflow only
+ passes `--close` when `AGENT_SHIN_ENABLED=true`.
+
+ Reconsider mode adds two extra safety guards on top of the regular
+ triage skip-internal-author check:
+
+ 1. **Bot-closed guard.** Only reopens if the most recent close was
+ performed by the bot identity (default `github-actions[bot]`).
+ This stops a contributor from using `@agent-shin reconsider` to
+ override a maintainer's close for non-rubric reasons.
+ 2. **Rate-limit guard.** If the bot has already posted a reconsider
+ verdict on this PR/issue within `RECONSIDER_RATE_LIMIT_SECONDS`,
+ skip — repeated triggers from the same contributor shouldn't burn
+ CI minutes or LLM budget.
+ """
+ fetcher = {"pr": fetch_pr, "issue": fetch_issue}[kind]
+ item = fetcher(repo, number)
+
+ title = item.get("title") or ""
+ body = item.get("body") or ""
+ login = (item.get("user") or {}).get("login") or ""
+ association = item.get("author_association") or ""
+ state = item.get("state") or ""
+
+ base_result = {
+ "kind": kind,
+ "number": number,
+ "title": title,
+ "author": login,
+ "author_association": association,
+ "state": state,
+ "reconsider": reconsider,
+ }
+
+ # Reconsider only makes sense on a closed PR/issue. A "reconsider on an
+ # open PR" is a no-op (the regular triage flow already evaluates open
+ # PRs); return a clear skip so the workflow can short-circuit.
+ if reconsider:
+ if state != "closed":
+ return {**base_result, "action": "skip-not-closed"}
+ else:
+ if state != "open":
+ return {**base_result, "action": "skip-not-open"}
+
+ if allowlist:
+ if login.lower() not in allowlist:
+ return {**base_result, "action": "skip-not-allowlisted"}
+ elif is_internal_contributor(item):
+ return {**base_result, "action": "skip-internal-author"}
+
+ # Reconsider-only guards — these run BEFORE the LLM call so a
+ # maintainer-closed PR / rate-limited trigger never spends LLM budget.
+ if reconsider:
+ if not was_closed_by_agent_shin(repo, number):
+ return {**base_result, "action": "skip-not-bot-closed"}
+ age = seconds_since_last_reconsider_verdict(repo, number)
+ if age is not None and age < RECONSIDER_RATE_LIMIT_SECONDS:
+ return {
+ **base_result,
+ "action": "skip-rate-limited",
+ "rate_limit_age_seconds": age,
+ "rate_limit_window_seconds": RECONSIDER_RATE_LIMIT_SECONDS,
+ }
+
+ if kind == "pr":
+ # Short-circuit: if body very clearly links a related issue, just pass.
+ if has_linked_issue(body):
+ base = {
+ **base_result,
+ "action": "pass-linked-issue",
+ "verdict": {
+ "verdict": "pass",
+ "linked_issue": True,
+ "explanation": "Linked-issue regex matched; LLM was not called.",
+ },
+ }
+ if reconsider:
+ # Pass-on-reconsider -> reopen the PR with a friendly comment.
+ reopen_body = format_reopen_comment(kind)
+ if not close:
+ return {
+ **base,
+ "action": "would-reopen",
+ "comment": reopen_body,
+ }
+ post_comment(repo, number, reopen_body)
+ reopen_pr(repo, number)
+ return {
+ **base,
+ "action": "reopened",
+ "comment": reopen_body,
+ }
+ return base
+ prompt = build_pr_prompt(title=title, body=body)
+ else:
+ prompt = build_issue_prompt(title=title, body=body)
+
+ if print_prompt:
+ return {**base_result, "action": "print-prompt", "prompt": prompt}
+
+ if judge is None:
+ api_key = os.environ.get("OPENAI_API_KEY")
+ if not api_key:
+ # No key configured — never take a destructive action. Report skip.
+ return {
+ **base_result,
+ "action": "skip-no-llm-key",
+ "prompt_preview": prompt[:200],
+ }
+ base_url = os.environ.get("OPENAI_BASE_URL") or None
+
+ def judge(p: str) -> str:
+ return call_llm_judge(p, model=model, api_key=api_key, base_url=base_url)
+
+ try:
+ raw = judge(prompt)
+ verdict = parse_verdict(raw)
+ except Exception as exc: # noqa: BLE001 - judge errors must never close PRs
+ return {**base_result, "action": "skip-llm-error", "error": str(exc)}
+
+ decision = (verdict.get("verdict") or "").lower()
+
+ if reconsider:
+ # Reconsider: an explicit `pass` -> reopen + post reopen comment;
+ # anything else (fail, missing/malformed verdict, typo) -> leave
+ # closed + post a "still failing" comment so the contributor can
+ # iterate again. Reopen is destructive, so a flaky/empty verdict
+ # must not satisfy the gate.
+ # In dry-run (`close=False`) we return `would-*` actions instead
+ # of touching GitHub state, mirroring the regular triage flow's
+ # `would-close`. This lets a local operator preview the outcome
+ # of `python triage_with_llm.py --reconsider --pr N` without
+ # risking accidental comments or reopens.
+ if decision == "pass":
+ reopen_body = format_reopen_comment(kind)
+ if not close:
+ return {
+ **base_result,
+ "action": "would-reopen",
+ "verdict": verdict,
+ "comment": reopen_body,
+ }
+ post_comment(repo, number, reopen_body)
+ if kind == "pr":
+ reopen_pr(repo, number)
+ else:
+ reopen_issue(repo, number)
+ return {
+ **base_result,
+ "action": "reopened",
+ "verdict": verdict,
+ "comment": reopen_body,
+ }
+ still_failing = format_reconsider_still_failing_comment(kind, verdict)
+ if not close:
+ return {
+ **base_result,
+ "action": "would-reconsider-still-failing",
+ "verdict": verdict,
+ "comment": still_failing,
+ }
+ post_comment(repo, number, still_failing)
+ return {
+ **base_result,
+ "action": "reconsider-still-failing",
+ "verdict": verdict,
+ "comment": still_failing,
+ }
+
+ if decision != "fail":
+ return {**base_result, "action": "pass-llm", "verdict": verdict}
+
+ # Grace-period flow: on the first low-quality detection, post a warning
+ # comment instead of closing immediately. On a subsequent triage run
+ # (manual re-trigger, or the daily `close_low_quality_prs.py` cron
+ # finding the same PR in its own pass), if `GRACE_PERIOD_SECONDS` has
+ # elapsed since the warning AND the PR still fails the rubric, close.
+ grace_age = seconds_since_last_grace_warning(repo, number)
+ if grace_age is None:
+ warning_body = (
+ format_grace_warning_pr_comment(verdict)
+ if kind == "pr"
+ else format_grace_warning_issue_comment(verdict)
+ )
+ if not close:
+ return {
+ **base_result,
+ "action": "would-warn-grace",
+ "verdict": verdict,
+ "comment": warning_body,
+ }
+ post_comment(repo, number, warning_body)
+ return {
+ **base_result,
+ "action": "warned-grace",
+ "verdict": verdict,
+ "comment": warning_body,
+ }
+ if grace_age < GRACE_PERIOD_SECONDS:
+ return {
+ **base_result,
+ "action": "skip-in-grace-period",
+ "verdict": verdict,
+ "grace_age_seconds": grace_age,
+ "grace_period_seconds": GRACE_PERIOD_SECONDS,
+ }
+
+ # The grace window has elapsed. `--close` still gates the destructive
+ # write so a dry-run preview never posts or closes — the workflow only
+ # passes `--close` when `AGENT_SHIN_ENABLED=true`, which keeps the bot
+ # inert by default.
+ if not close:
+ return {**base_result, "action": "would-close", "verdict": verdict}
+
+ comment_body = (
+ format_pr_close_comment(verdict)
+ if kind == "pr"
+ else format_issue_close_comment(verdict)
+ )
+ post_comment(repo, number, comment_body)
+ if kind == "pr":
+ close_pr(repo, number)
+ else:
+ close_issue(repo, number)
+
+ return {
+ **base_result,
+ "action": "closed",
+ "verdict": verdict,
+ "comment": comment_body,
+ }
+
+
+# ---------------------------------------------------------------------------
+# CLI
+
+
+def render_summary(result: dict) -> str:
+ """Render a human-readable summary block (used for stdout + step summary)."""
+ lines = ["## Agent Shin verdict", ""]
+ lines.append(
+ f"- **{result['kind'].upper()} #{result['number']}**: {result.get('title', '')}"
+ )
+ lines.append(
+ f"- **Author**: `{result.get('author', '')}` ({result.get('author_association', '')})"
+ )
+ lines.append(f"- **State**: {result.get('state', '')}")
+ lines.append(f"- **Action**: `{result['action']}`")
+ verdict = result.get("verdict")
+ if verdict:
+ lines.append("")
+ lines.append("```json")
+ lines.append(json.dumps(verdict, indent=2))
+ lines.append("```")
+ error = result.get("error")
+ if error:
+ lines.append("")
+ lines.append(f"_LLM error: {error}_")
+ comment = result.get("comment")
+ if comment:
+ lines.append("")
+ lines.append("### Posted comment:")
+ lines.append("")
+ lines.append("> " + comment.replace("\n", "\n> "))
+ return "\n".join(lines)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--repo", required=True, help="Repository (owner/repo).")
+ target = parser.add_mutually_exclusive_group(required=True)
+ target.add_argument("--pr", type=int, help="Pull request number to triage.")
+ target.add_argument("--issue", type=int, help="Issue number to triage.")
+ parser.add_argument(
+ "--close",
+ action="store_true",
+ help="Actually post comment + close on fail (default: dry run).",
+ )
+ parser.add_argument(
+ "--model",
+ # `os.environ.get("TRIAGE_MODEL", DEFAULT_MODEL)` would return "" when
+ # GitHub Actions exposes an unset repo variable as an empty-string env
+ # var, silently bypassing DEFAULT_MODEL and causing every call to fail
+ # as `skip-llm-error`. The `or` guard collapses empty -> default.
+ default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL,
+ help=f"OpenAI-compatible model name (default: {DEFAULT_MODEL}).",
+ )
+ parser.add_argument(
+ "--print-prompt",
+ action="store_true",
+ help="Print the prompt that would be sent to the judge and exit.",
+ )
+ parser.add_argument(
+ "--reconsider",
+ action="store_true",
+ help=(
+ "Re-run triage on a CLOSED PR/issue and reopen it on pass. "
+ "Used by the `@agent-shin reconsider` comment-trigger workflow. "
+ "Only invoke this from a workflow that has already gated on "
+ "AGENT_SHIN_ENABLED=true and verified the commenter is the "
+ "PR/issue author or an internal collaborator."
+ ),
+ )
+ parser.add_argument(
+ "--review-gate",
+ action="store_true",
+ help=(
+ "Reconcile the `ready for review` label for an OPEN PR: tag on "
+ "pass, remove the tag + comment on regression, close after the "
+ "grace window if it never passed. PR-only."
+ ),
+ )
+ parser.add_argument(
+ "--grace-days",
+ type=int,
+ default=DEFAULT_GRACE_DAYS,
+ help=(
+ "Review-gate only: hours/24 a failing, un-tagged PR may stay open "
+ f"before auto-close (default: {DEFAULT_GRACE_DAYS} = 24h)."
+ ),
+ )
+ parser.add_argument(
+ "--min-greptile-score",
+ type=int,
+ default=DEFAULT_MIN_GREPTILE_SCORE,
+ choices=range(1, 6),
+ help=(
+ "Review-gate only: Greptile score below which a PR counts as not "
+ f"passing (default: {DEFAULT_MIN_GREPTILE_SCORE} -> <4/5 regresses)."
+ ),
+ )
+ args = parser.parse_args()
+
+ kind = "pr" if args.pr is not None else "issue"
+ number = args.pr if args.pr is not None else args.issue
+
+ if args.review_gate:
+ if kind != "pr":
+ parser.error("--review-gate applies to pull requests only (use --pr).")
+ result = review_gate(
+ repo=args.repo,
+ number=number,
+ close=args.close,
+ model=args.model,
+ grace_days=args.grace_days,
+ min_greptile_score=args.min_greptile_score,
+ )
+ else:
+ result = triage(
+ repo=args.repo,
+ kind=kind,
+ number=number,
+ close=args.close,
+ model=args.model,
+ print_prompt=args.print_prompt,
+ reconsider=args.reconsider,
+ )
+
+ if result.get("action") == "print-prompt":
+ print(result["prompt"])
+ return 0
+
+ summary = render_summary(result)
+ print(summary)
+ write_step_summary(summary + "\n")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml
index eeb5545b15e..d8053c15683 100644
--- a/.github/workflows/check-ui-api-types.yml
+++ b/.github/workflows/check-ui-api-types.yml
@@ -54,7 +54,7 @@ jobs:
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Set up Node.js
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0
+ uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version: "20"
cache: "npm"
diff --git a/.github/workflows/close_low_quality_prs.yml b/.github/workflows/close_low_quality_prs.yml
new file mode 100644
index 00000000000..2401be84000
--- /dev/null
+++ b/.github/workflows/close_low_quality_prs.yml
@@ -0,0 +1,92 @@
+name: Close Low-Quality PRs
+
+# Auto-close any open PR (including drafts, regardless of age) authored by an
+# external OSS contributor that Greptile reviewed with a confidence score
+# below 4/5. Closures are explained in a comment that tells the contributor
+# to push fixes and open a fresh PR (since OSS authors cannot reopen a PR
+# closed by a bot/maintainer) or comment `@agent-shin reconsider` to have
+# Agent Shin re-evaluate.
+#
+# Manual one-off run:
+# gh workflow run "Close Low-Quality PRs" -f close=true
+#
+# Dry-run preview (no PRs are touched):
+# gh workflow run "Close Low-Quality PRs" -f close=false
+
+on:
+ schedule:
+ # Daily at 09:00 UTC. Pairs well with the stale-issue workflow at midnight.
+ - cron: "0 9 * * *"
+ workflow_dispatch:
+ inputs:
+ close:
+ description: "Actually close matching PRs (false = dry run)."
+ required: false
+ default: "false"
+ type: choice
+ options:
+ - "true"
+ - "false"
+ min_age_days:
+ description: "Minimum PR age in days (default 0 = no age filter)."
+ required: false
+ default: "0"
+ min_score:
+ description: "Greptile score below which a PR is closed (1-5)."
+ required: false
+ default: "4"
+ limit:
+ description: "Maximum number of PRs to close in a single run."
+ required: false
+ default: "25"
+
+permissions:
+ contents: read
+ pull-requests: write
+ issues: write
+
+jobs:
+ close-low-quality-prs:
+ if: github.repository == 'BerriAI/litellm'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout triage script
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ sparse-checkout: .github/scripts
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Run low-quality PR closer
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Scheduled runs are ALWAYS dry-run, even when AGENT_SHIN_ENABLED is
+ # "true", so the team can QA the closer's verdicts in step summaries
+ # before any contributor sees a PR closed. Real closures only happen
+ # on manual workflow_dispatch with close=true (and the variable set).
+ CLOSE_FLAG: ${{ github.event.inputs.close || 'false' }}
+ AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
+ MIN_AGE_DAYS: ${{ github.event.inputs.min_age_days || '0' }}
+ MIN_SCORE: ${{ github.event.inputs.min_score || '4' }}
+ LIMIT: ${{ github.event.inputs.limit || '25' }}
+ run: |
+ set -euo pipefail
+ ARGS=(
+ --repo "${{ github.repository }}"
+ --min-age-days "${MIN_AGE_DAYS}"
+ --min-score "${MIN_SCORE}"
+ --limit "${LIMIT}"
+ )
+ if [ "${AGENT_SHIN_ENABLED:-false}" != "true" ]; then
+ echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> forcing dry-run regardless of close input."
+ elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${CLOSE_FLAG}" = "true" ]; then
+ ARGS+=(--close)
+ echo "::notice::Running in close-on-fail mode."
+ else
+ echo "::notice::AGENT_SHIN_ENABLED is true but this trigger is dry-run (scheduled event or close=false)."
+ fi
+ python3 .github/scripts/close_low_quality_prs.py "${ARGS[@]}"
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index babe3b62933..d3a165a11da 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -43,14 +43,14 @@ jobs:
persist-credentials: false
- name: Initialize CodeQL
- uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
+ uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: ./.github/codeql/codeql-config.yml
- name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
+ uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
with:
category: "/language:${{ matrix.language }}"
output: sarif-results
@@ -77,7 +77,7 @@ jobs:
output: sarif-results/python.sarif
- name: Upload SARIF
- uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
+ uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
with:
sarif_file: sarif-results
category: "/language:${{ matrix.language }}"
diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml
index de7e1b68346..950d6ca31a6 100644
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@@ -14,7 +14,7 @@ permissions:
jobs:
lint:
runs-on: ubuntu-latest
- timeout-minutes: 10
+ timeout-minutes: 15
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -87,9 +87,11 @@ jobs:
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- - name: Run basedpyright type checking
+ - name: Check basedpyright budget (delta vs base)
+ env:
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
- (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py
+ (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
- name: Check for circular imports
run: |
diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml
index 68497b10dbb..b83119712a7 100644
--- a/.github/workflows/test-litellm-ui-build.yml
+++ b/.github/workflows/test-litellm-ui-build.yml
@@ -25,7 +25,7 @@ jobs:
persist-credentials: false
- name: Setup Node.js
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0
+ uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version: "20"
cache: "npm"
@@ -77,7 +77,7 @@ jobs:
- name: Setup Node.js
if: steps.changed.outputs.has_files == 'true'
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0
+ uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version: "20"
cache: "npm"
diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml
new file mode 100644
index 00000000000..3d0a159cdc7
--- /dev/null
+++ b/.github/workflows/test-rust.yml
@@ -0,0 +1,65 @@
+name: LiteLLM Rust
+
+on:
+ push:
+ paths:
+ - "litellm-rust/**"
+ - ".github/workflows/test-rust.yml"
+ pull_request:
+ branches:
+ - main
+ - litellm_internal_staging
+ - litellm_oss_branch
+ - "litellm_**"
+ paths:
+ - "litellm-rust/**"
+ - ".github/workflows/test-rust.yml"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ rust-checks:
+ name: rustfmt, clippy, test
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ defaults:
+ run:
+ working-directory: litellm-rust
+ env:
+ CARGO_TERM_COLOR: always
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Rust
+ run: |
+ rustup toolchain install stable --profile minimal --component clippy,rustfmt
+ rustup default stable
+
+ - name: Cache Cargo registry and target
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ litellm-rust/target
+ key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-
+
+ - name: Check Rust formatting
+ run: cargo fmt --check
+
+ - name: Run Clippy
+ run: cargo clippy --workspace --all-targets --locked -- -D warnings
+
+ - name: Run Rust tests
+ run: cargo test --workspace --locked
diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml
index a7363ac3b43..2226d519331 100644
--- a/.github/workflows/test-unit-misc.yml
+++ b/.github/workflows/test-unit-misc.yml
@@ -32,7 +32,9 @@ jobs:
tests/test_litellm/repositories
tests/test_litellm/images
tests/test_litellm/interactions
+ tests/test_litellm/ocr
tests/test_litellm/passthrough
+ tests/test_litellm/sandbox
tests/test_litellm/vector_stores
tests/test_litellm/test_*.py
workers: 2
diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml
index 0a9513ec024..d9b6a348b60 100644
--- a/.github/workflows/test-unit-proxy-endpoints.yml
+++ b/.github/workflows/test-unit-proxy-endpoints.yml
@@ -11,8 +11,6 @@ on:
permissions:
contents: read
- id-token: write
- pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -20,6 +18,10 @@ concurrency:
jobs:
proxy-endpoints:
+ permissions:
+ contents: read
+ id-token: write
+ pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
@@ -52,6 +54,10 @@ jobs:
# is independent and its coverage artifact is uploaded separately.
# See: https://www.notion.so/36c43b8acdab81ee845fd5365128a2fc
proxy-server:
+ permissions:
+ contents: read
+ id-token: write
+ pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: tests/test_litellm/proxy/proxy_server
diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml
index 57ff746c9c8..985653796c2 100644
--- a/.github/workflows/test_server_root_path.yml
+++ b/.github/workflows/test_server_root_path.yml
@@ -32,17 +32,16 @@ jobs:
df -h /
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12
+ uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
- name: Build Docker image
- uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14
+ uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 # v6.14.0
with:
context: .
file: ./docker/Dockerfile.non_root
tags: litellm-test:${{ github.sha }}
load: true
- cache-from: type=gha
- cache-to: type=gha,mode=max
+ push: false
- name: Start LiteLLM container with SERVER_ROOT_PATH
run: |
diff --git a/.github/workflows/triage_issue_with_llm.yml b/.github/workflows/triage_issue_with_llm.yml
new file mode 100644
index 00000000000..765453cf2c6
--- /dev/null
+++ b/.github/workflows/triage_issue_with_llm.yml
@@ -0,0 +1,96 @@
+name: Agent Shin — Issue triage
+
+# LLM-as-judge triage for external GitHub issues.
+#
+# DRY-RUN BY DEFAULT. See .github/workflows/triage_pr_with_llm.yml for the
+# enablement procedure — same repo variable (`AGENT_SHIN_ENABLED=true`)
+# unlocks the PR and issue triage flows together.
+
+on:
+ issues:
+ types: [opened, reopened]
+ workflow_dispatch:
+ inputs:
+ issue_number:
+ description: "Issue number to triage manually."
+ required: true
+ close:
+ description: "If true and AGENT_SHIN_ENABLED=true, actually close on fail."
+ required: false
+ default: "false"
+ type: choice
+ options:
+ - "true"
+ - "false"
+
+permissions:
+ contents: read
+ issues: write
+
+jobs:
+ triage:
+ if: github.repository == 'BerriAI/litellm'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout triage script
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ sparse-checkout: .github/scripts
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install LLM client
+ run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
+
+ - name: Run Agent Shin
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Only expose the LLM key when the bot is enabled or a collaborator
+ # triggers it manually, so an external user can't force paid LLM
+ # calls by churning issues while the bot is still in dry-run.
+ # The Python script calls the LLM whenever this var is set
+ # (regardless of `--close`); stripping `--close` doesn't suppress
+ # the API call, only the destructive side effects.
+ OPENAI_API_KEY: ${{ (vars.AGENT_SHIN_ENABLED == 'true' || github.event_name == 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}
+ OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
+ TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
+ AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
+ DISPATCH_CLOSE: ${{ github.event.inputs.close }}
+ ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }}
+ run: |
+ set -euo pipefail
+ ARGS=(--repo "${{ github.repository }}" --issue "${ISSUE_NUMBER}")
+ # Fail-safe gating: only the EXACT string "true" enables the
+ # destructive --close path. The workflow_dispatch input is a
+ # `choice` dropdown of "true"/"false" so the UI is constrained,
+ # but the API (`gh workflow run -f close=...`) accepts any
+ # string, and a `!= "false"` check would treat "True", "yes",
+ # "1", "TRUE", typos, and accidental whitespace as enabling
+ # closure. Mirror the Greptile closer's `= "true"` pattern.
+ if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ] && [ "${DISPATCH_CLOSE:-false}" = "true" ]; then
+ ARGS+=(--close)
+ echo "::notice::Agent Shin is ENABLED and running in close-on-fail mode."
+ elif [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
+ echo "::notice::Agent Shin is ENABLED but this trigger is dry-run (workflow_dispatch close != 'true')."
+ else
+ echo "::notice::Agent Shin is in DRY-RUN mode (AGENT_SHIN_ENABLED is not 'true'). No comments will be posted; no issues will be closed."
+ fi
+ # Automatic `issues` events stay dry-run regardless until the team
+ # explicitly invokes workflow_dispatch with close=true.
+ if [ "${GITHUB_EVENT_NAME:-}" = "issues" ]; then
+ # filter out --close rather than substituting to "" (which would
+ # leave an empty positional arg that argparse rejects)
+ FILTERED=()
+ for arg in "${ARGS[@]}"; do
+ if [ "${arg}" != "--close" ]; then
+ FILTERED+=("${arg}")
+ fi
+ done
+ ARGS=("${FILTERED[@]}")
+ echo "::notice::issues trigger -> forcing dry-run."
+ fi
+ python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"
diff --git a/.github/workflows/triage_reconsider.yml b/.github/workflows/triage_reconsider.yml
new file mode 100644
index 00000000000..f35f681d09a
--- /dev/null
+++ b/.github/workflows/triage_reconsider.yml
@@ -0,0 +1,172 @@
+name: Agent Shin — reconsider
+
+# Comment-trigger workflow: when the PR/issue author (or an internal
+# collaborator) comments `@agent-shin reconsider` on a CLOSED PR/issue,
+# Agent Shin re-runs LLM-judge triage on the current title+body and:
+#
+# - on PASS: posts a "re-evaluated and reopened" comment + reopens.
+# - on FAIL: posts a "still missing X" comment and leaves it closed,
+# so the contributor can iterate again.
+#
+# This exists because GitHub does NOT let an external (non-write-access)
+# OSS contributor reopen a PR/issue closed by a bot or maintainer. Without
+# this comment trigger, a contributor whose PR Agent Shin auto-closed
+# would have no path back into the review queue except opening a fresh PR
+# (which loses the original PR's history). The bot, on the other hand,
+# has write access via GH_TOKEN and can reopen on their behalf.
+#
+# DRY-RUN BY DEFAULT — gated on `vars.AGENT_SHIN_ENABLED == 'true'` just
+# like the other Agent Shin workflows. The workflow also gates on the
+# commenter being either the PR/issue author or an internal collaborator
+# (OWNER/MEMBER/COLLABORATOR) so random commenters cannot DOS the LLM
+# judge or force a reopen.
+
+on:
+ issue_comment:
+ types: [created]
+
+permissions:
+ contents: read
+ issues: write
+ pull-requests: write
+
+jobs:
+ reconsider:
+ if: |
+ github.repository == 'BerriAI/litellm'
+ && contains(github.event.comment.body, '@agent-shin reconsider')
+ runs-on: ubuntu-latest
+ steps:
+ - name: Authorize commenter
+ # Only the PR/issue author OR an internal collaborator may trigger
+ # a reconsider. Outside random commenters could otherwise spam the
+ # phrase to burn LLM budget or, if a fail-open bug were ever
+ # introduced, force a reopen on someone else's behalf.
+ #
+ # We expose the authorization decision as a step output and gate
+ # every subsequent (potentially destructive) step on it. A `run:`
+ # step with `exit 0` would NOT stop the job — only `if:` gating
+ # on a known-true output is safe here.
+ id: auth
+ env:
+ COMMENTER: ${{ github.event.comment.user.login }}
+ AUTHOR: ${{ github.event.issue.user.login }}
+ ASSOCIATION: ${{ github.event.comment.author_association }}
+ run: |
+ set -euo pipefail
+ if [ "${COMMENTER}" = "${AUTHOR}" ]; then
+ echo "::notice::Authorized: commenter is the PR/issue author."
+ echo "authorized=true" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ case "${ASSOCIATION}" in
+ OWNER|MEMBER|COLLABORATOR)
+ echo "::notice::Authorized: commenter is an internal collaborator (${ASSOCIATION})."
+ echo "authorized=true" >> "$GITHUB_OUTPUT"
+ ;;
+ *)
+ echo "::notice::Commenter '${COMMENTER}' (${ASSOCIATION}) is not authorized to trigger reconsider; skipping subsequent steps."
+ echo "authorized=false" >> "$GITHUB_OUTPUT"
+ ;;
+ esac
+
+ - name: React 👀 to acknowledge the reconsider
+ # Add an eyes reaction to the triggering comment the moment we accept
+ # it, so the contributor gets instant feedback that the bot saw their
+ # `@agent-shin reconsider` before the slower triage steps run. Gated on
+ # AGENT_SHIN_ENABLED so dry-run leaves no visible trace. Best-effort:
+ # a reactions API hiccup must never fail the actual reconsider.
+ if: steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ COMMENT_ID: ${{ github.event.comment.id }}
+ run: |
+ set -euo pipefail
+ gh api --method POST \
+ -H "Accept: application/vnd.github+json" \
+ "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \
+ -f content=eyes \
+ || echo "::warning::failed to add 👀 reaction (non-fatal)"
+
+ - name: Checkout triage script
+ if: steps.auth.outputs.authorized == 'true'
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ sparse-checkout: .github/scripts
+ persist-credentials: false
+
+ - name: Set up Python
+ if: steps.auth.outputs.authorized == 'true'
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install LLM client
+ if: steps.auth.outputs.authorized == 'true'
+ run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
+
+ - name: Run Agent Shin reconsider
+ if: steps.auth.outputs.authorized == 'true'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Only expose the LLM key when the bot is enabled, so a PR/issue
+ # author can't force paid LLM calls by spamming `@agent-shin
+ # reconsider` while the bot is still in dry-run. The Python script
+ # calls the LLM whenever this var is set (regardless of `--close`);
+ # stripping `--close` doesn't suppress the API call, only the
+ # destructive side effects. Mirror the gating used by every other
+ # Agent Shin workflow (triage_pr_with_llm.yml, review_gate.yml, ...).
+ OPENAI_API_KEY: ${{ vars.AGENT_SHIN_ENABLED == 'true' && secrets.OPENAI_API_KEY || '' }}
+ OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
+ TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
+ AGENT_SHIN_ENABLED: ${{ vars.AGENT_SHIN_ENABLED }}
+ # `issue_comment` events fire for both issues and PR comments.
+ # `issue.pull_request` is set iff this is a PR comment, so we use
+ # its presence to decide whether to invoke `--pr N` or `--issue N`.
+ IS_PR: ${{ github.event.issue.pull_request != null }}
+ NUMBER: ${{ github.event.issue.number }}
+ run: |
+ set -euo pipefail
+ if [ "${IS_PR}" = "true" ]; then
+ ARGS=(--repo "${{ github.repository }}" --pr "${NUMBER}" --reconsider)
+ else
+ ARGS=(--repo "${{ github.repository }}" --issue "${NUMBER}" --reconsider)
+ fi
+ # Reconsider's destructive actions (post comment + reopen) are
+ # gated on `--close`, mirroring the regular triage workflows.
+ # When AGENT_SHIN_ENABLED is not the EXACT string "true", we
+ # still run the script so its verdict + would-X action lands in
+ # the step summary for QA — but without `--close`, the script
+ # returns `would-reopen` / `would-reconsider-still-failing`
+ # instead of touching GitHub state.
+ #
+ # Use the positive `= "true"` gate (not `!= "true" -> exit`) so
+ # the workflow guardrails in
+ # tests/test_litellm/test_github_triage_workflows.py see the
+ # canonical fail-safe enable pattern. Unknown values like
+ # "True", "yes", "1", or typos fall through to the dry-run
+ # branch, which is the safe default.
+ if [ "${AGENT_SHIN_ENABLED:-false}" = "true" ]; then
+ ARGS+=(--close)
+ echo "::notice::Agent Shin reconsider ENABLED — running real triage (close=true)."
+ else
+ echo "::notice::AGENT_SHIN_ENABLED is not 'true' -> reconsider stays in dry-run (no comment, no reopen)."
+ fi
+ python3 .github/scripts/triage_with_llm.py "${ARGS[@]}"
+
+ - name: React 👍 when the reconsider finishes
+ # Once the reconsider run has completed successfully, add a thumbs-up so
+ # the contributor sees the bot is done (the 👀 stays, signalling
+ # seen -> handled). `success()` keeps this from firing if the run
+ # errored, and the AGENT_SHIN_ENABLED gate keeps dry-run inert.
+ if: success() && steps.auth.outputs.authorized == 'true' && vars.AGENT_SHIN_ENABLED == 'true'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ COMMENT_ID: ${{ github.event.comment.id }}
+ run: |
+ set -euo pipefail
+ gh api --method POST \
+ -H "Accept: application/vnd.github+json" \
+ "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}/reactions" \
+ -f content=+1 \
+ || echo "::warning::failed to add 👍 reaction (non-fatal)"
diff --git a/.github/workflows/triage_rollout_heads_up.yml b/.github/workflows/triage_rollout_heads_up.yml
new file mode 100644
index 00000000000..903960151e2
--- /dev/null
+++ b/.github/workflows/triage_rollout_heads_up.yml
@@ -0,0 +1,92 @@
+name: Agent Shin — rollout heads-up (one-shot)
+
+# Fires the 7-day heads-up comment on every open external PR/issue that the
+# new triage bot would auto-close. The real sweep is a deliberate one-shot:
+# trigger it at rollout via a manual `workflow_dispatch` with `dry_run=false`.
+# The script is idempotent (skips items that already carry the
+# `` marker), so a re-run is harmless.
+#
+# The automatic push trigger runs DRY-RUN only, so merging the script to
+# `litellm_internal_staging` never posts a comment; it just confirms the
+# workflow is wired up. Posting real comments requires the manual dispatch,
+# which is also the only trigger that exposes `OPENAI_API_KEY`. The heads-up
+# is intentionally NOT gated on `AGENT_SHIN_ENABLED`: it has to warn
+# contributors while that flag is still off, ahead of the flip that turns on
+# auto-closing.
+#
+# The workflow is a thin shell over `.github/scripts/triage_rollout_heads_up.py`.
+# Dry-run vs. real run differ in EXACTLY one CLI flag (`--close`), added only
+# on a manual dispatch with `dry_run=false`.
+
+on:
+ push:
+ branches:
+ - litellm_internal_staging
+ paths:
+ # The presence of this script on staging IS the rollout merge marker.
+ # Editing the file later would re-fire the workflow; that's safe because
+ # the script skips PRs/issues that already have the heads-up marker.
+ - ".github/scripts/triage_rollout_heads_up.py"
+ workflow_dispatch:
+ inputs:
+ dry_run:
+ description: "Dry run (true = preview only, false = actually post comments)."
+ required: false
+ default: "true"
+ type: choice
+ options:
+ - "true"
+ - "false"
+
+permissions:
+ contents: read
+ issues: write
+ pull-requests: write
+
+jobs:
+ heads-up:
+ if: github.repository == 'BerriAI/litellm'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout triage scripts
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ sparse-checkout: .github/scripts
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install LLM client
+ run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
+
+ - name: Run heads-up sweep
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Only the manual dispatch (the real-run trigger) needs the LLM key.
+ # The automatic push trigger runs dry-run and never posts, so it gets
+ # no key. Mirrors the sibling triage workflows, which expose the key
+ # only on an enabled/dispatched run rather than unconditionally.
+ OPENAI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.OPENAI_API_KEY || '' }}
+ OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
+ TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
+ # The real run is a deliberate manual dispatch with dry_run=false.
+ # Use the EXACT "false" comparison so any unexpected input value
+ # fail-closes to dry-run (mirrors the AGENT_SHIN_ENABLED pattern in
+ # the sibling workflows). The automatic push trigger always stays
+ # dry-run, so merging the script never posts.
+ DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }}
+ run: |
+ set -euo pipefail
+ ARGS=(--repo "${{ github.repository }}")
+ if [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${DRY_RUN_INPUT:-true}" = "false" ]; then
+ ARGS+=(--close)
+ echo "::notice::Manual rollout dispatch with dry_run=false -> heads-up comments WILL be posted."
+ elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ]; then
+ echo "::notice::Manual dispatch in dry-run mode -> previewing only, no comments will be posted."
+ else
+ echo "::notice::Automatic push trigger -> dry-run preview only. Fire the real rollout sweep with a manual workflow_dispatch (dry_run=false)."
+ fi
+ python3 .github/scripts/triage_rollout_heads_up.py "${ARGS[@]}"
diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml
index 9a1e899fed5..db79fe43038 100644
--- a/.github/workflows/zizmor.yml
+++ b/.github/workflows/zizmor.yml
@@ -2,9 +2,9 @@ name: GitHub Actions Security Analysis
on:
push:
- branches: [main]
+ branches: [main, litellm_internal_staging]
pull_request:
- branches: [main]
+ branches: [main, litellm_internal_staging]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -18,9 +18,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
- security-events: write
contents: read
- actions: read
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -28,4 +26,9 @@ jobs:
persist-credentials: false
- name: Run zizmor
- uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2
+ uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
+ with:
+ version: "1.24.1"
+ min-severity: medium
+ advanced-security: false
+ annotations: true
diff --git a/Makefile b/Makefile
index 6183dff1556..076eac0f4a7 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev format \
lint-basedpyright lint-basedpyright-budget-update \
- lint-ruff-budget lint-ruff-budget-update lint-budget-update \
+ lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety
@@ -28,6 +28,7 @@ help:
@echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
@echo " make lint-black - Check Black formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
+ @echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)"
@echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
@echo " make lint-budget-update - Re-capture all ratchet budgets (ruff + basedpyright)"
@echo " make check-circular-imports - Check for circular imports"
@@ -124,7 +125,8 @@ lint-ruff-FULL-dev: install-dev
else echo "No changed .py files to check."; fi
lint-basedpyright: install-dev
- ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py
+ git fetch origin litellm_internal_staging
+ ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
lint-basedpyright-budget-update: install-dev
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
@@ -134,6 +136,12 @@ lint-black: format-check
lint-ruff-budget: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py
+# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
+# means the CI check will pass too.
+lint-gate: install-dev
+ git fetch origin litellm_internal_staging
+ $(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging
+
lint-ruff-budget-update: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --update
diff --git a/README.md b/README.md
index d7dc665dcec..3d0f7282d7c 100644
--- a/README.md
+++ b/README.md
@@ -6,10 +6,10 @@
Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.
@@ -345,6 +345,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
+| [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ | | | | | | | |
| [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ | | | | | | | |
| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | |
| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | |
@@ -405,6 +406,140 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
+### Deploy on AWS or GCP with Terraform
+
+Run the LiteLLM proxy as a production-ready componentized stack (gateway, backend, UI on separate services; managed Postgres + Redis + object store) using the published Terraform modules. Both modules are on the [public Terraform Registry](https://registry.terraform.io/namespaces/BerriAI) — no auth needed.
+
+#### AWS — ECS Fargate + Aurora + ElastiCache + ALB
+
+[](https://console.aws.amazon.com/cloudshell/home) — opens an in-browser shell, already authenticated to your AWS account. Once inside, run:
+
+```bash
+git clone https://github.com/BerriAI/litellm.git
+cd litellm/terraform/litellm/aws/examples/default
+cp terraform.tfvars.example terraform.tfvars # edit region/tenant/env
+terraform init && terraform apply
+```
+
+[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest)
+
+Or call the module from your own root config:
+
+```hcl
+# main.tf
+terraform {
+ required_version = ">= 1.6.0"
+ required_providers {
+ aws = { source = "hashicorp/aws", version = "~> 5.60" }
+ }
+}
+
+provider "aws" {
+ region = "us-west-2"
+}
+
+module "litellm" {
+ source = "BerriAI/litellm/aws"
+ version = "~> 1.89"
+
+ region = "us-west-2"
+ azs = ["us-west-2a", "us-west-2b"]
+ tenant = "acme"
+ env = "prod"
+
+ # Production: provide an ACM cert. Without one, set allow_plaintext_alb = true
+ # (dev/trial only).
+ # acm_certificate_arn = "arn:aws:acm:us-west-2:111122223333:certificate/..."
+ allow_plaintext_alb = true
+}
+
+output "litellm_url" {
+ value = module.litellm.alb_dns_name
+}
+```
+
+```bash
+terraform init
+terraform apply
+```
+
+Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest?tab=inputs).
+
+#### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB
+
+[](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
+
+Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply.
+
+[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest)
+
+To call the module from your own config instead, Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR:
+
+```bash
+gcloud artifacts repositories create litellm \
+ --location=us-central1 \
+ --repository-format=docker \
+ --mode=remote-repository \
+ --remote-docker-repo=https://ghcr.io \
+ --project=my-gcp-project
+```
+
+Then:
+
+```hcl
+# main.tf
+terraform {
+ required_version = ">= 1.6.0"
+ required_providers {
+ google = { source = "hashicorp/google", version = "~> 6.10" }
+ google-beta = { source = "hashicorp/google-beta", version = "~> 6.10" }
+ }
+}
+
+provider "google" { project = "my-gcp-project"; region = "us-central1" }
+provider "google-beta" { project = "my-gcp-project"; region = "us-central1" }
+
+module "litellm" {
+ source = "BerriAI/litellm/google"
+ version = "~> 1.89"
+
+ project_id = "my-gcp-project"
+ region = "us-central1"
+ tenant = "acme"
+ env = "prod"
+
+ # Replace my-gcp-project with your GCP project ID (same value as project_id above).
+ image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai"
+
+ # Production: provide DNS already pointing at the LB IP for Google-managed certs.
+ # Without one, set allow_plaintext_lb = true (dev/trial only).
+ # lb_domains = ["proxy.example.com"]
+ allow_plaintext_lb = true
+}
+
+output "litellm_url" {
+ value = module.litellm.load_balancer_url
+}
+```
+
+```bash
+terraform init
+terraform apply
+```
+
+Provider API keys live in Secret Manager; reference resource IDs (e.g. `projects/my-gcp-project/secrets/openai-api-key`) via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/google/latest?tab=inputs).
+
+#### Both stacks include
+
+- The full componentized split (gateway / backend / UI as independent services)
+- Managed Postgres (writer + reader) and Redis
+- Versioned object store for proxy state + file uploads
+- An auto-generated `LITELLM_MASTER_KEY` in your cloud's secret manager
+- A one-off migration job that runs `prisma migrate deploy` before the proxy starts
+- The same `proxy_config` surface as the [Helm chart](./helm/litellm/) — pass YAML as a typed map
+
+The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws/) and [`terraform/litellm/gcp/`](./terraform/litellm/gcp/) in this repo; the registry entries are read-only mirrors updated on each release.
+
### Run in Developer Mode
#### Services
1. Setup .env file in root
diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py
index d1a576aeb33..2f65f99c292 100644
--- a/backend/routes/allowlist.py
+++ b/backend/routes/allowlist.py
@@ -120,6 +120,9 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/robots.txt",
# Health (k8s probes)
"/health",
+ # Plugin system
+ "/api/plugins",
+ "/plugin-proxy/",
)
BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 73bc5c47703..f5b0a9aaf81 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -1,31 +1,31 @@
{
"reportAny": {
- "baseline": 24954,
+ "baseline": 24989,
"slack": 2500
},
"reportArgumentType": {
- "baseline": 1863,
+ "baseline": 1934,
"slack": 180
},
"reportAssignmentType": {
"baseline": 220,
- "slack": 3
+ "slack": 22
},
"reportAttributeAccessIssue": {
- "baseline": 335,
- "slack": 3
+ "baseline": 346,
+ "slack": 35
},
"reportCallIssue": {
- "baseline": 77,
+ "baseline": 87,
"slack": 10
},
"reportConstantRedefinition": {
"baseline": 39,
- "slack": 3
+ "slack": 4
},
"reportDeprecated": {
"baseline": 217,
- "slack": 10
+ "slack": 22
},
"reportDuplicateImport": {
"baseline": 28,
@@ -41,11 +41,11 @@
},
"reportGeneralTypeIssues": {
"baseline": 151,
- "slack": 3
+ "slack": 15
},
"reportIncompatibleMethodOverride": {
"baseline": 52,
- "slack": 10
+ "slack": 5
},
"reportIncompatibleVariableOverride": {
"baseline": 8,
@@ -73,7 +73,7 @@
},
"reportMissingParameterType": {
"baseline": 3933,
- "slack": 10
+ "slack": 390
},
"reportMissingTypeArgument": {
"baseline": 10612,
@@ -97,7 +97,7 @@
},
"reportOptionalMemberAccess": {
"baseline": 724,
- "slack": 10
+ "slack": 72
},
"reportOptionalOperand": {
"baseline": 3,
@@ -120,8 +120,8 @@
"slack": 3
},
"reportReturnType": {
- "baseline": 118,
- "slack": 10
+ "baseline": 126,
+ "slack": 100
},
"reportTypedDictNotRequiredAccess": {
"baseline": 20,
@@ -136,19 +136,19 @@
"slack": 3000
},
"reportUnknownLambdaType": {
- "baseline": 76,
+ "baseline": 75,
"slack": 10
},
"reportUnknownMemberType": {
- "baseline": 27322,
+ "baseline": 27037,
"slack": 2500
},
"reportUnknownParameterType": {
- "baseline": 13636,
+ "baseline": 13612,
"slack": 1000
},
"reportUnknownVariableType": {
- "baseline": 21776,
+ "baseline": 21445,
"slack": 2000
},
"reportUnnecessaryCast": {
@@ -156,20 +156,20 @@
"slack": 10
},
"reportUnnecessaryComparison": {
- "baseline": 680,
- "slack": 10
+ "baseline": 683,
+ "slack": 100
},
"reportUnnecessaryContains": {
"baseline": 4,
"slack": 3
},
"reportUnnecessaryIsInstance": {
- "baseline": 807,
- "slack": 10
+ "baseline": 808,
+ "slack": 80
},
"reportUntypedBaseClass": {
"baseline": 110,
- "slack": 3
+ "slack": 11
},
"reportUntypedFunctionDecorator": {
"baseline": 22,
@@ -185,10 +185,10 @@
},
"reportUnusedImport": {
"baseline": 670,
- "slack": 10
+ "slack": 50
},
"reportUnusedVariable": {
"baseline": 865,
- "slack": 10
+ "slack": 50
}
}
diff --git a/docker/build_from_pip/litellm_config.yaml b/docker/build_from_pip/litellm_config.yaml
index 51223026170..f54647853ef 100644
--- a/docker/build_from_pip/litellm_config.yaml
+++ b/docker/build_from_pip/litellm_config.yaml
@@ -3,7 +3,7 @@ model_list:
litellm_params:
model: openai/fake
api_key: fake-key
- api_base: https://exampleopenaiendpoint-production.up.railway.app/
+ api_base: os.environ/FAKE_OPENAI_API_BASE
general_settings:
alerting: ["slack"]
\ No newline at end of file
diff --git a/docs/plugin_architecture.md b/docs/plugin_architecture.md
new file mode 100644
index 00000000000..8801761531d
--- /dev/null
+++ b/docs/plugin_architecture.md
@@ -0,0 +1,141 @@
+# LiteLLM Plugin Architecture
+
+Plugins let external services appear as selectable modes in the litellm UI sidebar alongside the AI Gateway.
+
+---
+
+## Quick start
+
+### 1. Configure the plugin
+
+Add a `plugins` block to your litellm `config.yaml`:
+
+```yaml
+general_settings:
+ master_key: sk-...
+ plugins:
+ - name: my-plugin # unique identifier (no spaces)
+ display_name: My Plugin # shown in the UI dropdown
+ url: "https://my-plugin.example.com"
+ plugin_key: "sk-..." # plugin's own auth credential
+```
+
+`plugin_key` is injected as `Authorization: Bearer ` on every
+request proxied through `/plugin-proxy/my-plugin/*`. The caller's litellm
+credential is stripped before forwarding so the plugin never receives a live
+litellm API key.
+
+### 2. Implement two endpoints on your service
+
+| Endpoint | Method | Purpose |
+|---|---|---|
+| `GET /api/plugin-manifest` | public | Returns plugin metadata for the UI |
+| `POST /api/plugin-auth` | public | Decrypts the identity claim for seamless sign-in |
+
+#### `GET /api/plugin-manifest`
+
+```json
+{
+ "name": "my-plugin",
+ "display_name": "My Plugin",
+ "version": "1.0.0",
+ "nav_items": [
+ { "key": "home", "label": "Home", "icon": "HomeOutlined", "path": "/" },
+ { "key": "reports", "label": "Reports", "icon": "BarChartOutlined", "path": "/reports" }
+ ],
+ "capabilities": ["reports", "data"]
+}
+```
+
+#### `POST /api/plugin-auth`
+
+Receives `{ "session_claim": "" }`.
+
+The proxy never shares `LITELLM_SALT_KEY` with your plugin. Each plugin is
+provisioned with its own dedicated key, derived as
+`HMAC-SHA256(LITELLM_SALT_KEY, plugin_name)`. Compute it once on the proxy
+host and hand the result to your plugin as a secret (e.g. `PLUGIN_AUTH_KEY`):
+
+```bash
+python -c 'import base64,hmac,hashlib,os; \
+print(base64.urlsafe_b64encode(hmac.new(os.environ["LITELLM_SALT_KEY"].encode(), b"my-plugin", hashlib.sha256).digest()).decode())'
+```
+
+A compromised plugin holding only this scoped key cannot recover
+`LITELLM_SALT_KEY` or decrypt any other litellm secret.
+
+Decrypt and validate the claim with that key:
+
+```python
+import json, os, time
+from cryptography.fernet import Fernet
+
+_CLAIM_TTL_SECONDS = 30
+
+def plugin_auth(session_claim: str) -> dict:
+ cipher = Fernet(os.environ["PLUGIN_AUTH_KEY"].encode())
+ claim = json.loads(cipher.decrypt(session_claim.encode(), ttl=_CLAIM_TTL_SECONDS))
+ if claim.get("plugin") != "my-plugin":
+ raise ValueError("claim audience mismatch")
+ if int(claim.get("exp", 0)) < int(time.time()):
+ raise ValueError("claim expired")
+ return claim
+```
+
+The claim is `{ "plugin", "user_id", "user_role", "exp" }`; it carries no
+litellm bearer token. Establish the plugin's own session from `user_id` /
+`user_role` and authenticate API calls back to litellm through the
+`/plugin-proxy/my-plugin/*` reverse proxy, which injects `plugin_key` for you.
+
+---
+
+## How iframe auth works
+
+```
+litellm UI
+ ├─ GET /api/plugins/auth-token -> { session_claim }
+ └─ postMessage({ type:"litellm-auth", session_claim }, pluginOrigin)
+ │
+ ▼
+Plugin iframe browser
+ └─ POST /api/plugin-auth { session_claim }
+ │
+ ▼
+Plugin server
+ ├─ decrypt(session_claim, PLUGIN_AUTH_KEY) -> { user_id, user_role, exp }
+ └─ establish plugin session -> stored in sessionStorage
+```
+
+No litellm bearer token ever leaves the proxy; the claim only conveys the
+caller's identity and expires after 30 seconds. A postMessage intercept
+yields ciphertext that is useless without the plugin's scoped key.
+
+---
+
+## Proxy routes
+
+- `GET /api/plugins` — list registered plugins (`name`, `display_name`, `url`). `plugin_key` is **never** returned; it stays server-side. Requires an authenticated caller.
+- `GET /api/plugins/auth-token?plugin_name=` — short-lived encrypted identity claim for the named plugin. Requires `LITELLM_SALT_KEY` to be set (503 otherwise) and the plugin to be registered (404 otherwise).
+- `ANY /plugin-proxy/{name}/{path}` — authenticated reverse proxy to the plugin backend. Restricted to `proxy_admin`.
+
+---
+
+## Reverse proxy behaviour
+
+When an admin (or server-to-server caller) hits `/plugin-proxy//`, the proxy authenticates the caller locally, then rewrites the request before forwarding it to the plugin's `url`:
+
+- **Every litellm credential header is stripped** — `Authorization`, `x-api-key`, `API-Key`, `x-goog-api-key`, `Ocp-Apim-Subscription-Key`, `x-litellm-api-key`, any configured `litellm_key_header_name`, plus `Cookie`. The plugin can never be handed the caller's live litellm key.
+- **`plugin_key` is injected** as `Authorization: Bearer ` — the only credential the plugin receives.
+- **Caller identity is forwarded** as `x-litellm-user-id` and `x-litellm-user-role` so the plugin can run its own authorization. These are informational, not credentials.
+- **Responses are sandboxed** — `Content-Security-Policy: sandbox` and `X-Content-Type-Options: nosniff` are set so plugin-controlled bytes served from the litellm origin cannot execute against the dashboard.
+
+---
+
+## Security checklist
+
+- [ ] `LITELLM_SALT_KEY` is set on the proxy and never shared with the plugin
+- [ ] The plugin holds only its derived `HMAC(LITELLM_SALT_KEY, plugin_name)` key, provisioned as a dedicated secret
+- [ ] `plugin_key` is a dedicated credential scoped to the plugin (not your litellm master key)
+- [ ] Plugin's `POST /api/plugin-auth` enforces the claim's `plugin` audience and `exp` (30s TTL)
+- [ ] Plugin treats `x-litellm-user-id` / `x-litellm-user-role` as identity hints, not as proof of authentication
+- [ ] Plugin service URL uses HTTPS in production
diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
index 6830147116d..8486e37384e 100644
--- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
@@ -1,9 +1,9 @@
# What is this?
## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id
-import asyncio
import base64
import json
+from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast
from fastapi import HTTPException
@@ -1472,8 +1472,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
error_message += f" (showing {MAX_BATCHES_IN_ERROR} most recent): {', '.join(batch_statuses)}. "
error_message += (
- f"To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. "
- f"Alternatively, wait for all batches to complete and for cost to be computed (batch_processed=true)."
+ "To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. "
+ "Alternatively, wait for all batches to complete and for cost to be computed (batch_processed=true)."
)
# Record blocked deletion metric
@@ -1550,9 +1550,22 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if specific_model_file_id_mapping:
exception_dict = {}
- for model_id, file_id in specific_model_file_id_mapping.items():
+ for model_id, provider_file_id in specific_model_file_id_mapping.items():
try:
- return await llm_router.afile_content(model=model_id, file_id=file_id, **data) # type: ignore
+ # Cloud-storage providers (e.g. Bedrock S3) validate file ids
+ # against the deployment's configured bucket, which they only
+ # trust from this immutable server-side snapshot, never from
+ # request params.
+ credentials = llm_router.get_deployment_credentials_with_provider(
+ model_id=model_id
+ )
+ if credentials is not None:
+ data["_litellm_internal_model_credentials"] = cast(
+ Dict, MappingProxyType(dict(credentials))
+ )
+ else:
+ data.pop("_litellm_internal_model_credentials", None)
+ return await llm_router.afile_content(model=model_id, file_id=provider_file_id, **data) # type: ignore
except Exception as e:
exception_dict[model_id] = str(e)
raise Exception(
diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index d0432448433..b032942427c 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
-version = "0.1.42"
+version = "0.1.43"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
-version = "0.1.42"
+version = "0.1.43"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",
diff --git a/litellm-rust/.gitignore b/litellm-rust/.gitignore
new file mode 100644
index 00000000000..b83d22266ac
--- /dev/null
+++ b/litellm-rust/.gitignore
@@ -0,0 +1 @@
+/target/
diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md
new file mode 100644
index 00000000000..1d2987e0a1a
--- /dev/null
+++ b/litellm-rust/CLAUDE.md
@@ -0,0 +1,88 @@
+# CLAUDE.md
+
+This file defines the rules for Rust work in LiteLLM.
+
+## Core Boundary
+
+The `core` and `providers` crates describe work; hosts execute work.
+
+Route-level Rust structure mirrors LiteLLM's Python responsibilities:
+- `core/src//` owns the route contract, shared types, and provider
+ template traits. For OCR, this means `core/src/ocr`.
+- `providers/src///transformation.rs` owns the
+ provider-specific transform. For Mistral OCR, this means
+ `providers/src/mistral/ocr/transformation.rs`.
+- Future network execution belongs in a host/transport layer such as
+ `llm_http_handler`, not inside `core` or `providers`.
+
+Allowed in `core` and `providers`:
+- Pure request transforms
+- Pure response transforms
+- Pure stream chunk normalization
+- Shared data types and validation errors
+- Deterministic token/cost helper logic
+
+Not allowed in `core` or `providers`:
+- Network calls
+- Environment variable or secret reads
+- Filesystem access
+- Database or cache access
+- Provider SDK signing or auth flows
+- Logging callbacks, spend writes, or custom callbacks
+- Global mutable runtime state
+
+Python owns rollout state and fallback while Rust is being introduced. Rust
+paths must be off by default until parity tests prove equivalence with Python.
+
+## Production Bar
+
+Rust code in this workspace is held to a strict parity and robustness bar from
+the first PR:
+
+- Correctness parity is proven with tests. Do not rely on README claims or
+ manual inspection for a port that mirrors Python behavior.
+- Every provider transform must have unit tests for supported-parameter
+ filtering, request body shape, response normalization, missing/null fields,
+ and bad-input errors.
+- When Rust is exposed through Python, add Python tests that prove disabled,
+ enabled, and unavailable-bridge fallback behavior.
+- Avoid panics on user/provider input. Return typed errors and let the host map
+ them to Python exceptions or HTTP responses.
+- OCR handles documents that often contain personal data. Do not log document
+ contents, base64 payloads, provider response bodies, or secrets.
+- Error messages must be useful but data-minimized. Truncate or sanitize any
+ upstream body before it crosses a host boundary.
+- Treat empty or whitespace-only credentials, URLs, and config values as absent
+ at the host/config resolution layer.
+- Preserve Python output shape intentionally. If a field is always serialized as
+ `null` for Python parity, leave a short comment explaining that parity choice.
+
+## Host I/O Rules
+
+These rules apply when adding future crates or modules that execute network I/O,
+such as `ai-gateway`, router hosts, or standalone servers:
+
+- Set connect and full-request timeouts. No unbounded waits.
+- Reuse HTTP clients; do not construct clients per request.
+- Prefer rustls TLS for portable Python wheels and Linux images unless there is
+ a documented reason not to.
+- Add request IDs and structured tracing at the host layer, without logging OCR
+ document contents or secrets.
+- Do not echo raw upstream response bodies to callers. Sanitize and bound them.
+- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
+ impossible by construction and documented.
+
+## Checks
+
+Run these before pushing Rust changes. The same checks run in GitHub Actions
+for changes under `litellm-rust/`.
+
+```bash
+cd litellm-rust
+cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```
+
+When a Rust path is exposed through Python, add Python parity tests that compare
+the existing Python output with the Rust-backed output.
diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
new file mode 100644
index 00000000000..0fb2fcb2921
--- /dev/null
+++ b/litellm-rust/Cargo.lock
@@ -0,0 +1,1498 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
+name = "bitflags"
+version = "2.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "bytes"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
+
+[[package]]
+name = "cc"
+version = "1.2.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
+[[package]]
+name = "displaydoc"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
+[[package]]
+name = "form_urlencoded"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
+dependencies = [
+ "percent-encoding",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+
+[[package]]
+name = "futures-io"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
+
+[[package]]
+name = "futures-sink"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
+
+[[package]]
+name = "futures-task"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+
+[[package]]
+name = "futures-util"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+dependencies = [
+ "futures-core",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "slab",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "libc",
+ "wasi",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "libc",
+ "r-efi",
+ "wasip2",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "http"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+dependencies = [
+ "bytes",
+ "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
+name = "hyper"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "http",
+ "http-body",
+ "httparse",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
+dependencies = [
+ "http",
+ "hyper",
+ "hyper-util",
+ "rustls",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+ "webpki-roots",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "ipnet",
+ "libc",
+ "percent-encoding",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "icu_collections"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
+dependencies = [
+ "displaydoc",
+ "potential_utf",
+ "utf8_iter",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locale_core"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
+dependencies = [
+ "displaydoc",
+ "litemap",
+ "tinystr",
+ "writeable",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
+dependencies = [
+ "icu_collections",
+ "icu_normalizer_data",
+ "icu_properties",
+ "icu_provider",
+ "smallvec",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer_data"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
+
+[[package]]
+name = "icu_properties"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
+dependencies = [
+ "icu_collections",
+ "icu_locale_core",
+ "icu_properties_data",
+ "icu_provider",
+ "zerotrie",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_properties_data"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
+
+[[package]]
+name = "icu_provider"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
+dependencies = [
+ "displaydoc",
+ "icu_locale_core",
+ "writeable",
+ "yoke",
+ "zerofrom",
+ "zerotrie",
+ "zerovec",
+]
+
+[[package]]
+name = "idna"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
+dependencies = [
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
+dependencies = [
+ "icu_normalizer",
+ "icu_properties",
+]
+
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "ipnet"
+version = "2.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "js-sys"
+version = "0.3.102"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "litellm-core"
+version = "0.1.0"
+dependencies = [
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
+[[package]]
+name = "litellm-providers"
+version = "0.1.0"
+dependencies = [
+ "litellm-core",
+ "reqwest",
+ "serde_json",
+]
+
+[[package]]
+name = "litellm-python-bridge"
+version = "0.1.0"
+dependencies = [
+ "litellm-core",
+ "litellm-providers",
+ "pyo3",
+ "serde_json",
+]
+
+[[package]]
+name = "litemap"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
+
+[[package]]
+name = "log"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+
+[[package]]
+name = "lru-slab"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+
+[[package]]
+name = "memchr"
+version = "2.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "mio"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
+dependencies = [
+ "libc",
+ "wasi",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
+[[package]]
+name = "potential_utf"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
+dependencies = [
+ "zerovec",
+]
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quinn"
+version = "0.11.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8"
+dependencies = [
+ "bytes",
+ "cfg_aliases",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls",
+ "socket2",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e"
+dependencies = [
+ "bytes",
+ "getrandom 0.3.4",
+ "lru-slab",
+ "rand",
+ "ring",
+ "rustc-hash",
+ "rustls",
+ "rustls-pki-types",
+ "slab",
+ "thiserror",
+ "tinyvec",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
+dependencies = [
+ "cfg_aliases",
+ "libc",
+ "once_cell",
+ "socket2",
+ "tracing",
+ "windows-sys 0.60.2",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "rand"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+dependencies = [
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom 0.3.4",
+]
+
+[[package]]
+name = "reqwest"
+version = "0.12.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "js-sys",
+ "log",
+ "percent-encoding",
+ "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "tokio",
+ "tokio-rustls",
+ "tower",
+ "tower-http",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "webpki-roots",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.17",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+
+[[package]]
+name = "rustls"
+version = "0.23.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
+dependencies = [
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+dependencies = [
+ "web-time",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "ryu"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.150"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
+[[package]]
+name = "smallvec"
+version = "1.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+
+[[package]]
+name = "socket2"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
+[[package]]
+name = "syn"
+version = "2.0.118"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sync_wrapper"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
+dependencies = [
+ "futures-core",
+]
+
+[[package]]
+name = "synstructure"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
+[[package]]
+name = "thiserror"
+version = "2.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tinystr"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
+dependencies = [
+ "displaydoc",
+ "zerovec",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
+name = "tokio"
+version = "1.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+dependencies = [
+ "bytes",
+ "libc",
+ "mio",
+ "pin-project-lite",
+ "socket2",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "tokio-rustls"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
+dependencies = [
+ "rustls",
+ "tokio",
+]
+
+[[package]]
+name = "tower"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
+dependencies = [
+ "bitflags",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "pin-project-lite",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "url",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
+
+[[package]]
+name = "tower-service"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
+
+[[package]]
+name = "tracing"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
+dependencies = [
+ "pin-project-lite",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "url"
+version = "2.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+ "serde",
+]
+
+[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.125"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.125"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.125"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.125"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.102"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
+dependencies = [
+ "windows-targets 0.53.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm 0.52.6",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.53.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
+dependencies = [
+ "windows-link",
+ "windows_aarch64_gnullvm 0.53.1",
+ "windows_aarch64_msvc 0.53.1",
+ "windows_i686_gnu 0.53.1",
+ "windows_i686_gnullvm 0.53.1",
+ "windows_i686_msvc 0.53.1",
+ "windows_x86_64_gnu 0.53.1",
+ "windows_x86_64_gnullvm 0.53.1",
+ "windows_x86_64_msvc 0.53.1",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
+[[package]]
+name = "writeable"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
+
+[[package]]
+name = "yoke"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
+dependencies = [
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
+
+[[package]]
+name = "zerocopy"
+version = "0.8.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "zerofrom"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
+
+[[package]]
+name = "zeroize"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
+
+[[package]]
+name = "zerotrie"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+]
+
+[[package]]
+name = "zerovec"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
+dependencies = [
+ "yoke",
+ "zerofrom",
+ "zerovec-derive",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "zmij"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
new file mode 100644
index 00000000000..fdc5f5efde1
--- /dev/null
+++ b/litellm-rust/Cargo.toml
@@ -0,0 +1,21 @@
+[workspace]
+members = [
+ "crates/core",
+ "crates/providers",
+ "crates/python-bridge",
+]
+resolver = "2"
+
+[workspace.package]
+edition = "2021"
+license = "MIT"
+repository = "https://github.com/BerriAI/litellm"
+
+[workspace.dependencies]
+litellm-core = { path = "crates/core" }
+litellm-providers = { path = "crates/providers" }
+pyo3 = "0.23.5"
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "2.0"
diff --git a/litellm-rust/README.md b/litellm-rust/README.md
new file mode 100644
index 00000000000..15ad1855420
--- /dev/null
+++ b/litellm-rust/README.md
@@ -0,0 +1,34 @@
+# LiteLLM Rust
+
+This workspace contains the staged Rust implementation for LiteLLM.
+
+Rust starts as a pure transform core used by the existing Python host. Python
+continues to own auth, configuration, network I/O, retries, routing, logging,
+callbacks, spend tracking, and customer plugins until each Rust path has parity
+coverage and production evidence.
+
+## Layout
+
+```text
+crates/
+ core/ Route contracts, shared pure types, errors, and templates.
+ src/ocr/
+ providers/ Provider-specific pure transforms.
+ src/mistral/ocr/transformation.rs
+ python-bridge/ PyO3 bridge for Python LiteLLM.
+```
+
+The folder shape should follow the Python provider tree:
+`providers/src///transformation.rs`. The bridge should expose
+one function per top-level route, starting with `ocr(payload)`.
+
+## Checks
+
+Run these before pushing Rust changes. GitHub Actions runs the same checks for
+changes under `litellm-rust/`.
+
+```bash
+cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```
diff --git a/litellm-rust/crates/core/CLAUDE.md b/litellm-rust/crates/core/CLAUDE.md
new file mode 100644
index 00000000000..51e0d215e6a
--- /dev/null
+++ b/litellm-rust/crates/core/CLAUDE.md
@@ -0,0 +1,38 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/core`.
+
+## Responsibility
+
+`core` owns shared data types, typed errors, and deterministic helper contracts.
+It must stay pure and host-independent.
+
+Allowed:
+- Shared request/response structs.
+- Typed errors with stable, non-sensitive messages.
+- Deterministic validation helpers.
+- Serialization helpers that intentionally mirror Python output shape.
+- Route templates that match Python base config responsibilities, such as
+ `ocr::transformation::OcrProviderConfig`.
+
+Not allowed:
+- Network, filesystem, database, cache, or environment access.
+- Secret reads or auth/header construction.
+- Logging callbacks, tracing spans, spend writes, or customer callbacks.
+- Provider-specific branching that belongs in `providers`.
+- Panics for user/provider-controlled input.
+
+## Structure
+
+Use route names directly under `src/`: `ocr`, future `messages`,
+`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not
+invent broad names like `engine` for route contracts.
+
+## Parity Rules
+
+- Every shared type used by a provider transform needs unit tests for
+ serialization shape.
+- If Python parity requires always emitting a `null` field instead of omitting
+ it, document that in code and pin it with a test.
+- Error enums should preserve enough detail for Python/HTTP hosts to map errors
+ consistently without exposing document contents or upstream bodies.
diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml
new file mode 100644
index 00000000000..e54002fe5e8
--- /dev/null
+++ b/litellm-rust/crates/core/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "litellm-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs
new file mode 100644
index 00000000000..645e261f76d
--- /dev/null
+++ b/litellm-rust/crates/core/src/error.rs
@@ -0,0 +1,33 @@
+use thiserror::Error;
+
+pub type CoreResult = Result;
+
+#[derive(Debug, Error, PartialEq, Eq)]
+pub enum CoreError {
+ #[error("expected {expected}, got {actual}")]
+ InvalidType {
+ expected: &'static str,
+ actual: &'static str,
+ },
+ #[error("missing required field: {0}")]
+ MissingField(&'static str),
+ #[error("invalid response: {0}")]
+ InvalidResponse(String),
+ #[error("{0}")]
+ Auth(String),
+ #[error("OCR request failed with status {status}: {body}")]
+ Http { status: u16, body: String },
+ #[error("OCR network error: {0}")]
+ Network(String),
+}
+
+pub fn json_type_name(value: &serde_json::Value) -> &'static str {
+ match value {
+ serde_json::Value::Null => "null",
+ serde_json::Value::Bool(_) => "bool",
+ serde_json::Value::Number(_) => "number",
+ serde_json::Value::String(_) => "string",
+ serde_json::Value::Array(_) => "array",
+ serde_json::Value::Object(_) => "object",
+ }
+}
diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs
new file mode 100644
index 00000000000..a88204867c4
--- /dev/null
+++ b/litellm-rust/crates/core/src/lib.rs
@@ -0,0 +1,4 @@
+pub mod error;
+pub mod ocr;
+
+pub use error::{CoreError, CoreResult};
diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs
new file mode 100644
index 00000000000..ec2fbb969a6
--- /dev/null
+++ b/litellm-rust/crates/core/src/ocr/mod.rs
@@ -0,0 +1,2 @@
+pub mod transformation;
+pub mod types;
diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs
new file mode 100644
index 00000000000..7353d9d22c4
--- /dev/null
+++ b/litellm-rust/crates/core/src/ocr/transformation.rs
@@ -0,0 +1,32 @@
+use serde_json::{Map, Value};
+
+use crate::CoreResult;
+
+use super::types::{OcrRequestData, OcrResponseData};
+
+pub trait OcrProviderConfig {
+ fn supported_ocr_params(&self) -> &'static [&'static str];
+
+ fn map_ocr_params(&self, non_default_params: &Map) -> Map {
+ let mut mapped_params = Map::new();
+ for (param, value) in non_default_params {
+ if self.supported_ocr_params().contains(¶m.as_str()) {
+ mapped_params.insert(param.clone(), value.clone());
+ }
+ }
+ mapped_params
+ }
+
+ fn transform_ocr_request(
+ &self,
+ model: &str,
+ document: Value,
+ optional_params: Map,
+ ) -> CoreResult;
+
+ fn transform_ocr_response(
+ &self,
+ model: &str,
+ response_json: Value,
+ ) -> CoreResult;
+}
diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs
new file mode 100644
index 00000000000..1a72b8f1d66
--- /dev/null
+++ b/litellm-rust/crates/core/src/ocr/types.rs
@@ -0,0 +1,29 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrRequestData {
+ pub data: Value,
+ pub files: Option,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrResponseData {
+ pub pages: Vec,
+ pub model: String,
+ pub document_annotation: Option,
+ pub usage_info: Option,
+ pub object: String,
+}
+
+impl OcrResponseData {
+ pub fn into_json(self) -> Value {
+ serde_json::json!({
+ "pages": self.pages,
+ "model": self.model,
+ "document_annotation": self.document_annotation,
+ "usage_info": self.usage_info,
+ "object": self.object,
+ })
+ }
+}
diff --git a/litellm-rust/crates/providers/CLAUDE.md b/litellm-rust/crates/providers/CLAUDE.md
new file mode 100644
index 00000000000..0f7fdcda2aa
--- /dev/null
+++ b/litellm-rust/crates/providers/CLAUDE.md
@@ -0,0 +1,53 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/providers`.
+
+## Responsibility
+
+`providers` owns provider-specific pure transforms. It mirrors the existing
+Python provider modules closely enough that parity review is mechanical.
+
+Provider files should map to the Python provider tree:
+
+```text
+providers/src///transformation.rs
+```
+
+For example, Mistral OCR lives at
+`providers/src/mistral/ocr/transformation.rs`, matching
+`litellm/llms/mistral/ocr/transformation.py`.
+
+Allowed:
+- Provider request transforms.
+- Provider response normalization.
+- Supported-parameter filtering.
+- Provider-specific validation that does not require I/O or secrets.
+
+Not allowed:
+- HTTP clients or provider SDK calls.
+- Environment variable reads.
+- API key resolution or auth header construction.
+- Logging, callbacks, spend tracking, retries, routing, cooldowns, or fallbacks.
+- Panics on bad user/provider input.
+
+## Required Tests
+
+Every provider transform must include focused unit tests for:
+- Supported params matching the Python provider config.
+- Unknown params being dropped or transformed the same way as Python.
+- Request body shape matching Python output.
+- Response normalization with complete, missing, null, and extra fields.
+- Bad input returning typed errors.
+
+For OCR specifically, assume documents can contain personal data. Tests should
+prove transforms do not copy document contents into error messages.
+
+## Implementation Rules
+
+- Prefer static supported-parameter lists over allocating strings on every call.
+- Keep transforms deterministic and allocation-conscious, but choose clarity over
+ premature micro-optimization for tiny parameter lists.
+- Use typed errors from `core`; avoid stringly-typed error plumbing.
+- Add comments only when they explain Python-parity decisions or provider quirks.
+- Put route-level provider dispatch in a route file such as `providers/src/ocr.rs`.
+ Do not move provider-specific transform logic into the Python bridge.
diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/providers/Cargo.toml
new file mode 100644
index 00000000000..b8ed2066079
--- /dev/null
+++ b/litellm-rust/crates/providers/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "litellm-providers"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-core.workspace = true
+reqwest.workspace = true
+serde_json.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/providers/src/lib.rs
new file mode 100644
index 00000000000..1a0ca5b7e42
--- /dev/null
+++ b/litellm-rust/crates/providers/src/lib.rs
@@ -0,0 +1,2 @@
+pub mod mistral;
+pub mod ocr;
diff --git a/litellm-rust/crates/providers/src/mistral/mod.rs b/litellm-rust/crates/providers/src/mistral/mod.rs
new file mode 100644
index 00000000000..3621ff6a2fd
--- /dev/null
+++ b/litellm-rust/crates/providers/src/mistral/mod.rs
@@ -0,0 +1 @@
+pub mod ocr;
diff --git a/litellm-rust/crates/providers/src/mistral/ocr/mod.rs b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs
new file mode 100644
index 00000000000..f239b6921fa
--- /dev/null
+++ b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs
@@ -0,0 +1 @@
+pub mod transformation;
diff --git a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs
new file mode 100644
index 00000000000..fd691177783
--- /dev/null
+++ b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs
@@ -0,0 +1,292 @@
+use litellm_core::error::{json_type_name, CoreError, CoreResult};
+use litellm_core::ocr::transformation::OcrProviderConfig;
+use litellm_core::ocr::types::{OcrRequestData, OcrResponseData};
+use serde_json::{Map, Value};
+
+const SUPPORTED_OCR_PARAMS: &[&str] = &[
+ "pages",
+ "include_image_base64",
+ "image_limit",
+ "image_min_size",
+ "bbox_annotation_format",
+ "document_annotation_format",
+ "document_annotation_prompt",
+ "extract_header",
+ "extract_footer",
+ "table_format",
+ "confidence_scores_granularity",
+ "id",
+];
+
+/// Default Mistral API base, used when the caller does not override `api_base`.
+pub const MISTRAL_DEFAULT_API_BASE: &str = "https://api.mistral.ai/v1";
+
+/// Environment variable holding the Mistral API key.
+pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY";
+
+/// Error message raised when no Mistral API key can be resolved.
+pub const MISSING_KEY_MESSAGE: &str = "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params";
+
+/// Build the complete OCR endpoint URL, de-duplicating a trailing `/v1`.
+///
+/// Blank/whitespace `api_base` is treated as absent (guard at resolution time).
+pub fn complete_url(api_base: Option<&str>) -> String {
+ let base = api_base
+ .map(str::trim)
+ .filter(|base| !base.is_empty())
+ .unwrap_or(MISTRAL_DEFAULT_API_BASE)
+ .trim_end_matches('/');
+
+ if base.ends_with("/v1") {
+ format!("{base}/ocr")
+ } else {
+ format!("{base}/v1/ocr")
+ }
+}
+
+/// Resolve the Mistral API key from the explicit param or the environment.
+///
+/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth`
+/// when no usable key is available.
+///
+/// Note: the env fallback only reads the process environment. Secret-manager
+/// backends (AWS/Azure/GCP/Vault) are resolved on the Python side and passed in
+/// via `api_key`; this fallback is a last resort for direct/standalone use.
+pub fn resolve_api_key(
+ api_key: Option<&str>,
+ env_lookup: &dyn Fn(&str) -> Option,
+) -> CoreResult {
+ api_key
+ .map(str::trim)
+ .filter(|key| !key.is_empty())
+ .map(str::to_string)
+ .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty()))
+ .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+pub struct MistralOcrConfig;
+
+pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig;
+
+impl OcrProviderConfig for MistralOcrConfig {
+ fn supported_ocr_params(&self) -> &'static [&'static str] {
+ SUPPORTED_OCR_PARAMS
+ }
+
+ fn transform_ocr_request(
+ &self,
+ model: &str,
+ document: Value,
+ optional_params: Map,
+ ) -> CoreResult {
+ if !document.is_object() {
+ return Err(CoreError::InvalidType {
+ expected: "object",
+ actual: json_type_name(&document),
+ });
+ }
+
+ let mut data = Map::new();
+ data.insert("model".to_string(), Value::String(model.to_string()));
+ data.insert("document".to_string(), document);
+ for (param, value) in optional_params {
+ data.insert(param, value);
+ }
+
+ Ok(OcrRequestData {
+ data: Value::Object(data),
+ files: None,
+ })
+ }
+
+ fn transform_ocr_response(
+ &self,
+ model: &str,
+ response_json: Value,
+ ) -> CoreResult {
+ let response_object = response_json
+ .as_object()
+ .ok_or_else(|| CoreError::InvalidType {
+ expected: "object",
+ actual: json_type_name(&response_json),
+ })?;
+
+ let pages = response_object
+ .get("pages")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let model = response_object
+ .get("model")
+ .and_then(Value::as_str)
+ .unwrap_or(model)
+ .to_string();
+ let document_annotation = response_object.get("document_annotation").cloned();
+ let usage_info = response_object.get("usage_info").cloned();
+
+ Ok(OcrResponseData {
+ pages,
+ model,
+ document_annotation,
+ usage_info,
+ object: "ocr".to_string(),
+ })
+ }
+}
+
+pub fn supported_ocr_params() -> &'static [&'static str] {
+ MISTRAL_OCR_CONFIG.supported_ocr_params()
+}
+
+pub fn map_ocr_params(non_default_params: &Map) -> Map {
+ MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params)
+}
+
+pub fn transform_ocr_request(
+ model: &str,
+ document: Value,
+ optional_params: Map,
+) -> CoreResult {
+ MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params)
+}
+
+pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult {
+ MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn supported_params_match_python_mistral_ocr_config() {
+ assert_eq!(
+ supported_ocr_params(),
+ &[
+ "pages",
+ "include_image_base64",
+ "image_limit",
+ "image_min_size",
+ "bbox_annotation_format",
+ "document_annotation_format",
+ "document_annotation_prompt",
+ "extract_header",
+ "extract_footer",
+ "table_format",
+ "confidence_scores_granularity",
+ "id",
+ ]
+ );
+ }
+
+ #[test]
+ fn map_ocr_params_drops_unknown_params() {
+ let params = json!({
+ "extract_header": true,
+ "unsupported_param": "value",
+ "pages": [0, 1]
+ });
+ let mapped = map_ocr_params(params.as_object().unwrap());
+
+ assert_eq!(mapped.get("extract_header"), Some(&json!(true)));
+ assert_eq!(mapped.get("pages"), Some(&json!([0, 1])));
+ assert!(!mapped.contains_key("unsupported_param"));
+ }
+
+ #[test]
+ fn transform_ocr_request_builds_mistral_body() {
+ let document = json!({
+ "type": "document_url",
+ "document_url": "https://example.com/doc.pdf"
+ });
+ let optional_params = json!({
+ "include_image_base64": true,
+ "table_format": "html"
+ })
+ .as_object()
+ .unwrap()
+ .clone();
+
+ let result = transform_ocr_request("mistral-ocr-latest", document.clone(), optional_params)
+ .expect("request should transform");
+
+ assert_eq!(
+ result.data,
+ json!({
+ "model": "mistral-ocr-latest",
+ "document": document,
+ "include_image_base64": true,
+ "table_format": "html"
+ })
+ );
+ assert_eq!(result.files, None);
+ }
+
+ #[test]
+ fn transform_ocr_request_rejects_non_object_document() {
+ let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new())
+ .expect_err("string document should be rejected");
+
+ assert_eq!(
+ err,
+ CoreError::InvalidType {
+ expected: "object",
+ actual: "string",
+ }
+ );
+ }
+
+ #[test]
+ fn transform_ocr_response_normalizes_mistral_json() {
+ let response = json!({
+ "pages": [{"index": 0, "markdown": "hello"}],
+ "model": "mistral-ocr-2505-completion",
+ "document_annotation": null,
+ "usage_info": {"pages_processed": 1}
+ });
+
+ let result = transform_ocr_response("mistral-ocr-latest", response)
+ .expect("response should transform");
+
+ assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]);
+ assert_eq!(result.model, "mistral-ocr-2505-completion");
+ assert_eq!(result.document_annotation, Some(Value::Null));
+ assert_eq!(result.usage_info, Some(json!({"pages_processed": 1})));
+ assert_eq!(result.object, "ocr");
+ }
+
+ #[test]
+ fn complete_url_defaults_and_dedupes_v1() {
+ assert_eq!(complete_url(None), "https://api.mistral.ai/v1/ocr");
+ assert_eq!(complete_url(Some(" ")), "https://api.mistral.ai/v1/ocr");
+ assert_eq!(
+ complete_url(Some("https://proxy.internal")),
+ "https://proxy.internal/v1/ocr"
+ );
+ assert_eq!(
+ complete_url(Some("https://proxy.internal/v1/")),
+ "https://proxy.internal/v1/ocr"
+ );
+ }
+
+ #[test]
+ fn resolve_api_key_prefers_param_then_env() {
+ let no_env = |_: &str| None;
+ assert_eq!(
+ resolve_api_key(Some("sk-param"), &no_env).unwrap(),
+ "sk-param"
+ );
+
+ let with_env = |key: &str| (key == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string());
+ assert_eq!(resolve_api_key(None, &with_env).unwrap(), "sk-env");
+ // Blank param falls through to the environment.
+ assert_eq!(resolve_api_key(Some(" "), &with_env).unwrap(), "sk-env");
+ }
+
+ #[test]
+ fn resolve_api_key_errors_when_absent() {
+ let err = resolve_api_key(None, &|_| None).expect_err("missing key should error");
+ assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string()));
+ }
+}
diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs
new file mode 100644
index 00000000000..dcd56a5f0b4
--- /dev/null
+++ b/litellm-rust/crates/providers/src/ocr.rs
@@ -0,0 +1,127 @@
+//! End-to-end OCR orchestration.
+//!
+//! Owns the whole Mistral OCR call so the Python side stays a thin bridge:
+//! resolve the API key, build the URL + body via the pure transforms, POST it,
+//! and normalize the response. The HTTP client is built once and reused.
+
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use litellm_core::error::CoreError;
+use litellm_core::ocr::transformation::OcrProviderConfig;
+use litellm_core::CoreResult;
+use serde_json::{Map, Value};
+
+use crate::mistral::ocr::transformation as mistral;
+use crate::mistral::ocr::transformation::MISTRAL_OCR_CONFIG;
+
+/// OCR over large documents can take a while; bound it generously rather than
+/// hanging forever on an unresponsive upstream. The client-level limit is the
+/// outer ceiling; callers can tighten it per request via ``run_ocr``'s ``timeout``.
+const OCR_TIMEOUT_SECS: u64 = 600;
+
+/// Maximum upstream body characters retained in error messages. OCR responses
+/// can echo document contents and prompts; keep enough for debugging without
+/// forwarding sensitive payloads across the host boundary.
+const ERROR_BODY_MAX_CHARS: usize = 256;
+
+/// Process-wide blocking HTTP client (connection pool + TLS reused across calls).
+fn http_client() -> &'static reqwest::blocking::Client {
+ static CLIENT: OnceLock = OnceLock::new();
+ CLIENT.get_or_init(|| {
+ reqwest::blocking::Client::builder()
+ .timeout(Duration::from_secs(OCR_TIMEOUT_SECS))
+ .build()
+ .expect("failed to build reqwest client")
+ })
+}
+
+fn truncate_error_body(body: &str) -> String {
+ if body.chars().count() <= ERROR_BODY_MAX_CHARS {
+ return body.to_string();
+ }
+ let truncated: String = body.chars().take(ERROR_BODY_MAX_CHARS).collect();
+ format!("{truncated}... (truncated)")
+}
+
+/// Perform a Mistral OCR call end to end and return the normalized response as
+/// JSON (the shape the Python `OCRResponse` model expects).
+///
+/// Blocking: intended to be called with the GIL released from the Python bridge.
+pub fn run_ocr(
+ model: &str,
+ document: Value,
+ api_key: Option<&str>,
+ api_base: Option<&str>,
+ optional_params: Map,
+ timeout: Option,
+) -> CoreResult {
+ let config = &MISTRAL_OCR_CONFIG;
+
+ let api_key = mistral::resolve_api_key(api_key, &|key| std::env::var(key).ok())?;
+ let url = mistral::complete_url(api_base);
+ let filtered_params = config.map_ocr_params(&optional_params);
+ let body = config
+ .transform_ocr_request(model, document, filtered_params)?
+ .data;
+
+ let mut request = http_client().post(&url).bearer_auth(&api_key).json(&body);
+ if let Some(duration) = timeout {
+ request = request.timeout(duration);
+ }
+
+ let response = request
+ .send()
+ .map_err(|err| CoreError::Network(err.to_string()))?;
+
+ let status = response.status();
+ let text = response
+ .text()
+ .map_err(|err| CoreError::Network(err.to_string()))?;
+
+ if !status.is_success() {
+ return Err(CoreError::Http {
+ status: status.as_u16(),
+ body: truncate_error_body(&text),
+ });
+ }
+
+ let response_json: Value = serde_json::from_str(&text)
+ .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?;
+
+ Ok(config
+ .transform_ocr_response(model, response_json)?
+ .into_json())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn truncate_error_body_passes_short_strings_through() {
+ let body = "Unauthorized";
+ assert_eq!(truncate_error_body(body), "Unauthorized");
+ }
+
+ #[test]
+ fn truncate_error_body_caps_long_payloads() {
+ let body = "x".repeat(ERROR_BODY_MAX_CHARS + 50);
+ let truncated = truncate_error_body(&body);
+
+ assert!(truncated.ends_with("... (truncated)"));
+ let prefix_chars = truncated
+ .strip_suffix("... (truncated)")
+ .expect("truncated marker present")
+ .chars()
+ .count();
+ assert_eq!(prefix_chars, ERROR_BODY_MAX_CHARS);
+ }
+
+ #[test]
+ fn truncate_error_body_does_not_split_multibyte_chars() {
+ let body = "é".repeat(ERROR_BODY_MAX_CHARS + 10);
+ let truncated = truncate_error_body(&body);
+ assert!(truncated.is_char_boundary(truncated.len()));
+ }
+}
diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md
new file mode 100644
index 00000000000..efa1a554c9c
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/CLAUDE.md
@@ -0,0 +1,36 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/python-bridge`.
+
+## Responsibility
+
+`python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms.
+Keep this crate thin. It adapts Python objects to Rust payloads and returns
+Python-compatible dictionaries.
+
+## Bridge Shape
+
+- Prefer one stable method per top-level LiteLLM route, for example
+ `ocr(payload)`.
+- Do not add one exported PyO3 function per provider helper unless there is a
+ measured reason.
+- Provider dispatch belongs in Rust route modules such as
+ `litellm_providers::ocr`, not in this PyO3 crate.
+- Python owns rollout state and fallback. Rust should return errors; Python
+ decides whether to raise or fall back.
+
+## Data Handling
+
+- OCR payloads can contain personal data and large base64 images. Do not log
+ payloads or provider responses.
+- Avoid copying large payloads more than needed. The current JSON round-trip is
+ acceptable for the first scaffold, but future performance work should evaluate
+ direct PyO3 conversion before expanding Rust coverage to image-heavy paths.
+- Do not expose raw Rust errors that include document contents or upstream
+ bodies.
+
+## Tests
+
+- `cargo test --workspace` must compile this crate.
+- Python tests must cover bridge disabled, bridge enabled, and module-missing
+ fallback behavior for every exposed route.
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
new file mode 100644
index 00000000000..80b6478daac
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "litellm-python-bridge"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[lib]
+name = "litellm_python_bridge"
+crate-type = ["cdylib"]
+
+[dependencies]
+litellm-core.workspace = true
+litellm-providers.workspace = true
+pyo3 = { workspace = true, features = ["extension-module"] }
+serde_json.workspace = true
diff --git a/litellm-rust/crates/python-bridge/src/gil.rs b/litellm-rust/crates/python-bridge/src/gil.rs
new file mode 100644
index 00000000000..dc1b591735c
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/src/gil.rs
@@ -0,0 +1,32 @@
+//! GIL accounting.
+//!
+//! A single chokepoint for releasing the GIL around blocking work. Every
+//! blocking call in the bridge goes through [`release_gil`] instead of calling
+//! `Python::allow_threads` directly, so the release count stays accurate and we
+//! have one place to extend later (timing histograms, per-call labels, etc.).
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use pyo3::prelude::*;
+
+/// Number of times the bridge has released the GIL since process start.
+static GIL_RELEASES: AtomicU64 = AtomicU64::new(0);
+
+/// Release the GIL around `f`, recording the release.
+///
+/// `f` must not touch any Python state — that is what makes releasing the GIL
+/// safe. Returning the value back to Python re-acquires the GIL at the call
+/// site, after `f` has finished.
+pub fn release_gil(py: Python<'_>, f: F) -> T
+where
+ F: FnOnce() -> T + Send,
+ T: Send,
+{
+ GIL_RELEASES.fetch_add(1, Ordering::Relaxed);
+ py.allow_threads(f)
+}
+
+/// Total GIL releases performed by the bridge so far.
+pub fn release_count() -> u64 {
+ GIL_RELEASES.load(Ordering::Relaxed)
+}
diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs
new file mode 100644
index 00000000000..15e93f7b00c
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/src/lib.rs
@@ -0,0 +1,100 @@
+use std::time::Duration;
+
+use litellm_core::error::CoreError;
+use litellm_providers::ocr::run_ocr;
+use pyo3::exceptions::{PyRuntimeError, PyValueError};
+use pyo3::prelude::*;
+use pyo3::types::{PyAny, PyDict};
+use serde_json::{Map, Value};
+
+mod gil;
+
+fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult {
+ let json = py.import("json")?;
+ let encoded: String = json.call_method1("dumps", (value,))?.extract()?;
+ serde_json::from_str(&encoded).map_err(|err| PyValueError::new_err(err.to_string()))
+}
+
+fn json_to_py(py: Python<'_>, value: Value) -> PyResult> {
+ let json = py.import("json")?;
+ let encoded =
+ serde_json::to_string(&value).map_err(|err| PyValueError::new_err(err.to_string()))?;
+ Ok(json.call_method1("loads", (encoded,))?.unbind())
+}
+
+/// Map a core error to the closest Python exception. Caller-input problems
+/// (auth, bad types, missing fields) -> `ValueError`; everything else
+/// (network, upstream status, parse failures) -> `RuntimeError`.
+fn core_error_to_pyerr(err: CoreError) -> PyErr {
+ match err {
+ CoreError::Auth(message) => PyValueError::new_err(message),
+ CoreError::InvalidType { .. } | CoreError::MissingField(_) => {
+ PyValueError::new_err(err.to_string())
+ }
+ other => PyRuntimeError::new_err(other.to_string()),
+ }
+}
+
+/// Perform a Mistral OCR call end to end and return the response as a dict.
+#[pyfunction]
+#[pyo3(signature = (model, document, api_key=None, api_base=None, optional_params=None, timeout_seconds=None))]
+fn ocr(
+ py: Python<'_>,
+ model: String,
+ document: Py,
+ api_key: Option,
+ api_base: Option,
+ optional_params: Option>,
+ timeout_seconds: Option,
+) -> PyResult> {
+ let document = py_to_json(py, document.bind(py))?;
+
+ let optional_params = match optional_params {
+ Some(params) => match py_to_json(py, params.bind(py))? {
+ Value::Object(map) => map,
+ _ => return Err(PyValueError::new_err("optional_params must be a dict")),
+ },
+ None => Map::new(),
+ };
+
+ let timeout = timeout_seconds.and_then(|secs| {
+ if secs.is_finite() && secs > 0.0 {
+ Some(Duration::from_secs_f64(secs))
+ } else {
+ None
+ }
+ });
+
+ // Release the GIL during the blocking HTTP call (counted for observability).
+ let result = gil::release_gil(py, || {
+ run_ocr(
+ &model,
+ document,
+ api_key.as_deref(),
+ api_base.as_deref(),
+ optional_params,
+ timeout,
+ )
+ });
+
+ match result {
+ Ok(value) => json_to_py(py, value),
+ Err(err) => Err(core_error_to_pyerr(err)),
+ }
+}
+
+/// Bridge GIL accounting, e.g. `{"releases": 12}`. Lets the Python side observe
+/// how often the bridge has dropped the GIL for blocking work.
+#[pyfunction]
+fn gil_stats(py: Python<'_>) -> PyResult> {
+ let stats = PyDict::new(py);
+ stats.set_item("releases", gil::release_count())?;
+ Ok(stats.into_any().unbind())
+}
+
+#[pymodule]
+fn litellm_python_bridge(module: &Bound<'_, PyModule>) -> PyResult<()> {
+ module.add_function(wrap_pyfunction!(ocr, module)?)?;
+ module.add_function(wrap_pyfunction!(gil_stats, module)?)?;
+ Ok(())
+}
diff --git a/litellm/__init__.py b/litellm/__init__.py
index 0d6a788e368..d0513f77b35 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -80,6 +80,7 @@ from litellm.constants import (
WANDB_MODELS,
REPEATED_STREAMING_CHUNK_LIMIT,
request_timeout,
+ request_timeout_explicitly_set as request_timeout_explicitly_set,
open_ai_embedding_models,
cohere_embedding_models,
bedrock_embedding_models,
@@ -213,6 +214,15 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
+# When True (default — preserves historical behavior), the Router appends
+# internal config names (model_group, fallback model groups, deployment
+# timeouts, fallback failure details) onto exception messages and surfaces
+# them to clients via ProxyException.message. Set to False if you do NOT
+# want the proxy's internal model_name / fallback wiring visible to clients.
+# Deprecation: planned to flip to False (redact by default) in a future
+# major release; opt in early with `litellm.expose_router_debug_in_errors
+# = False`.
+expose_router_debug_in_errors: bool = True
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
@@ -235,6 +245,17 @@ modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))
use_chat_completions_url_for_anthropic_messages: bool = bool(
os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API
+# When True, strip the OpenAI-flavored `usage.total_tokens` field that
+# LiteLLM injects into non-streaming /v1/messages responses, bringing the
+# wire response into line with the Anthropic spec (matches the streaming
+# SSE path, which already omits total_tokens). Default False to preserve
+# backward compatibility for clients that read the LiteLLM-shaped
+# `usage.total_tokens` today. Planned to flip to True in a future major
+# release; opt in early via Python:
+# `litellm.strip_anthropic_total_tokens = True`
+# Or via `litellm_settings.strip_anthropic_total_tokens: true` in
+# config.yaml.
+strip_anthropic_total_tokens: bool = False
route_all_chat_openai_to_responses: bool = (
os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true"
) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge
@@ -413,7 +434,7 @@ anthropic_beta_headers_url: str = os.getenv(
"LITELLM_ANTHROPIC_BETA_HEADERS_URL",
"https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json",
)
-suppress_debug_info = False
+suppress_debug_info: bool = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
s3_audit_callback_params: Optional[Dict] = None
@@ -653,6 +674,7 @@ elevenlabs_models: Set = set()
dashscope_models: Set = set()
moonshot_models: Set = set()
publicai_models: Set = set()
+darkbloom_models: Set = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
@@ -907,6 +929,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
moonshot_models.add(key)
elif value.get("litellm_provider") == "publicai":
publicai_models.add(key)
+ elif value.get("litellm_provider") == "darkbloom":
+ darkbloom_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
@@ -1055,6 +1079,7 @@ model_list = list(
| dashscope_models
| moonshot_models
| publicai_models
+ | darkbloom_models
| v0_models
| morph_models
| lambda_ai_models
@@ -1159,6 +1184,7 @@ models_by_provider: dict = {
"modelscope": modelscope_models,
"moonshot": moonshot_models,
"publicai": publicai_models,
+ "darkbloom": darkbloom_models,
"v0": v0_models,
"morph": morph_models,
"lambda_ai": lambda_ai_models,
@@ -1380,7 +1406,9 @@ from .skills.main import (
)
from .containers.main import *
from .ocr.main import *
+from .ocr.rust_bridge import use_litellm_rust
from .rag.main import *
+from .sandbox.main import *
from .search.main import *
from .realtime_api.main import (
_arealtime,
@@ -1901,9 +1929,6 @@ if TYPE_CHECKING:
from .llms.fireworks_ai.completion.transformation import (
FireworksAITextCompletionConfig as FireworksAITextCompletionConfig,
)
- from .llms.fireworks_ai.audio_transcription.transformation import (
- FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig,
- )
from .llms.fireworks_ai.embed.fireworks_ai_transformation import (
FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig,
)
diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py
index e653b40fd04..4f131354d2e 100644
--- a/litellm/_lazy_imports_registry.py
+++ b/litellm/_lazy_imports_registry.py
@@ -260,7 +260,6 @@ LLM_CONFIG_NAMES = (
"SambaNovaEmbeddingConfig",
"FireworksAIConfig",
"FireworksAITextCompletionConfig",
- "FireworksAIAudioTranscriptionConfig",
"FireworksAIEmbeddingConfig",
"FriendliaiChatConfig",
"JinaAIEmbeddingConfig",
@@ -1027,10 +1026,6 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.fireworks_ai.completion.transformation",
"FireworksAITextCompletionConfig",
),
- "FireworksAIAudioTranscriptionConfig": (
- ".llms.fireworks_ai.audio_transcription.transformation",
- "FireworksAIAudioTranscriptionConfig",
- ),
"FireworksAIEmbeddingConfig": (
".llms.fireworks_ai.embed.fireworks_ai_transformation",
"FireworksAIEmbeddingConfig",
diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py
index 997ad10bc33..cb122e90102 100644
--- a/litellm/caching/caching.py
+++ b/litellm/caching/caching.py
@@ -100,6 +100,8 @@ class Cache:
gcs_path: Optional[str] = None,
redis_semantic_cache_embedding_model: str = "text-embedding-ada-002",
redis_semantic_cache_index_name: Optional[str] = None,
+ valkey_semantic_cache_embedding_model: str = "text-embedding-ada-002",
+ valkey_semantic_cache_index_name: str | None = None,
redis_flush_size: Optional[int] = None,
redis_startup_nodes: Optional[List] = None,
disk_cache_dir: Optional[str] = None,
@@ -208,6 +210,21 @@ class Cache:
index_name=redis_semantic_cache_index_name,
**kwargs,
)
+ elif type == LiteLLMCacheType.VALKEY_SEMANTIC:
+ # Imported here, not at module top, so the optional redis dependency
+ # is only required when this backend is actually selected.
+ from .valkey_semantic_cache import ValkeySemanticCache
+
+ self.cache = ValkeySemanticCache(
+ host=host,
+ port=port,
+ password=password,
+ similarity_threshold=similarity_threshold,
+ embedding_model=valkey_semantic_cache_embedding_model,
+ index_name=valkey_semantic_cache_index_name,
+ startup_nodes=redis_startup_nodes,
+ **kwargs,
+ )
elif type == LiteLLMCacheType.QDRANT_SEMANTIC:
self.cache = QdrantSemanticCache(
qdrant_api_base=qdrant_api_base,
@@ -267,12 +284,50 @@ class Cache:
if (
self.type == LiteLLMCacheType.REDIS
or self.type == LiteLLMCacheType.REDIS_SEMANTIC
+ or self.type == LiteLLMCacheType.VALKEY_SEMANTIC
) and default_in_redis_ttl is not None:
self.ttl = default_in_redis_ttl
if self.namespace is not None and isinstance(self.cache, RedisCache):
self.cache.namespace = self.namespace
+ # Params whose values carry prompt content. Excluded from semantic-cache
+ # scope keys so differently worded prompts share a bucket and match via
+ # vector similarity rather than being split into per-wording buckets.
+ _SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS: frozenset = frozenset(
+ {"messages", "prompt", "input"}
+ )
+
+ # Server-set identity (from proxy auth) used to isolate semantic-cache
+ # buckets per tenant. Required once the prompt is out of the scope key, so a
+ # similar prompt from another key/team/org stays in a separate bucket.
+ _SEMANTIC_CACHE_TENANT_SCOPE_FIELDS: tuple[str, ...] = (
+ "user_api_key",
+ "user_api_key_team_id",
+ "user_api_key_org_id",
+ )
+
+ def _is_semantic_cache(self) -> bool:
+ return self.type in (
+ LiteLLMCacheType.REDIS_SEMANTIC,
+ LiteLLMCacheType.QDRANT_SEMANTIC,
+ LiteLLMCacheType.VALKEY_SEMANTIC,
+ )
+
+ def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
+ metadata: dict = kwargs.get("metadata") or {}
+ litellm_params: dict = kwargs.get("litellm_params") or {}
+ metadata_in_litellm_params: dict = litellm_params.get("metadata") or {}
+
+ scope = ""
+ for field in self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS:
+ value = metadata.get(field)
+ if value is None:
+ value = metadata_in_litellm_params.get(field)
+ if value is not None:
+ scope += f"{field}: {value}"
+ return scope
+
def get_cache_key(self, **kwargs) -> str:
"""
Get the cache key for the given arguments.
@@ -293,7 +348,15 @@ class Cache:
combined_kwargs = ModelParamHelper._get_all_llm_api_params()
litellm_param_kwargs = all_litellm_params
+ is_semantic_cache = self._is_semantic_cache()
+ scope_excluded_params = (
+ self._SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS
+ if is_semantic_cache
+ else frozenset()
+ )
for param in kwargs:
+ if param in scope_excluded_params:
+ continue
if param in combined_kwargs:
param_value: Optional[str] = self._get_param_value(param, kwargs)
if param_value is not None:
@@ -309,6 +372,9 @@ class Cache:
param_value = kwargs[param]
cache_key += f"{str(param)}: {str(param_value)}"
+ if is_semantic_cache:
+ cache_key += self._get_semantic_cache_tenant_scope(kwargs)
+
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
diff --git a/litellm/caching/gcs_cache.py b/litellm/caching/gcs_cache.py
index 3327e094bc2..0e6a111eb2b 100644
--- a/litellm/caching/gcs_cache.py
+++ b/litellm/caching/gcs_cache.py
@@ -5,6 +5,7 @@ Supports syncing responses to Google Cloud Storage Buckets using HTTP requests.
import json
import asyncio
from typing import Optional
+from urllib.parse import quote
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
@@ -48,7 +49,7 @@ class GCSCache(BaseCache):
headers = self._construct_headers()
object_name = self.key_prefix + key
bucket_name = self.bucket_name
- url = f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}"
+ url = f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={quote(object_name, safe='')}"
data = json.dumps(value)
self.sync_client.post(url=url, data=data, headers=headers)
except Exception as e:
@@ -59,7 +60,7 @@ class GCSCache(BaseCache):
headers = self._construct_headers()
object_name = self.key_prefix + key
bucket_name = self.bucket_name
- url = f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}"
+ url = f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={quote(object_name, safe='')}"
data = json.dumps(value)
await self.async_client.post(url=url, data=data, headers=headers)
except Exception as e:
@@ -72,7 +73,7 @@ class GCSCache(BaseCache):
headers = self._construct_headers()
object_name = self.key_prefix + key
bucket_name = self.bucket_name
- url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}?alt=media"
+ url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{quote(object_name, safe='')}?alt=media"
response = self.sync_client.get(url=url, headers=headers)
if response.status_code == 200:
cached_response = json.loads(response.text)
@@ -91,7 +92,7 @@ class GCSCache(BaseCache):
headers = self._construct_headers()
object_name = self.key_prefix + key
bucket_name = self.bucket_name
- url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}?alt=media"
+ url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{quote(object_name, safe='')}?alt=media"
response = await self.async_client.get(url=url, headers=headers)
if response.status_code == 200:
return json.loads(response.text)
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index 263e1df2ee7..ba07511448a 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -903,6 +903,43 @@ class RedisCache(BaseCache):
)
raise e
+ @_redis_circuit_breaker_guard
+ async def async_set_max(
+ self,
+ key: str,
+ value: float,
+ ttl: int | None = None,
+ ) -> float | None:
+ """Atomically set ``key`` to ``value`` only when ``value`` is greater
+ than the stored value (or the key is unset), refreshing the TTL.
+
+ Monotonic by construction: it never lowers the stored value, so a repair
+ that writes an authoritative-but-slightly-stale total cannot clobber a
+ concurrent increment that has already pushed the counter higher. The
+ GET/compare/SET runs in a single Lua call, so it is also atomic across
+ racing callers and pods. Returns the resulting value.
+ """
+ _redis_client = self.init_async_client()
+ _used_ttl = self.get_ttl(ttl=ttl)
+ key = self.check_and_fix_namespace(key=key)
+ lua = (
+ "local cur = redis.call('GET', KEYS[1]) "
+ "if cur == false or tonumber(cur) < tonumber(ARGV[1]) then "
+ "redis.call('SET', KEYS[1], ARGV[1]) "
+ "if tonumber(ARGV[2]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]) end "
+ "return ARGV[1] end "
+ "return cur"
+ )
+ result = cast(
+ "str | bytes | int | float | None",
+ await _redis_client.eval(lua, 1, key, str(value), str(int(_used_ttl or 0))),
+ )
+ if result is None:
+ return None
+ if isinstance(result, bytes):
+ result = result.decode()
+ return float(result)
+
async def flush_cache_buffer(self):
print_verbose(
f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}"
diff --git a/litellm/caching/valkey_semantic_cache.py b/litellm/caching/valkey_semantic_cache.py
new file mode 100644
index 00000000000..bf368b74d07
--- /dev/null
+++ b/litellm/caching/valkey_semantic_cache.py
@@ -0,0 +1,353 @@
+"""
+Valkey Semantic Cache implementation for LiteLLM
+
+Backs semantic caching with Valkey (for example AWS ElastiCache for Valkey)
+running the valkey-search module.
+
+RedisVL cannot drive valkey-search: it gates on a RediSearch module version
+that valkey-search does not report, and its SemanticCache index uses a TEXT
+field that valkey-search does not implement. This backend therefore talks to
+valkey-search directly over redis-py, building a vector index from the field
+types valkey-search does support (TAG for cache-key isolation and VECTOR for
+the prompt embedding) and running KNN queries for retrieval. Prompt extraction,
+embedding generation, and cached-response parsing are reused from
+RedisSemanticCache since those are backend agnostic.
+"""
+
+import asyncio
+import hashlib
+import os
+import struct
+from dataclasses import dataclass
+from typing import Any
+
+from redis import Redis
+from redis.asyncio import Redis as AsyncRedis
+from redis.commands.search.field import TagField, VectorField
+from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.query import Query
+
+from litellm._logging import print_verbose
+from litellm._uuid import uuid
+
+from .redis_semantic_cache import RedisSemanticCache
+
+
+@dataclass(frozen=True, slots=True)
+class _ValkeyCacheHit:
+ response: str
+ distance: float
+
+
+class ValkeySemanticCache(RedisSemanticCache):
+ """Valkey-backed semantic cache for LLM responses."""
+
+ DEFAULT_VALKEY_INDEX_NAME: str = "litellm_semantic_cache_index"
+ EMBEDDING_FIELD_NAME: str = "embedding"
+ PROMPT_FIELD_NAME: str = "prompt"
+ RESPONSE_FIELD_NAME: str = "response"
+ DISTANCE_FIELD_NAME: str = "vector_distance"
+
+ def __init__(
+ self,
+ host: str | None = None,
+ port: str | None = None,
+ password: str | None = None,
+ redis_url: str | None = None,
+ similarity_threshold: float | None = None,
+ embedding_model: str = "text-embedding-ada-002",
+ index_name: str | None = None,
+ ssl: bool = False,
+ startup_nodes: list | None = None,
+ sync_client: Redis | None = None,
+ async_client: AsyncRedis | None = None,
+ **kwargs: Any,
+ ):
+ if similarity_threshold is None:
+ raise ValueError("similarity_threshold must be provided, passed None")
+
+ if startup_nodes:
+ raise ValueError(
+ "valkey-semantic does not support cluster-mode-enabled (multi-shard) "
+ "endpoints. The async cluster client cannot route the FT.* search "
+ "commands reliably. Point it at a cluster-mode-disabled endpoint "
+ "instead (a primary with replicas is fine; only horizontal sharding "
+ "is unsupported), or pass a single redis_url. On AWS, vector search "
+ "needs ElastiCache for Valkey 8.2+ on a node-based cluster."
+ )
+
+ self.similarity_threshold = similarity_threshold
+ self.embedding_model = embedding_model
+ self.index_name = index_name or self.DEFAULT_VALKEY_INDEX_NAME
+ self.key_prefix = f"{self.index_name}:"
+ self._index_dim: int | None = None
+
+ resolved_url = None
+ if sync_client is None or async_client is None:
+ resolved_url = redis_url or self._build_valkey_url(
+ host, port, password, ssl
+ )
+ self.sync_client = (
+ sync_client if sync_client is not None else Redis.from_url(resolved_url) # type: ignore[arg-type]
+ )
+ self.async_client = (
+ async_client
+ if async_client is not None
+ else AsyncRedis.from_url(resolved_url) # type: ignore[arg-type]
+ )
+
+ print_verbose(f"Valkey semantic-cache initializing index - {self.index_name}")
+
+ @staticmethod
+ def _build_valkey_url(
+ host: str | None, port: str | None, password: str | None, ssl: bool = False
+ ) -> str:
+ host = host or os.environ.get("VALKEY_HOST") or os.environ.get("REDIS_HOST")
+ port = port or os.environ.get("VALKEY_PORT") or os.environ.get("REDIS_PORT")
+ password = (
+ password
+ or os.environ.get("VALKEY_PASSWORD")
+ or os.environ.get("REDIS_PASSWORD")
+ )
+
+ if not host or not port:
+ raise ValueError(
+ "Missing required Valkey configuration. Provide host and port "
+ "(or VALKEY_HOST/VALKEY_PORT), or pass redis_url."
+ )
+
+ credentials = f":{password}@" if password else ""
+ scheme = "rediss" if ssl else "redis"
+ return f"{scheme}://{credentials}{host}:{port}"
+
+ @classmethod
+ def _scope_tag(cls, key: str) -> str:
+ # valkey-search TAG fields tokenize on punctuation and do not honour
+ # backslash escaping, so an arbitrary cache key cannot be matched
+ # verbatim. Hashing to hex yields a token that is always exact-match
+ # safe and still uniquely isolates a caller's scope.
+ return hashlib.sha256(str(key).encode("utf-8")).hexdigest()
+
+ @staticmethod
+ def _embedding_to_bytes(embedding: list[float]) -> bytes:
+ return struct.pack(f"<{len(embedding)}f", *embedding)
+
+ def _index_schema(self, dim: int) -> tuple[TagField, VectorField]:
+ return (
+ TagField(self.CACHE_KEY_FIELD_NAME),
+ VectorField(
+ self.EMBEDDING_FIELD_NAME,
+ "HNSW",
+ {"TYPE": "FLOAT32", "DIM": dim, "DISTANCE_METRIC": "COSINE"},
+ ),
+ )
+
+ def _index_definition(self) -> IndexDefinition:
+ return IndexDefinition(prefix=[self.key_prefix], index_type=IndexType.HASH)
+
+ @staticmethod
+ def _is_index_exists_error(exc: Exception) -> bool:
+ return "already exists" in str(exc).lower()
+
+ @staticmethod
+ def _extract_index_dim(info: dict) -> int | None:
+ # FT.INFO nests the vector field's "dimensions" one level inside its
+ # "index" block, so flatten each field descriptor a single level and
+ # scan for the dimensions marker.
+ for field in info.get("attributes") or []:
+ if not isinstance(field, (list, tuple)):
+ continue
+ flat = [
+ sub
+ for item in field
+ for sub in (item if isinstance(item, (list, tuple)) else [item])
+ ]
+ for i, marker in enumerate(flat):
+ if marker in (b"dimensions", "dimensions") and i + 1 < len(flat):
+ return int(flat[i + 1])
+ return None
+
+ def _assert_dim_matches(self, info: dict, dim: int) -> None:
+ existing_dim = self._extract_index_dim(info)
+ if existing_dim is not None and existing_dim != dim:
+ raise ValueError(
+ f"Valkey semantic-cache index '{self.index_name}' already exists with "
+ f"embedding dimension {existing_dim}, but the configured embedding "
+ f"model produced dimension {dim}. Use a different "
+ f"valkey_semantic_cache_index_name or drop the existing index."
+ )
+
+ def _ensure_index_sync(self, dim: int) -> None:
+ if self._index_dim == dim:
+ return
+ try:
+ self.sync_client.ft(self.index_name).create_index(
+ self._index_schema(dim), definition=self._index_definition()
+ )
+ except Exception as exc:
+ if not self._is_index_exists_error(exc):
+ raise
+ self._assert_dim_matches(self.sync_client.ft(self.index_name).info(), dim)
+ self._index_dim = dim
+
+ async def _ensure_index_async(self, dim: int) -> None:
+ if self._index_dim == dim:
+ return
+ try:
+ await self.async_client.ft(self.index_name).create_index(
+ self._index_schema(dim), definition=self._index_definition()
+ )
+ except Exception as exc:
+ if not self._is_index_exists_error(exc):
+ raise
+ info = await self.async_client.ft(self.index_name).info()
+ self._assert_dim_matches(info, dim)
+ self._index_dim = dim
+
+ def _doc_key(self, key: str) -> str:
+ return f"{self.key_prefix}{self._scope_tag(key)}:{uuid.uuid4()}"
+
+ def _doc_mapping(
+ self, key: str, prompt: str, value_str: str, embedding: list[float]
+ ) -> dict:
+ return {
+ self.CACHE_KEY_FIELD_NAME: self._scope_tag(key),
+ self.PROMPT_FIELD_NAME: prompt,
+ self.RESPONSE_FIELD_NAME: value_str,
+ self.EMBEDDING_FIELD_NAME: self._embedding_to_bytes(embedding),
+ }
+
+ def _knn_query(self, key: str) -> Query:
+ scope = self._scope_tag(key)
+ query_string = (
+ f"(@{self.CACHE_KEY_FIELD_NAME}:{{{scope}}})"
+ f"=>[KNN 1 @{self.EMBEDDING_FIELD_NAME} $vec AS {self.DISTANCE_FIELD_NAME}]"
+ )
+ return (
+ Query(query_string)
+ .return_fields(self.RESPONSE_FIELD_NAME, self.DISTANCE_FIELD_NAME)
+ .dialect(2)
+ )
+
+ @classmethod
+ def _first_hit(cls, search_result: Any) -> _ValkeyCacheHit | None:
+ docs = getattr(search_result, "docs", [])
+ if not docs:
+ return None
+ doc = docs[0]
+ return _ValkeyCacheHit(
+ response=str(getattr(doc, cls.RESPONSE_FIELD_NAME)),
+ distance=float(getattr(doc, cls.DISTANCE_FIELD_NAME)),
+ )
+
+ def _resolve_hit(self, hit: _ValkeyCacheHit | None, key: str, **kwargs: Any) -> Any:
+ if hit is None:
+ kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+ return None
+
+ similarity = 1 - hit.distance
+ kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
+
+ if similarity < self.similarity_threshold:
+ return None
+ return self._get_cache_logic(cached_response=hit.response)
+
+ def set_cache(self, key: str, value: Any, **kwargs: Any) -> None:
+ print_verbose(f"Valkey semantic-cache set_cache, kwargs: {kwargs}")
+ try:
+ prompt = self._get_prompt_from_kwargs(**kwargs)
+ if prompt is None:
+ print_verbose("No prompt provided for semantic caching")
+ return
+
+ embedding = self._get_embedding(prompt)
+ self._ensure_index_sync(len(embedding))
+
+ doc_key = self._doc_key(key)
+ self.sync_client.hset(
+ doc_key, mapping=self._doc_mapping(key, prompt, str(value), embedding)
+ )
+ ttl = self._get_ttl(**kwargs)
+ if ttl is not None:
+ self.sync_client.expire(doc_key, ttl)
+ except Exception as e:
+ print_verbose(f"Error in Valkey semantic-cache set_cache: {str(e)}")
+
+ def get_cache(self, key: str, **kwargs: Any) -> Any:
+ print_verbose(f"Valkey semantic-cache get_cache, kwargs: {kwargs}")
+ try:
+ prompt = self._get_prompt_from_kwargs(**kwargs)
+ if prompt is None:
+ kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+ return None
+
+ embedding = self._get_embedding(prompt)
+ self._ensure_index_sync(len(embedding))
+
+ search_result = self.sync_client.ft(self.index_name).search(
+ self._knn_query(key),
+ query_params={"vec": self._embedding_to_bytes(embedding)},
+ )
+ return self._resolve_hit(self._first_hit(search_result), key, **kwargs)
+ except Exception as e:
+ print_verbose(f"Error in Valkey semantic-cache get_cache: {str(e)}")
+ kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+
+ async def async_set_cache(self, key: str, value: Any, **kwargs: Any) -> None:
+ print_verbose(f"Async Valkey semantic-cache set_cache, kwargs: {kwargs}")
+ try:
+ prompt = self._get_prompt_from_kwargs(**kwargs)
+ if prompt is None:
+ print_verbose("No prompt provided for semantic caching")
+ return
+
+ embedding = await self._get_async_embedding(prompt, **kwargs)
+ await self._ensure_index_async(len(embedding))
+
+ doc_key = self._doc_key(key)
+ await self.async_client.hset(
+ doc_key, mapping=self._doc_mapping(key, prompt, str(value), embedding)
+ )
+ ttl = self._get_ttl(**kwargs)
+ if ttl is not None:
+ await self.async_client.expire(doc_key, ttl)
+ except Exception as e:
+ print_verbose(f"Error in async Valkey semantic-cache set_cache: {str(e)}")
+
+ async def async_get_cache(self, key: str, **kwargs: Any) -> Any:
+ print_verbose(f"Async Valkey semantic-cache get_cache, kwargs: {kwargs}")
+ try:
+ prompt = self._get_prompt_from_kwargs(**kwargs)
+ if prompt is None:
+ kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+ return None
+
+ embedding = await self._get_async_embedding(prompt, **kwargs)
+ await self._ensure_index_async(len(embedding))
+
+ search_result = await self.async_client.ft(self.index_name).search(
+ self._knn_query(key),
+ query_params={"vec": self._embedding_to_bytes(embedding)},
+ )
+ return self._resolve_hit(self._first_hit(search_result), key, **kwargs)
+ except Exception as e:
+ print_verbose(f"Error in async Valkey semantic-cache get_cache: {str(e)}")
+ kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+
+ async def async_set_cache_pipeline(
+ self, cache_list: list[tuple[str, Any]], **kwargs: Any
+ ) -> None:
+ try:
+ await asyncio.gather(
+ *[
+ self.async_set_cache(key, value, **kwargs)
+ for key, value in cache_list
+ ]
+ )
+ except Exception as e:
+ print_verbose(
+ f"Error in Valkey semantic-cache async_set_cache_pipeline: {str(e)}"
+ )
+
+ async def _index_info(self) -> dict:
+ return await self.async_client.ft(self.index_name).info()
diff --git a/litellm/constants.py b/litellm/constants.py
index a3ea68c7949..212d34357f8 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -201,6 +201,18 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
# Provider-specific API base URLs
XAI_API_BASE = "https://api.x.ai/v1"
+OPEN_SANDBOX_API_BASE_ENV_VAR = "OPEN_SANDBOX_API_BASE"
+OPEN_SANDBOX_API_KEY_ENV_VAR = "OPEN_SANDBOX_API_KEY"
+OPEN_SANDBOX_DEFAULT_TEMPLATE = "opensandbox/code-interpreter:v1.1.0"
+_OPEN_SANDBOX_FALLBACK_ENTRYPOINT = "/opt/code-interpreter/code-interpreter.sh"
+OPEN_SANDBOX_DEFAULT_ENTRYPOINT = (_OPEN_SANDBOX_FALLBACK_ENTRYPOINT,)
+OPEN_SANDBOX_DEFAULT_LANGUAGE = "python"
+OPEN_SANDBOX_DEFAULT_CPU_LIMIT = "1"
+OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT = "2Gi"
+OPEN_SANDBOX_EXECD_PORT = 44772
+OPEN_SANDBOX_DEFAULT_TIMEOUT = 300
+OPEN_SANDBOX_READY_TIMEOUT = 30.0
+OPEN_SANDBOX_POLL_INTERVAL = 0.2
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024)
@@ -456,6 +468,7 @@ HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: float = 5.0
request_timeout: float = float(
os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS)))
)
+request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ
DEFAULT_A2A_AGENT_TIMEOUT: float = float(
os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)
) # 10 minutes
@@ -802,6 +815,7 @@ openai_compatible_endpoints: List = [
"https://api.inference.wandb.ai/v1",
"https://api.clarifai.com/v2/ext/openai/v1",
"https://api.libertai.io/v1",
+ "https://pinstripes.io/v1",
]
@@ -865,6 +879,8 @@ openai_compatible_providers: List = [
"clarifai",
"docker_model_runner",
"ragflow",
+ "pinstripes", # Pinstripes - JSON-configured provider
+ "darkbloom",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index bb5b778d02e..27a146df7bf 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -888,6 +888,23 @@ def _map_traffic_type_to_service_tier(traffic_type: Optional[str]) -> Optional[s
return service_tier
+def _normalize_service_tier(service_tier: object) -> str | None:
+ """
+ Reduce a service_tier value to a concrete billable tier string or None.
+
+ "auto" is a routing preference and any non-string value is not a billable
+ tier, so both defer to standard pricing (or to the tier the provider reports
+ on the response usage) instead of crashing the downstream cost-key lookup,
+ which calls service_tier.lower()
+ """
+ if (
+ not isinstance(service_tier, str)
+ or service_tier.lower() == ServiceTier.AUTO.value
+ ):
+ return None
+ return service_tier
+
+
def _get_usage_object(
completion_response: Any,
) -> Optional[Usage]:
@@ -1227,15 +1244,7 @@ def completion_cost(
if service_tier is None and optional_params is not None:
service_tier = optional_params.get("service_tier")
- # A request-level service_tier only prices the request when it is a
- # concrete billable tier string. "auto" is a routing preference and any
- # non-string value is not a billable tier, so defer to the tier the
- # provider reports on the response/usage instead of crashing or mispricing
- if (
- not isinstance(service_tier, str)
- or service_tier.lower() == ServiceTier.AUTO.value
- ):
- service_tier = None
+ service_tier = _normalize_service_tier(service_tier)
# Extract service_tier from completion_response if not provided
if service_tier is None and completion_response is not None:
@@ -1244,6 +1253,8 @@ def completion_cost(
elif isinstance(completion_response, dict):
service_tier = completion_response.get("service_tier")
+ service_tier = _normalize_service_tier(service_tier)
+
# Extract service_tier from usage object if not provided
if service_tier is None and cost_per_token_usage_object is not None:
if isinstance(cost_per_token_usage_object, BaseModel):
@@ -1253,6 +1264,8 @@ def completion_cost(
elif isinstance(cost_per_token_usage_object, dict):
service_tier = cost_per_token_usage_object.get("service_tier")
+ service_tier = _normalize_service_tier(service_tier)
+
selected_model = _select_model_name_for_cost_calc(
model=model,
completion_response=completion_response,
diff --git a/litellm/integrations/code_interpreter_interception/__init__.py b/litellm/integrations/code_interpreter_interception/__init__.py
new file mode 100644
index 00000000000..2256356b6f5
--- /dev/null
+++ b/litellm/integrations/code_interpreter_interception/__init__.py
@@ -0,0 +1,15 @@
+"""
+Code Interpreter Interception Module
+
+Converts the native OpenAI Responses ``code_interpreter`` tool into a function
+tool, runs the model-emitted code in a sandbox, and feeds the result back into
+the agentic loop.
+"""
+
+from litellm.integrations.code_interpreter_interception.handler import (
+ CodeInterpreterInterceptionLogger,
+)
+
+__all__ = [
+ "CodeInterpreterInterceptionLogger",
+]
diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py
new file mode 100644
index 00000000000..362581937d7
--- /dev/null
+++ b/litellm/integrations/code_interpreter_interception/handler.py
@@ -0,0 +1,839 @@
+"""
+Code Interpreter Interception Handler
+
+CustomLogger that swaps the native OpenAI Responses ``code_interpreter`` tool for
+a function tool, executes the code the model emits inside a sandbox, and feeds the
+captured stdout back through the typed agentic loop plan.
+"""
+
+import json
+import time
+import uuid
+from typing import Any, Literal, TypedDict, cast
+
+import litellm
+from pydantic import ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.integrations.code_interpreter_interception import (
+ CodeInterpreterInterceptionConfig,
+)
+from litellm.types.integrations.custom_logger import (
+ AgenticLoopPlan,
+ AgenticLoopRequestPatch,
+ CHAT_COMPLETION_AGENTIC_SURFACE,
+ NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
+ is_interception_internal_key,
+)
+from litellm.types.llms.openai import (
+ ChatCompletionAssistantMessage,
+ ChatCompletionAssistantToolCall,
+ ChatCompletionToolMessage,
+)
+from litellm.types.utils import (
+ CallTypes,
+ ChatCompletionMessageToolCall,
+ ModelResponse,
+)
+
+LITELLM_CODE_EXECUTION_TOOL_NAME = "litellm_code_execution"
+_INTERCEPTION_ACTIVE_KEY = "_code_interpreter_interception_active"
+_SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
+_CONVERTED_STREAM_KEY = "_code_interpreter_interception_converted_stream"
+_LITELLM_METADATA_KEY = "litellm_metadata"
+_CACHE_TTL_SECONDS = 15 * 60
+
+
+class CodeExecutionToolCall(TypedDict, total=False):
+ id: str | None
+ call_id: str | None
+ type: Literal["function"]
+ name: str
+ arguments: str
+
+
+class CodeInterpreterLogOutput(TypedDict):
+ type: Literal["logs"]
+ logs: str
+
+
+class CodeInterpreterCall(TypedDict):
+ id: str
+ type: Literal["code_interpreter_call"]
+ status: Literal["completed"]
+ code: str
+ container_id: str | None
+ outputs: list[CodeInterpreterLogOutput]
+
+
+class CodeExecutionFunctionParameters(TypedDict):
+ type: Literal["object"]
+ properties: dict[str, dict[str, str]]
+ required: list[str]
+
+
+class ResponsesFunctionTool(TypedDict):
+ type: Literal["function"]
+ name: str
+ description: str
+ parameters: CodeExecutionFunctionParameters
+
+
+class ChatCompletionFunctionDefinition(TypedDict):
+ name: str
+ description: str
+ parameters: CodeExecutionFunctionParameters
+
+
+class ChatCompletionFunctionTool(TypedDict):
+ type: Literal["function"]
+ function: ChatCompletionFunctionDefinition
+
+
+CodeExecutionFunctionTool = ResponsesFunctionTool | ChatCompletionFunctionTool
+
+
+class ResponsesFunctionToolChoice(TypedDict):
+ type: Literal["function"]
+ name: str
+
+
+class ChatCompletionFunctionToolChoice(TypedDict):
+ type: Literal["function"]
+ function: dict[str, str]
+
+
+CodeExecutionFunctionToolChoice = (
+ ResponsesFunctionToolChoice | ChatCompletionFunctionToolChoice
+)
+
+
+def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> dict[str, Any] | None:
+ try:
+ from litellm.sandbox.sandbox_tools import resolve_sandbox_tool
+ except ImportError:
+ return None
+ return resolve_sandbox_tool(sandbox_tool_name)
+
+
+class CodeInterpreterInterceptionLogger(CustomLogger):
+ """
+ CustomLogger that implements transparent code-interpreter execution loops.
+
+ Flow:
+ 1. Replace the native ``code_interpreter`` tool with a function tool in the
+ pre-call hook so the model emits code as function-call arguments.
+ 2. Detect ``litellm_code_execution`` function calls in the model response.
+ 3. Run the emitted code in a sandbox (reused per request via a server-minted
+ sandbox key) and build a typed rerun plan that appends the
+ function_call_output.
+ """
+
+ def __init__(
+ self,
+ enabled: bool = True,
+ enabled_providers: list[str] | None = None,
+ sandbox_tool_name: str | None = None,
+ sandbox_config: Any | None = None,
+ ):
+ super().__init__()
+ self.enabled = enabled
+ self.enabled_providers = enabled_providers
+ self.sandbox_tool_name = sandbox_tool_name
+ self.sandbox_config = sandbox_config
+ self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float]] = {}
+
+ @classmethod
+ def from_config_yaml(
+ cls, config: CodeInterpreterInterceptionConfig
+ ) -> "CodeInterpreterInterceptionLogger":
+ return cls(
+ enabled=bool(config.get("enabled", True)),
+ enabled_providers=config.get("enabled_providers"),
+ sandbox_tool_name=config.get("sandbox_tool_name"),
+ )
+
+ @staticmethod
+ def initialize_from_proxy_config(
+ litellm_settings: dict[str, Any],
+ callback_specific_params: dict[str, Any],
+ ) -> "CodeInterpreterInterceptionLogger":
+ params: CodeInterpreterInterceptionConfig = {}
+ if "code_interpreter_interception_params" in litellm_settings:
+ params = litellm_settings["code_interpreter_interception_params"]
+ elif "code_interpreter_interception" in callback_specific_params and isinstance(
+ callback_specific_params["code_interpreter_interception"], dict
+ ):
+ params = cast(
+ CodeInterpreterInterceptionConfig,
+ callback_specific_params["code_interpreter_interception"],
+ )
+ return CodeInterpreterInterceptionLogger.from_config_yaml(params)
+
+ async def async_pre_call_deployment_hook(
+ self, kwargs: dict[str, Any], call_type: CallTypes | None
+ ) -> dict | None:
+ if not kwargs.get("_agentic_loop_depth"):
+ kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None)
+ kwargs.pop(_SANDBOX_KEY, None)
+ self._strip_interception_metadata(kwargs)
+ if not self.enabled:
+ return None
+ if call_type not in (
+ CallTypes.responses,
+ CallTypes.aresponses,
+ CallTypes.completion,
+ CallTypes.acompletion,
+ ):
+ return None
+ if (
+ self.enabled_providers is not None
+ and self._resolve_provider(kwargs) not in self.enabled_providers
+ ):
+ return None
+
+ tools = kwargs.get("tools")
+ if not isinstance(tools, list):
+ return None
+ if not any(
+ isinstance(tool, dict) and tool.get("type") == "code_interpreter"
+ for tool in tools
+ ):
+ return None
+
+ kwargs[_INTERCEPTION_ACTIVE_KEY] = True
+ kwargs[_SANDBOX_KEY] = uuid.uuid4().hex
+ if kwargs.get("stream"):
+ kwargs["stream"] = False
+ kwargs[_CONVERTED_STREAM_KEY] = True
+ self._write_interception_metadata(kwargs)
+
+ function_tool = self._get_function_tool(call_type=call_type)
+ kwargs["tools"] = [
+ (
+ function_tool
+ if isinstance(tool, dict) and tool.get("type") == "code_interpreter"
+ else tool
+ )
+ for tool in tools
+ ]
+ if self._tool_choice_targets_code_interpreter(kwargs.get("tool_choice")):
+ kwargs["tool_choice"] = self._get_function_tool_choice(call_type=call_type)
+ return kwargs
+
+ @staticmethod
+ def _strip_interception_metadata(kwargs: dict[str, Any]) -> None:
+ metadata = kwargs.get(_LITELLM_METADATA_KEY)
+ if not isinstance(metadata, dict):
+ return
+ filtered_metadata = {
+ key: value
+ for key, value in metadata.items()
+ if not is_interception_internal_key(key)
+ and not key.startswith("_agentic_loop")
+ and key != "max_agentic_loops"
+ }
+ if filtered_metadata:
+ kwargs[_LITELLM_METADATA_KEY] = filtered_metadata
+ else:
+ kwargs.pop(_LITELLM_METADATA_KEY, None)
+
+ @staticmethod
+ def _write_interception_metadata(kwargs: dict[str, Any]) -> None:
+ metadata = kwargs.get(_LITELLM_METADATA_KEY)
+ metadata = dict(metadata) if isinstance(metadata, dict) else {}
+ for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _CONVERTED_STREAM_KEY):
+ if key in kwargs:
+ metadata[key] = kwargs[key]
+ kwargs[_LITELLM_METADATA_KEY] = metadata
+
+ @staticmethod
+ def _get_function_parameters() -> CodeExecutionFunctionParameters:
+ return {
+ "type": "object",
+ "properties": {"code": {"type": "string"}},
+ "required": ["code"],
+ }
+
+ def _get_function_tool(
+ self, call_type: CallTypes | None
+ ) -> CodeExecutionFunctionTool:
+ description = "Execute python code in a sandbox and return stdout."
+ if call_type in (CallTypes.completion, CallTypes.acompletion):
+ return {
+ "type": "function",
+ "function": {
+ "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+ "description": description,
+ "parameters": self._get_function_parameters(),
+ },
+ }
+ return {
+ "type": "function",
+ "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+ "description": description,
+ "parameters": self._get_function_parameters(),
+ }
+
+ @staticmethod
+ def _get_function_tool_choice(
+ call_type: CallTypes | None,
+ ) -> CodeExecutionFunctionToolChoice:
+ if call_type in (CallTypes.completion, CallTypes.acompletion):
+ return {
+ "type": "function",
+ "function": {"name": LITELLM_CODE_EXECUTION_TOOL_NAME},
+ }
+ return {
+ "type": "function",
+ "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+ }
+
+ @staticmethod
+ def _tool_choice_targets_code_interpreter(tool_choice: Any) -> bool:
+ if not isinstance(tool_choice, dict):
+ return False
+ function = tool_choice.get("function")
+ return (
+ tool_choice.get("type") == "code_interpreter"
+ or tool_choice.get("name") == "code_interpreter"
+ or tool_choice.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+ or (
+ isinstance(function, dict)
+ and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+ )
+ )
+
+ def _resolve_provider(self, kwargs: dict[str, Any]) -> str | None:
+ provider = kwargs.get("custom_llm_provider")
+ if provider:
+ return provider
+ model = kwargs.get("model")
+ if not isinstance(model, str):
+ return None
+ try:
+ return litellm.get_llm_provider(model=model)[1]
+ except Exception:
+ return None
+
+ async def async_should_run_agentic_loop(
+ self,
+ response: Any,
+ model: str,
+ messages: list[dict],
+ tools: list[dict] | None,
+ stream: bool,
+ custom_llm_provider: str,
+ kwargs: dict,
+ ) -> tuple[bool, dict]:
+ if not self.enabled:
+ return False, {}
+ if not kwargs.get(_INTERCEPTION_ACTIVE_KEY):
+ return False, {}
+ if (
+ self.enabled_providers is not None
+ and custom_llm_provider not in self.enabled_providers
+ ):
+ return False, {}
+
+ tool_calls = (
+ self._extract_chat_completion_code_execution_tool_calls(response=response)
+ if kwargs.get("_agentic_loop_api_surface")
+ == CHAT_COMPLETION_AGENTIC_SURFACE
+ else self._extract_code_execution_tool_calls(response=response)
+ )
+ if not tool_calls:
+ return False, {}
+
+ return True, {"tool_calls": tool_calls}
+
+ async def async_build_agentic_loop_plan(
+ self,
+ tools: dict,
+ model: str,
+ messages: list[dict],
+ response: Any,
+ anthropic_messages_provider_config: Any,
+ anthropic_messages_optional_request_params: dict,
+ logging_obj: Any,
+ stream: bool,
+ kwargs: dict,
+ ) -> AgenticLoopPlan:
+ if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE:
+ return await self._build_chat_completion_agentic_loop_plan(
+ tools=tools,
+ model=model,
+ messages=messages,
+ optional_params=anthropic_messages_optional_request_params,
+ kwargs=kwargs,
+ )
+
+ await self._prune_expired_cache()
+ tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
+ sandbox_key = kwargs.get(_SANDBOX_KEY)
+ container, params = await self._get_or_create_container(cache_key=sandbox_key)
+
+ try:
+ container_id = cast(str | None, getattr(container, "id", None))
+ input_list = self._normalize_messages(messages)
+ code_interpreter_calls: list[CodeInterpreterCall] = []
+ for tool_call in tool_calls:
+ arguments = tool_call.get("arguments", "")
+ code = self._parse_code(arguments)
+ stdout = await self._run_tool_call(
+ container=container, params=params, arguments=arguments
+ )
+ input_list.append(
+ {
+ "type": "function_call",
+ "call_id": tool_call.get("call_id"),
+ "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+ "arguments": arguments,
+ }
+ )
+ input_list.append(
+ {
+ "type": "function_call_output",
+ "call_id": tool_call.get("call_id"),
+ "output": stdout,
+ }
+ )
+ code_interpreter_calls.append(
+ {
+ "id": f"ci_{uuid.uuid4().hex}",
+ "type": "code_interpreter_call",
+ "status": "completed",
+ "code": code,
+ "container_id": container_id,
+ "outputs": (
+ [{"type": "logs", "logs": stdout}] if stdout else []
+ ),
+ }
+ )
+ except Exception:
+ await self._delete_container_for_cache_key(sandbox_key)
+ raise
+
+ optional_params = anthropic_messages_optional_request_params
+ request_patch = AgenticLoopRequestPatch(
+ model=model,
+ messages=input_list,
+ tools=self._get_followup_tools(
+ tools=optional_params.get("tools"),
+ call_type=CallTypes.responses,
+ ),
+ optional_params=self._get_followup_optional_params(optional_params),
+ kwargs=self._filter_agentic_loop_kwargs(kwargs),
+ )
+
+ return AgenticLoopPlan(
+ run_agentic_loop=True,
+ request_patch=request_patch,
+ metadata={
+ "tool_type": "code_interpreter",
+ "sandbox_key": sandbox_key or "",
+ "code_interpreter_calls": code_interpreter_calls,
+ },
+ )
+
+ async def _build_chat_completion_agentic_loop_plan(
+ self,
+ tools: dict[str, object],
+ model: str,
+ messages: list[dict],
+ optional_params: dict[str, object],
+ kwargs: dict[str, object],
+ ) -> AgenticLoopPlan:
+ await self._prune_expired_cache()
+ tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
+ sandbox_key = cast(str | None, kwargs.get(_SANDBOX_KEY))
+ container, params = await self._get_or_create_container(cache_key=sandbox_key)
+
+ try:
+ container_id = cast(str | None, getattr(container, "id", None))
+ tool_results = [
+ await self._build_chat_completion_tool_result(
+ container=container,
+ params=params,
+ tool_call=tool_call,
+ container_id=container_id,
+ )
+ for tool_call in tool_calls
+ ]
+ except Exception:
+ await self._delete_container_for_cache_key(sandbox_key)
+ raise
+ tool_messages = [result[0] for result in tool_results]
+ code_interpreter_calls = [result[1] for result in tool_results]
+
+ request_patch = AgenticLoopRequestPatch(
+ model=model,
+ messages=list(messages)
+ + [self._build_chat_completion_assistant_message(tool_calls)]
+ + tool_messages,
+ tools=self._get_followup_tools(
+ tools=optional_params.get("tools"),
+ call_type=CallTypes.completion,
+ ),
+ optional_params=self._get_followup_optional_params(optional_params),
+ kwargs=self._filter_agentic_loop_kwargs(kwargs),
+ )
+
+ return AgenticLoopPlan(
+ run_agentic_loop=True,
+ request_patch=request_patch,
+ metadata={
+ "tool_type": "code_interpreter",
+ "sandbox_key": sandbox_key or "",
+ "code_interpreter_calls": code_interpreter_calls,
+ "response_format": "openai",
+ },
+ )
+
+ async def _build_chat_completion_tool_result(
+ self,
+ container: object,
+ params: dict[str, Any] | None,
+ tool_call: CodeExecutionToolCall,
+ container_id: str | None,
+ ) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]:
+ arguments = tool_call.get("arguments", "")
+ code = self._parse_code(arguments)
+ stdout = await self._run_tool_call(
+ container=container, params=params, arguments=arguments
+ )
+ tool_call_id = (
+ tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex
+ )
+ return (
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": stdout,
+ },
+ {
+ "id": f"ci_{uuid.uuid4().hex}",
+ "type": "code_interpreter_call",
+ "status": "completed",
+ "code": code,
+ "container_id": container_id,
+ "outputs": [{"type": "logs", "logs": stdout}] if stdout else [],
+ },
+ )
+
+ async def async_agentic_loop_cleanup_hook(
+ self, plan: AgenticLoopPlan, kwargs: dict
+ ) -> None:
+ metadata = plan.metadata or {} if plan else {}
+ await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
+
+ @staticmethod
+ def _filter_agentic_loop_kwargs(kwargs: dict[str, object]) -> dict[str, object]:
+ return {
+ k: v
+ for k, v in kwargs.items()
+ if k not in {"litellm_logging_obj", "acompletion"}
+ and not is_interception_internal_key(
+ k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES
+ )
+ }
+
+ def _get_followup_tools(
+ self, tools: object, call_type: CallTypes | None
+ ) -> list[dict[str, Any]] | None:
+ if not isinstance(tools, list):
+ return None
+ return [
+ (
+ self._get_function_tool(call_type=call_type)
+ if isinstance(tool, dict) and tool.get("type") == "code_interpreter"
+ else tool
+ )
+ for tool in tools
+ ]
+
+ def _get_followup_optional_params(
+ self, optional_params: dict[str, object]
+ ) -> dict[str, object]:
+ drop_tool_choice = self._tool_choice_targets_code_interpreter(
+ optional_params.get("tool_choice")
+ )
+ return {
+ k: v
+ for k, v in optional_params.items()
+ if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
+ }
+
+ async def async_post_agentic_loop_response_hook(
+ self, response: Any, plan: AgenticLoopPlan, kwargs: dict
+ ) -> Any:
+ metadata = plan.metadata or {} if plan else {}
+ await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
+
+ calls = metadata.get("code_interpreter_calls")
+ if not calls:
+ return response
+
+ is_dict = isinstance(response, dict)
+ output = (
+ response.get("output") if is_dict else getattr(response, "output", None)
+ )
+ if not isinstance(output, list):
+ return response
+
+ def _item_type(item: Any) -> Any:
+ return (
+ item.get("type")
+ if isinstance(item, dict)
+ else getattr(item, "type", None)
+ )
+
+ insert_at = next(
+ (i for i, item in enumerate(output) if _item_type(item) == "message"),
+ len(output),
+ )
+ new_output = output[:insert_at] + list(calls) + output[insert_at:]
+ if is_dict:
+ response["output"] = new_output
+ else:
+ response.output = new_output
+ return response
+
+ @staticmethod
+ def _parse_code(arguments: str) -> str:
+ try:
+ return json.loads(arguments).get("code", "") if arguments else ""
+ except (json.JSONDecodeError, TypeError, AttributeError):
+ return ""
+
+ async def _run_tool_call(
+ self, container: Any, params: dict[str, Any] | None, arguments: str
+ ) -> str:
+ try:
+ code = json.loads(arguments).get("code", "") if arguments else ""
+ except (json.JSONDecodeError, TypeError):
+ return "[invalid tool arguments: could not parse code]"
+
+ result = await self._run_code(container=container, params=params, code=code)
+ if getattr(result, "error", None):
+ error = result.error
+ message = (
+ error.get("value") or error.get("name")
+ if isinstance(error, dict)
+ else str(error)
+ )
+ return f"[execution error] {message}"
+ return getattr(result, "stdout", "") or ""
+
+ async def _get_or_create_container(
+ self, cache_key: str | None
+ ) -> tuple[Any, dict[str, Any] | None]:
+ if cache_key:
+ cached = self._container_cache.get(cache_key)
+ if cached is not None:
+ return cached[0], cached[1]
+
+ container, params = await self._create_container()
+ if cache_key:
+ self._container_cache[cache_key] = (container, params, time.time())
+ return container, params
+
+ async def _create_container(self) -> tuple[Any, dict[str, Any] | None]:
+ if self.sandbox_config is not None:
+ return await self.sandbox_config.acreate_sandbox(), None
+
+ params = _resolve_sandbox_tool(self.sandbox_tool_name)
+ if params is None:
+ raise ValueError(
+ "CodeInterpreterInterception: no sandbox available. Provide a "
+ "sandbox_config or configure a sandbox tool resolvable via "
+ "sandbox_tool_name."
+ )
+ container = await litellm.acreate_sandbox(
+ provider=params["sandbox_provider"],
+ api_key=params.get("api_key"),
+ api_base=params.get("api_base"),
+ )
+ return container, params
+
+ async def _run_code(
+ self, container: Any, params: dict[str, Any] | None, code: str
+ ) -> Any:
+ if self.sandbox_config is not None:
+ return await self.sandbox_config.arun_code(container=container, code=code)
+ if params is None:
+ raise ValueError(
+ "CodeInterpreterInterception: no sandbox available to run code."
+ )
+ return await litellm.arun_code(
+ provider=params["sandbox_provider"],
+ container=container,
+ code=code,
+ api_key=params.get("api_key"),
+ )
+
+ async def _delete_container(
+ self, container: Any, params: dict[str, Any] | None
+ ) -> None:
+ try:
+ if self.sandbox_config is not None:
+ await self.sandbox_config.adelete_sandbox(container=container)
+ return
+ if params is None:
+ return
+ await litellm.adelete_sandbox(
+ provider=params["sandbox_provider"],
+ container=container,
+ api_key=params.get("api_key"),
+ api_base=params.get("api_base"),
+ )
+ except Exception:
+ verbose_logger.exception(
+ "CodeInterpreterInterception: failed to delete sandbox container"
+ )
+
+ async def _delete_container_for_cache_key(self, cache_key: str | None) -> None:
+ if not cache_key:
+ return
+ cached = self._container_cache.pop(cache_key, None)
+ if cached is None:
+ return
+ await self._delete_container(container=cached[0], params=cached[1])
+
+ def _normalize_messages(self, messages: Any) -> list[dict[str, Any]]:
+ if isinstance(messages, str):
+ return [{"role": "user", "content": messages}]
+ if isinstance(messages, list):
+ return list(messages)
+ return []
+
+ def _extract_code_execution_tool_calls(
+ self, response: object
+ ) -> list[CodeExecutionToolCall]:
+ if isinstance(response, dict):
+ output = response.get("output", [])
+ else:
+ output = getattr(response, "output", []) or []
+ if not isinstance(output, list):
+ return []
+
+ return [
+ {
+ "call_id": (
+ item.get("call_id")
+ if isinstance(item, dict)
+ else getattr(item, "call_id", None)
+ ),
+ "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+ "arguments": (
+ item.get("arguments")
+ if isinstance(item, dict)
+ else getattr(item, "arguments", "")
+ ),
+ }
+ for item in output
+ if self._is_code_execution_call(item)
+ ]
+
+ def _extract_chat_completion_code_execution_tool_calls(
+ self, response: ModelResponse | dict[str, Any]
+ ) -> list[CodeExecutionToolCall]:
+ model_response = self._to_model_response(response)
+ if model_response is None:
+ return []
+ choices = model_response.choices or []
+ if not choices:
+ return []
+ message = choices[0].message
+ tool_calls = message.tool_calls or []
+
+ return [
+ normalized
+ for tool_call in tool_calls
+ if (normalized := self._normalize_chat_completion_tool_call(tool_call))
+ is not None
+ ]
+
+ @staticmethod
+ def _normalize_chat_completion_tool_call(
+ tool_call: ChatCompletionMessageToolCall,
+ ) -> CodeExecutionToolCall | None:
+ if (
+ tool_call.type != "function"
+ or tool_call.function.name != LITELLM_CODE_EXECUTION_TOOL_NAME
+ ):
+ return None
+
+ arguments = tool_call.function.arguments
+ if isinstance(arguments, dict):
+ arguments = json.dumps(arguments)
+ elif not isinstance(arguments, str):
+ arguments = "" if arguments is None else str(arguments)
+
+ return {
+ "id": tool_call.id,
+ "call_id": tool_call.id,
+ "type": "function",
+ "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+ "arguments": arguments,
+ }
+
+ @staticmethod
+ def _build_chat_completion_assistant_message(
+ tool_calls: list[CodeExecutionToolCall],
+ ) -> ChatCompletionAssistantMessage:
+ return {
+ "role": "assistant",
+ "tool_calls": [
+ cast(
+ ChatCompletionAssistantToolCall,
+ {
+ "id": tool_call.get("id"),
+ "type": "function",
+ "function": {
+ "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+ "arguments": tool_call.get("arguments", ""),
+ },
+ },
+ )
+ for tool_call in tool_calls
+ ],
+ }
+
+ @staticmethod
+ def _to_model_response(
+ response: ModelResponse | dict[str, Any],
+ ) -> ModelResponse | None:
+ if isinstance(response, ModelResponse):
+ return response
+ try:
+ return ModelResponse(**response)
+ except (TypeError, ValidationError):
+ return None
+
+ def _is_code_execution_call(self, item: Any) -> bool:
+ if isinstance(item, dict):
+ return (
+ item.get("type") == "function_call"
+ and item.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+ )
+ return (
+ getattr(item, "type", None) == "function_call"
+ and getattr(item, "name", None) == LITELLM_CODE_EXECUTION_TOOL_NAME
+ )
+
+ async def _prune_expired_cache(self) -> None:
+ now = time.time()
+ expired = [
+ (cache_key, container, params)
+ for cache_key, (
+ container,
+ params,
+ created_at,
+ ) in self._container_cache.items()
+ if now - created_at > _CACHE_TTL_SECONDS
+ ]
+ for cache_key, container, params in expired:
+ self._container_cache.pop(cache_key, None)
+ await self._delete_container(container=container, params=params)
diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py
index 481cf7fce8e..94fb97dff53 100644
--- a/litellm/integrations/custom_logger.py
+++ b/litellm/integrations/custom_logger.py
@@ -718,6 +718,24 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
"""
return response
+ async def async_agentic_loop_cleanup_hook(
+ self,
+ plan: AgenticLoopPlan,
+ kwargs: dict,
+ ) -> None:
+ """
+ Release resources held for an agentic-loop iteration.
+
+ Runs in a ``finally`` around the follow-up provider call, so it fires
+ whether the rerun returns normally, hits a loop safety abort, or raises
+ an upstream error. Implementations must be idempotent because the
+ post-response hook may already have released the same resource on the
+ success path. Use ``plan.metadata`` to locate what to clean up.
+
+ Default does nothing.
+ """
+ return None
+
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,
diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py
index 1869e9ca388..79931c0796c 100644
--- a/litellm/integrations/otel/logger.py
+++ b/litellm/integrations/otel/logger.py
@@ -3,7 +3,7 @@
from collections import OrderedDict
from contextlib import contextmanager
from datetime import datetime
-from typing import TYPE_CHECKING, Any, Iterator, Mapping, cast
+from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, cast
from opentelemetry.context import attach, get_current
from opentelemetry.sdk.trace import TracerProvider
@@ -546,6 +546,58 @@ class OpenTelemetryV2(CustomLogger):
return span
+def select_global_otel_v2_logger(
+ in_memory_loggers: Sequence[object],
+ registered: "OpenTelemetryV2 | None" = None,
+) -> "OpenTelemetryV2":
+ """The single ``OpenTelemetryV2`` whose provider should become the OTel global.
+
+ The callback factory designates one logger as canonical the moment it builds
+ the first one (``_init_otel_logger_on_litellm_proxy`` sets
+ ``proxy_server.open_telemetry_logger``), and every other v2 entry point —
+ guardrail, identity seeding, phase spans — already routes through that same
+ ``registered`` owner. Reuse it here too so the global provider has one source
+ of truth instead of a second, independently-derived guess; this is the logger
+ a preset (arize, langfuse, …) folds the ``OTEL_*`` base exporter and its own
+ exporter into, so the FastAPI server span and the gen-ai spans share one
+ provider and one trace.
+
+ Fall back to ``in_memory_loggers`` for the SDK path, where no proxy global is
+ set (selecting from there, not ``service_callback``, which a preset logger does
+ not always reach), and build a generic logger from ``OTEL_*`` only when none was
+ configured at all. Each fallback still avoids the second generic logger that
+ orphaned the gen-ai spans onto a different backend than the server span.
+ """
+ if registered is not None:
+ return registered
+ existing = next(
+ (cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)), None
+ )
+ return existing if existing is not None else OpenTelemetryV2()
+
+
+def publish_global_otel_v2_provider(
+ in_memory_loggers: Sequence[object],
+ set_global_provider: Callable[[TracerProvider], None],
+ registered: "OpenTelemetryV2 | None" = None,
+) -> "OpenTelemetryV2":
+ """Select the single v2 logger and publish its provider as the OTel global.
+
+ The proxy calls this once at startup, after callbacks are initialized, so the
+ preset logger already exists; it passes ``registered`` (the canonical owner the
+ factory designated as ``proxy_server.open_telemetry_logger``) so the global
+ provider reuses the same logger the rest of the v2 code emits through (see
+ :func:`select_global_otel_v2_logger`). Both ``registered`` and
+ ``set_global_provider`` (the proxy passes
+ ``opentelemetry.trace.set_tracer_provider``) are injected so the publish step is
+ unit-testable without reading or mutating real global OTel state. Returns the
+ logger whose provider was published.
+ """
+ logger = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
+ set_global_provider(logger._tracer_provider)
+ return logger
+
+
def _registered_v2_logger() -> "OpenTelemetryV2 | None":
try:
from litellm.proxy import proxy_server
diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py
index 4f7c3277ebb..991b156ae64 100644
--- a/litellm/integrations/otel/model/config.py
+++ b/litellm/integrations/otel/model/config.py
@@ -1,5 +1,7 @@
"""Typed configuration for the OpenTelemetry instrumentation."""
+from enum import Enum
+from functools import lru_cache
from typing import Any, List
from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
@@ -23,13 +25,35 @@ class CaptureMessageContent(str):
SPAN_AND_EVENT = "span_and_event"
+class ExporterOwner(str, Enum):
+ """The preset that contributed an exporter. Values match the callback names
+ in ``presets.PRESET_BY_CALLBACK`` so per-request dynamic-credential routing
+ can match an exporter's owner against the credential source's callback name.
+ A ``str`` enum so the value compares equal to the bare callback-name string."""
+
+ # Arize AX (the hosted platform) and Arize Phoenix (the open-source / Phoenix
+ # Cloud tracer) are distinct backends with separate config and auth, so they
+ # are separate owners. The member value stays the public callback name.
+ ARIZE_AX = "arize"
+ ARIZE_PHOENIX = "arize_phoenix"
+ LANGFUSE_OTEL = "langfuse_otel"
+ WEAVE_OTEL = "weave_otel"
+ LEVO = "levo"
+ AGENTOPS = "agentops"
+
+
class _OTelV2Flag(BaseSettings):
model_config = SettingsConfigDict(extra="ignore")
enabled: bool = Field(default=False, validation_alias=AliasChoices(OTEL_V2_ENV))
+@lru_cache(maxsize=1)
def is_otel_v2_enabled() -> bool:
+ # Resolved once at startup and cached: constructing the pydantic-settings
+ # model re-scans the environment and cost ~28us, which on the proxy hot path
+ # (auth, logging-callback setup) compounded into a measurable throughput
+ # regression. Tests that toggle the env must call ``is_otel_v2_enabled.cache_clear()``.
return _OTelV2Flag().enabled
@@ -49,6 +73,15 @@ class ExporterSpec(BaseModel):
)
endpoint: str | None = None
headers: str | None = None
+ owner: ExporterOwner | None = Field(
+ default=None,
+ description=(
+ "The preset that contributed this exporter. Per-request dynamic OTLP "
+ "credentials are applied only to the exporter whose owner matches the "
+ "credential source, so one tenant's vendor key never lands on a "
+ "different backend's exporter."
+ ),
+ )
options: dict[str, str] | None = Field(
default=None,
description=(
diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py
index 4d0943a263a..1f2f1b202d9 100644
--- a/litellm/integrations/otel/plumbing/routing.py
+++ b/litellm/integrations/otel/plumbing/routing.py
@@ -88,13 +88,23 @@ class TenantTracerCache:
return get_tracer(provider, self._tracer_name)
def _config_with_headers(self, headers: Mapping[str, str]) -> OpenTelemetryV2Config:
- """Clone the config, replacing OTLP exporter headers with ``headers``."""
+ """Clone the config, stamping ``headers`` onto the credential's own exporter.
+
+ ``headers`` are the per-request credentials of ``self._callback_name`` (the
+ integration that built this cache), so they apply only to the exporter that
+ integration contributed (``spec.owner``). A request that carries one
+ tenant's Arize key must never rewrite the headers of a co-configured
+ Langfuse or self-hosted collector exporter, which would leak that key to a
+ different backend.
+ """
header_str = ",".join(f"{key}={value}" for key, value in headers.items())
+ header_update: dict[str, str] = {"headers": header_str}
exporters = [
(
- spec
- if spec.kind.lower() in _NON_OTLP_KINDS
- else spec.model_copy(update={"headers": header_str})
+ spec.model_copy(update=header_update)
+ if spec.owner == self._callback_name
+ and spec.kind.lower() not in _NON_OTLP_KINDS
+ else spec
)
for spec in self._config.exporters
]
diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py
index 5a12818fd99..7b0783935ac 100644
--- a/litellm/integrations/otel/presets/agentops.py
+++ b/litellm/integrations/otel/presets/agentops.py
@@ -16,7 +16,11 @@ from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm._logging import verbose_logger
-from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import (
+ ExporterOwner,
+ ExporterSpec,
+ OpenTelemetryV2Config,
+)
from litellm.integrations.otel.plumbing.providers import register_exporter_factory
_AGENTOPS_ENDPOINT = "https://otlp.agentops.cloud/v1/traces"
@@ -59,6 +63,7 @@ def agentops_preset(
options=(
{"api_key": settings.api_key} if settings.api_key else None
),
+ owner=ExporterOwner.AGENTOPS,
),
],
"resource_attributes": {
diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py
index 4df15125f5a..b6af88c6b34 100644
--- a/litellm/integrations/otel/presets/arize.py
+++ b/litellm/integrations/otel/presets/arize.py
@@ -4,7 +4,11 @@ from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm.integrations.arize.arize import ArizeLogger as _V1ArizeLogger
-from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import (
+ ExporterOwner,
+ ExporterSpec,
+ OpenTelemetryV2Config,
+)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
@@ -34,6 +38,7 @@ def arize_preset(
kind=arize_cfg.protocol or "otlp_grpc",
endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1",
headers=headers,
+ owner=ExporterOwner.ARIZE_AX,
),
],
"mapper_names": ensure_mappers(base.mapper_names, "openinference"),
diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py
index 011545384b9..5631da6429f 100644
--- a/litellm/integrations/otel/presets/langfuse.py
+++ b/litellm/integrations/otel/presets/langfuse.py
@@ -3,7 +3,11 @@
from litellm.integrations.langfuse.langfuse_otel import (
LangfuseOtelLogger as _V1Langfuse,
)
-from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import (
+ ExporterOwner,
+ ExporterSpec,
+ OpenTelemetryV2Config,
+)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
@@ -23,6 +27,7 @@ def langfuse_preset(
kind=kind,
endpoint=cfg.endpoint,
headers=cfg.headers,
+ owner=ExporterOwner.LANGFUSE_OTEL,
),
],
"mapper_names": ensure_mappers(base.mapper_names, "langfuse"),
diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py
index 4c4cba982a4..74a95b100cb 100644
--- a/litellm/integrations/otel/presets/levo.py
+++ b/litellm/integrations/otel/presets/levo.py
@@ -1,7 +1,11 @@
"""Levo preset — OTLP/HTTP to a Levo collector with org+workspace headers."""
from litellm.integrations.levo.levo import LevoLogger as _V1Levo
-from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import (
+ ExporterOwner,
+ ExporterSpec,
+ OpenTelemetryV2Config,
+)
def levo_preset(
@@ -18,6 +22,7 @@ def levo_preset(
kind="otlp_http",
endpoint=cfg.endpoint,
headers=cfg.otlp_auth_headers,
+ owner=ExporterOwner.LEVO,
),
],
}
diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py
index 4c2b165ffca..5485b599321 100644
--- a/litellm/integrations/otel/presets/phoenix.py
+++ b/litellm/integrations/otel/presets/phoenix.py
@@ -6,7 +6,11 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm.integrations.arize.arize_phoenix import (
ArizePhoenixLogger as _V1Phoenix,
)
-from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import (
+ ExporterOwner,
+ ExporterSpec,
+ OpenTelemetryV2Config,
+)
from litellm.integrations.otel.presets.utils import ensure_mappers
@@ -37,6 +41,7 @@ def phoenix_preset(
kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http",
endpoint=cfg.endpoint,
headers=headers,
+ owner=ExporterOwner.ARIZE_PHOENIX,
),
],
"mapper_names": ensure_mappers(base.mapper_names, "openinference"),
diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py
index 9fc03c84a6d..d22f7641289 100644
--- a/litellm/integrations/otel/presets/weave.py
+++ b/litellm/integrations/otel/presets/weave.py
@@ -1,6 +1,10 @@
"""Weave (W&B) preset."""
-from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import (
+ ExporterOwner,
+ ExporterSpec,
+ OpenTelemetryV2Config,
+)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.integrations.weave.weave_otel import (
_get_weave_authorization_header,
@@ -23,6 +27,7 @@ def weave_preset(
kind=weave_cfg.protocol or "otlp_http",
endpoint=weave_cfg.endpoint,
headers=weave_cfg.otlp_auth_headers,
+ owner=ExporterOwner.WEAVE_OTEL,
),
],
# Weave consumes OpenInference + a small Weave-specific overlay.
diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py
index 173d4ca8764..0ff1a97cd0b 100644
--- a/litellm/interactions/litellm_responses_transformation/transformation.py
+++ b/litellm/interactions/litellm_responses_transformation/transformation.py
@@ -300,9 +300,6 @@ class LiteLLMResponsesInteractionsConfig:
"total_output_tokens": getattr(usage, "output_tokens", 0),
}
- # Add role
- interactions_response_dict["role"] = "model"
-
# Add updated (same as created for now)
interactions_response_dict["updated"] = created
diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py
new file mode 100644
index 00000000000..938e892bd50
--- /dev/null
+++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py
@@ -0,0 +1,332 @@
+# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
+
+import json
+from typing import cast
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.integrations.custom_logger import (
+ CHAT_COMPLETION_AGENTIC_SURFACE,
+ NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
+ AgenticLoopPlan,
+ AgenticLoopRequestPatch,
+ is_interception_internal_key,
+)
+from litellm.types.utils import ModelResponse
+from litellm.utils import CustomStreamWrapper
+
+_FOLLOWUP_INTERNAL_PARAMS = frozenset(
+ (
+ "acompletion",
+ "litellm_logging_obj",
+ "custom_llm_provider",
+ "model_alias_map",
+ "stream_response",
+ "custom_prompt_dict",
+ "_agentic_loop_api_surface",
+ )
+)
+
+
+def _gate_overridden(callback: CustomLogger) -> bool:
+ base = CustomLogger.async_should_run_agentic_loop
+ func = type(callback).async_should_run_agentic_loop
+ return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
+
+
+def _build_plan_overridden(callback: CustomLogger) -> bool:
+ base = CustomLogger.async_build_agentic_loop_plan
+ func = type(callback).async_build_agentic_loop_plan
+ return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
+
+
+def _post_hook_overridden(callback: CustomLogger) -> bool:
+ base = CustomLogger.async_post_agentic_loop_response_hook
+ func = type(callback).async_post_agentic_loop_response_hook
+ return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
+
+
+def _coerce_int(value: object, default: int) -> int:
+ return int(value) if isinstance(value, (int, str)) else default
+
+
+def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
+ depth = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
+ max_loops = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
+ raw_fingerprints = kwargs.get("_agentic_loop_fingerprints")
+ fingerprints = (
+ [str(fp) for fp in raw_fingerprints]
+ if isinstance(raw_fingerprints, list)
+ else []
+ )
+ return depth, max_loops, fingerprints
+
+
+def _fingerprint_tools(tool_calls: object) -> str:
+ try:
+ return json.dumps(tool_calls, sort_keys=True, default=str)
+ except Exception:
+ return str(tool_calls)
+
+
+def _check_agentic_loop_safety(
+ tool_calls: object,
+ fingerprints: list[str],
+ depth: int,
+ max_loops: int,
+ model: str,
+) -> str:
+ fingerprint = _fingerprint_tools(tool_calls)
+ if fingerprint in fingerprints:
+ raise ValueError(
+ "Agentic loop detected repeated tool-call fingerprint; aborting rerun"
+ )
+ if depth >= max_loops:
+ raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
+ return fingerprint
+
+
+def _wrap_response_as_fake_stream(response: object) -> object:
+ if getattr(response, "object", None) == "chat.completion.chunk":
+ return response
+ if not hasattr(response, "choices"):
+ return response
+ from litellm.llms.base_llm.base_model_iterator import (
+ convert_model_response_to_streaming,
+ )
+
+ return convert_model_response_to_streaming(cast(ModelResponse, response))
+
+
+def _add_agentic_loop_metadata(kwargs_for_followup: dict[str, object]) -> None:
+ metadata = kwargs_for_followup.get("litellm_metadata")
+ metadata = dict(metadata) if isinstance(metadata, dict) else {}
+ for key, value in kwargs_for_followup.items():
+ if (
+ key.startswith("_agentic_loop")
+ or key == "max_agentic_loops"
+ or is_interception_internal_key(key)
+ ):
+ metadata[key] = value
+ kwargs_for_followup["litellm_metadata"] = metadata
+
+
+def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]:
+ return {
+ k: v
+ for k, v in source.items()
+ if not is_interception_internal_key(
+ k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES
+ )
+ and k not in _FOLLOWUP_INTERNAL_PARAMS
+ }
+
+
+async def _execute_chat_completion_agentic_plan(
+ *,
+ plan: AgenticLoopPlan,
+ callback: CustomLogger,
+ model: str,
+ optional_params: dict[str, object],
+ kwargs: dict[str, object],
+ logging_obj: object,
+ custom_llm_provider: str,
+ depth: int,
+ max_loops: int,
+ fingerprints: list[str],
+ fingerprint: str,
+) -> object:
+ import litellm
+
+ patch = plan.request_patch or AgenticLoopRequestPatch()
+ if patch.messages is None:
+ raise ValueError("Agentic loop plan missing patched messages")
+
+ full_model_name = patch.model or model
+ if "/" not in full_model_name:
+ full_model_name = f"{custom_llm_provider}/{full_model_name}"
+
+ optional_params_for_followup = {**optional_params, **patch.optional_params}
+ if patch.tools is not None:
+ optional_params_for_followup["tools"] = patch.tools
+ if "tool_choice" not in patch.optional_params:
+ optional_params_for_followup.pop("tool_choice", None)
+
+ kwargs_for_followup = _filter_followup_kwargs(kwargs)
+ kwargs_for_followup.update(
+ {
+ k: v
+ for k, v in _filter_followup_kwargs(patch.kwargs).items()
+ if k not in optional_params_for_followup
+ }
+ )
+ kwargs_for_followup["_agentic_loop_depth"] = depth + 1
+ kwargs_for_followup["max_agentic_loops"] = max_loops
+ kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
+ _add_agentic_loop_metadata(kwargs_for_followup)
+
+ try:
+ response_followup = await litellm.acompletion(
+ model=full_model_name,
+ messages=patch.messages,
+ **optional_params_for_followup,
+ **kwargs_for_followup,
+ )
+ if _post_hook_overridden(callback):
+ try:
+ response_followup = (
+ await callback.async_post_agentic_loop_response_hook(
+ response=response_followup, plan=plan, kwargs=kwargs
+ )
+ )
+ except Exception as e:
+ _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+ verbose_logger.exception(
+ "LiteLLM.AgenticHookError: Exception in "
+ "async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s",
+ _call_id,
+ model,
+ str(e),
+ )
+ if kwargs.get("_code_interpreter_interception_converted_stream") and not depth:
+ return _wrap_response_as_fake_stream(response_followup)
+ return response_followup
+ finally:
+ try:
+ await callback.async_agentic_loop_cleanup_hook(plan=plan, kwargs=kwargs)
+ except Exception as e:
+ _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+ verbose_logger.exception(
+ "LiteLLM.AgenticHookError: Exception in "
+ "async_agentic_loop_cleanup_hook [call_id=%s model=%s]: %s",
+ _call_id,
+ model,
+ str(e),
+ )
+
+
+async def maybe_run_chat_completion_agentic_loop(
+ *,
+ response: ModelResponse,
+ model: str,
+ messages: list,
+ optional_params: dict,
+ kwargs: dict,
+ logging_obj: object,
+ custom_llm_provider: str,
+ stream: bool,
+) -> ModelResponse | CustomStreamWrapper | None:
+ import litellm
+
+ callbacks = litellm.callbacks + (
+ getattr(logging_obj, "dynamic_success_callbacks", None) or []
+ )
+ depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
+ tools = optional_params.get("tools", [])
+
+ for callback in callbacks:
+ if not isinstance(callback, CustomLogger):
+ continue
+ if not _gate_overridden(callback):
+ continue
+
+ gate_kwargs = {
+ **kwargs,
+ "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
+ "custom_llm_provider": custom_llm_provider,
+ }
+ try:
+ should_run, tool_calls = await callback.async_should_run_agentic_loop(
+ response=response,
+ model=model,
+ messages=messages,
+ tools=tools,
+ stream=stream,
+ custom_llm_provider=custom_llm_provider,
+ kwargs=gate_kwargs,
+ )
+ except Exception as e:
+ verbose_logger.exception(
+ "LiteLLM.AgenticHookError: Exception in chat completion agentic gate: %s",
+ str(e),
+ )
+ continue
+
+ if not should_run:
+ continue
+
+ fingerprint = _check_agentic_loop_safety(
+ tool_calls=tool_calls,
+ fingerprints=fingerprints,
+ depth=depth,
+ max_loops=max_loops,
+ model=model,
+ )
+
+ try:
+ plan_kwargs = {
+ **kwargs,
+ "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
+ "custom_llm_provider": custom_llm_provider,
+ }
+ if not _build_plan_overridden(callback):
+ return await callback.async_run_agentic_loop(
+ tools=tool_calls,
+ model=model,
+ messages=messages,
+ response=response,
+ anthropic_messages_provider_config=None,
+ anthropic_messages_optional_request_params=optional_params,
+ logging_obj=logging_obj,
+ stream=stream,
+ kwargs=plan_kwargs,
+ )
+
+ plan = await callback.async_build_agentic_loop_plan(
+ tools=tool_calls,
+ model=model,
+ messages=messages,
+ response=response,
+ anthropic_messages_provider_config=None,
+ anthropic_messages_optional_request_params=optional_params,
+ logging_obj=logging_obj,
+ stream=stream,
+ kwargs=plan_kwargs,
+ )
+
+ if plan.response_override is not None:
+ return plan.response_override
+ if plan.terminate:
+ return response
+ if not plan.run_agentic_loop:
+ continue
+
+ return await _execute_chat_completion_agentic_plan(
+ plan=plan,
+ callback=callback,
+ model=model,
+ optional_params=optional_params,
+ kwargs=kwargs,
+ logging_obj=logging_obj,
+ custom_llm_provider=custom_llm_provider,
+ depth=depth,
+ max_loops=max_loops,
+ fingerprints=fingerprints,
+ fingerprint=fingerprint,
+ )
+ except Exception as e:
+ verbose_logger.exception(
+ "LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s",
+ str(e),
+ )
+
+ if (
+ kwargs.get("_code_interpreter_interception_converted_stream")
+ and not depth
+ and hasattr(response, "choices")
+ ):
+ return cast(
+ "ModelResponse | CustomStreamWrapper",
+ _wrap_response_as_fake_stream(response),
+ )
+ return None
diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py
index daa3dc60320..a75d1178d5a 100644
--- a/litellm/litellm_core_utils/cloud_storage_security.py
+++ b/litellm/litellm_core_utils/cloud_storage_security.py
@@ -15,8 +15,23 @@ BEDROCK_MANAGED_S3_PREFIXES = (
BEDROCK_MANAGED_S3_UPLOAD_PREFIX,
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
)
+MANAGED_CLOUD_STORAGE_SCHEMES = ("s3://", "gs://")
_MAPPING_PROXY_TYPE: type = type(MappingProxyType({}))
+
+def is_managed_cloud_storage_uri(file_id: str) -> bool:
+ """
+ True if file_id is a raw cloud-storage object URI (e.g. ``s3://bucket/key``).
+
+ These are internal provider artifacts. On the multi-tenant proxy they must be
+ retrieved through their managed unified file id so owner/team access is enforced;
+ a raw URI supplied by a caller bypasses that check.
+ """
+ return isinstance(file_id, str) and file_id.startswith(
+ MANAGED_CLOUD_STORAGE_SCHEMES
+ )
+
+
_SAFE_OBJECT_COMPONENT_PATTERN = re.compile(r"[^A-Za-z0-9._-]+")
diff --git a/litellm/litellm_core_utils/completion_timeout.py b/litellm/litellm_core_utils/completion_timeout.py
index 5350d88e593..70c6896a323 100644
--- a/litellm/litellm_core_utils/completion_timeout.py
+++ b/litellm/litellm_core_utils/completion_timeout.py
@@ -6,10 +6,7 @@ from typing import Callable, Optional, Union
import httpx
-from litellm.constants import (
- COMPLETION_HTTP_FALLBACK_SECONDS,
- DEFAULT_REQUEST_TIMEOUT_SECONDS,
-)
+from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
class CompletionTimeout:
@@ -22,17 +19,13 @@ class CompletionTimeout:
"""
Used when ``model_timeout`` and kwargs timeouts are all unset.
- ``global_timeout`` is :attr:`litellm.request_timeout` (numeric / string), not
- :class:`httpx.Timeout`.
-
- If it equals :data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS` (6000),
- return :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`. Same if
- ``None``. Otherwise return ``float(global_timeout)``.
+ ``global_timeout`` is the explicitly-configured ``litellm.request_timeout``
+ (numeric / string) or ``None`` when it was never set. ``None`` falls back to
+ :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`; any explicit value
+ (including ``6000``) is honored.
"""
if global_timeout is None:
return COMPLETION_HTTP_FALLBACK_SECONDS
- if float(global_timeout) == float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
- return COMPLETION_HTTP_FALLBACK_SECONDS
return float(global_timeout)
@staticmethod
@@ -50,11 +43,10 @@ class CompletionTimeout:
1. ``model_timeout`` (call argument / merged ``litellm_params``)
2. ``kwargs["timeout"]``
3. ``kwargs["request_timeout"]``
- 4. Fallback from ``global_timeout`` (:attr:`litellm.request_timeout`) — if it is
- the package default (6000), use 600 instead.
+ 4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``),
+ or 600 when nothing was configured.
Coerce :class:`httpx.Timeout` when the provider does not support it.
- Explicit ``6000`` on the model or in kwargs is kept as ``6000``.
"""
resolved: Union[float, str, httpx.Timeout]
if model_timeout is not None:
diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py
index 6087e55b136..9b2a9af4126 100644
--- a/litellm/litellm_core_utils/exception_mapping_utils.py
+++ b/litellm/litellm_core_utils/exception_mapping_utils.py
@@ -1,7 +1,7 @@
import json
import re
import traceback
-from typing import Any, Optional
+from typing import Any, Optional, Protocol, cast
import httpx
@@ -170,6 +170,16 @@ def get_error_message(error_obj) -> Optional[str]:
####### EXCEPTION MAPPING ################
+def _get_body_error_code(error_str: str) -> int | None:
+ """Return error.code from a JSON error body, or None if not parseable."""
+ try:
+ body = json.loads(error_str)
+ code = body.get("error", {}).get("code")
+ return int(code) if code is not None else None
+ except Exception:
+ return None
+
+
def _get_response_headers(original_exception: Exception) -> Optional[httpx.Headers]:
"""
Extract and return the response headers from an exception, if present.
@@ -234,6 +244,2011 @@ def extract_and_raise_litellm_exception(
)
+class _ProviderHTTPException(Protocol):
+ status_code: int
+ message: str
+ response: httpx.Response
+ request: httpx.Request
+ body: object
+ code: str
+ llm_provider: str
+
+
+def _map_openai_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ # custom_llm_provider is openai, make it OpenAI
+ message = get_error_message(error_obj=original_exception)
+ if message is None:
+ if hasattr(original_exception, "message"):
+ message = original_exception.message
+ else:
+ message = str(original_exception)
+
+ if message is not None and isinstance(
+ message, str
+ ): # done to prevent user-confusion. Relevant issue - https://github.com/BerriAI/litellm/issues/1414
+ message = message.replace("OPENAI", custom_llm_provider.upper())
+ message = message.replace(
+ "openai.OpenAIError",
+ "{}.{}Error".format(custom_llm_provider, custom_llm_provider),
+ )
+ if custom_llm_provider == "openai":
+ exception_provider = "OpenAI" + "Exception"
+ else:
+ exception_provider = (
+ custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception"
+ )
+
+ if ExceptionCheckers.is_error_str_rate_limit(error_str):
+ raise RateLimitError(
+ message=f"RateLimitError: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ )
+ elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
+ raise ContextWindowExceededError(
+ message=f"ContextWindowExceededError: {exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif "invalid_request_error" in error_str and "model_not_found" in error_str:
+ raise NotFoundError(
+ message=f"{exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif "A timeout occurred" in error_str:
+ raise Timeout(
+ message=f"{exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif (
+ (
+ "invalid_request_error" in error_str
+ and "content_policy_violation" in error_str
+ )
+ or ("Invalid prompt" in error_str and "violating our usage policy" in error_str)
+ or (
+ "request was rejected as a result of the safety system" in error_str.lower()
+ )
+ ):
+ raise ContentPolicyViolationError(
+ message=f"ContentPolicyViolationError: {exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif (
+ "invalid_encrypted_content" in error_str or "could not be verified" in error_str
+ ):
+ helpful_message = (
+ f"{exception_provider} - {message}\n\n"
+ " This error occurs when load balancing Responses API across deployments with different API keys.\n"
+ " Encrypted content is tied to the organization that created it and cannot be decrypted by other organizations.\n\n"
+ " Solution: Enable 'encrypted_content_affinity' to route follow-up requests to the correct deployment:\n\n"
+ " router_settings:\n"
+ " enable_pre_call_checks: true\n"
+ " optional_pre_call_checks:\n"
+ " - encrypted_content_affinity\n\n"
+ " Learn more: https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing"
+ )
+ raise BadRequestError(
+ message=helpful_message,
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ body=getattr(original_exception, "body", None),
+ )
+ elif (
+ "invalid_request_error" in error_str
+ and "Incorrect API key provided" not in error_str
+ ):
+ raise BadRequestError(
+ message=f"{exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ body=getattr(original_exception, "body", None),
+ )
+ elif (
+ "Web server is returning an unknown error" in error_str
+ or "The server had an error processing your request." in error_str
+ ):
+ raise litellm.InternalServerError(
+ message=f"{exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ )
+ elif "Request too large" in error_str:
+ raise RateLimitError(
+ message=f"RateLimitError: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif (
+ "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"
+ in error_str
+ ):
+ raise AuthenticationError(
+ message=f"AuthenticationError: {exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif "Mistral API raised a streaming error" in error_str:
+ _request = httpx.Request(method="POST", url="https://api.openai.com/v1")
+ raise APIError(
+ status_code=500,
+ message=f"{exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ request=_request,
+ litellm_debug_info=extra_information,
+ )
+ elif hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"{exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"AuthenticationError: {exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 404:
+ raise NotFoundError(
+ message=f"NotFoundError: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"Timeout Error: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 422:
+ raise BadRequestError(
+ message=f"{exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ body=getattr(original_exception, "body", None),
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"RateLimitError: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 500:
+ raise InternalServerError(
+ message=f"InternalServerError: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 502:
+ raise BadGatewayError(
+ message=f"BadGatewayError: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"ServiceUnavailableError: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 504: # gateway timeout error
+ raise Timeout(
+ message=f"Timeout Error: {exception_provider} - {message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ exception_status_code=original_exception.status_code,
+ )
+ else:
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"APIError: {exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ request=getattr(original_exception, "request", None),
+ litellm_debug_info=extra_information,
+ )
+ else:
+ # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
+ # exception_mapping_worked = True
+ raise APIConnectionError(
+ message=f"APIConnectionError: {exception_provider} - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ litellm_debug_info=extra_information,
+ request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
+ )
+
+
+def _map_anthropic_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if (
+ "prompt is too long" in error_str
+ or "prompt: length" in error_str
+ or ExceptionCheckers.is_error_str_context_window_exceeded(error_str)
+ ):
+ raise ContextWindowExceededError(
+ message="AnthropicError - {}".format(error_str),
+ model=model,
+ llm_provider="anthropic",
+ )
+ elif "overloaded_error" in error_str or "Overloaded" in error_str:
+ raise InternalServerError(
+ message="AnthropicError - {}".format(error_str),
+ model=model,
+ llm_provider="anthropic",
+ )
+ if "Invalid API Key" in error_str:
+ raise AuthenticationError(
+ message="AnthropicError - {}".format(error_str),
+ model=model,
+ llm_provider="anthropic",
+ )
+ if "content filtering policy" in error_str:
+ raise ContentPolicyViolationError(
+ message="AnthropicError - {}".format(error_str),
+ model=model,
+ llm_provider="anthropic",
+ )
+ if "Client error '400 Bad Request'" in error_str:
+ raise BadRequestError(
+ message="AnthropicError - {}".format(error_str),
+ model=model,
+ llm_provider="anthropic",
+ )
+ if hasattr(original_exception, "status_code"):
+ verbose_logger.debug(f"status_code: {original_exception.status_code}")
+ if original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"AnthropicException - {error_str}",
+ llm_provider="anthropic",
+ model=model,
+ )
+ elif (
+ original_exception.status_code == 400
+ or original_exception.status_code == 413
+ ):
+ raise BadRequestError(
+ message=f"AnthropicException - {error_str}",
+ model=model,
+ llm_provider="anthropic",
+ )
+ elif original_exception.status_code == 404:
+ raise NotFoundError(
+ message=f"AnthropicException - {error_str}",
+ model=model,
+ llm_provider="anthropic",
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"AnthropicException - {error_str}",
+ model=model,
+ llm_provider="anthropic",
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"AnthropicException - {error_str}",
+ llm_provider="anthropic",
+ model=model,
+ )
+ elif (
+ original_exception.status_code == 500
+ or original_exception.status_code == 529
+ ):
+ raise litellm.InternalServerError(
+ message=f"AnthropicException - {error_str}. Handle with `litellm.InternalServerError`.",
+ llm_provider="anthropic",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 502:
+ raise BadGatewayError(
+ message=f"AnthropicException BadGatewayError - {error_str}",
+ llm_provider="anthropic",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 503:
+ raise litellm.ServiceUnavailableError(
+ message=f"AnthropicException - {error_str}. Handle with `litellm.ServiceUnavailableError`.",
+ llm_provider="anthropic",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 504: # gateway timeout error
+ raise Timeout(
+ message=f"AnthropicException Timeout - {error_str}",
+ model=model,
+ llm_provider="anthropic",
+ exception_status_code=original_exception.status_code,
+ )
+
+
+def _map_replicate_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "Incorrect authentication token" in error_str:
+ raise AuthenticationError(
+ message=f"ReplicateException - {error_str}",
+ llm_provider="replicate",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "input is too long" in error_str:
+ raise ContextWindowExceededError(
+ message=f"ReplicateException - {error_str}",
+ model=model,
+ llm_provider="replicate",
+ response=getattr(original_exception, "response", None),
+ )
+ elif exception_type == "ModelError":
+ raise BadRequestError(
+ message=f"ReplicateException - {error_str}",
+ model=model,
+ llm_provider="replicate",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Request was throttled" in error_str:
+ raise RateLimitError(
+ message=f"ReplicateException - {error_str}",
+ llm_provider="replicate",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"ReplicateException - {original_exception.message}",
+ llm_provider="replicate",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ original_exception.status_code == 400
+ or original_exception.status_code == 413
+ ):
+ raise BadRequestError(
+ message=f"ReplicateException - {original_exception.message}",
+ model=model,
+ llm_provider="replicate",
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 422:
+ raise UnprocessableEntityError(
+ message=f"ReplicateException - {original_exception.message}",
+ model=model,
+ llm_provider="replicate",
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"ReplicateException - {original_exception.message}",
+ model=model,
+ llm_provider="replicate",
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"ReplicateException - {original_exception.message}",
+ llm_provider="replicate",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 500:
+ raise ServiceUnavailableError(
+ message=f"ReplicateException - {original_exception.message}",
+ llm_provider="replicate",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ raise APIError(
+ status_code=500,
+ message=f"ReplicateException - {str(original_exception)}",
+ llm_provider="replicate",
+ model=model,
+ request=httpx.Request(
+ method="POST",
+ url="https://api.replicate.com/v1/deployments",
+ ),
+ )
+
+
+def _map_openai_like_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "authorization denied for" in error_str:
+
+ # Predibase returns the raw API Key in the response - this block ensures it's not returned in the exception
+ if (
+ error_str is not None
+ and isinstance(error_str, str)
+ and "bearer" in error_str.lower()
+ ):
+ # only keep the first 10 chars after the occurnence of "bearer"
+ _bearer_token_start_index = error_str.lower().find("bearer")
+ error_str = error_str[: _bearer_token_start_index + 14]
+ error_str += "XXXXXXX" + '"'
+
+ raise AuthenticationError(
+ message=f"{custom_llm_provider.capitalize()}Exception: Authentication Error - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
+ raise ContextWindowExceededError(
+ message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif "token_quota_reached" in error_str:
+ raise RateLimitError(
+ message=f"{custom_llm_provider.capitalize()}Exception: Rate Limit Errror - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ "The server received an invalid response from an upstream server." in error_str
+ ):
+ raise litellm.InternalServerError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ elif "model_no_support_for_function" in error_str:
+ raise BadRequestError(
+ message=f"{custom_llm_provider.capitalize()}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ elif hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 500:
+ raise litellm.InternalServerError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ elif (
+ original_exception.status_code == 401
+ or original_exception.status_code == 403
+ ):
+ raise AuthenticationError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ elif original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ elif original_exception.status_code == 404:
+ raise NotFoundError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif (
+ original_exception.status_code == 422
+ or original_exception.status_code == 424
+ ):
+ raise BadRequestError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 504: # gateway timeout error
+ raise Timeout(
+ message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ exception_status_code=original_exception.status_code,
+ )
+
+
+def _map_bedrock_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if (
+ "too many tokens" in error_str
+ or "expected maxLength:" in error_str
+ or "Input is too long" in error_str
+ or "prompt is too long" in error_str
+ or "prompt: length: 1.." in error_str
+ or "Too many input tokens" in error_str
+ ):
+ raise ContextWindowExceededError(
+ message=f"BedrockException: Context Window Error - {error_str}",
+ model=model,
+ llm_provider="bedrock",
+ )
+ elif (
+ "Conversation blocks and tool result blocks cannot be provided in the same turn."
+ in error_str
+ ):
+ raise BadRequestError(
+ message=f"BedrockException - {error_str}\n. Enable 'litellm.modify_params=True' (for PROXY do: `litellm_settings::modify_params: True`) to insert a dummy assistant message and fix this error.",
+ model=model,
+ llm_provider="bedrock",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Malformed input request" in error_str:
+ raise BadRequestError(
+ message=f"BedrockException - {error_str}",
+ model=model,
+ llm_provider="bedrock",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "A conversation must start with a user message." in error_str:
+ raise BadRequestError(
+ message=f"BedrockException - {error_str}\n. Pass in default user message via `completion(..,user_continue_message=)` or enable `litellm.modify_params=True`.\nFor Proxy: do via `litellm_settings::modify_params: True` or user_continue_message under `litellm_params`",
+ model=model,
+ llm_provider="bedrock",
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ "Unable to locate credentials" in error_str
+ or "The security token included in the request is invalid" in error_str
+ ):
+ raise AuthenticationError(
+ message=f"BedrockException Invalid Authentication - {error_str}",
+ model=model,
+ llm_provider="bedrock",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "AccessDeniedException" in error_str:
+ raise PermissionDeniedError(
+ message=f"BedrockException PermissionDeniedError - {error_str}",
+ model=model,
+ llm_provider="bedrock",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "throttlingException" in error_str or "ThrottlingException" in error_str:
+ raise RateLimitError(
+ message=f"BedrockException: Rate Limit Error - {error_str}",
+ model=model,
+ llm_provider="bedrock",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Connect timeout on endpoint URL" in error_str or "timed out" in error_str:
+ raise Timeout(
+ message=f"BedrockException: Timeout Error - {error_str}",
+ model=model,
+ llm_provider="bedrock",
+ )
+ elif "Could not process image" in error_str:
+ raise litellm.InternalServerError(
+ message=f"BedrockException - {error_str}",
+ model=model,
+ llm_provider="bedrock",
+ )
+ elif hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 500:
+ raise ServiceUnavailableError(
+ message=f"BedrockException - {original_exception.message}",
+ llm_provider="bedrock",
+ model=model,
+ response=httpx.Response(
+ status_code=500,
+ request=httpx.Request(
+ method="POST", url="https://api.openai.com/v1/"
+ ),
+ ),
+ )
+ elif original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"BedrockException - {original_exception.message}",
+ llm_provider="bedrock",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"BedrockException - {original_exception.message}",
+ llm_provider="bedrock",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 404:
+ raise NotFoundError(
+ message=f"BedrockException - {original_exception.message}",
+ llm_provider="bedrock",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"BedrockException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 422:
+ raise BadRequestError(
+ message=f"BedrockException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"BedrockException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"BedrockException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 504: # gateway timeout error
+ raise Timeout(
+ message=f"BedrockException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ exception_status_code=original_exception.status_code,
+ )
+
+
+def _map_sagemaker_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "Unable to locate credentials" in error_str:
+ raise BadRequestError(
+ message=f"litellm.BadRequestError: SagemakerException - {error_str}",
+ model=model,
+ llm_provider="sagemaker",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Input validation error: `best_of` must be > 0 and <= 2" in error_str:
+ raise BadRequestError(
+ message="SagemakerException - the value of 'n' must be > 0 and <= 2 for sagemaker endpoints",
+ model=model,
+ llm_provider="sagemaker",
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ "`inputs` tokens + `max_new_tokens` must be <=" in error_str
+ or "instance type with more CPU capacity or memory" in error_str
+ ):
+ raise ContextWindowExceededError(
+ message=f"SagemakerException - {error_str}",
+ model=model,
+ llm_provider="sagemaker",
+ response=getattr(original_exception, "response", None),
+ )
+ elif hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 500:
+ raise ServiceUnavailableError(
+ message=f"SagemakerException - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=httpx.Response(
+ status_code=500,
+ request=httpx.Request(
+ method="POST", url="https://api.openai.com/v1/"
+ ),
+ ),
+ )
+ elif original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"SagemakerException - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"SagemakerException - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 404:
+ raise NotFoundError(
+ message=f"SagemakerException - {original_exception.message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"SagemakerException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif (
+ original_exception.status_code == 422
+ or original_exception.status_code == 424
+ ):
+ raise BadRequestError(
+ message=f"SagemakerException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"SagemakerException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"SagemakerException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 504: # gateway timeout error
+ raise Timeout(
+ message=f"SagemakerException - {original_exception.message}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ exception_status_code=original_exception.status_code,
+ )
+
+
+def _map_vertex_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if (
+ "Vertex AI API has not been used in project" in error_str
+ or "Unable to find your project" in error_str
+ ):
+ raise BadRequestError(
+ message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=httpx.Response(
+ status_code=400,
+ request=httpx.Request(
+ method="POST",
+ url=" https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ litellm_debug_info=extra_information,
+ )
+ if "400 Request payload size exceeds" in error_str:
+ raise ContextWindowExceededError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ )
+ elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
+ raise ContextWindowExceededError(
+ message=f"ContextWindowExceededError: {custom_llm_provider.capitalize()}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif "None Unknown Error." in error_str or "Content has no parts." in error_str:
+ raise litellm.InternalServerError(
+ message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=httpx.Response(
+ status_code=500,
+ content=str(original_exception),
+ request=httpx.Request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
+ ),
+ litellm_debug_info=extra_information,
+ )
+ elif "API key not valid." in error_str:
+ raise AuthenticationError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif "403" in error_str:
+ raise BadRequestError(
+ message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=httpx.Response(
+ status_code=403,
+ request=httpx.Request(
+ method="POST",
+ url=" https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ litellm_debug_info=extra_information,
+ )
+ elif (
+ "The response was blocked." in error_str
+ or "Output blocked by content filtering policy"
+ in error_str # anthropic on vertex ai
+ ):
+ raise ContentPolicyViolationError(
+ message=f"{custom_llm_provider.capitalize()}Exception ContentPolicyViolationError - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ response=httpx.Response(
+ status_code=400,
+ request=httpx.Request(
+ method="POST",
+ url=" https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ )
+ elif (
+ "429 Quota exceeded" in error_str
+ or "Quota exceeded for" in error_str
+ or "Resource exhausted" in error_str
+ or "IndexError: list index out of range" in error_str
+ or "429 Unable to submit request because the service is temporarily out of capacity."
+ in error_str
+ ):
+ raise RateLimitError(
+ message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ response=httpx.Response(
+ status_code=429,
+ request=httpx.Request(
+ method="POST",
+ url=" https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ )
+ elif (
+ isinstance(getattr(original_exception, "status_code", None), int)
+ and 500 <= original_exception.status_code < 600
+ and _get_body_error_code(error_str) == 429
+ ):
+ # upstream gateway wraps a 429 inside a 5xx envelope
+ # e.g. HTTP 500/503 with {"error":{"code":429,...}}.
+ # Scoped to 5xx so HTTP 400/401 with body code:429
+ # still maps to BadRequestError / AuthenticationError.
+ raise RateLimitError(
+ message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ response=httpx.Response(
+ status_code=429,
+ request=httpx.Request(
+ method="POST",
+ url=" https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ )
+ elif (
+ "500 Internal Server Error" in error_str
+ or "The model is overloaded." in error_str
+ ):
+ raise litellm.InternalServerError(
+ message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ if hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ response=httpx.Response(
+ status_code=400,
+ request=httpx.Request(
+ method="POST",
+ url="https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ )
+ if original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ if original_exception.status_code == 403:
+ raise PermissionDeniedError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=httpx.Response(
+ status_code=403,
+ request=httpx.Request(
+ method="POST",
+ url="https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ )
+ if original_exception.status_code == 404:
+ raise NotFoundError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ if original_exception.status_code == 408:
+ raise Timeout(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+
+ if original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"litellm.RateLimitError: {custom_llm_provider.capitalize()}Exception - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ response=httpx.Response(
+ status_code=429,
+ request=httpx.Request(
+ method="POST",
+ url=" https://cloud.google.com/vertex-ai/",
+ ),
+ ),
+ )
+ if original_exception.status_code == 500:
+ raise litellm.InternalServerError(
+ message=f"{custom_llm_provider.capitalize()}Exception InternalServerError - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ response=httpx.Response(
+ status_code=500,
+ content=str(original_exception),
+ request=httpx.Request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
+ ),
+ )
+ if original_exception.status_code == 502:
+ raise APIConnectionError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+ if original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ )
+
+
+def _map_cloudflare_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "Authentication error" in error_str:
+ raise AuthenticationError(
+ message=f"Cloudflare Exception - {original_exception.message}",
+ llm_provider="cloudflare",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ if "must have required property" in error_str:
+ raise BadRequestError(
+ message=f"Cloudflare Exception - {original_exception.message}",
+ llm_provider="cloudflare",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+
+
+def _map_cohere_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "invalid api token" in error_str or "No API key provided." in error_str:
+ raise AuthenticationError(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "invalid type: parameter" in error_str:
+ raise BadRequestError(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "too many tokens" in error_str:
+ raise ContextWindowExceededError(
+ message=f"CohereException - {original_exception.message}",
+ model=model,
+ llm_provider="cohere",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "internal server error" in error_str.lower():
+ raise InternalServerError(
+ message=f"CohereException - {error_str}",
+ model=model,
+ llm_provider="cohere",
+ response=getattr(original_exception, "response", None),
+ )
+ elif hasattr(original_exception, "status_code"):
+ if (
+ original_exception.status_code == 400
+ or original_exception.status_code == 498
+ ):
+ raise BadRequestError(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ )
+ elif original_exception.status_code == 500:
+ raise InternalServerError(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ "CohereConnectionError" in exception_type
+ ): # cohere seems to fire these errors when we load test it (1k+ messages / min)
+ raise RateLimitError(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "invalid type:" in error_str:
+ raise BadRequestError(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Unexpected server error" in error_str:
+ raise InternalServerError(
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ else:
+ if hasattr(original_exception, "status_code"):
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"CohereException - {original_exception.message}",
+ llm_provider="cohere",
+ model=model,
+ request=getattr(original_exception, "request", None),
+ )
+ raise cast(Exception, original_exception)
+
+
+def _map_huggingface_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "length limit exceeded" in error_str:
+ raise ContextWindowExceededError(
+ message=error_str,
+ model=model,
+ llm_provider="huggingface",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "A valid user token is required" in error_str:
+ raise BadRequestError(
+ message=error_str,
+ llm_provider="huggingface",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Rate limit reached" in error_str:
+ raise RateLimitError(
+ message=error_str,
+ llm_provider="huggingface",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ if hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"HuggingfaceException - {original_exception.message}",
+ llm_provider="huggingface",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"HuggingfaceException - {original_exception.message}",
+ model=model,
+ llm_provider="huggingface",
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"HuggingfaceException - {original_exception.message}",
+ model=model,
+ llm_provider="huggingface",
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"HuggingfaceException - {original_exception.message}",
+ llm_provider="huggingface",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"HuggingfaceException - {original_exception.message}",
+ llm_provider="huggingface",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ else:
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"HuggingfaceException - {original_exception.message}",
+ llm_provider="huggingface",
+ model=model,
+ request=getattr(original_exception, "request", None),
+ )
+
+
+def _map_ai21_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if hasattr(original_exception, "message"):
+ if "Prompt has too many tokens" in original_exception.message:
+ raise ContextWindowExceededError(
+ message=f"AI21Exception - {original_exception.message}",
+ model=model,
+ llm_provider="ai21",
+ response=getattr(original_exception, "response", None),
+ )
+ if "Bad or missing API token." in original_exception.message:
+ raise BadRequestError(
+ message=f"AI21Exception - {original_exception.message}",
+ model=model,
+ llm_provider="ai21",
+ response=getattr(original_exception, "response", None),
+ )
+ if hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"AI21Exception - {original_exception.message}",
+ llm_provider="ai21",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"AI21Exception - {original_exception.message}",
+ model=model,
+ llm_provider="ai21",
+ )
+ if original_exception.status_code == 422:
+ raise BadRequestError(
+ message=f"AI21Exception - {original_exception.message}",
+ model=model,
+ llm_provider="ai21",
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"AI21Exception - {original_exception.message}",
+ llm_provider="ai21",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ else:
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"AI21Exception - {original_exception.message}",
+ llm_provider="ai21",
+ model=model,
+ request=getattr(original_exception, "request", None),
+ )
+
+
+def _map_nlp_cloud_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "detail" in error_str:
+ if "Input text length should not exceed" in error_str:
+ raise ContextWindowExceededError(
+ message=f"NLPCloudException - {error_str}",
+ model=model,
+ llm_provider="nlp_cloud",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "value is not a valid" in error_str:
+ raise BadRequestError(
+ message=f"NLPCloudException - {error_str}",
+ model=model,
+ llm_provider="nlp_cloud",
+ response=getattr(original_exception, "response", None),
+ )
+ else:
+ raise APIError(
+ status_code=500,
+ message=f"NLPCloudException - {error_str}",
+ model=model,
+ llm_provider="nlp_cloud",
+ request=getattr(original_exception, "request", None),
+ )
+ if hasattr(
+ original_exception, "status_code"
+ ): # https://docs.nlpcloud.com/?shell#errors
+ if (
+ original_exception.status_code == 400
+ or original_exception.status_code == 406
+ or original_exception.status_code == 413
+ or original_exception.status_code == 422
+ ):
+ raise BadRequestError(
+ message=f"NLPCloudException - {original_exception.message}",
+ llm_provider="nlp_cloud",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ original_exception.status_code == 401
+ or original_exception.status_code == 403
+ ):
+ raise AuthenticationError(
+ message=f"NLPCloudException - {original_exception.message}",
+ llm_provider="nlp_cloud",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ original_exception.status_code == 522
+ or original_exception.status_code == 524
+ ):
+ raise Timeout(
+ message=f"NLPCloudException - {original_exception.message}",
+ model=model,
+ llm_provider="nlp_cloud",
+ )
+ elif (
+ original_exception.status_code == 429
+ or original_exception.status_code == 402
+ ):
+ raise RateLimitError(
+ message=f"NLPCloudException - {original_exception.message}",
+ llm_provider="nlp_cloud",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ original_exception.status_code == 500
+ or original_exception.status_code == 503
+ ):
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"NLPCloudException - {original_exception.message}",
+ llm_provider="nlp_cloud",
+ model=model,
+ request=getattr(original_exception, "request", None),
+ )
+ elif (
+ original_exception.status_code == 504
+ or original_exception.status_code == 520
+ ):
+ raise ServiceUnavailableError(
+ message=f"NLPCloudException - {original_exception.message}",
+ model=model,
+ llm_provider="nlp_cloud",
+ response=getattr(original_exception, "response", None),
+ )
+ else:
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"NLPCloudException - {original_exception.message}",
+ llm_provider="nlp_cloud",
+ model=model,
+ request=getattr(original_exception, "request", None),
+ )
+
+
+def _map_together_ai_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ try:
+ error_response = json.loads(error_str)
+ except Exception:
+ error_response = {"error": error_str}
+ if (
+ "error" in error_response
+ and "`inputs` tokens + `max_new_tokens` must be <=" in error_response["error"]
+ ):
+ raise ContextWindowExceededError(
+ message=f"TogetherAIException - {error_response['error']}",
+ model=model,
+ llm_provider="together_ai",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "error" in error_response and "invalid private key" in error_response["error"]:
+ raise AuthenticationError(
+ message=f"TogetherAIException - {error_response['error']}",
+ llm_provider="together_ai",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "error" in error_response and "INVALID_ARGUMENT" in error_response["error"]:
+ raise BadRequestError(
+ message=f"TogetherAIException - {error_response['error']}",
+ model=model,
+ llm_provider="together_ai",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "A timeout occurred" in error_str:
+ raise Timeout(
+ message=f"TogetherAIException - {error_str}",
+ model=model,
+ llm_provider="together_ai",
+ )
+ elif (
+ "error" in error_response
+ and "API key doesn't match expected format." in error_response["error"]
+ ):
+ raise BadRequestError(
+ message=f"TogetherAIException - {error_response['error']}",
+ model=model,
+ llm_provider="together_ai",
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ "error_type" in error_response and error_response["error_type"] == "validation"
+ ):
+ raise BadRequestError(
+ message=f"TogetherAIException - {error_response['error']}",
+ model=model,
+ llm_provider="together_ai",
+ response=getattr(original_exception, "response", None),
+ )
+ if hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 408:
+ raise Timeout(
+ message=f"TogetherAIException - {original_exception.message}",
+ model=model,
+ llm_provider="together_ai",
+ )
+ elif original_exception.status_code == 422:
+ raise BadRequestError(
+ message=f"TogetherAIException - {error_response['error']}",
+ model=model,
+ llm_provider="together_ai",
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"TogetherAIException - {original_exception.message}",
+ llm_provider="together_ai",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 524:
+ raise Timeout(
+ message=f"TogetherAIException - {original_exception.message}",
+ llm_provider="together_ai",
+ model=model,
+ )
+ else:
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"TogetherAIException - {original_exception.message}",
+ llm_provider="together_ai",
+ model=model,
+ request=getattr(original_exception, "request", None),
+ )
+
+
+def _map_aleph_alpha_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if "This is longer than the model's maximum context length" in error_str:
+ raise ContextWindowExceededError(
+ message=f"AlephAlphaException - {original_exception.message}",
+ llm_provider="aleph_alpha",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "InvalidToken" in error_str or "No token provided" in error_str:
+ raise BadRequestError(
+ message=f"AlephAlphaException - {original_exception.message}",
+ llm_provider="aleph_alpha",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif hasattr(original_exception, "status_code"):
+ verbose_logger.debug(f"status code: {original_exception.status_code}")
+ if original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"AlephAlphaException - {original_exception.message}",
+ llm_provider="aleph_alpha",
+ model=model,
+ )
+ elif original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"AlephAlphaException - {original_exception.message}",
+ llm_provider="aleph_alpha",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"AlephAlphaException - {original_exception.message}",
+ llm_provider="aleph_alpha",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 500:
+ raise ServiceUnavailableError(
+ message=f"AlephAlphaException - {original_exception.message}",
+ llm_provider="aleph_alpha",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ raise cast(Exception, original_exception)
+ raise cast(Exception, original_exception)
+
+
+def _map_ollama_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if isinstance(original_exception, dict):
+ error_str = original_exception.get("error", "")
+ else:
+ error_str = str(original_exception)
+ if "no such file or directory" in error_str:
+ raise BadRequestError(
+ message=f"OllamaException: Invalid Model/Model not loaded - {original_exception}",
+ model=model,
+ llm_provider="ollama",
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Failed to establish a new connection" in error_str:
+ raise ServiceUnavailableError(
+ message=f"OllamaException: {original_exception}",
+ llm_provider="ollama",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Invalid response object from API" in error_str:
+ raise BadRequestError(
+ message=f"OllamaException: {original_exception}",
+ llm_provider="ollama",
+ model=model,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Read timed out" in error_str:
+ raise Timeout(
+ message=f"OllamaException: {original_exception}",
+ llm_provider="ollama",
+ model=model,
+ )
+
+
+def _map_vllm_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 0:
+ raise APIConnectionError(
+ message=f"VLLMException - {original_exception.message}",
+ llm_provider="vllm",
+ model=model,
+ request=getattr(original_exception, "request", None),
+ )
+
+
+def _map_azure_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ message = get_error_message(error_obj=original_exception)
+ if message is None:
+ if hasattr(original_exception, "message"):
+ message = original_exception.message
+ else:
+ message = str(original_exception)
+
+ # Azure OpenAI (especially Images) often nests error details under
+ # body["error"]. Detect content policy violations using the structured
+ # payload in addition to string matching.
+ azure_error_code: Optional[str] = None
+ try:
+ body_dict = getattr(original_exception, "body", None) or {}
+ if isinstance(body_dict, dict):
+ if isinstance(body_dict.get("error"), dict):
+ azure_error_code = body_dict["error"].get("code") # type: ignore[index]
+ # Also check inner_error for
+ # ResponsibleAIPolicyViolation which indicates a
+ # content policy violation even when the top-level
+ # code is generic (e.g. "invalid_request_error").
+ if azure_error_code != "content_policy_violation":
+ _inner = body_dict["error"].get(
+ "inner_error"
+ ) or body_dict[ # type: ignore[index]
+ "error"
+ ].get(
+ "innererror"
+ ) # type: ignore[index]
+ if (
+ isinstance(_inner, dict)
+ and _inner.get("code") == "ResponsibleAIPolicyViolation"
+ ):
+ azure_error_code = "content_policy_violation"
+ else:
+ azure_error_code = body_dict.get("code")
+ except Exception:
+ azure_error_code = None
+
+ if "Internal server error" in error_str:
+ raise litellm.InternalServerError(
+ message=f"AzureException Internal server error - {message}",
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "This model's maximum context length is" in error_str:
+ raise ContextWindowExceededError(
+ message=f"AzureException ContextWindowExceededError - {message}",
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "DeploymentNotFound" in error_str:
+ raise NotFoundError(
+ message=f"AzureException NotFoundError - {message}",
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif (
+ azure_error_code == "content_policy_violation"
+ or ExceptionCheckers.is_azure_content_policy_violation_error(error_str)
+ ):
+ from litellm.llms.azure.exception_mapping import (
+ AzureOpenAIExceptionMapping,
+ )
+
+ raise AzureOpenAIExceptionMapping.create_content_policy_violation_error(
+ message=message,
+ model=model,
+ extra_information=extra_information,
+ original_exception=original_exception,
+ )
+ elif (
+ azure_error_code == "invalid_encrypted_content"
+ or "could not be verified" in error_str
+ ):
+ helpful_message = (
+ f"AzureException - {message}\n\n"
+ "This error occurs when load balancing Responses API across deployments with different API keys.\n"
+ " Encrypted content is tied to the organization that created it and cannot be decrypted by other organizations.\n\n"
+ " Solution: Enable 'encrypted_content_affinity' to route follow-up requests to the correct deployment:\n\n"
+ " router_settings:\n"
+ " enable_pre_call_checks: true\n"
+ " optional_pre_call_checks:\n"
+ " - encrypted_content_affinity\n\n"
+ " Learn more: https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing"
+ )
+ raise BadRequestError(
+ message=helpful_message,
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ body=getattr(original_exception, "body", None),
+ )
+ elif "invalid_request_error" in error_str:
+ raise BadRequestError(
+ message=f"AzureException BadRequestError - {message}",
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ body=getattr(original_exception, "body", None),
+ )
+ elif (
+ "The api_key client option must be set either by passing api_key to the client or by setting"
+ in error_str
+ ):
+ raise AuthenticationError(
+ message=f"{exception_provider} AuthenticationError - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif "Connection error" in error_str:
+ raise APIConnectionError(
+ message=f"{exception_provider} APIConnectionError - {message}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ litellm_debug_info=extra_information,
+ )
+ elif hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"AzureException - {message}",
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ body=getattr(original_exception, "body", None),
+ )
+ elif original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"AzureException AuthenticationError - {message}",
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"AzureException Timeout - {message}",
+ model=model,
+ litellm_debug_info=extra_information,
+ llm_provider="azure",
+ )
+ elif original_exception.status_code == 422:
+ raise BadRequestError(
+ message=f"AzureException BadRequestError - {message}",
+ model=model,
+ llm_provider="azure",
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"AzureException RateLimitError - {message}",
+ model=model,
+ llm_provider="azure",
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 502:
+ raise BadGatewayError(
+ message=f"AzureException BadGatewayError - {message}",
+ model=model,
+ llm_provider="azure",
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"AzureException ServiceUnavailableError - {message}",
+ model=model,
+ llm_provider="azure",
+ litellm_debug_info=extra_information,
+ response=getattr(original_exception, "response", None),
+ )
+ elif original_exception.status_code == 504: # gateway timeout error
+ raise Timeout(
+ message=f"AzureException Timeout - {message}",
+ model=model,
+ litellm_debug_info=extra_information,
+ llm_provider="azure",
+ exception_status_code=original_exception.status_code,
+ )
+ else:
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"AzureException APIError - {message}",
+ llm_provider="azure",
+ litellm_debug_info=extra_information,
+ model=model,
+ request=httpx.Request(method="POST", url="https://openai.com/"),
+ )
+ else:
+ # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
+ raise APIConnectionError(
+ message=f"{exception_provider} APIConnectionError - {message}\n{_redact_string(traceback.format_exc())}",
+ llm_provider="azure",
+ model=model,
+ litellm_debug_info=extra_information,
+ request=httpx.Request(method="POST", url="https://openai.com/"),
+ )
+
+
+def _map_openrouter_exception(
+ *,
+ model: str,
+ original_exception: _ProviderHTTPException,
+ custom_llm_provider: str,
+ error_str: str,
+ exception_type: str,
+ exception_provider: str,
+ extra_information: str,
+) -> None:
+ if hasattr(original_exception, "status_code"):
+ if original_exception.status_code == 400:
+ raise BadRequestError(
+ message=f"{exception_provider} - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 401:
+ raise AuthenticationError(
+ message=f"AuthenticationError: {exception_provider} - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 404:
+ raise NotFoundError(
+ message=f"NotFoundError: {exception_provider} - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 408:
+ raise Timeout(
+ message=f"Timeout Error: {exception_provider} - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 422:
+ raise BadRequestError(
+ message=f"BadRequestError: {exception_provider} - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 429:
+ raise RateLimitError(
+ message=f"RateLimitError: {exception_provider} - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 503:
+ raise ServiceUnavailableError(
+ message=f"ServiceUnavailableError: {exception_provider} - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ response=getattr(original_exception, "response", None),
+ litellm_debug_info=extra_information,
+ )
+ elif original_exception.status_code == 504: # gateway timeout error
+ raise Timeout(
+ message=f"Timeout Error: {exception_provider} - {error_str}",
+ model=model,
+ llm_provider=custom_llm_provider,
+ litellm_debug_info=extra_information,
+ exception_status_code=original_exception.status_code,
+ )
+ else:
+ raise APIError(
+ status_code=original_exception.status_code,
+ message=f"APIError: {exception_provider} - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ request=getattr(original_exception, "request", None),
+ litellm_debug_info=extra_information,
+ )
+ else:
+ # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
+ raise APIConnectionError(
+ message=f"APIConnectionError: {exception_provider} - {error_str}",
+ llm_provider=custom_llm_provider,
+ model=model,
+ litellm_debug_info=extra_information,
+ request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
+ )
+
+
def exception_type( # type: ignore
model,
original_exception,
@@ -249,6 +2264,9 @@ def exception_type( # type: ignore
return original_exception
exception_mapping_worked = False
exception_provider = custom_llm_provider
+ mappable_exception: _ProviderHTTPException = cast(
+ "_ProviderHTTPException", original_exception
+ )
if litellm.suppress_debug_info is False:
print() # noqa: T201
print( # noqa: T201
@@ -372,2055 +2390,199 @@ def exception_type( # type: ignore
or custom_llm_provider in litellm.openai_compatible_providers
or custom_llm_provider == "mistral"
):
- # custom_llm_provider is openai, make it OpenAI
- message = get_error_message(error_obj=original_exception)
- if message is None:
- if hasattr(original_exception, "message"):
- message = original_exception.message
- else:
- message = str(original_exception)
-
- if message is not None and isinstance(
- message, str
- ): # done to prevent user-confusion. Relevant issue - https://github.com/BerriAI/litellm/issues/1414
- message = message.replace("OPENAI", custom_llm_provider.upper())
- message = message.replace(
- "openai.OpenAIError",
- "{}.{}Error".format(custom_llm_provider, custom_llm_provider),
- )
- if custom_llm_provider == "openai":
- exception_provider = "OpenAI" + "Exception"
- else:
- exception_provider = (
- custom_llm_provider[0].upper()
- + custom_llm_provider[1:]
- + "Exception"
- )
-
- if ExceptionCheckers.is_error_str_rate_limit(error_str):
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"RateLimitError: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- )
- elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"ContextWindowExceededError: {exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif (
- "invalid_request_error" in error_str
- and "model_not_found" in error_str
- ):
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"{exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif "A timeout occurred" in error_str:
- exception_mapping_worked = True
- raise Timeout(
- message=f"{exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif (
- (
- "invalid_request_error" in error_str
- and "content_policy_violation" in error_str
- )
- or (
- "Invalid prompt" in error_str
- and "violating our usage policy" in error_str
- )
- or (
- "request was rejected as a result of the safety system"
- in error_str.lower()
- )
- ):
- exception_mapping_worked = True
- raise ContentPolicyViolationError(
- message=f"ContentPolicyViolationError: {exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif (
- "invalid_encrypted_content" in error_str
- or "could not be verified" in error_str
- ):
- exception_mapping_worked = True
- helpful_message = (
- f"{exception_provider} - {message}\n\n"
- " This error occurs when load balancing Responses API across deployments with different API keys.\n"
- " Encrypted content is tied to the organization that created it and cannot be decrypted by other organizations.\n\n"
- " Solution: Enable 'encrypted_content_affinity' to route follow-up requests to the correct deployment:\n\n"
- " router_settings:\n"
- " enable_pre_call_checks: true\n"
- " optional_pre_call_checks:\n"
- " - encrypted_content_affinity\n\n"
- " Learn more: https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing"
- )
- raise BadRequestError(
- message=helpful_message,
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- body=getattr(original_exception, "body", None),
- )
- elif (
- "invalid_request_error" in error_str
- and "Incorrect API key provided" not in error_str
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- body=getattr(original_exception, "body", None),
- )
- elif (
- "Web server is returning an unknown error" in error_str
- or "The server had an error processing your request." in error_str
- ):
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"{exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- )
- elif "Request too large" in error_str:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"RateLimitError: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif (
- "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"
- in error_str
- ):
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"AuthenticationError: {exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif "Mistral API raised a streaming error" in error_str:
- exception_mapping_worked = True
- _request = httpx.Request(
- method="POST", url="https://api.openai.com/v1"
- )
- raise APIError(
- status_code=500,
- message=f"{exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- request=_request,
- litellm_debug_info=extra_information,
- )
- elif hasattr(original_exception, "status_code"):
- exception_mapping_worked = True
- if original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"AuthenticationError: {exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 404:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"NotFoundError: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"Timeout Error: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 422:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- body=getattr(original_exception, "body", None),
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"RateLimitError: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 500:
- exception_mapping_worked = True
- raise InternalServerError(
- message=f"InternalServerError: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 502:
- exception_mapping_worked = True
- raise BadGatewayError(
- message=f"BadGatewayError: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"ServiceUnavailableError: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 504: # gateway timeout error
- exception_mapping_worked = True
- raise Timeout(
- message=f"Timeout Error: {exception_provider} - {message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- exception_status_code=original_exception.status_code,
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"APIError: {exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- request=getattr(original_exception, "request", None),
- litellm_debug_info=extra_information,
- )
- else:
- # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
- # exception_mapping_worked = True
- raise APIConnectionError(
- message=f"APIConnectionError: {exception_provider} - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- litellm_debug_info=extra_information,
- request=httpx.Request(
- method="POST", url="https://api.openai.com/v1/"
- ),
- )
+ _map_openai_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif (
custom_llm_provider == "anthropic"
or custom_llm_provider == "anthropic_text"
): # one of the anthropics
- if (
- "prompt is too long" in error_str
- or "prompt: length" in error_str
- or ExceptionCheckers.is_error_str_context_window_exceeded(error_str)
- ):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message="AnthropicError - {}".format(error_str),
- model=model,
- llm_provider="anthropic",
- )
- elif "overloaded_error" in error_str or "Overloaded" in error_str:
- exception_mapping_worked = True
- raise InternalServerError(
- message="AnthropicError - {}".format(error_str),
- model=model,
- llm_provider="anthropic",
- )
- if "Invalid API Key" in error_str:
- exception_mapping_worked = True
- raise AuthenticationError(
- message="AnthropicError - {}".format(error_str),
- model=model,
- llm_provider="anthropic",
- )
- if "content filtering policy" in error_str:
- exception_mapping_worked = True
- raise ContentPolicyViolationError(
- message="AnthropicError - {}".format(error_str),
- model=model,
- llm_provider="anthropic",
- )
- if "Client error '400 Bad Request'" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message="AnthropicError - {}".format(error_str),
- model=model,
- llm_provider="anthropic",
- )
- if hasattr(original_exception, "status_code"):
- verbose_logger.debug(
- f"status_code: {original_exception.status_code}"
- )
- if original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"AnthropicException - {error_str}",
- llm_provider="anthropic",
- model=model,
- )
- elif (
- original_exception.status_code == 400
- or original_exception.status_code == 413
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AnthropicException - {error_str}",
- model=model,
- llm_provider="anthropic",
- )
- elif original_exception.status_code == 404:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"AnthropicException - {error_str}",
- model=model,
- llm_provider="anthropic",
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"AnthropicException - {error_str}",
- model=model,
- llm_provider="anthropic",
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"AnthropicException - {error_str}",
- llm_provider="anthropic",
- model=model,
- )
- elif (
- original_exception.status_code == 500
- or original_exception.status_code == 529
- ):
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"AnthropicException - {error_str}. Handle with `litellm.InternalServerError`.",
- llm_provider="anthropic",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 502:
- exception_mapping_worked = True
- raise BadGatewayError(
- message=f"AnthropicException BadGatewayError - {error_str}",
- llm_provider="anthropic",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise litellm.ServiceUnavailableError(
- message=f"AnthropicException - {error_str}. Handle with `litellm.ServiceUnavailableError`.",
- llm_provider="anthropic",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 504: # gateway timeout error
- exception_mapping_worked = True
- raise Timeout(
- message=f"AnthropicException Timeout - {error_str}",
- model=model,
- llm_provider="anthropic",
- exception_status_code=original_exception.status_code,
- )
- elif custom_llm_provider == "replicate":
- if "Incorrect authentication token" in error_str:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"ReplicateException - {error_str}",
- llm_provider="replicate",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "input is too long" in error_str:
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"ReplicateException - {error_str}",
- model=model,
- llm_provider="replicate",
- response=getattr(original_exception, "response", None),
- )
- elif exception_type == "ModelError":
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"ReplicateException - {error_str}",
- model=model,
- llm_provider="replicate",
- response=getattr(original_exception, "response", None),
- )
- elif "Request was throttled" in error_str:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"ReplicateException - {error_str}",
- llm_provider="replicate",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif hasattr(original_exception, "status_code"):
- if original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"ReplicateException - {original_exception.message}",
- llm_provider="replicate",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif (
- original_exception.status_code == 400
- or original_exception.status_code == 413
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"ReplicateException - {original_exception.message}",
- model=model,
- llm_provider="replicate",
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 422:
- exception_mapping_worked = True
- raise UnprocessableEntityError(
- message=f"ReplicateException - {original_exception.message}",
- model=model,
- llm_provider="replicate",
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"ReplicateException - {original_exception.message}",
- model=model,
- llm_provider="replicate",
- )
- elif original_exception.status_code == 422:
- exception_mapping_worked = True
- raise UnprocessableEntityError(
- message=f"ReplicateException - {original_exception.message}",
- llm_provider="replicate",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"ReplicateException - {original_exception.message}",
- llm_provider="replicate",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 500:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"ReplicateException - {original_exception.message}",
- llm_provider="replicate",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- exception_mapping_worked = True
- raise APIError(
- status_code=500,
- message=f"ReplicateException - {str(original_exception)}",
- llm_provider="replicate",
+ _map_anthropic_exception(
model=model,
- request=httpx.Request(
- method="POST",
- url="https://api.replicate.com/v1/deployments",
- ),
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
+ elif custom_llm_provider == "replicate":
+ _map_replicate_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
)
elif custom_llm_provider in litellm._openai_like_providers:
- if "authorization denied for" in error_str:
- exception_mapping_worked = True
-
- # Predibase returns the raw API Key in the response - this block ensures it's not returned in the exception
- if (
- error_str is not None
- and isinstance(error_str, str)
- and "bearer" in error_str.lower()
- ):
- # only keep the first 10 chars after the occurnence of "bearer"
- _bearer_token_start_index = error_str.lower().find("bearer")
- error_str = error_str[: _bearer_token_start_index + 14]
- error_str += "XXXXXXX" + '"'
-
- raise AuthenticationError(
- message=f"{custom_llm_provider.capitalize()}Exception: Authentication Error - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif "token_quota_reached" in error_str:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"{custom_llm_provider.capitalize()}Exception: Rate Limit Errror - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif (
- "The server received an invalid response from an upstream server."
- in error_str
- ):
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- elif "model_no_support_for_function" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{custom_llm_provider.capitalize()}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- elif hasattr(original_exception, "status_code"):
- if original_exception.status_code == 500:
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- elif (
- original_exception.status_code == 401
- or original_exception.status_code == 403
- ):
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- elif original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- elif original_exception.status_code == 404:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif (
- original_exception.status_code == 422
- or original_exception.status_code == 424
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 504: # gateway timeout error
- exception_mapping_worked = True
- raise Timeout(
- message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- exception_status_code=original_exception.status_code,
- )
+ _map_openai_like_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "bedrock":
- if (
- "too many tokens" in error_str
- or "expected maxLength:" in error_str
- or "Input is too long" in error_str
- or "prompt is too long" in error_str
- or "prompt: length: 1.." in error_str
- or "Too many input tokens" in error_str
- ):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"BedrockException: Context Window Error - {error_str}",
- model=model,
- llm_provider="bedrock",
- )
- elif (
- "Conversation blocks and tool result blocks cannot be provided in the same turn."
- in error_str
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"BedrockException - {error_str}\n. Enable 'litellm.modify_params=True' (for PROXY do: `litellm_settings::modify_params: True`) to insert a dummy assistant message and fix this error.",
- model=model,
- llm_provider="bedrock",
- response=getattr(original_exception, "response", None),
- )
- elif "Malformed input request" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"BedrockException - {error_str}",
- model=model,
- llm_provider="bedrock",
- response=getattr(original_exception, "response", None),
- )
- elif "A conversation must start with a user message." in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"BedrockException - {error_str}\n. Pass in default user message via `completion(..,user_continue_message=)` or enable `litellm.modify_params=True`.\nFor Proxy: do via `litellm_settings::modify_params: True` or user_continue_message under `litellm_params`",
- model=model,
- llm_provider="bedrock",
- response=getattr(original_exception, "response", None),
- )
- elif (
- "Unable to locate credentials" in error_str
- or "The security token included in the request is invalid"
- in error_str
- ):
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"BedrockException Invalid Authentication - {error_str}",
- model=model,
- llm_provider="bedrock",
- response=getattr(original_exception, "response", None),
- )
- elif "AccessDeniedException" in error_str:
- exception_mapping_worked = True
- raise PermissionDeniedError(
- message=f"BedrockException PermissionDeniedError - {error_str}",
- model=model,
- llm_provider="bedrock",
- response=getattr(original_exception, "response", None),
- )
- elif (
- "throttlingException" in error_str
- or "ThrottlingException" in error_str
- ):
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"BedrockException: Rate Limit Error - {error_str}",
- model=model,
- llm_provider="bedrock",
- response=getattr(original_exception, "response", None),
- )
- elif (
- "Connect timeout on endpoint URL" in error_str
- or "timed out" in error_str
- ):
- exception_mapping_worked = True
- raise Timeout(
- message=f"BedrockException: Timeout Error - {error_str}",
- model=model,
- llm_provider="bedrock",
- )
- elif "Could not process image" in error_str:
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"BedrockException - {error_str}",
- model=model,
- llm_provider="bedrock",
- )
- elif hasattr(original_exception, "status_code"):
- if original_exception.status_code == 500:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"BedrockException - {original_exception.message}",
- llm_provider="bedrock",
- model=model,
- response=httpx.Response(
- status_code=500,
- request=httpx.Request(
- method="POST", url="https://api.openai.com/v1/"
- ),
- ),
- )
- elif original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"BedrockException - {original_exception.message}",
- llm_provider="bedrock",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"BedrockException - {original_exception.message}",
- llm_provider="bedrock",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 404:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"BedrockException - {original_exception.message}",
- llm_provider="bedrock",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"BedrockException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 422:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"BedrockException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"BedrockException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"BedrockException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 504: # gateway timeout error
- exception_mapping_worked = True
- raise Timeout(
- message=f"BedrockException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- exception_status_code=original_exception.status_code,
- )
+ _map_bedrock_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif (
custom_llm_provider == "sagemaker"
or custom_llm_provider == "sagemaker_chat"
):
- if "Unable to locate credentials" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"litellm.BadRequestError: SagemakerException - {error_str}",
- model=model,
- llm_provider="sagemaker",
- response=getattr(original_exception, "response", None),
- )
- elif (
- "Input validation error: `best_of` must be > 0 and <= 2"
- in error_str
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message="SagemakerException - the value of 'n' must be > 0 and <= 2 for sagemaker endpoints",
- model=model,
- llm_provider="sagemaker",
- response=getattr(original_exception, "response", None),
- )
- elif (
- "`inputs` tokens + `max_new_tokens` must be <=" in error_str
- or "instance type with more CPU capacity or memory" in error_str
- ):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"SagemakerException - {error_str}",
- model=model,
- llm_provider="sagemaker",
- response=getattr(original_exception, "response", None),
- )
- elif hasattr(original_exception, "status_code"):
- if original_exception.status_code == 500:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"SagemakerException - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=httpx.Response(
- status_code=500,
- request=httpx.Request(
- method="POST", url="https://api.openai.com/v1/"
- ),
- ),
- )
- elif original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"SagemakerException - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"SagemakerException - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 404:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"SagemakerException - {original_exception.message}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"SagemakerException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif (
- original_exception.status_code == 422
- or original_exception.status_code == 424
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"SagemakerException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"SagemakerException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"SagemakerException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 504: # gateway timeout error
- exception_mapping_worked = True
- raise Timeout(
- message=f"SagemakerException - {original_exception.message}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- exception_status_code=original_exception.status_code,
- )
+ _map_sagemaker_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif (
custom_llm_provider == LlmProviders.VERTEX_AI
or custom_llm_provider == LlmProviders.VERTEX_AI_BETA
or custom_llm_provider == LlmProviders.GEMINI
):
- if (
- "Vertex AI API has not been used in project" in error_str
- or "Unable to find your project" in error_str
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=httpx.Response(
- status_code=400,
- request=httpx.Request(
- method="POST",
- url=" https://cloud.google.com/vertex-ai/",
- ),
- ),
- litellm_debug_info=extra_information,
- )
- if "400 Request payload size exceeds" in error_str:
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- )
- elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"ContextWindowExceededError: {custom_llm_provider.capitalize()}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif (
- "None Unknown Error." in error_str
- or "Content has no parts." in error_str
- ):
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=httpx.Response(
- status_code=500,
- content=str(original_exception),
- request=httpx.Request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
- ),
- litellm_debug_info=extra_information,
- )
- elif "API key not valid." in error_str:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif "403" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=httpx.Response(
- status_code=403,
- request=httpx.Request(
- method="POST",
- url=" https://cloud.google.com/vertex-ai/",
- ),
- ),
- litellm_debug_info=extra_information,
- )
- elif (
- "The response was blocked." in error_str
- or "Output blocked by content filtering policy"
- in error_str # anthropic on vertex ai
- ):
- exception_mapping_worked = True
- raise ContentPolicyViolationError(
- message=f"{custom_llm_provider.capitalize()}Exception ContentPolicyViolationError - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- response=httpx.Response(
- status_code=400,
- request=httpx.Request(
- method="POST",
- url=" https://cloud.google.com/vertex-ai/",
- ),
- ),
- )
- elif (
- "429 Quota exceeded" in error_str
- or "Quota exceeded for" in error_str
- or "Resource exhausted" in error_str
- or "IndexError: list index out of range" in error_str
- or "429 Unable to submit request because the service is temporarily out of capacity."
- in error_str
- ):
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- response=httpx.Response(
- status_code=429,
- request=httpx.Request(
- method="POST",
- url=" https://cloud.google.com/vertex-ai/",
- ),
- ),
- )
- elif (
- "500 Internal Server Error" in error_str
- or "The model is overloaded." in error_str
- ):
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- if hasattr(original_exception, "status_code"):
- if original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- response=httpx.Response(
- status_code=400,
- request=httpx.Request(
- method="POST",
- url="https://cloud.google.com/vertex-ai/",
- ),
- ),
- )
- if original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- if original_exception.status_code == 403:
- exception_mapping_worked = True
- raise PermissionDeniedError(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- response=httpx.Response(
- status_code=403,
- request=httpx.Request(
- method="POST",
- url="https://cloud.google.com/vertex-ai/",
- ),
- ),
- )
- if original_exception.status_code == 404:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- if original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- )
-
- if original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"litellm.RateLimitError: {custom_llm_provider.capitalize()}Exception - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- response=httpx.Response(
- status_code=429,
- request=httpx.Request(
- method="POST",
- url=" https://cloud.google.com/vertex-ai/",
- ),
- ),
- )
- if original_exception.status_code == 500:
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"{custom_llm_provider.capitalize()}Exception InternalServerError - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- response=httpx.Response(
- status_code=500,
- content=str(original_exception),
- request=httpx.Request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
- ),
- )
- if original_exception.status_code == 502:
- exception_mapping_worked = True
- raise APIConnectionError(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- )
- if original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- )
+ _map_vertex_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "cloudflare":
- if "Authentication error" in error_str:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"Cloudflare Exception - {original_exception.message}",
- llm_provider="cloudflare",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- if "must have required property" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"Cloudflare Exception - {original_exception.message}",
- llm_provider="cloudflare",
- model=model,
- response=getattr(original_exception, "response", None),
- )
+ _map_cloudflare_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif (
custom_llm_provider == "cohere" or custom_llm_provider == "cohere_chat"
): # Cohere
- if (
- "invalid api token" in error_str
- or "No API key provided." in error_str
- ):
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "invalid type: parameter" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "too many tokens" in error_str:
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"CohereException - {original_exception.message}",
- model=model,
- llm_provider="cohere",
- response=getattr(original_exception, "response", None),
- )
- elif "internal server error" in error_str.lower():
- exception_mapping_worked = True
- raise InternalServerError(
- message=f"CohereException - {error_str}",
- model=model,
- llm_provider="cohere",
- response=getattr(original_exception, "response", None),
- )
- elif hasattr(original_exception, "status_code"):
- if (
- original_exception.status_code == 400
- or original_exception.status_code == 498
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- )
- elif original_exception.status_code == 500:
- exception_mapping_worked = True
- raise InternalServerError(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif (
- "CohereConnectionError" in exception_type
- ): # cohere seems to fire these errors when we load test it (1k+ messages / min)
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "invalid type:" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "Unexpected server error" in error_str:
- exception_mapping_worked = True
- raise InternalServerError(
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- else:
- if hasattr(original_exception, "status_code"):
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"CohereException - {original_exception.message}",
- llm_provider="cohere",
- model=model,
- request=getattr(original_exception, "request", None),
- )
- raise original_exception
+ _map_cohere_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "huggingface":
- if "length limit exceeded" in error_str:
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=error_str,
- model=model,
- llm_provider="huggingface",
- response=getattr(original_exception, "response", None),
- )
- elif "A valid user token is required" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=error_str,
- llm_provider="huggingface",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "Rate limit reached" in error_str:
- exception_mapping_worked = True
- raise RateLimitError(
- message=error_str,
- llm_provider="huggingface",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- if hasattr(original_exception, "status_code"):
- if original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"HuggingfaceException - {original_exception.message}",
- llm_provider="huggingface",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"HuggingfaceException - {original_exception.message}",
- model=model,
- llm_provider="huggingface",
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"HuggingfaceException - {original_exception.message}",
- model=model,
- llm_provider="huggingface",
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"HuggingfaceException - {original_exception.message}",
- llm_provider="huggingface",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"HuggingfaceException - {original_exception.message}",
- llm_provider="huggingface",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"HuggingfaceException - {original_exception.message}",
- llm_provider="huggingface",
- model=model,
- request=getattr(original_exception, "request", None),
- )
+ _map_huggingface_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "ai21":
- if hasattr(original_exception, "message"):
- if "Prompt has too many tokens" in original_exception.message:
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"AI21Exception - {original_exception.message}",
- model=model,
- llm_provider="ai21",
- response=getattr(original_exception, "response", None),
- )
- if "Bad or missing API token." in original_exception.message:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AI21Exception - {original_exception.message}",
- model=model,
- llm_provider="ai21",
- response=getattr(original_exception, "response", None),
- )
- if hasattr(original_exception, "status_code"):
- if original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"AI21Exception - {original_exception.message}",
- llm_provider="ai21",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"AI21Exception - {original_exception.message}",
- model=model,
- llm_provider="ai21",
- )
- if original_exception.status_code == 422:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AI21Exception - {original_exception.message}",
- model=model,
- llm_provider="ai21",
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"AI21Exception - {original_exception.message}",
- llm_provider="ai21",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"AI21Exception - {original_exception.message}",
- llm_provider="ai21",
- model=model,
- request=getattr(original_exception, "request", None),
- )
+ _map_ai21_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "nlp_cloud":
- if "detail" in error_str:
- if "Input text length should not exceed" in error_str:
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"NLPCloudException - {error_str}",
- model=model,
- llm_provider="nlp_cloud",
- response=getattr(original_exception, "response", None),
- )
- elif "value is not a valid" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"NLPCloudException - {error_str}",
- model=model,
- llm_provider="nlp_cloud",
- response=getattr(original_exception, "response", None),
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=500,
- message=f"NLPCloudException - {error_str}",
- model=model,
- llm_provider="nlp_cloud",
- request=getattr(original_exception, "request", None),
- )
- if hasattr(
- original_exception, "status_code"
- ): # https://docs.nlpcloud.com/?shell#errors
- if (
- original_exception.status_code == 400
- or original_exception.status_code == 406
- or original_exception.status_code == 413
- or original_exception.status_code == 422
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"NLPCloudException - {original_exception.message}",
- llm_provider="nlp_cloud",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif (
- original_exception.status_code == 401
- or original_exception.status_code == 403
- ):
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"NLPCloudException - {original_exception.message}",
- llm_provider="nlp_cloud",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif (
- original_exception.status_code == 522
- or original_exception.status_code == 524
- ):
- exception_mapping_worked = True
- raise Timeout(
- message=f"NLPCloudException - {original_exception.message}",
- model=model,
- llm_provider="nlp_cloud",
- )
- elif (
- original_exception.status_code == 429
- or original_exception.status_code == 402
- ):
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"NLPCloudException - {original_exception.message}",
- llm_provider="nlp_cloud",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif (
- original_exception.status_code == 500
- or original_exception.status_code == 503
- ):
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"NLPCloudException - {original_exception.message}",
- llm_provider="nlp_cloud",
- model=model,
- request=getattr(original_exception, "request", None),
- )
- elif (
- original_exception.status_code == 504
- or original_exception.status_code == 520
- ):
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"NLPCloudException - {original_exception.message}",
- model=model,
- llm_provider="nlp_cloud",
- response=getattr(original_exception, "response", None),
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"NLPCloudException - {original_exception.message}",
- llm_provider="nlp_cloud",
- model=model,
- request=getattr(original_exception, "request", None),
- )
+ _map_nlp_cloud_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "together_ai":
- try:
- error_response = json.loads(error_str)
- except Exception:
- error_response = {"error": error_str}
- if (
- "error" in error_response
- and "`inputs` tokens + `max_new_tokens` must be <="
- in error_response["error"]
- ):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"TogetherAIException - {error_response['error']}",
- model=model,
- llm_provider="together_ai",
- response=getattr(original_exception, "response", None),
- )
- elif (
- "error" in error_response
- and "invalid private key" in error_response["error"]
- ):
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"TogetherAIException - {error_response['error']}",
- llm_provider="together_ai",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif (
- "error" in error_response
- and "INVALID_ARGUMENT" in error_response["error"]
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"TogetherAIException - {error_response['error']}",
- model=model,
- llm_provider="together_ai",
- response=getattr(original_exception, "response", None),
- )
- elif "A timeout occurred" in error_str:
- exception_mapping_worked = True
- raise Timeout(
- message=f"TogetherAIException - {error_str}",
- model=model,
- llm_provider="together_ai",
- )
- elif (
- "error" in error_response
- and "API key doesn't match expected format."
- in error_response["error"]
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"TogetherAIException - {error_response['error']}",
- model=model,
- llm_provider="together_ai",
- response=getattr(original_exception, "response", None),
- )
- elif (
- "error_type" in error_response
- and error_response["error_type"] == "validation"
- ):
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"TogetherAIException - {error_response['error']}",
- model=model,
- llm_provider="together_ai",
- response=getattr(original_exception, "response", None),
- )
- if hasattr(original_exception, "status_code"):
- if original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"TogetherAIException - {original_exception.message}",
- model=model,
- llm_provider="together_ai",
- )
- elif original_exception.status_code == 422:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"TogetherAIException - {error_response['error']}",
- model=model,
- llm_provider="together_ai",
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"TogetherAIException - {original_exception.message}",
- llm_provider="together_ai",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 524:
- exception_mapping_worked = True
- raise Timeout(
- message=f"TogetherAIException - {original_exception.message}",
- llm_provider="together_ai",
- model=model,
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"TogetherAIException - {original_exception.message}",
- llm_provider="together_ai",
- model=model,
- request=getattr(original_exception, "request", None),
- )
+ _map_together_ai_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "aleph_alpha":
- if (
- "This is longer than the model's maximum context length"
- in error_str
- ):
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"AlephAlphaException - {original_exception.message}",
- llm_provider="aleph_alpha",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "InvalidToken" in error_str or "No token provided" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AlephAlphaException - {original_exception.message}",
- llm_provider="aleph_alpha",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif hasattr(original_exception, "status_code"):
- verbose_logger.debug(
- f"status code: {original_exception.status_code}"
- )
- if original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"AlephAlphaException - {original_exception.message}",
- llm_provider="aleph_alpha",
- model=model,
- )
- elif original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AlephAlphaException - {original_exception.message}",
- llm_provider="aleph_alpha",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"AlephAlphaException - {original_exception.message}",
- llm_provider="aleph_alpha",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 500:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"AlephAlphaException - {original_exception.message}",
- llm_provider="aleph_alpha",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- raise original_exception
- raise original_exception
+ _map_aleph_alpha_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif (
custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat"
):
- if isinstance(original_exception, dict):
- error_str = original_exception.get("error", "")
- else:
- error_str = str(original_exception)
- if "no such file or directory" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"OllamaException: Invalid Model/Model not loaded - {original_exception}",
- model=model,
- llm_provider="ollama",
- response=getattr(original_exception, "response", None),
- )
- elif "Failed to establish a new connection" in error_str:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"OllamaException: {original_exception}",
- llm_provider="ollama",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "Invalid response object from API" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"OllamaException: {original_exception}",
- llm_provider="ollama",
- model=model,
- response=getattr(original_exception, "response", None),
- )
- elif "Read timed out" in error_str:
- exception_mapping_worked = True
- raise Timeout(
- message=f"OllamaException: {original_exception}",
- llm_provider="ollama",
- model=model,
- )
+ _map_ollama_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "vllm":
- if hasattr(original_exception, "status_code"):
- if original_exception.status_code == 0:
- exception_mapping_worked = True
- raise APIConnectionError(
- message=f"VLLMException - {original_exception.message}",
- llm_provider="vllm",
- model=model,
- request=getattr(original_exception, "request", None),
- )
+ _map_vllm_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
elif custom_llm_provider == "azure" or custom_llm_provider == "azure_text":
- message = get_error_message(error_obj=original_exception)
- if message is None:
- if hasattr(original_exception, "message"):
- message = original_exception.message
- else:
- message = str(original_exception)
-
- # Azure OpenAI (especially Images) often nests error details under
- # body["error"]. Detect content policy violations using the structured
- # payload in addition to string matching.
- azure_error_code: Optional[str] = None
- try:
- body_dict = getattr(original_exception, "body", None) or {}
- if isinstance(body_dict, dict):
- if isinstance(body_dict.get("error"), dict):
- azure_error_code = body_dict["error"].get("code") # type: ignore[index]
- # Also check inner_error for
- # ResponsibleAIPolicyViolation which indicates a
- # content policy violation even when the top-level
- # code is generic (e.g. "invalid_request_error").
- if azure_error_code != "content_policy_violation":
- _inner = body_dict["error"].get(
- "inner_error"
- ) or body_dict[ # type: ignore[index]
- "error"
- ].get(
- "innererror"
- ) # type: ignore[index]
- if (
- isinstance(_inner, dict)
- and _inner.get("code")
- == "ResponsibleAIPolicyViolation"
- ):
- azure_error_code = "content_policy_violation"
- else:
- azure_error_code = body_dict.get("code")
- except Exception:
- azure_error_code = None
-
- if "Internal server error" in error_str:
- exception_mapping_worked = True
- raise litellm.InternalServerError(
- message=f"AzureException Internal server error - {message}",
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif "This model's maximum context length is" in error_str:
- exception_mapping_worked = True
- raise ContextWindowExceededError(
- message=f"AzureException ContextWindowExceededError - {message}",
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif "DeploymentNotFound" in error_str:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"AzureException NotFoundError - {message}",
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif (
- azure_error_code == "content_policy_violation"
- or ExceptionCheckers.is_azure_content_policy_violation_error(
- error_str
- )
- ):
- exception_mapping_worked = True
- from litellm.llms.azure.exception_mapping import (
- AzureOpenAIExceptionMapping,
- )
-
- raise AzureOpenAIExceptionMapping.create_content_policy_violation_error(
- message=message,
- model=model,
- extra_information=extra_information,
- original_exception=original_exception,
- )
- elif (
- azure_error_code == "invalid_encrypted_content"
- or "could not be verified" in error_str
- ):
- exception_mapping_worked = True
- helpful_message = (
- f"AzureException - {message}\n\n"
- "This error occurs when load balancing Responses API across deployments with different API keys.\n"
- " Encrypted content is tied to the organization that created it and cannot be decrypted by other organizations.\n\n"
- " Solution: Enable 'encrypted_content_affinity' to route follow-up requests to the correct deployment:\n\n"
- " router_settings:\n"
- " enable_pre_call_checks: true\n"
- " optional_pre_call_checks:\n"
- " - encrypted_content_affinity\n\n"
- " Learn more: https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing"
- )
- raise BadRequestError(
- message=helpful_message,
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- body=getattr(original_exception, "body", None),
- )
- elif "invalid_request_error" in error_str:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AzureException BadRequestError - {message}",
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- body=getattr(original_exception, "body", None),
- )
- elif (
- "The api_key client option must be set either by passing api_key to the client or by setting"
- in error_str
- ):
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"{exception_provider} AuthenticationError - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif "Connection error" in error_str:
- exception_mapping_worked = True
- raise APIConnectionError(
- message=f"{exception_provider} APIConnectionError - {message}",
- llm_provider=custom_llm_provider,
- model=model,
- litellm_debug_info=extra_information,
- )
- elif hasattr(original_exception, "status_code"):
- exception_mapping_worked = True
- if original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AzureException - {message}",
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- body=getattr(original_exception, "body", None),
- )
- elif original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"AzureException AuthenticationError - {message}",
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"AzureException Timeout - {message}",
- model=model,
- litellm_debug_info=extra_information,
- llm_provider="azure",
- )
- elif original_exception.status_code == 422:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"AzureException BadRequestError - {message}",
- model=model,
- llm_provider="azure",
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"AzureException RateLimitError - {message}",
- model=model,
- llm_provider="azure",
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 502:
- exception_mapping_worked = True
- raise BadGatewayError(
- message=f"AzureException BadGatewayError - {message}",
- model=model,
- llm_provider="azure",
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"AzureException ServiceUnavailableError - {message}",
- model=model,
- llm_provider="azure",
- litellm_debug_info=extra_information,
- response=getattr(original_exception, "response", None),
- )
- elif original_exception.status_code == 504: # gateway timeout error
- exception_mapping_worked = True
- raise Timeout(
- message=f"AzureException Timeout - {message}",
- model=model,
- litellm_debug_info=extra_information,
- llm_provider="azure",
- exception_status_code=original_exception.status_code,
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"AzureException APIError - {message}",
- llm_provider="azure",
- litellm_debug_info=extra_information,
- model=model,
- request=httpx.Request(
- method="POST", url="https://openai.com/"
- ),
- )
- else:
- # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
- raise APIConnectionError(
- message=f"{exception_provider} APIConnectionError - {message}\n{_redact_string(traceback.format_exc())}",
- llm_provider="azure",
- model=model,
- litellm_debug_info=extra_information,
- request=httpx.Request(method="POST", url="https://openai.com/"),
- )
+ _map_azure_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
if custom_llm_provider == "openrouter":
- if hasattr(original_exception, "status_code"):
- exception_mapping_worked = True
- if original_exception.status_code == 400:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"{exception_provider} - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 401:
- exception_mapping_worked = True
- raise AuthenticationError(
- message=f"AuthenticationError: {exception_provider} - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 404:
- exception_mapping_worked = True
- raise NotFoundError(
- message=f"NotFoundError: {exception_provider} - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 408:
- exception_mapping_worked = True
- raise Timeout(
- message=f"Timeout Error: {exception_provider} - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 422:
- exception_mapping_worked = True
- raise BadRequestError(
- message=f"BadRequestError: {exception_provider} - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 429:
- exception_mapping_worked = True
- raise RateLimitError(
- message=f"RateLimitError: {exception_provider} - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 503:
- exception_mapping_worked = True
- raise ServiceUnavailableError(
- message=f"ServiceUnavailableError: {exception_provider} - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- response=getattr(original_exception, "response", None),
- litellm_debug_info=extra_information,
- )
- elif original_exception.status_code == 504: # gateway timeout error
- exception_mapping_worked = True
- raise Timeout(
- message=f"Timeout Error: {exception_provider} - {error_str}",
- model=model,
- llm_provider=custom_llm_provider,
- litellm_debug_info=extra_information,
- exception_status_code=original_exception.status_code,
- )
- else:
- exception_mapping_worked = True
- raise APIError(
- status_code=original_exception.status_code,
- message=f"APIError: {exception_provider} - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- request=getattr(original_exception, "request", None),
- litellm_debug_info=extra_information,
- )
- else:
- # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
- raise APIConnectionError(
- message=f"APIConnectionError: {exception_provider} - {error_str}",
- llm_provider=custom_llm_provider,
- model=model,
- litellm_debug_info=extra_information,
- request=httpx.Request(
- method="POST", url="https://api.openai.com/v1/"
- ),
- )
+ _map_openrouter_exception(
+ model=model,
+ original_exception=mappable_exception,
+ custom_llm_provider=custom_llm_provider,
+ error_str=error_str,
+ exception_type=exception_type,
+ exception_provider=exception_provider,
+ extra_information=extra_information,
+ )
if (
"BadRequestError.__init__() missing 1 required positional argument: 'param'"
in str(original_exception)
diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py
index 4941d52d7d6..bb8b1a82996 100644
--- a/litellm/litellm_core_utils/get_llm_provider_logic.py
+++ b/litellm/litellm_core_utils/get_llm_provider_logic.py
@@ -388,6 +388,9 @@ def get_llm_provider(
elif endpoint == "https://api.inference.wandb.ai/v1":
custom_llm_provider = "wandb"
dynamic_api_key = get_secret_str("WANDB_API_KEY")
+ elif endpoint == "https://pinstripes.io/v1":
+ custom_llm_provider = "pinstripes"
+ dynamic_api_key = get_secret_str("PINSTRIPES_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception(
@@ -641,7 +644,7 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info(
- api_base, api_key, litellm_params=litellm_params
+ api_base, api_key, litellm_params=litellm_params, model=model
)
elif custom_llm_provider == "nvidia_nim":
# nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py
index e87042b9101..c22d3b99705 100644
--- a/litellm/litellm_core_utils/get_supported_openai_params.py
+++ b/litellm/litellm_core_utils/get_supported_openai_params.py
@@ -86,9 +86,7 @@ def get_supported_openai_params(
model=model
)
elif request_type == "transcription":
- return litellm.FireworksAIAudioTranscriptionConfig().get_supported_openai_params(
- model=model
- )
+ return None
else:
return litellm.FireworksAIConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "nvidia_nim":
@@ -191,7 +189,9 @@ def get_supported_openai_params(
)
elif custom_llm_provider == "sambanova":
if request_type == "embeddings":
- litellm.SambaNovaEmbeddingConfig().get_supported_openai_params(model=model)
+ return litellm.SambaNovaEmbeddingConfig().get_supported_openai_params(
+ model=model
+ )
else:
return litellm.SambanovaConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "nebius":
diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py
index 9e972f1910b..5a29ea73a74 100644
--- a/litellm/litellm_core_utils/health_check_helpers.py
+++ b/litellm/litellm_core_utils/health_check_helpers.py
@@ -44,8 +44,8 @@ class HealthCheckHelpers:
model_params["litellm_logging_obj"] = litellm_logging_obj
model_params["fallbacks"] = fallback_models
model_params["max_tokens"] = model_params.get(
- "max_tokens", 10
- ) # gpt-5-nano throws errors for max_tokens=1
+ "max_tokens", 16
+ ) # GPT-5 models require max_output_tokens >= 16
await acompletion(**model_params)
return {}
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index d241c501797..d750a509054 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -2975,7 +2975,12 @@ class Logging(LiteLLMLoggingBaseClass):
)
self.model_call_details["end_time"] = end_time
self.model_call_details.setdefault("original_response", None)
- self.model_call_details["response_cost"] = 0
+ # A stream interrupted mid-flight still billed the provider for the
+ # chunks already delivered; the router stashes that recovered usage as
+ # ``combined_usage_object`` and pre-computes its cost, so preserve it
+ # here instead of zeroing the spend on an otherwise-failed request.
+ if self.model_call_details.get("combined_usage_object") is None:
+ self.model_call_details["response_cost"] = 0
if hasattr(exception, "headers") and isinstance(exception.headers, dict):
self.model_call_details.setdefault("litellm_params", {})
@@ -5416,19 +5421,20 @@ class StandardLoggingPayloadSetup:
tb_lines[:MAXIMUM_TRACEBACK_LINES_TO_LOG]
) # Limit to first 100 lines
+ # Prefer the `.message` attribute (set by ProxyException and every
+ # litellm.exceptions.* class) over str(exc); ProxyException does not
+ # call super().__init__() nor define __str__, so str() on it returns
+ # an empty string, which used to silently strip the human-readable
+ # message from spend_logs.metadata.error_information.
+ # Use isinstance, not truthiness: an explicit empty string on
+ # `.message` is a deliberate value and must not be replaced by
+ # `str(exc)`.
explicit_message = getattr(original_exception, "message", None)
- error_message = (
- explicit_message
- if isinstance(explicit_message, str) and explicit_message
- else str(original_exception)
- )
+ if isinstance(explicit_message, str):
+ error_message = explicit_message
+ else:
+ error_message = str(original_exception) if original_exception else ""
- # Duck-typed read so bare-Exception subclasses like
- # `litellm.BudgetExceededError` can participate without joining the
- # RateLimitError hierarchy (which would break `except BudgetExceededError`).
- # Validated against the enum value sets so a third-party exception that
- # happens to declare a `.category` or `.rate_limit_type` string attribute
- # can't leak garbage into the payload or Prometheus label cardinality.
rate_limit_category = validate_rate_limit_category(
getattr(original_exception, "category", None)
)
@@ -5441,7 +5447,7 @@ class StandardLoggingPayloadSetup:
error_class=error_class,
llm_provider=_llm_provider_in_exception,
traceback=traceback_info,
- error_message=error_message if original_exception else "",
+ error_message=error_message,
error_rate_limit_category=rate_limit_category,
error_rate_limit_type=rate_limit_type,
)
diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py
index 6c6b8611da6..7a7fde3087e 100644
--- a/litellm/litellm_core_utils/llm_cost_calc/utils.py
+++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py
@@ -191,6 +191,11 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st
return base_key
+def _parse_above_token_threshold(key: str) -> float:
+ threshold_str = key.split("_above_")[1].split("_tokens")[0]
+ return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1)
+
+
def _get_token_base_cost(
model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None
) -> Tuple[float, float, float, float, float]:
@@ -256,15 +261,13 @@ def _get_token_base_cost(
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
threshold: Optional[float] = None
- for key in sorted(threshold_keys, reverse=True):
+ for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True):
value = model_info.get(key)
if value is not None:
try:
# Handle both formats: _above_128k_tokens and _above_128_tokens
threshold_str = key.split("_above_")[1].split("_tokens")[0]
- threshold = float(threshold_str.replace("k", "")) * (
- 1000 if "k" in threshold_str else 1
- )
+ threshold = _parse_above_token_threshold(key)
if usage.prompt_tokens > threshold:
# Prefer a service_tier-specific above-threshold key when available,
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
diff --git a/litellm/litellm_core_utils/request_timeout_resolver.py b/litellm/litellm_core_utils/request_timeout_resolver.py
new file mode 100644
index 00000000000..146c39ce9f3
--- /dev/null
+++ b/litellm/litellm_core_utils/request_timeout_resolver.py
@@ -0,0 +1,29 @@
+"""Single source of truth for whether ``litellm.request_timeout`` was configured.
+
+``litellm.request_timeout`` always holds a value (the package default,
+:data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS`), so a bare read can't
+tell "user asked for this" from "nobody set it". This resolver answers that:
+
+* ``request_timeout_explicitly_set`` is the authoritative signal, set when the
+ value comes from the ``REQUEST_TIMEOUT`` env var or ``litellm_settings``.
+* A runtime value that differs from the package default (e.g. ``litellm.request_timeout
+ = 300`` in SDK code) is also treated as explicit, for backwards compatibility.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
+
+
+def get_configured_request_timeout() -> Optional[float]:
+ """Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
+ import litellm
+
+ timeout = float(litellm.request_timeout)
+ if litellm.request_timeout_explicitly_set:
+ return timeout
+ if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
+ return timeout
+ return None
diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py
index 4928dd08386..b14e12de7cd 100644
--- a/litellm/litellm_core_utils/sensitive_data_masker.py
+++ b/litellm/litellm_core_utils/sensitive_data_masker.py
@@ -12,6 +12,7 @@ class SensitiveDataMasker:
visible_prefix: int = 4,
visible_suffix: int = 4,
mask_char: str = "*",
+ mask_short_values: bool = True,
):
self.sensitive_patterns = sensitive_patterns or {
"password",
@@ -38,12 +39,17 @@ class SensitiveDataMasker:
self.visible_prefix = visible_prefix
self.visible_suffix = visible_suffix
self.mask_char = mask_char
+ self.mask_short_values = mask_short_values
def _mask_value(self, value: str) -> str:
- if not value or len(str(value)) < (self.visible_prefix + self.visible_suffix):
- return value
-
value_str = str(value)
+ if not value_str:
+ return value
+ if len(value_str) <= (self.visible_prefix + self.visible_suffix):
+ return (
+ self.mask_char * len(value_str) if self.mask_short_values else value_str
+ )
+
masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix)
# Handle the case where visible_suffix is 0 to avoid showing the entire string
diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py
index 888a9658396..e278483d689 100644
--- a/litellm/litellm_core_utils/streaming_handler.py
+++ b/litellm/litellm_core_utils/streaming_handler.py
@@ -6,6 +6,7 @@ import logging
import threading
import time
import traceback
+from dataclasses import dataclass
from typing import (
Any,
AsyncIterator,
@@ -97,6 +98,19 @@ def print_verbose(print_statement):
pass
+@dataclass(frozen=True, slots=True)
+class _ProviderChunkParsed:
+ response_obj: dict[str, Any]
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderChunkEarlyReturn:
+ value: Any
+
+
+_ProviderChunkResult = Union[_ProviderChunkParsed, _ProviderChunkEarlyReturn]
+
+
class CustomStreamWrapper:
def __init__(
self,
@@ -1145,381 +1159,392 @@ class CustomStreamWrapper:
del model_response.choices[0].delta.reasoning_content
return
+ def _dispatch_provider_chunk(
+ self,
+ chunk: Any,
+ model_response: ModelResponseStream,
+ completion_obj: dict[str, Any],
+ ) -> _ProviderChunkResult:
+ response_obj: dict[str, Any] = {}
+ if (
+ isinstance(chunk, ModelResponseStream)
+ and self.custom_llm_provider is not None
+ and self.custom_llm_provider in litellm._custom_providers
+ ):
+ _has_content = bool(
+ chunk.choices
+ and chunk.choices[0].delta is not None
+ and (
+ chunk.choices[0].delta.content or chunk.choices[0].delta.tool_calls
+ )
+ )
+ if self.received_finish_reason is not None:
+ if not _has_content:
+ raise StopIteration
+ if chunk.choices and chunk.choices[0].finish_reason:
+ self.received_finish_reason = chunk.choices[0].finish_reason
+ if not _has_content:
+ return _ProviderChunkEarlyReturn(None)
+ # Strip finish_reason from the content chunk so it appears
+ # only on the trailing empty-delta chunk (OpenAI spec).
+ # finish_reason_handler() will emit the proper terminal chunk.
+ chunk.choices[0].finish_reason = None # type: ignore[assignment]
+ return _ProviderChunkEarlyReturn(chunk)
+
+ if (
+ isinstance(chunk, dict)
+ and generic_chunk_has_all_required_fields(
+ chunk=chunk
+ ) # check if chunk is a generic streaming chunk
+ ) or (
+ self.custom_llm_provider
+ and self.custom_llm_provider in litellm._custom_providers
+ ):
+ if self.received_finish_reason is not None:
+ _chunk_has_content = isinstance(chunk, dict) and (
+ bool(chunk.get("text", ""))
+ or chunk.get("tool_use") is not None
+ # Usage-only final chunks are valid and needed to surface
+ # finish_reason/usage to downstream translators.
+ or chunk.get("usage") is not None
+ )
+ if not _chunk_has_content and (
+ not isinstance(chunk, dict)
+ or "provider_specific_fields" not in chunk
+ ):
+ raise StopIteration
+ anthropic_response_obj: GChunk = cast(GChunk, chunk)
+ completion_obj["content"] = anthropic_response_obj["text"]
+ if anthropic_response_obj["is_finished"]:
+ self.received_finish_reason = anthropic_response_obj["finish_reason"]
+
+ if anthropic_response_obj["finish_reason"]:
+ self.intermittent_finish_reason = anthropic_response_obj[
+ "finish_reason"
+ ]
+
+ if anthropic_response_obj["usage"] is not None:
+ setattr(
+ model_response,
+ "usage",
+ litellm.Usage(**anthropic_response_obj["usage"]),
+ )
+
+ if (
+ "tool_use" in anthropic_response_obj
+ and anthropic_response_obj["tool_use"] is not None
+ ):
+ completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]]
+
+ if (
+ "provider_specific_fields" in anthropic_response_obj
+ and anthropic_response_obj["provider_specific_fields"] is not None
+ ):
+ for key, value in anthropic_response_obj[
+ "provider_specific_fields"
+ ].items():
+ setattr(model_response, key, value)
+
+ response_obj = cast(dict[str, Any], anthropic_response_obj)
+ elif self.model == "replicate" or self.custom_llm_provider == "replicate":
+ response_obj = self.handle_replicate_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ elif self.custom_llm_provider and self.custom_llm_provider == "predibase":
+ response_obj = self.handle_predibase_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ elif (
+ self.custom_llm_provider and self.custom_llm_provider == "baseten"
+ ): # baseten doesn't provide streaming
+ completion_obj["content"] = self.handle_baseten_chunk(chunk)
+ elif (
+ self.custom_llm_provider and self.custom_llm_provider == "ai21"
+ ): # ai21 doesn't provide streaming
+ response_obj = self.handle_ai21_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ elif self.custom_llm_provider and self.custom_llm_provider == "maritalk":
+ response_obj = self.handle_maritalk_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ elif self.custom_llm_provider and self.custom_llm_provider == "vllm":
+ completion_obj["content"] = chunk[0].outputs[0].text
+ elif (
+ self.custom_llm_provider and self.custom_llm_provider == "aleph_alpha"
+ ): # aleph alpha doesn't provide streaming
+ response_obj = self.handle_aleph_alpha_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ elif self.custom_llm_provider == "nlp_cloud":
+ try:
+ response_obj = self.handle_nlp_cloud_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ except Exception as e:
+ if self.received_finish_reason:
+ raise e
+ else:
+ if self.sent_first_chunk is False:
+ raise Exception("An unknown error occurred with the stream")
+ self.received_finish_reason = "stop"
+ elif self.custom_llm_provider == "vertex_ai" and not isinstance(
+ chunk, ModelResponseStream
+ ):
+ chunk = cast(Any, chunk)
+ import proto # type: ignore
+
+ if hasattr(chunk, "candidates") is True:
+ try:
+ try:
+ completion_obj["content"] = chunk.text # type: ignore
+ except Exception as e:
+ original_exception = e
+ if "Part has no text." in str(e):
+ ## check for function calling
+ function_call = (
+ chunk.candidates[0].content.parts[0].function_call # type: ignore
+ )
+
+ args_dict = {}
+
+ # Check if it's a RepeatedComposite instance
+ for key, val in function_call.args.items():
+ if isinstance(
+ val,
+ proto.marshal.collections.repeated.RepeatedComposite, # type: ignore
+ ):
+ # If so, convert to list
+ args_dict[key] = [v for v in val]
+ else:
+ args_dict[key] = val
+
+ try:
+ args_str = json.dumps(args_dict)
+ except Exception as e:
+ raise e
+ _delta_obj = litellm.utils.Delta(
+ content=None,
+ tool_calls=[
+ {
+ "id": f"call_{str(uuid.uuid4())}",
+ "function": {
+ "arguments": args_str,
+ "name": function_call.name,
+ },
+ "type": "function",
+ }
+ ],
+ )
+ _streaming_response = StreamingChoices(delta=_delta_obj)
+ _model_response = ModelResponseStream()
+ _model_response.choices = [_streaming_response]
+ response_obj = {"original_chunk": _model_response}
+ else:
+ raise original_exception
+ if (
+ hasattr(chunk.candidates[0], "finish_reason") # type: ignore
+ and chunk.candidates[0].finish_reason.name # type: ignore
+ != "FINISH_REASON_UNSPECIFIED"
+ ): # every non-final chunk in vertex ai has this
+ self.received_finish_reason = map_finish_reason( # type: ignore
+ chunk.candidates[0].finish_reason.name
+ )
+ except Exception:
+ if chunk.candidates[0].finish_reason.name == "SAFETY": # type: ignore
+ raise Exception(
+ f"The response was blocked by VertexAI. {str(chunk)}"
+ )
+ else:
+ completion_obj["content"] = str(chunk)
+ elif self.custom_llm_provider == "petals":
+ if self.completion_stream is None or len(self.completion_stream) == 0:
+ if self.received_finish_reason is not None:
+ raise StopIteration
+ else:
+ self.received_finish_reason = "stop"
+ chunk_size = 30
+ stream = cast(Any, self.completion_stream)
+ new_chunk = stream[:chunk_size]
+ completion_obj["content"] = new_chunk
+ self.completion_stream = stream[chunk_size:]
+ elif self.custom_llm_provider == "palm":
+ # fake streaming
+ response_obj = {}
+ if self.completion_stream is None or len(self.completion_stream) == 0:
+ if self.received_finish_reason is not None:
+ raise StopIteration
+ else:
+ self.received_finish_reason = "stop"
+ chunk_size = 30
+ stream = cast(Any, self.completion_stream)
+ new_chunk = stream[:chunk_size]
+ completion_obj["content"] = new_chunk
+ self.completion_stream = stream[chunk_size:]
+ elif self.custom_llm_provider == "triton":
+ response_obj = self.handle_triton_stream(chunk)
+ completion_obj["content"] = response_obj["text"]
+ print_verbose(f"completion obj content: {completion_obj['content']}")
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ elif self.custom_llm_provider == "text-completion-openai":
+ response_obj = self.handle_openai_text_completion_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ print_verbose(f"completion obj content: {completion_obj['content']}")
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ if response_obj["usage"] is not None:
+ setattr(
+ model_response,
+ "usage",
+ litellm.Usage(
+ prompt_tokens=response_obj["usage"].prompt_tokens,
+ completion_tokens=response_obj["usage"].completion_tokens,
+ total_tokens=response_obj["usage"].total_tokens,
+ ),
+ )
+ elif self.custom_llm_provider == "text-completion-codestral":
+ if not isinstance(chunk, str):
+ raise ValueError(f"chunk is not a string: {chunk}")
+ response_obj = cast(
+ dict[str, Any],
+ litellm.CodestralTextCompletionConfig()._chunk_parser(chunk),
+ )
+ completion_obj["content"] = response_obj["text"]
+ print_verbose(f"completion obj content: {completion_obj['content']}")
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ if "usage" in response_obj is not None:
+ setattr(
+ model_response,
+ "usage",
+ litellm.Usage(
+ prompt_tokens=response_obj["usage"].prompt_tokens,
+ completion_tokens=response_obj["usage"].completion_tokens,
+ total_tokens=response_obj["usage"].total_tokens,
+ ),
+ )
+ elif self.custom_llm_provider == "azure_text":
+ response_obj = self.handle_azure_text_completion_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ print_verbose(f"completion obj content: {completion_obj['content']}")
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ elif self.custom_llm_provider == "cached_response":
+ chunk = cast(ModelResponseStream, chunk)
+ response_obj = {
+ "text": chunk.choices[0].delta.content,
+ "is_finished": True,
+ "finish_reason": chunk.choices[0].finish_reason,
+ "original_chunk": chunk,
+ "tool_calls": (
+ chunk.choices[0].delta.tool_calls
+ if hasattr(chunk.choices[0].delta, "tool_calls")
+ else None
+ ),
+ }
+
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["tool_calls"] is not None:
+ completion_obj["tool_calls"] = response_obj["tool_calls"]
+ print_verbose(f"completion obj content: {completion_obj['content']}")
+ if hasattr(chunk, "id"):
+ model_response.id = chunk.id
+ self.response_id = chunk.id
+ if hasattr(chunk, "system_fingerprint"):
+ self.system_fingerprint = chunk.system_fingerprint
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ else: # openai / azure chat model
+ if self.custom_llm_provider in [
+ LlmProviders.AZURE.value,
+ LlmProviders.AZURE_AI.value,
+ ]:
+ if isinstance(chunk, BaseModel) and hasattr(chunk, "model"):
+ # for azure, we need to pass the model from the original chunk
+ self.model = getattr(chunk, "model", self.model)
+ response_obj = self.handle_openai_chat_completion_chunk(chunk)
+ if response_obj is None:
+ return _ProviderChunkEarlyReturn(None)
+ completion_obj["content"] = response_obj["text"]
+ self.intermittent_finish_reason = response_obj.get("finish_reason", None)
+ if response_obj["is_finished"]:
+ if response_obj["finish_reason"] == "error":
+ raise Exception(
+ "{} raised a streaming error - finish_reason: error, no content string given. Received Chunk={}".format(
+ self.custom_llm_provider, response_obj
+ )
+ )
+ self.received_finish_reason = response_obj["finish_reason"]
+ if response_obj.get("original_chunk", None) is not None:
+ if hasattr(response_obj["original_chunk"], "id"):
+ model_response = self.set_model_id(
+ response_obj["original_chunk"].id, model_response
+ )
+ if hasattr(response_obj["original_chunk"], "system_fingerprint"):
+ model_response.system_fingerprint = response_obj[
+ "original_chunk"
+ ].system_fingerprint
+ self.system_fingerprint = response_obj[
+ "original_chunk"
+ ].system_fingerprint
+ if response_obj["logprobs"] is not None:
+ model_response.choices[0].logprobs = response_obj["logprobs"]
+
+ if response_obj["usage"] is not None:
+ if isinstance(response_obj["usage"], dict):
+ setattr(
+ model_response,
+ "usage",
+ litellm.Usage(
+ prompt_tokens=response_obj["usage"].get(
+ "prompt_tokens", None
+ )
+ or None,
+ completion_tokens=response_obj["usage"].get(
+ "completion_tokens", None
+ )
+ or None,
+ total_tokens=response_obj["usage"].get("total_tokens", None)
+ or None,
+ ),
+ )
+ elif isinstance(response_obj["usage"], Usage):
+ setattr(
+ model_response,
+ "usage",
+ response_obj["usage"],
+ )
+ elif isinstance(response_obj["usage"], BaseModel):
+ setattr(
+ model_response,
+ "usage",
+ litellm.Usage(**response_obj["usage"].model_dump()),
+ )
+ return _ProviderChunkParsed(response_obj)
+
def chunk_creator(self, chunk: Any): # type: ignore
if hasattr(chunk, "id"):
self.response_id = chunk.id
model_response = self.model_response_creator()
- response_obj: Dict[str, Any] = {}
+ response_obj: dict[str, Any] = {}
try:
# return this for all models
- completion_obj: Dict[str, Any] = {"content": ""}
- from litellm.types.utils import GenericStreamingChunk as GChunk
-
- if (
- isinstance(chunk, ModelResponseStream)
- and self.custom_llm_provider is not None
- and self.custom_llm_provider in litellm._custom_providers
- ):
- _has_content = bool(
- chunk.choices
- and chunk.choices[0].delta is not None
- and (
- chunk.choices[0].delta.content
- or chunk.choices[0].delta.tool_calls
- )
- )
- if self.received_finish_reason is not None:
- if not _has_content:
- raise StopIteration
- if chunk.choices and chunk.choices[0].finish_reason:
- self.received_finish_reason = chunk.choices[0].finish_reason
- if not _has_content:
- return None
- # Strip finish_reason from the content chunk so it appears
- # only on the trailing empty-delta chunk (OpenAI spec).
- # finish_reason_handler() will emit the proper terminal chunk.
- chunk.choices[0].finish_reason = None # type: ignore[assignment]
- return chunk
-
- if (
- isinstance(chunk, dict)
- and generic_chunk_has_all_required_fields(
- chunk=chunk
- ) # check if chunk is a generic streaming chunk
- ) or (
- self.custom_llm_provider
- and self.custom_llm_provider in litellm._custom_providers
- ):
- if self.received_finish_reason is not None:
- _chunk_has_content = isinstance(chunk, dict) and (
- bool(chunk.get("text", ""))
- or chunk.get("tool_use") is not None
- # Usage-only final chunks are valid and needed to surface
- # finish_reason/usage to downstream translators.
- or chunk.get("usage") is not None
- )
- if not _chunk_has_content and (
- not isinstance(chunk, dict)
- or "provider_specific_fields" not in chunk
- ):
- raise StopIteration
- anthropic_response_obj: GChunk = cast(GChunk, chunk)
- completion_obj["content"] = anthropic_response_obj["text"]
- if anthropic_response_obj["is_finished"]:
- self.received_finish_reason = anthropic_response_obj[
- "finish_reason"
- ]
-
- if anthropic_response_obj["finish_reason"]:
- self.intermittent_finish_reason = anthropic_response_obj[
- "finish_reason"
- ]
-
- if anthropic_response_obj["usage"] is not None:
- setattr(
- model_response,
- "usage",
- litellm.Usage(**anthropic_response_obj["usage"]),
- )
-
- if (
- "tool_use" in anthropic_response_obj
- and anthropic_response_obj["tool_use"] is not None
- ):
- completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]]
-
- if (
- "provider_specific_fields" in anthropic_response_obj
- and anthropic_response_obj["provider_specific_fields"] is not None
- ):
- for key, value in anthropic_response_obj[
- "provider_specific_fields"
- ].items():
- setattr(model_response, key, value)
-
- response_obj = cast(Dict[str, Any], anthropic_response_obj)
- elif self.model == "replicate" or self.custom_llm_provider == "replicate":
- response_obj = self.handle_replicate_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- elif self.custom_llm_provider and self.custom_llm_provider == "predibase":
- response_obj = self.handle_predibase_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- elif (
- self.custom_llm_provider and self.custom_llm_provider == "baseten"
- ): # baseten doesn't provide streaming
- completion_obj["content"] = self.handle_baseten_chunk(chunk)
- elif (
- self.custom_llm_provider and self.custom_llm_provider == "ai21"
- ): # ai21 doesn't provide streaming
- response_obj = self.handle_ai21_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- elif self.custom_llm_provider and self.custom_llm_provider == "maritalk":
- response_obj = self.handle_maritalk_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- elif self.custom_llm_provider and self.custom_llm_provider == "vllm":
- completion_obj["content"] = chunk[0].outputs[0].text
- elif (
- self.custom_llm_provider and self.custom_llm_provider == "aleph_alpha"
- ): # aleph alpha doesn't provide streaming
- response_obj = self.handle_aleph_alpha_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- elif self.custom_llm_provider == "nlp_cloud":
- try:
- response_obj = self.handle_nlp_cloud_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- except Exception as e:
- if self.received_finish_reason:
- raise e
- else:
- if self.sent_first_chunk is False:
- raise Exception("An unknown error occurred with the stream")
- self.received_finish_reason = "stop"
- elif self.custom_llm_provider == "vertex_ai" and not isinstance(
- chunk, ModelResponseStream
- ):
- import proto # type: ignore
-
- if hasattr(chunk, "candidates") is True:
- try:
- try:
- completion_obj["content"] = chunk.text # type: ignore
- except Exception as e:
- original_exception = e
- if "Part has no text." in str(e):
- ## check for function calling
- function_call = (
- chunk.candidates[0].content.parts[0].function_call # type: ignore
- )
-
- args_dict = {}
-
- # Check if it's a RepeatedComposite instance
- for key, val in function_call.args.items():
- if isinstance(
- val,
- proto.marshal.collections.repeated.RepeatedComposite, # type: ignore
- ):
- # If so, convert to list
- args_dict[key] = [v for v in val]
- else:
- args_dict[key] = val
-
- try:
- args_str = json.dumps(args_dict)
- except Exception as e:
- raise e
- _delta_obj = litellm.utils.Delta(
- content=None,
- tool_calls=[
- {
- "id": f"call_{str(uuid.uuid4())}",
- "function": {
- "arguments": args_str,
- "name": function_call.name,
- },
- "type": "function",
- }
- ],
- )
- _streaming_response = StreamingChoices(delta=_delta_obj)
- _model_response = ModelResponseStream()
- _model_response.choices = [_streaming_response]
- response_obj = {"original_chunk": _model_response}
- else:
- raise original_exception
- if (
- hasattr(chunk.candidates[0], "finish_reason") # type: ignore
- and chunk.candidates[0].finish_reason.name # type: ignore
- != "FINISH_REASON_UNSPECIFIED"
- ): # every non-final chunk in vertex ai has this
- self.received_finish_reason = map_finish_reason( # type: ignore
- chunk.candidates[0].finish_reason.name
- )
- except Exception:
- if chunk.candidates[0].finish_reason.name == "SAFETY": # type: ignore
- raise Exception(
- f"The response was blocked by VertexAI. {str(chunk)}"
- )
- else:
- completion_obj["content"] = str(chunk)
- elif self.custom_llm_provider == "petals":
- if self.completion_stream is None or len(self.completion_stream) == 0:
- if self.received_finish_reason is not None:
- raise StopIteration
- else:
- self.received_finish_reason = "stop"
- chunk_size = 30
- new_chunk = self.completion_stream[:chunk_size] # type: ignore[index]
- completion_obj["content"] = new_chunk
- self.completion_stream = self.completion_stream[chunk_size:] # type: ignore[index]
- elif self.custom_llm_provider == "palm":
- # fake streaming
- response_obj = {}
- if self.completion_stream is None or len(self.completion_stream) == 0:
- if self.received_finish_reason is not None:
- raise StopIteration
- else:
- self.received_finish_reason = "stop"
- chunk_size = 30
- new_chunk = self.completion_stream[:chunk_size] # type: ignore[index]
- completion_obj["content"] = new_chunk
- self.completion_stream = self.completion_stream[chunk_size:] # type: ignore[index]
- elif self.custom_llm_provider == "triton":
- response_obj = self.handle_triton_stream(chunk)
- completion_obj["content"] = response_obj["text"]
- print_verbose(f"completion obj content: {completion_obj['content']}")
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- elif self.custom_llm_provider == "text-completion-openai":
- response_obj = self.handle_openai_text_completion_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- print_verbose(f"completion obj content: {completion_obj['content']}")
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- if response_obj["usage"] is not None:
- setattr(
- model_response,
- "usage",
- litellm.Usage(
- prompt_tokens=response_obj["usage"].prompt_tokens,
- completion_tokens=response_obj["usage"].completion_tokens,
- total_tokens=response_obj["usage"].total_tokens,
- ),
- )
- elif self.custom_llm_provider == "text-completion-codestral":
- if not isinstance(chunk, str):
- raise ValueError(f"chunk is not a string: {chunk}")
- response_obj = cast(
- Dict[str, Any],
- litellm.CodestralTextCompletionConfig()._chunk_parser(chunk),
- )
- completion_obj["content"] = response_obj["text"]
- print_verbose(f"completion obj content: {completion_obj['content']}")
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- if "usage" in response_obj is not None:
- setattr(
- model_response,
- "usage",
- litellm.Usage(
- prompt_tokens=response_obj["usage"].prompt_tokens,
- completion_tokens=response_obj["usage"].completion_tokens,
- total_tokens=response_obj["usage"].total_tokens,
- ),
- )
- elif self.custom_llm_provider == "azure_text":
- response_obj = self.handle_azure_text_completion_chunk(chunk)
- completion_obj["content"] = response_obj["text"]
- print_verbose(f"completion obj content: {completion_obj['content']}")
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- elif self.custom_llm_provider == "cached_response":
- chunk = cast(ModelResponseStream, chunk)
- response_obj = {
- "text": chunk.choices[0].delta.content,
- "is_finished": True,
- "finish_reason": chunk.choices[0].finish_reason,
- "original_chunk": chunk,
- "tool_calls": (
- chunk.choices[0].delta.tool_calls
- if hasattr(chunk.choices[0].delta, "tool_calls")
- else None
- ),
- }
-
- completion_obj["content"] = response_obj["text"]
- if response_obj["tool_calls"] is not None:
- completion_obj["tool_calls"] = response_obj["tool_calls"]
- print_verbose(f"completion obj content: {completion_obj['content']}")
- if hasattr(chunk, "id"):
- model_response.id = chunk.id
- self.response_id = chunk.id
- if hasattr(chunk, "system_fingerprint"):
- self.system_fingerprint = chunk.system_fingerprint
- if response_obj["is_finished"]:
- self.received_finish_reason = response_obj["finish_reason"]
- else: # openai / azure chat model
- if self.custom_llm_provider in [
- LlmProviders.AZURE.value,
- LlmProviders.AZURE_AI.value,
- ]:
- if isinstance(chunk, BaseModel) and hasattr(chunk, "model"):
- # for azure, we need to pass the model from the original chunk
- self.model = getattr(chunk, "model", self.model)
- response_obj = self.handle_openai_chat_completion_chunk(chunk)
- if response_obj is None:
- return
- completion_obj["content"] = response_obj["text"]
- self.intermittent_finish_reason = response_obj.get(
- "finish_reason", None
- )
- if response_obj["is_finished"]:
- if response_obj["finish_reason"] == "error":
- raise Exception(
- "{} raised a streaming error - finish_reason: error, no content string given. Received Chunk={}".format(
- self.custom_llm_provider, response_obj
- )
- )
- self.received_finish_reason = response_obj["finish_reason"]
- if response_obj.get("original_chunk", None) is not None:
- if hasattr(response_obj["original_chunk"], "id"):
- model_response = self.set_model_id(
- response_obj["original_chunk"].id, model_response
- )
- if hasattr(response_obj["original_chunk"], "system_fingerprint"):
- model_response.system_fingerprint = response_obj[
- "original_chunk"
- ].system_fingerprint
- self.system_fingerprint = response_obj[
- "original_chunk"
- ].system_fingerprint
- if response_obj["logprobs"] is not None:
- model_response.choices[0].logprobs = response_obj["logprobs"]
-
- if response_obj["usage"] is not None:
- if isinstance(response_obj["usage"], dict):
- setattr(
- model_response,
- "usage",
- litellm.Usage(
- prompt_tokens=response_obj["usage"].get(
- "prompt_tokens", None
- )
- or None,
- completion_tokens=response_obj["usage"].get(
- "completion_tokens", None
- )
- or None,
- total_tokens=response_obj["usage"].get(
- "total_tokens", None
- )
- or None,
- ),
- )
- elif isinstance(response_obj["usage"], Usage):
- setattr(
- model_response,
- "usage",
- response_obj["usage"],
- )
- elif isinstance(response_obj["usage"], BaseModel):
- setattr(
- model_response,
- "usage",
- litellm.Usage(**response_obj["usage"].model_dump()),
- )
+ completion_obj: dict[str, Any] = {"content": ""}
+ dispatch_result = self._dispatch_provider_chunk(
+ chunk=chunk,
+ model_response=model_response,
+ completion_obj=completion_obj,
+ )
+ if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
+ return dispatch_result.value
+ response_obj = dispatch_result.response_obj
model_response.model = self.model
## FUNCTION CALL PARSING
@@ -1980,11 +2005,29 @@ class CustomStreamWrapper:
except StopIteration:
if self.sent_last_chunk is True:
- complete_streaming_response = litellm.stream_chunk_builder(
- chunks=self.chunks,
- messages=self.messages,
- logging_obj=self.logging_obj,
- )
+ try:
+ complete_streaming_response = litellm.stream_chunk_builder(
+ chunks=self.chunks,
+ messages=self.messages,
+ logging_obj=self.logging_obj,
+ )
+ except Exception as e:
+ # stream_chunk_builder can re-raise (as APIError) on large agentic
+ # streams. The raise originates inside this except-StopIteration block,
+ # so the sibling `except Exception` below does not catch it; it would
+ # escape __next__ and drop the request from SpendLogs. Recover
+ # best-effort usage from the raw chunks so cost is still tracked
+ verbose_logger.warning(
+ "stream_chunk_builder raised at end-of-stream (%s); logging "
+ "best-effort usage from chunks.",
+ str(e),
+ )
+ try:
+ complete_streaming_response = self.model_response_creator(
+ chunk={"usage": calculate_total_usage(chunks=self.chunks)}
+ )
+ except Exception:
+ complete_streaming_response = None
response = self.model_response_creator()
if complete_streaming_response is not None:
@@ -2209,11 +2252,27 @@ class CustomStreamWrapper:
except (StopAsyncIteration, StopIteration):
if self.sent_last_chunk is True:
# log the final chunk with accurate streaming values
- complete_streaming_response = litellm.stream_chunk_builder(
- chunks=self.chunks,
- messages=self.messages,
- logging_obj=self.logging_obj,
- )
+ try:
+ complete_streaming_response = litellm.stream_chunk_builder(
+ chunks=self.chunks,
+ messages=self.messages,
+ logging_obj=self.logging_obj,
+ )
+ except Exception as e:
+ # see sync __next__: a raise from stream_chunk_builder inside this
+ # except handler escapes __anext__ and drops the request from SpendLogs.
+ # Recover best-effort usage from the raw chunks so cost is still tracked
+ verbose_logger.warning(
+ "stream_chunk_builder raised at end-of-stream (%s); logging "
+ "best-effort usage from chunks.",
+ str(e),
+ )
+ try:
+ complete_streaming_response = self.model_response_creator(
+ chunk={"usage": calculate_total_usage(chunks=self.chunks)}
+ )
+ except Exception:
+ complete_streaming_response = None
response = self.model_response_creator()
if complete_streaming_response is not None:
@@ -2290,6 +2349,7 @@ class CustomStreamWrapper:
litellm.request_timeout
)
if self.logging_obj is not None:
+ self._record_partial_usage_for_failure()
## LOGGING
threading.Thread(
target=self.logging_obj.failure_handler,
@@ -2303,6 +2363,7 @@ class CustomStreamWrapper:
except Exception as e:
traceback_exception = traceback.format_exc()
if self.logging_obj is not None:
+ self._record_partial_usage_for_failure()
## LOGGING
threading.Thread(
target=self.logging_obj.failure_handler,
@@ -2314,6 +2375,33 @@ class CustomStreamWrapper:
)
self._handle_stream_fallback_error(e)
+ def _record_partial_usage_for_failure(self) -> None:
+ """
+ A stream that breaks mid-flight still billed the provider for the chunks
+ already delivered. Recover that partial usage from the chunks seen so
+ far and stash it, with its cost, on the logging object so the failure
+ handler records the real partial spend instead of zero. A request that
+ later recovers via a router fallback overwrites this with the combined
+ success log on the same request id, so this never double counts.
+ """
+ if self.logging_obj is None or not self.chunks:
+ return
+ try:
+ partial_response = litellm.stream_chunk_builder(chunks=self.chunks)
+ usage = cast(Optional[Usage], getattr(partial_response, "usage", None))
+ if usage is None:
+ return
+ self.logging_obj.model_call_details["combined_usage_object"] = usage
+ self.logging_obj.model_call_details["response_cost"] = (
+ self.logging_obj._response_cost_calculator(result=partial_response)
+ or 0.0
+ )
+ except Exception as recover_error:
+ verbose_logger.debug(
+ "could not recover partial usage for interrupted stream: %s",
+ recover_error,
+ )
+
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
index bf425637b56..75a8acdfcc3 100644
--- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
+++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
@@ -859,7 +859,17 @@ class LiteLLMAnthropicMessagesAdapter:
"""
new_tools: List[ChatCompletionToolParam] = []
tool_name_mapping: Dict[str, str] = {}
- mapped_tool_params = ["name", "input_schema", "description", "cache_control"]
+ # "type" is the Anthropic tool type (e.g. "custom"); it must not be
+ # merged into the OpenAI function `parameters` schema below, or it
+ # overwrites the real parameters.type ("object") and the provider
+ # rejects the request. See #30557.
+ mapped_tool_params = [
+ "name",
+ "input_schema",
+ "description",
+ "cache_control",
+ "type",
+ ]
for idx, tool in enumerate(tools):
# Check if this is an Anthropic-native tool that should be kept as-is
diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py
index c7c110ff3e3..8714939f025 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py
@@ -84,8 +84,14 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
)
# Optional routing overrides for the advisor sub-call (e.g. proxy routing).
# If not set in the tool definition, litellm resolves from env vars.
- advisor_api_key: Optional[str] = advisor_tool.get("api_key")
- advisor_api_base: Optional[str] = advisor_tool.get("api_base")
+ # The advisor tool is caller-controlled; only honor a client-supplied
+ # api_base/api_key when the proxy has enabled clientside credentials,
+ # otherwise let litellm resolve from server config.
+ advisor_api_key: Optional[str] = None
+ advisor_api_base: Optional[str] = None
+ if _allow_client_side_advisor_credentials():
+ advisor_api_key = advisor_tool.get("api_key")
+ advisor_api_base = advisor_tool.get("api_base")
# Build the synthetic tool definition the provider will receive.
synthetic_advisor_tool = _make_synthetic_advisor_tool()
@@ -181,6 +187,20 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
# ---------------------------------------------------------------------------
+def _allow_client_side_advisor_credentials() -> bool:
+ """Whether a caller-supplied advisor api_base/api_key may be honored.
+
+ Gated on the proxy's ``allow_client_side_credentials`` opt-in. When the
+ interceptor runs outside the proxy (SDK use), there is no admin boundary
+ to protect, so client-supplied routing is allowed.
+ """
+ try:
+ from litellm.proxy.proxy_server import general_settings
+ except (ImportError, ModuleNotFoundError):
+ return True
+ return general_settings.get("allow_client_side_credentials") is True
+
+
def _make_synthetic_advisor_tool() -> Dict:
"""Build a regular tool definition the executor provider can understand."""
return {
diff --git a/litellm/llms/apiserpent/search/transformation.py b/litellm/llms/apiserpent/search/transformation.py
index 1eb7d34c875..bc11875ba12 100644
--- a/litellm/llms/apiserpent/search/transformation.py
+++ b/litellm/llms/apiserpent/search/transformation.py
@@ -53,7 +53,13 @@ class APISerpentSearchConfig(BaseSearchConfig):
api_base: Optional[str] = None,
**kwargs,
) -> Dict:
- api_key = api_key or get_secret_str("APISERPENT_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("APISERPENT_API_KEY",),
+ base_env_var="APISERPENT_API_BASE",
+ default_api_base=APISERPENT_BASE,
+ )
if not api_key:
raise ValueError(
"APISERPENT_API_KEY is not set. Set `APISERPENT_API_KEY` environment variable."
diff --git a/litellm/llms/base_llm/sandbox/__init__.py b/litellm/llms/base_llm/sandbox/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/llms/base_llm/sandbox/transformation.py b/litellm/llms/base_llm/sandbox/transformation.py
new file mode 100644
index 00000000000..1c012a15fdb
--- /dev/null
+++ b/litellm/llms/base_llm/sandbox/transformation.py
@@ -0,0 +1,96 @@
+"""
+Base Sandbox transformation configuration.
+
+A sandbox provider runs an executable string inside an isolated container and
+returns whatever the sandbox produced. The lifecycle is create container ->
+run code -> delete container; `code_interpreter_tool` combines all three.
+"""
+
+from typing import Any, Union
+
+import httpx
+
+from pydantic import Field, PrivateAttr
+
+from litellm.types.llms.base import LiteLLMPydanticObjectBase
+
+SANDBOX_MAX_OUTPUT_BYTES = 10 * 1024 * 1024
+
+
+class ContainerHandle(LiteLLMPydanticObjectBase):
+ """A live sandbox container. Carries everything needed to reach it again."""
+
+ id: str
+ provider: str
+ domain: str | None = None
+
+ model_config = {"extra": "allow"}
+
+ _hidden_params: dict = PrivateAttr(default_factory=dict)
+
+
+class CodeExecutionResult(LiteLLMPydanticObjectBase):
+ """Passthrough of the sandbox's own execution output."""
+
+ stdout: str = ""
+ stderr: str = ""
+ results: list[dict[str, Any]] = Field(default_factory=list)
+ error: dict[str, Any] | None = None
+ execution_count: int | None = None
+ object: str = "code_execution"
+
+ model_config = {"extra": "allow"}
+
+ _hidden_params: dict = PrivateAttr(default_factory=dict)
+
+
+class BaseSandboxConfig:
+ """Provider-agnostic sandbox operations."""
+
+ def validate_environment(self, api_key: str | None = None, **kwargs) -> str:
+ raise NotImplementedError(
+ "validate_environment must be implemented by provider"
+ )
+
+ async def acreate_sandbox(
+ self,
+ *,
+ template: str | None = None,
+ timeout: int | None = None,
+ allow_internet_access: bool | None = None,
+ api_key: str | None = None,
+ **kwargs,
+ ) -> ContainerHandle:
+ raise NotImplementedError("acreate_sandbox must be implemented by provider")
+
+ async def arun_code(
+ self,
+ *,
+ container: Union[ContainerHandle, str],
+ code: str,
+ api_key: str | None = None,
+ **kwargs,
+ ) -> CodeExecutionResult:
+ raise NotImplementedError("arun_code must be implemented by provider")
+
+ async def adelete_sandbox(
+ self,
+ *,
+ container: Union[ContainerHandle, str],
+ api_key: str | None = None,
+ **kwargs,
+ ) -> bool:
+ raise NotImplementedError("adelete_sandbox must be implemented by provider")
+
+ async def _read_capped_lines(self, response: httpx.Response) -> list[str]:
+ lines: list[str] = []
+ total = 0
+ async for line in response.aiter_lines():
+ total += len(line.encode("utf-8"))
+ if total > SANDBOX_MAX_OUTPUT_BYTES:
+ raise ValueError(
+ f"Sandbox output exceeded {SANDBOX_MAX_OUTPUT_BYTES} bytes; aborting "
+ "to avoid unbounded memory use."
+ )
+ lines.append(line)
+ return lines
diff --git a/litellm/llms/base_llm/search/transformation.py b/litellm/llms/base_llm/search/transformation.py
index 4dfe86685fb..1581d8bb064 100644
--- a/litellm/llms/base_llm/search/transformation.py
+++ b/litellm/llms/base_llm/search/transformation.py
@@ -3,11 +3,13 @@ Base Search transformation configuration.
"""
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
+from urllib.parse import urlsplit
import httpx
from pydantic import PrivateAttr
from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.base import LiteLLMPydanticObjectBase
if TYPE_CHECKING:
@@ -16,6 +18,29 @@ else:
LiteLLMLoggingObj = Any
+def _search_host(url: str) -> str:
+ return urlsplit(url).netloc.lower()
+
+
+def _is_trusted_search_api_base(
+ caller_api_base: str,
+ default_api_base: str | None,
+ base_env_var: str | None,
+) -> bool:
+ candidate = _search_host(caller_api_base)
+ if not candidate:
+ return False
+ trusted = {
+ _search_host(base)
+ for base in (
+ default_api_base,
+ get_secret_str(base_env_var) if base_env_var else None,
+ )
+ if base
+ }
+ return candidate in trusted
+
+
class SearchResult(LiteLLMPydanticObjectBase):
"""Single search result."""
@@ -86,6 +111,60 @@ class BaseSearchConfig:
"max_tokens_per_page",
}
+ def _assert_trusted_api_base_for_server_credential(
+ self,
+ caller_api_base: str | None,
+ default_api_base: str | None,
+ base_env_var: str | None,
+ credential_name: str,
+ ) -> None:
+ """
+ Block sending a server-managed credential to a caller-chosen host.
+
+ A caller-supplied api_base is honored when constructing the request URL, so
+ falling back to a server-configured secret while the caller controls the host
+ leaks that secret. The provider default and the operator's own api_base
+ override are the only trusted destinations for a server-managed credential.
+ """
+ if not caller_api_base:
+ return
+ if _is_trusted_search_api_base(caller_api_base, default_api_base, base_env_var):
+ return
+ raise ValueError(
+ f"Refusing to send the server-configured {credential_name} to the "
+ f"caller-supplied api_base '{caller_api_base}'. Pass an explicit api_key "
+ f"when overriding api_base for this search provider."
+ )
+
+ def resolve_server_api_key(
+ self,
+ *,
+ caller_api_key: str | None,
+ caller_api_base: str | None,
+ key_env_vars: tuple[str, ...],
+ base_env_var: str | None,
+ default_api_base: str | None,
+ ) -> str | None:
+ """
+ Resolve a single-secret search API key, falling back to a server-managed
+ secret only when the request targets a trusted host.
+
+ Returns the caller's key when provided, otherwise the first set
+ server-managed secret (or None when none is set, for keyless providers).
+ """
+ if caller_api_key:
+ return caller_api_key
+ server_key = next(
+ (key for key in (get_secret_str(var) for var in key_env_vars) if key),
+ None,
+ )
+ if server_key is None:
+ return None
+ self._assert_trusted_api_base_for_server_credential(
+ caller_api_base, default_api_base, base_env_var, key_env_vars[0]
+ )
+ return server_key
+
def validate_environment(
self,
headers: Dict,
diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py
index 2c9ea187912..c31462a735b 100644
--- a/litellm/llms/bedrock/base_aws_llm.py
+++ b/litellm/llms/bedrock/base_aws_llm.py
@@ -10,7 +10,6 @@ from typing import (
Callable,
ClassVar,
Dict,
- List,
Literal,
Optional,
Tuple,
@@ -210,32 +209,11 @@ class BaseAWSLLM:
"""
Return a boto3.Credentials object
"""
- ## CHECK IS 'os.environ/' passed in
- params_to_check: List[Optional[str]] = [
- aws_access_key_id,
- aws_secret_access_key,
- aws_session_token,
- aws_region_name,
- aws_session_name,
- aws_profile_name,
- aws_role_name,
- aws_web_identity_token,
- aws_sts_endpoint,
- aws_external_id,
- ]
-
- # Iterate over parameters and update if needed
- for i, param in enumerate(params_to_check):
- if param and param.startswith("os.environ/"):
- _v = get_secret(param)
- if _v is not None and isinstance(_v, str):
- params_to_check[i] = _v
- elif param is None: # check if uppercase value in env
- key = self.aws_authentication_params[i]
- if key.upper() in os.environ:
- params_to_check[i] = os.getenv(key.upper())
-
- # Assign updated values back to parameters
+ # Only config-sourced credentials are expanded against the environment.
+ # os.environ/ references in the model config are resolved at load time,
+ # so any reference still present at this point is caller-supplied input and is
+ # left as-is rather than expanded into a process environment variable. Each
+ # unset param falls back to its matching fixed AWS_* ambient env var.
(
aws_access_key_id,
aws_secret_access_key,
@@ -247,7 +225,21 @@ class BaseAWSLLM:
aws_web_identity_token,
aws_sts_endpoint,
aws_external_id,
- ) = params_to_check
+ ) = tuple(
+ value if value is not None else os.getenv(env_var)
+ for value, env_var in (
+ (aws_access_key_id, "AWS_ACCESS_KEY_ID"),
+ (aws_secret_access_key, "AWS_SECRET_ACCESS_KEY"),
+ (aws_session_token, "AWS_SESSION_TOKEN"),
+ (aws_region_name, "AWS_REGION_NAME"),
+ (aws_session_name, "AWS_SESSION_NAME"),
+ (aws_profile_name, "AWS_PROFILE_NAME"),
+ (aws_role_name, "AWS_ROLE_NAME"),
+ (aws_web_identity_token, "AWS_WEB_IDENTITY_TOKEN"),
+ (aws_sts_endpoint, "AWS_STS_ENDPOINT"),
+ (aws_external_id, "AWS_EXTERNAL_ID"),
+ )
+ )
verbose_logger.debug(
"in get credentials\n"
@@ -845,6 +837,20 @@ class BaseAWSLLM:
f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}"
)
+ # get_secret() expands environment-variable references (an os.environ/
+ # prefix, or a bare name matching an environment variable). Config-sourced
+ # references are expanded at load time, so such a reference reaching here is
+ # caller-supplied input; reject it rather than expanding a process-environment
+ # value for use as the token.
+ if (
+ aws_web_identity_token.startswith("os.environ/")
+ or aws_web_identity_token in os.environ
+ ):
+ raise AwsAuthError(
+ message="Invalid web identity token reference.",
+ status_code=400,
+ )
+
oidc_token = get_secret(aws_web_identity_token)
if oidc_token is None:
diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py
index 44ba1ce3c86..3cd3a249c33 100644
--- a/litellm/llms/bedrock/chat/agentcore/transformation.py
+++ b/litellm/llms/bedrock/chat/agentcore/transformation.py
@@ -218,8 +218,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
- Qualifier goes as query parameter
- Only the payload goes in the request body
+ Payload shape:
+ - ``prompt`` is always present and contains the text-only flatten of the
+ last message's content (existing behavior).
+ - ``content`` is added ONLY when the ``forward_multimodal_content`` litellm
+ param is truthy AND the last message's ``content`` is a list containing a
+ non-text block (e.g. ``image_url``, ``file``, ``input_audio``). The list is
+ forwarded verbatim so the agent's ``@app.entrypoint`` handler can parse the
+ OpenAI-shaped multimodal blocks. This is opt-in because an AgentCore agent
+ must be explicitly written to read ``payload["content"]``; by default the
+ payload stays byte-identical to the legacy ``{"prompt": "..."}`` shape.
+
Returns:
- dict: Payload dict containing the prompt
+ dict: Payload dict containing the prompt and (optionally) the OpenAI
+ content list.
"""
verbose_logger.debug(
f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}"
@@ -231,6 +243,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
# Create the payload - this is what goes in the body (raw JSON)
payload: dict = {"prompt": prompt}
+ # Opt-in: when forward_multimodal_content is set, forward the OpenAI content
+ # list verbatim under "content" so an attachment-aware agent can read the raw
+ # blocks (image_url, file, etc.). Default off keeps the payload byte-identical
+ # to the legacy {"prompt": "..."} shape for agents that only read the prompt.
+ if self._should_forward_multimodal_content(optional_params, litellm_params):
+ last_content = messages[-1].get("content")
+ if isinstance(last_content, list) and any(
+ isinstance(block, dict) and block.get("type") not in (None, "text")
+ for block in last_content
+ ):
+ # Copy so the payload never aliases messages[-1]["content"]; shallow,
+ # not deep, to avoid cloning large base64 media on the request path.
+ payload["content"] = list(last_content)
+
# Get or generate session ID - this goes in the header
runtime_session_id = self._get_runtime_session_id(optional_params)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = runtime_session_id
@@ -246,6 +272,29 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
verbose_logger.debug(f"PAYLOAD: {payload}")
return payload
+ @staticmethod
+ def _should_forward_multimodal_content(
+ optional_params: dict, litellm_params: dict
+ ) -> bool:
+ """Whether to forward raw OpenAI content blocks under ``payload["content"]``.
+
+ Opt-in via the ``forward_multimodal_content`` litellm param (default ``False``)
+ because AgentCore agents must be explicitly written to read the field. The
+ value may arrive as a bool or a config/env string ("true", "1", ...). Checks
+ ``optional_params`` first (where other AgentCore params land), then
+ ``litellm_params``.
+ """
+ for source in (optional_params, litellm_params):
+ if not isinstance(source, dict):
+ continue
+ value = source.get("forward_multimodal_content")
+ if value is None:
+ continue
+ if isinstance(value, str):
+ return value.strip().lower() in ("1", "true", "yes", "on")
+ return bool(value)
+ return False
+
def _extract_sse_json(self, line: str) -> Optional[Dict]:
"""Extract and parse JSON from an SSE data line."""
if not line.startswith("data:"):
diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py
index 75b560b4d6d..9fca7bc61af 100644
--- a/litellm/llms/bedrock/chat/invoke_handler.py
+++ b/litellm/llms/bedrock/chat/invoke_handler.py
@@ -70,6 +70,7 @@ from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
BedrockError,
ModelResponseIterator,
+ build_bedrock_stream_error,
get_bedrock_response_stream_shape,
get_bedrock_tool_name,
)
@@ -1841,23 +1842,7 @@ class AWSEventStreamDecoder:
parsed_response = self.parser.parse(response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
- decoded_body = response_dict["body"].decode()
- if isinstance(decoded_body, dict):
- error_message = decoded_body.get("message")
- elif isinstance(decoded_body, str):
- error_message = decoded_body
- else:
- error_message = ""
- exception_status = response_dict["headers"].get(":exception-type")
- error_message = exception_status + " " + error_message
- raise BedrockError(
- status_code=response_dict["status_code"],
- message=(
- json.dumps(error_message)
- if isinstance(error_message, dict)
- else error_message
- ),
- )
+ raise build_bedrock_stream_error(response_dict, response_stream_shape)
if "chunk" in parsed_response:
chunk = parsed_response.get("chunk")
if not chunk:
diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py
index bdc5da321c6..9f58e5c0f1c 100644
--- a/litellm/llms/bedrock/common_utils.py
+++ b/litellm/llms/bedrock/common_utils.py
@@ -7,9 +7,21 @@ Common utilities used across bedrock chat/embedding/image generation
import functools
import json
import os
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ List,
+ Literal,
+ Mapping,
+ Optional,
+ TypedDict,
+ Union,
+)
if TYPE_CHECKING:
+ from botocore.model import Shape
+
from litellm.types.llms.bedrock import BedrockCreateBatchRequest
import httpx
@@ -1132,6 +1144,39 @@ def get_bedrock_response_stream_shape():
return _load_bedrock_response_stream_shape()
+class BedrockEventStreamResponseDict(TypedDict):
+ status_code: int
+ headers: Mapping[str, str]
+ body: bytes
+
+
+def build_bedrock_stream_error(
+ response_dict: BedrockEventStreamResponseDict,
+ response_stream_shape: Shape | None,
+) -> BedrockError:
+ """Build a BedrockError for a non-200 event-stream error event.
+
+ botocore hard-codes HTTP 400 on every mid-stream error event, so the modeled
+ ResponseStream member's httpStatusCode is the real status. Resolve it from the
+ shape and fall back to the raw status when the type is not modeled.
+ """
+ exception_type = response_dict["headers"].get(":exception-type")
+ decoded_body = response_dict["body"].decode()
+ message = f"{exception_type} {decoded_body}" if exception_type else decoded_body
+
+ status_code = response_dict["status_code"]
+ if exception_type is not None and response_stream_shape is not None:
+ member = response_stream_shape.members.get(exception_type)
+ if member is not None:
+ modeled_status = (
+ (member.metadata or {}).get("error", {}).get("httpStatusCode")
+ )
+ if modeled_status is not None:
+ status_code = int(modeled_status)
+
+ return BedrockError(status_code=status_code, message=message)
+
+
class BedrockEventStreamDecoderBase:
"""
Base class for event stream decoding for Bedrock
@@ -1156,23 +1201,7 @@ class BedrockEventStreamDecoderBase:
parsed_response = self.parser.parse(response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
- decoded_body = response_dict["body"].decode()
- if isinstance(decoded_body, dict):
- error_message = decoded_body.get("message")
- elif isinstance(decoded_body, str):
- error_message = decoded_body
- else:
- error_message = ""
- exception_status = response_dict["headers"].get(":exception-type")
- error_message = exception_status + " " + error_message
- raise BedrockError(
- status_code=response_dict["status_code"],
- message=(
- json.dumps(error_message)
- if isinstance(error_message, dict)
- else error_message
- ),
- )
+ raise build_bedrock_stream_error(response_dict, response_stream_shape)
if "chunk" in parsed_response:
chunk = parsed_response.get("chunk")
if not chunk:
diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py
index ecf157e12ee..b6aae2159c1 100644
--- a/litellm/llms/bedrock/files/handler.py
+++ b/litellm/llms/bedrock/files/handler.py
@@ -1,8 +1,6 @@
import asyncio
-import base64
-import os
-from types import MappingProxyType
-from typing import Any, Coroutine, Mapping, Optional, Tuple, Union, cast
+from collections.abc import Mapping
+from typing import Any, Coroutine, Optional, Tuple, Union
import httpx
@@ -17,7 +15,6 @@ from litellm.types.llms.openai import (
FileContentRequest,
HttpxBinaryResponseContent,
)
-from litellm.types.utils import SpecialEnums
from ..base_aws_llm import BaseAWSLLM
@@ -37,40 +34,9 @@ class BedrockFilesHandler(BaseAWSLLM):
)
def _extract_s3_uri_from_file_id(self, file_id: str) -> str:
- """
- Extract S3 URI from encoded file ID.
+ from .transformation import extract_s3_uri_from_file_id
- The file ID can be in two formats:
- 1. Base64-encoded unified file ID containing: llm_output_file_id,s3://bucket/path
- 2. Direct S3 URI: s3://bucket/litellm-managed-prefix/path
-
- Args:
- file_id: Encoded file ID or direct S3 URI
-
- Returns:
- S3 URI (e.g., "s3://bucket-name/path/to/file")
- """
- # First, try to decode if it's a base64-encoded unified file ID
- try:
- # Add padding if needed
- padded = file_id + "=" * (-len(file_id) % 4)
- decoded = base64.urlsafe_b64decode(padded).decode()
-
- # Check if it's a unified file ID format
- if decoded.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value):
- # Extract llm_output_file_id from the decoded string
- if "llm_output_file_id," in decoded:
- s3_uri = decoded.split("llm_output_file_id,")[1].split(";")[0]
- return s3_uri
- except Exception:
- pass
-
- # If not base64 encoded or doesn't contain llm_output_file_id, accept only
- # explicit S3 URIs. Bucket and key validation happens before any S3 call.
- if file_id.startswith("s3://"):
- return file_id
-
- raise ValueError("file_id must be a managed LiteLLM S3 file id")
+ return extract_s3_uri_from_file_id(file_id)
def _parse_s3_uri(
self,
@@ -95,26 +61,12 @@ class BedrockFilesHandler(BaseAWSLLM):
allow_legacy_cloud_file_ids=allow_legacy_cloud_file_ids,
)
- def _get_configured_s3_bucket_name(self, litellm_params: dict) -> str:
- trusted_model_credentials = litellm_params.get(
- "_litellm_internal_model_credentials"
- )
- bucket_name = None
- if isinstance(trusted_model_credentials, type(MappingProxyType({}))):
- trusted_model_credentials_mapping = cast(
- Mapping[str, Any], trusted_model_credentials
- )
- candidate_bucket_name = trusted_model_credentials_mapping.get(
- "s3_bucket_name"
- )
- if isinstance(candidate_bucket_name, str):
- bucket_name = candidate_bucket_name
- bucket_name = bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
- if not bucket_name:
- raise ValueError(
- "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
- )
- return bucket_name
+ def _get_configured_s3_bucket_name(
+ self, litellm_params: Mapping[str, object]
+ ) -> str:
+ from .transformation import get_configured_s3_bucket_name
+
+ return get_configured_s3_bucket_name(litellm_params)
async def afile_content(
self,
diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py
index cec2e934af8..6cfaa88275d 100644
--- a/litellm/llms/bedrock/files/transformation.py
+++ b/litellm/llms/bedrock/files/transformation.py
@@ -1,23 +1,37 @@
+import base64
import json
import os
import time
-from typing import Any, Dict, List, Optional, Tuple, Union
+from collections.abc import Mapping, MutableMapping
+from types import MappingProxyType
+from typing import (
+ Any,
+ Dict,
+ List,
+ Optional,
+ Tuple,
+ Union,
+)
from urllib.parse import unquote
import httpx
from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
+from pydantic import BaseModel, ConfigDict
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.cloud_storage_security import (
BEDROCK_MANAGED_S3_BATCH_PREFIX,
+ BEDROCK_MANAGED_S3_PREFIXES,
BEDROCK_MANAGED_S3_UPLOAD_PREFIX,
build_managed_cloud_object_name,
encode_s3_object_key_for_url,
sanitize_cloud_object_component,
+ should_allow_legacy_cloud_file_ids,
split_configured_cloud_bucket_name,
+ validate_managed_cloud_file_id,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -28,18 +42,98 @@ from litellm.llms.base_llm.files.transformation import (
from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
+ FileContentRequest,
FileTypes,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
PathLike,
)
-from litellm.types.utils import ExtractedFileData, LlmProviders
+from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums
from litellm.utils import get_llm_provider
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError
+# litellm_params key used to hand the SigV4-signed GET headers from
+# `transform_file_content_request` to `validate_environment` (the only hook
+# the shared file-content HTTP handler exposes for setting request headers).
+# Same pattern as the `upload_url` handoff in `transform_create_file_request`.
+S3_SIGNED_GET_HEADERS_PARAM = "_s3_signed_get_headers"
+
+
+class _BedrockS3RequestParams(BaseModel):
+ """Typed view of the credential/region params the S3 GetObject path reads."""
+
+ model_config = ConfigDict(extra="ignore")
+
+ aws_access_key_id: str | None = None
+ aws_secret_access_key: str | None = None
+ aws_session_token: str | None = None
+ aws_region_name: str | None = None
+ aws_session_name: str | None = None
+ aws_profile_name: str | None = None
+ aws_role_name: str | None = None
+ aws_web_identity_token: str | None = None
+ aws_sts_endpoint: str | None = None
+ s3_region_name: str | None = None
+ s3_endpoint_url: str | None = None
+
+
+class _TrustedS3ModelCredentials(BaseModel):
+ """The S3 bucket the server trusts file ids against, from the deployment snapshot."""
+
+ model_config = ConfigDict(extra="ignore")
+
+ s3_bucket_name: str | None = None
+
+
+def extract_s3_uri_from_file_id(file_id: str) -> str:
+ """
+ Resolve a Bedrock file id to its S3 URI.
+
+ Accepts either a base64-encoded LiteLLM unified file id (whose decoded
+ form carries `llm_output_file_id,s3://...`) or a direct `s3://` URI.
+ """
+ try:
+ padded = file_id + "=" * (-len(file_id) % 4)
+ decoded = base64.urlsafe_b64decode(padded).decode()
+
+ if decoded.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value):
+ if "llm_output_file_id," in decoded:
+ return decoded.split("llm_output_file_id,")[1].split(";")[0]
+ except Exception:
+ pass
+
+ if file_id.startswith("s3://"):
+ return file_id
+
+ raise ValueError("file_id must be a managed LiteLLM S3 file id")
+
+
+def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str:
+ """
+ Resolve the server-configured S3 bucket for Bedrock file operations.
+
+ Only trusts the immutable server-side credential snapshot or the
+ environment; never a request-supplied param, since the bucket is what
+ `validate_managed_cloud_file_id` checks file ids against.
+ """
+ trusted_model_credentials = litellm_params.get(
+ "_litellm_internal_model_credentials"
+ )
+ bucket_name: str | None = None
+ if isinstance(trusted_model_credentials, MappingProxyType):
+ snapshot: dict[str, object] = {}
+ snapshot.update(trusted_model_credentials) # any-ok: untyped snapshot
+ bucket_name = _TrustedS3ModelCredentials.model_validate(snapshot).s3_bucket_name
+ bucket_name = bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
+ if not bucket_name:
+ raise ValueError(
+ "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
+ )
+ return bucket_name
+
class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
"""
@@ -63,16 +157,21 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
def validate_environment(
self,
- headers: dict,
+ headers: MutableMapping[str, object],
model: str,
messages: List[AllMessageValues],
optional_params: dict,
- litellm_params: dict,
- api_key: Optional[str] = None,
- api_base: Optional[str] = None,
+ litellm_params: MutableMapping[str, object],
+ api_key: str | None = None,
+ api_base: str | None = None,
) -> dict:
- # No additional headers needed for S3 uploads - AWS credentials handled by BaseAWSLLM
- return headers
+ result: dict[str, object] = {}
+ result.update(headers)
+ signed_headers = litellm_params.pop(S3_SIGNED_GET_HEADERS_PARAM, None)
+ if isinstance(signed_headers, Mapping):
+ result.update(signed_headers) # any-ok: untyped handoff headers
+ # otherwise no extra headers - AWS credentials are handled by BaseAWSLLM
+ return result
def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
"""
@@ -927,23 +1026,114 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
def transform_file_content_request(
self,
- file_content_request,
- optional_params: dict,
- litellm_params: dict,
- ) -> tuple[str, dict]:
- raise NotImplementedError(
- "BedrockFilesConfig does not support file content retrieval"
+ file_content_request: FileContentRequest,
+ optional_params: Mapping[str, object],
+ litellm_params: MutableMapping[str, object],
+ ) -> tuple[str, dict[str, str]]:
+ """
+ Build a SigV4-signed S3 GetObject request for a Bedrock batch file.
+
+ Bedrock batch file ids are `s3://bucket/key` URIs (or unified ids
+ that decode to one); the bucket and key are validated against the
+ server-configured bucket before any request is signed.
+ """
+ file_id = file_content_request.get("file_id")
+ if not file_id:
+ raise ValueError("file_id is required for Bedrock file content retrieval")
+
+ s3_uri = extract_s3_uri_from_file_id(file_id)
+ bucket_name, object_key = validate_managed_cloud_file_id(
+ file_id=s3_uri,
+ scheme="s3://",
+ configured_bucket_name=get_configured_s3_bucket_name(litellm_params),
+ allowed_object_prefixes=BEDROCK_MANAGED_S3_PREFIXES,
+ allow_legacy_cloud_file_ids=should_allow_legacy_cloud_file_ids(
+ litellm_params
+ ),
)
+ # The shared file-content handler passes optional_params={}, so AWS
+ # credentials/region arrive via litellm_params here (unlike the upload
+ # path). s3_region_name wins over aws_region_name, same priority as
+ # get_complete_file_url above.
+ merged_params: dict[str, object] = {}
+ merged_params.update(litellm_params)
+ merged_params.update(optional_params)
+ request_params = _BedrockS3RequestParams.model_validate(merged_params)
+
+ region_preference = (
+ request_params.s3_region_name or request_params.aws_region_name
+ )
+ region_params: dict[str, str | None] = {"aws_region_name": region_preference}
+ aws_region_name = self._get_aws_region_name(
+ optional_params=region_params, model=""
+ )
+
+ s3_endpoint_url = (
+ request_params.s3_endpoint_url
+ or f"https://s3.{aws_region_name}.amazonaws.com"
+ ).rstrip("/")
+ url = f"{s3_endpoint_url}/{bucket_name}/{encode_s3_object_key_for_url(object_key)}"
+
+ litellm_params[S3_SIGNED_GET_HEADERS_PARAM] = self._sign_s3_get_request(
+ api_base=url,
+ aws_region_name=aws_region_name,
+ request_params=request_params,
+ )
+ return url, {}
+
+ def _sign_s3_get_request(
+ self,
+ api_base: str,
+ aws_region_name: str,
+ request_params: _BedrockS3RequestParams,
+ ) -> dict[str, str]:
+ """
+ SigV4-sign an S3 GetObject request, mirroring `_sign_s3_request` (PUT).
+ """
+ try:
+ import hashlib
+
+ from botocore.auth import SigV4Auth
+ from botocore.awsrequest import AWSRequest
+ except ImportError:
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
+
+ credentials = self.get_credentials( # any-ok: boto3 Credentials is untyped
+ aws_access_key_id=request_params.aws_access_key_id,
+ aws_secret_access_key=request_params.aws_secret_access_key,
+ aws_session_token=request_params.aws_session_token,
+ aws_region_name=aws_region_name,
+ aws_session_name=request_params.aws_session_name,
+ aws_profile_name=request_params.aws_profile_name,
+ aws_role_name=request_params.aws_role_name,
+ aws_web_identity_token=request_params.aws_web_identity_token,
+ aws_sts_endpoint=request_params.aws_sts_endpoint,
+ )
+
+ empty_body_hash = hashlib.sha256(b"").hexdigest()
+ aws_request = AWSRequest( # any-ok: botocore AWSRequest is untyped
+ method="GET",
+ url=api_base,
+ headers={"x-amz-content-sha256": empty_body_hash},
+ )
+ auth = SigV4Auth(credentials, "s3", aws_region_name) # any-ok: botocore untyped
+ auth.add_auth(aws_request) # any-ok: botocore request mutation is untyped
+ return dict(aws_request.headers) # any-ok: botocore headers are untyped
+
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
- raise NotImplementedError(
- "BedrockFilesConfig does not support file content retrieval"
- )
+ if raw_response.status_code >= 400:
+ raise BedrockError(
+ status_code=raw_response.status_code,
+ message=raw_response.text,
+ headers=raw_response.headers,
+ )
+ return HttpxBinaryResponseContent(response=raw_response)
class BedrockJsonlFilesTransformation:
diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py
index 18f051f8524..f688cea10f1 100644
--- a/litellm/llms/bedrock_mantle/chat/transformation.py
+++ b/litellm/llms/bedrock_mantle/chat/transformation.py
@@ -4,8 +4,10 @@ Amazon Bedrock Mantle - OpenAI-compatible inference engine in Amazon Bedrock.
API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html
Base URL: https://bedrock-mantle.{region}.api.aws/v1
-Auth: AWS Bedrock API key as Bearer token (set via BEDROCK_MANTLE_API_KEY env var)
- or region-aware key via BEDROCK_MANTLE_{REGION}_API_KEY.
+Auth: Bearer token (litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the
+ standard AWS_BEARER_TOKEN_BEDROCK) when present; otherwise AWS SigV4
+ (service "bedrock") over the standard credential chain. See
+ BedrockMantleAuthMixin in common_utils.
"""
from typing import Iterator, AsyncIterator, Any, List, Optional, Tuple, Union
@@ -13,20 +15,27 @@ from typing import Iterator, AsyncIterator, Any, List, Optional, Tuple, Union
import litellm
from litellm._logging import verbose_logger
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
+from litellm.llms.bedrock_mantle.common_utils import (
+ BEDROCK_MANTLE_DEFAULT_REGION,
+ BedrockMantleAuthMixin,
+)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
+from ..common_utils import mantle_base_segment
from ...openai_like.chat.transformation import OpenAILikeChatConfig
-BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1"
-
-class BedrockMantleChatConfig(OpenAILikeChatConfig):
+class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
"""
Transformation config for Amazon Bedrock Mantle OpenAI-compatible API.
"""
+ def __init__(self, aws_signer: BaseAWSLLM | None = None):
+ super().__init__()
+ self._aws_signer = aws_signer or BaseAWSLLM()
+
@property
def custom_llm_provider(self) -> Optional[str]:
return "bedrock_mantle"
@@ -40,6 +49,7 @@ class BedrockMantleChatConfig(OpenAILikeChatConfig):
api_base: Optional[str],
api_key: Optional[str],
litellm_params: Optional[GenericLiteLLMParams] = None,
+ model: str | None = None,
) -> Tuple[Optional[str], Optional[str]]:
region = (
(litellm_params.aws_region_name if litellm_params else None)
@@ -49,12 +59,15 @@ class BedrockMantleChatConfig(OpenAILikeChatConfig):
or BEDROCK_MANTLE_DEFAULT_REGION
)
BaseAWSLLM._validate_aws_region_name(region)
+ # The base path segment is data-driven per model (use_openai_responses_path
+ # flag): gemma-4-* and gpt-5.x are served on /openai/v1, everything else on
+ # /v1. An explicit api_base still wins over the derived default.
api_base = (
api_base
or get_secret_str("BEDROCK_MANTLE_API_BASE")
- or f"https://bedrock-mantle.{region}.api.aws/v1"
+ or f"https://bedrock-mantle.{region}.api.aws/{mantle_base_segment(model, litellm.model_cost)}"
)
- dynamic_api_key = api_key or get_secret_str("BEDROCK_MANTLE_API_KEY")
+ dynamic_api_key = self._resolve_bearer_token(api_key)
return api_base, dynamic_api_key
def validate_environment(
diff --git a/litellm/llms/bedrock_mantle/common_utils.py b/litellm/llms/bedrock_mantle/common_utils.py
new file mode 100644
index 00000000000..d517ab940ce
--- /dev/null
+++ b/litellm/llms/bedrock_mantle/common_utils.py
@@ -0,0 +1,147 @@
+"""Shared auth, region resolution, and routing helpers for the Amazon Bedrock Mantle provider.
+
+Mantle authenticates with a Bearer token when one is available
+(litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the standard
+AWS_BEARER_TOKEN_BEDROCK); otherwise it falls back to AWS SigV4 (service
+"bedrock") over the standard credential chain (IAM role / access key / profile /
+web identity). The Chat Completions and Responses backends share this behaviour
+through BedrockMantleAuthMixin so the two paths can never drift apart.
+
+The two routing helpers (mantle_supports_responses, mantle_base_segment) are
+pure functions of (model, model_cost) so they can be unit-tested without patching
+global state.
+"""
+
+import re
+from typing import Tuple
+
+from botocore.exceptions import (
+ CredentialRetrievalError,
+ NoCredentialsError,
+ PartialCredentialsError,
+ ProfileNotFound,
+)
+
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
+from litellm.secret_managers.main import get_secret_str
+
+BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1"
+
+# Standard Mantle host: https://bedrock-mantle..api.aws (group 1 = region).
+MANTLE_HOST_RE = re.compile(
+ r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE
+)
+
+
+class BedrockMantleAuthMixin:
+ _aws_signer: BaseAWSLLM
+
+ @staticmethod
+ def _resolve_bearer_token(api_key: str | None) -> str | None:
+ return (
+ api_key
+ or get_secret_str("BEDROCK_MANTLE_API_KEY")
+ or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+ )
+
+ @staticmethod
+ def _resolve_region(params: dict) -> str:
+ region = params.get("aws_region_name")
+ if region:
+ BaseAWSLLM._validate_aws_region_name(region)
+ return region
+ base = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
+ if base:
+ match = MANTLE_HOST_RE.match(base.rstrip("/"))
+ if match:
+ return match.group(1)
+ return (
+ get_secret_str("BEDROCK_MANTLE_REGION")
+ or get_secret_str("AWS_REGION_NAME")
+ or get_secret_str("AWS_REGION")
+ or BEDROCK_MANTLE_DEFAULT_REGION
+ )
+
+ def sign_request(
+ self,
+ headers: dict,
+ optional_params: dict,
+ request_data: dict,
+ api_base: str,
+ api_key: str | None = None,
+ model: str | None = None,
+ stream: bool | None = None,
+ fake_stream: bool | None = None,
+ ) -> Tuple[dict, bytes | None]:
+ bearer = self._resolve_bearer_token(api_key)
+ if not bearer:
+ # Pin the credential-scope region to the region of the actual signing URL
+ # so the SigV4 scope and URL host can never disagree, even when a stale
+ # api_base and aws_region_name point at different regions.
+ host_match = MANTLE_HOST_RE.match(api_base.rstrip("/"))
+ optional_params = {
+ **optional_params,
+ "aws_region_name": (
+ host_match.group(1)
+ if host_match
+ else self._resolve_region({**optional_params, "api_base": api_base})
+ ),
+ }
+ headers = {k: v for k, v in headers.items() if k.lower() != "authorization"}
+ try:
+ return self._aws_signer._sign_request(
+ service_name="bedrock",
+ headers=headers,
+ optional_params=optional_params,
+ request_data=request_data,
+ api_base=api_base,
+ api_key=bearer,
+ model=model,
+ stream=stream,
+ fake_stream=fake_stream,
+ )
+ except (
+ NoCredentialsError,
+ PartialCredentialsError,
+ ProfileNotFound,
+ CredentialRetrievalError,
+ ) as e:
+ raise ValueError(
+ "Bedrock Mantle auth failed: no Bearer token and no usable AWS "
+ "credentials. Set BEDROCK_MANTLE_API_KEY (or AWS_BEARER_TOKEN_BEDROCK) "
+ "or pass api_key for Bearer auth, or provide AWS credentials "
+ "(IAM role / access key / profile / web identity) for SigV4."
+ ) from e
+
+
+def mantle_supports_responses(model: str | None, model_cost: dict) -> bool:
+ """Whether a Bedrock Mantle model can serve the native Responses API.
+
+ Purely data-driven from the model's price-map capability signal -- either
+ /v1/responses in supported_endpoints, or mode=responses -- both overridable
+ via register_model and proxy model_info, so onboarding a model is a JSON
+ change, never a code change. There is deliberately NO model-name match here:
+ capability is per-model, not per-family (openai.gpt-oss-120b supports
+ Responses while openai.gpt-oss-safeguard-120b does not, despite sharing the
+ gpt-oss substring), so a substring gate would be wrong. A model absent from
+ model_cost simply has no signal and returns False (chat-completions emulation).
+ """
+ entry = model_cost.get(f"bedrock_mantle/{model}", {})
+ if "/v1/responses" in (entry.get("supported_endpoints") or []):
+ return True
+ return entry.get("mode") == "responses"
+
+
+def mantle_base_segment(model: str | None, model_cost: dict) -> str:
+ """Return the base path segment for a Bedrock Mantle model's OpenAI surface.
+
+ Data-driven from the model's price-map use_openai_responses_path flag
+ (overridable via register_model / proxy model_info). Per the AWS model cards,
+ gpt-5.x and the google gemma-4-* family carry that flag and are served on the
+ /openai/v1 base (.../openai/v1/responses and .../openai/v1/chat/completions);
+ every other model including gpt-oss uses the standard /v1 base. The segment is
+ the base for the model's whole OpenAI-compatible surface, so both the chat and
+ responses configs derive from it -- there is no separate model-name rule.
+ """
+ entry = model_cost.get(f"bedrock_mantle/{model}", {})
+ return "openai/v1" if entry.get("use_openai_responses_path") is True else "v1"
diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py
index b409666a967..2e30f85fd0e 100644
--- a/litellm/llms/bedrock_mantle/responses/transformation.py
+++ b/litellm/llms/bedrock_mantle/responses/transformation.py
@@ -15,26 +15,20 @@ role / access key / profile / web identity), signed via the shared
BaseAWSLLM._sign_request after the request body is finalized.
"""
-import re
-from typing import Any, Dict, List, Optional, Tuple
-
-from botocore.exceptions import (
- CredentialRetrievalError,
- NoCredentialsError,
- PartialCredentialsError,
- ProfileNotFound,
-)
+from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
+from litellm.llms.bedrock_mantle.common_utils import (
+ MANTLE_HOST_RE,
+ BedrockMantleAuthMixin,
+)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
-BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1"
-
# Checked longest/most-specific first so a full endpoint URL collapses to host
# in one pass and the appended path never doubles.
_BASE_SUFFIXES_TO_STRIP = (
@@ -45,18 +39,13 @@ _BASE_SUFFIXES_TO_STRIP = (
"/v1",
)
-# Standard Mantle host: https://bedrock-mantle..api.aws (group 1 = region).
-_MANTLE_HOST_RE = re.compile(
- r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE
-)
-
# Per Bedrock Mantle Responses API validation errors.
_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES = frozenset(
{"function", "mcp", "custom", "namespace", "tool_search"}
)
-class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
+class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
def __init__(
self,
aws_signer: Optional[BaseAWSLLM] = None,
@@ -70,24 +59,6 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK_MANTLE
- @staticmethod
- def _resolve_region(params: dict) -> str:
- region = params.get("aws_region_name")
- if region:
- BaseAWSLLM._validate_aws_region_name(region)
- return region
- base = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
- if base:
- match = _MANTLE_HOST_RE.match(base.rstrip("/"))
- if match:
- return match.group(1)
- return (
- get_secret_str("BEDROCK_MANTLE_REGION")
- or get_secret_str("AWS_REGION_NAME")
- or get_secret_str("AWS_REGION")
- or BEDROCK_MANTLE_DEFAULT_REGION
- )
-
def get_complete_url(
self,
api_base: Optional[str],
@@ -107,7 +78,7 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
# For the standard Mantle host (including the default-region base that
# responses/main.py auto-injects into litellm_params.api_base), pin to the
# single resolved region so aws_region_name wins; preserve custom proxy hosts.
- if _MANTLE_HOST_RE.match(base):
+ if MANTLE_HOST_RE.match(base):
base = f"https://bedrock-mantle.{region}.api.aws"
path = "/openai/v1/responses" if self.use_openai_path else "/v1/responses"
return f"{base}{path}"
@@ -116,13 +87,9 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()
- api_key = (
- litellm_params.api_key
- or get_secret_str("BEDROCK_MANTLE_API_KEY")
- or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
- )
- if api_key:
- headers["Authorization"] = f"Bearer {api_key}"
+ bearer = self._resolve_bearer_token(litellm_params.api_key)
+ if bearer:
+ headers["Authorization"] = f"Bearer {bearer}"
if litellm_params.aws_bedrock_project_id:
headers["OpenAI-Project"] = litellm_params.aws_bedrock_project_id
return headers
@@ -182,58 +149,3 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
params.pop("tools", None)
return params
-
- def sign_request(
- self,
- headers: dict,
- optional_params: dict,
- request_data: dict,
- api_base: str,
- api_key: Optional[str] = None,
- model: Optional[str] = None,
- stream: Optional[bool] = None,
- fake_stream: Optional[bool] = None,
- ) -> Tuple[dict, Optional[bytes]]:
- bearer = (
- api_key
- or get_secret_str("BEDROCK_MANTLE_API_KEY")
- or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
- )
- if not bearer:
- # SigV4 path. Pin the credential-scope region to the region of the actual
- # signing URL (api_base, already region-resolved by get_complete_url) so the
- # SigV4 scope and the URL host can never disagree. Resolve from api_base first,
- # then fall back to the regular precedence. Also drop any caller Authorization
- # so _sign_request's restore-original-Authorization step cannot override the
- # SigV4 header.
- optional_params = {
- **optional_params,
- "aws_region_name": self._resolve_region(
- {**optional_params, "api_base": api_base}
- ),
- }
- headers = {k: v for k, v in headers.items() if k.lower() != "authorization"}
- try:
- return self._aws_signer._sign_request(
- service_name="bedrock",
- headers=headers,
- optional_params=optional_params,
- request_data=request_data,
- api_base=api_base,
- api_key=bearer,
- model=model,
- stream=stream,
- fake_stream=fake_stream,
- )
- except (
- NoCredentialsError,
- PartialCredentialsError,
- ProfileNotFound,
- CredentialRetrievalError,
- ) as e:
- raise ValueError(
- "Bedrock Mantle auth failed: no Bearer token and no usable AWS "
- "credentials. Set BEDROCK_MANTLE_API_KEY (or AWS_BEARER_TOKEN_BEDROCK) "
- "or pass api_key for Bearer auth, or provide AWS credentials "
- "(IAM role / access key / profile / web identity) for SigV4."
- ) from e
diff --git a/litellm/llms/brave/search/transformation.py b/litellm/llms/brave/search/transformation.py
index 9dfcd6bc75a..8ffe7dcb126 100644
--- a/litellm/llms/brave/search/transformation.py
+++ b/litellm/llms/brave/search/transformation.py
@@ -115,7 +115,13 @@ class BraveSearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("BRAVE_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("BRAVE_API_KEY",),
+ base_env_var="BRAVE_API_BASE",
+ default_api_base=self.BRAVE_API_BASE,
+ )
if not api_key:
raise ValueError(
diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py
index 66e253f304d..68f08741cc5 100644
--- a/litellm/llms/cloudflare/chat/transformation.py
+++ b/litellm/llms/cloudflare/chat/transformation.py
@@ -1,26 +1,15 @@
-import json
-import time
-from typing import AsyncIterator, Iterator, List, Optional, Union
+from typing import List, Optional, Union
import httpx
-import litellm
-from litellm.litellm_core_utils.url_utils import encode_url_path_segments
-from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
-from litellm.llms.base_llm.chat.transformation import (
- BaseConfig,
- BaseLLMException,
- LiteLLMLoggingObj,
+from litellm._logging import verbose_logger
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.secret_managers.main import (
+ get_secret_str,
+ normalize_nonempty_secret_str,
)
-from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import (
- ChatCompletionToolCallChunk,
- ChatCompletionUsageBlock,
- GenericStreamingChunk,
- ModelResponse,
- Usage,
-)
class CloudflareError(BaseLLMException):
@@ -34,26 +23,46 @@ class CloudflareError(BaseLLMException):
message=message,
request=self.request,
response=self.response,
- ) # Call the base class constructor with the parameters it needs
+ )
-class CloudflareChatConfig(BaseConfig):
- max_tokens: Optional[int] = None
- stream: Optional[bool] = None
-
- def __init__(
+class CloudflareChatConfig(OpenAIGPTConfig):
+ def get_complete_url(
self,
- max_tokens: Optional[int] = None,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
stream: Optional[bool] = None,
- ) -> None:
- locals_ = locals().copy()
- for key, value in locals_.items():
- if key != "self" and value is not None:
- setattr(self.__class__, key, value)
+ ) -> str:
+ return super().get_complete_url(
+ api_base=self._resolve_api_base(api_base),
+ api_key=api_key,
+ model=model,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ stream=stream,
+ )
- @classmethod
- def get_config(cls):
- return super().get_config()
+ @staticmethod
+ def _resolve_api_base(api_base: Optional[str]) -> str:
+ if not api_base:
+ account_id = normalize_nonempty_secret_str(
+ get_secret_str("CLOUDFLARE_ACCOUNT_ID")
+ )
+ if account_id is None:
+ raise ValueError(
+ "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID in the environment or pass api_base explicitly"
+ )
+ return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1"
+ trimmed = api_base.rstrip("/")
+ if trimmed.endswith("/ai/run"):
+ verbose_logger.warning(
+ "Cloudflare api_base ending in '/ai/run' is the legacy Workers AI path and no longer serves OpenAI-compatible requests; rewriting to the '/ai/v1' endpoint"
+ )
+ return f"{trimmed[: -len('/ai/run')]}/ai/v1"
+ return api_base
def validate_environment(
self,
@@ -67,107 +76,18 @@ class CloudflareChatConfig(BaseConfig):
) -> dict:
if api_key is None:
raise ValueError(
- "Missing CloudflareError API Key - A call is being made to cloudflare but no key is set either in the environment variables or via params"
+ "Missing Cloudflare API Key - A call is being made to cloudflare but no key is set either in the environment variables or via params"
)
- headers = {
- "accept": "application/json",
- "content-type": "apbplication/json",
- "Authorization": "Bearer " + api_key,
- }
- return headers
-
- def get_complete_url(
- self,
- api_base: Optional[str],
- api_key: Optional[str],
- model: str,
- optional_params: dict,
- litellm_params: dict,
- stream: Optional[bool] = None,
- ) -> str:
- if api_base is None:
- account_id = get_secret_str("CLOUDFLARE_ACCOUNT_ID")
- api_base = (
- f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/"
- )
- encoded_model = encode_url_path_segments(model, field_name="model")
- return api_base + encoded_model
-
- def get_supported_openai_params(self, model: str) -> List[str]:
- return [
- "stream",
- "max_tokens",
- ]
-
- def map_openai_params(
- self,
- non_default_params: dict,
- optional_params: dict,
- model: str,
- drop_params: bool,
- ) -> dict:
- supported_openai_params = self.get_supported_openai_params(model=model)
- for param, value in non_default_params.items():
- if param == "max_completion_tokens":
- optional_params["max_tokens"] = value
- elif param in supported_openai_params:
- optional_params[param] = value
- return optional_params
-
- def transform_request(
- self,
- model: str,
- messages: List[AllMessageValues],
- optional_params: dict,
- litellm_params: dict,
- headers: dict,
- ) -> dict:
- config = litellm.CloudflareChatConfig.get_config()
- for k, v in config.items():
- if k not in optional_params:
- optional_params[k] = v
-
- data = {
- "messages": messages,
- **optional_params,
- }
- return data
-
- def transform_response(
- self,
- model: str,
- raw_response: httpx.Response,
- model_response: ModelResponse,
- logging_obj: LiteLLMLoggingObj,
- request_data: dict,
- messages: List[AllMessageValues],
- optional_params: dict,
- litellm_params: dict,
- encoding: str,
- api_key: Optional[str] = None,
- json_mode: Optional[bool] = None,
- ) -> ModelResponse:
- completion_response = raw_response.json()
-
- # Support both "response" and "response_text" keys (newer models like Nemotron use "response_text")
- result = completion_response["result"]
- model_response.choices[0].message.content = result.get("response") if result.get("response") is not None else result.get("response_text", "") # type: ignore
-
- prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model)
- completion_tokens = len(
- encoding.encode(model_response["choices"][0]["message"].get("content", ""))
+ return super().validate_environment(
+ headers=headers,
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ api_key=api_key,
+ api_base=api_base,
)
- model_response.created = int(time.time())
- model_response.model = "cloudflare/" + model
- usage = Usage(
- prompt_tokens=prompt_tokens,
- completion_tokens=completion_tokens,
- total_tokens=prompt_tokens + completion_tokens,
- )
- setattr(model_response, "usage", usage)
- return model_response
-
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
@@ -175,48 +95,3 @@ class CloudflareChatConfig(BaseConfig):
status_code=status_code,
message=error_message,
)
-
- def get_model_response_iterator(
- self,
- streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
- sync_stream: bool,
- json_mode: Optional[bool] = False,
- ):
- return CloudflareChatResponseIterator(
- streaming_response=streaming_response,
- sync_stream=sync_stream,
- json_mode=json_mode,
- )
-
-
-class CloudflareChatResponseIterator(BaseModelResponseIterator):
- def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
- try:
- text = ""
- tool_use: Optional[ChatCompletionToolCallChunk] = None
- is_finished = False
- finish_reason = ""
- usage: Optional[ChatCompletionUsageBlock] = None
- provider_specific_fields = None
-
- index = int(chunk.get("index", 0))
-
- if "response" in chunk and chunk["response"] is not None:
- text = chunk["response"]
- elif "response_text" in chunk and chunk["response_text"] is not None:
- text = chunk["response_text"]
-
- returned_chunk = GenericStreamingChunk(
- text=text,
- tool_use=tool_use,
- is_finished=is_finished,
- finish_reason=finish_reason,
- usage=usage,
- index=index,
- provider_specific_fields=provider_specific_fields,
- )
-
- return returned_chunk
-
- except json.JSONDecodeError:
- raise ValueError(f"Failed to decode JSON from chunk: {chunk}")
diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py
index 62f707b3622..b97a59a93a6 100644
--- a/litellm/llms/custom_httpx/aiohttp_transport.py
+++ b/litellm/llms/custom_httpx/aiohttp_transport.py
@@ -116,6 +116,16 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
# For other exceptions, use the normal mapping
with map_aiohttp_exceptions():
raise
+ finally:
+ # Release the aiohttp connection when iteration ends for any
+ # reason (read timeout, cancellation from a client disconnect,
+ # GeneratorExit). Without this, abnormally terminated streams
+ # permanently hold a slot in the TCPConnector pool; once the
+ # pool is exhausted every request to that host times out (408)
+ # until the proxy is restarted, even after the backend recovers.
+ # On a fully-read response the connection was already released
+ # at EOF and close() is a no-op.
+ self._aiohttp_response.close()
async def aclose(self) -> None:
with map_aiohttp_exceptions():
diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py
index 01c94476431..1000ab12803 100644
--- a/litellm/llms/custom_httpx/http_handler.py
+++ b/litellm/llms/custom_httpx/http_handler.py
@@ -42,6 +42,9 @@ from litellm.constants import (
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
)
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
+from litellm.litellm_core_utils.request_timeout_resolver import (
+ get_configured_request_timeout,
+)
from litellm.types.llms.custom_http import *
if TYPE_CHECKING:
@@ -134,6 +137,18 @@ _DEFAULT_TIMEOUT = httpx.Timeout(
timeout=COMPLETION_HTTP_FALLBACK_SECONDS,
connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
)
+
+
+def _default_cached_client_timeout() -> httpx.Timeout:
+ """Timeout for cached default httpx clients; honors an explicit litellm.request_timeout."""
+ configured = get_configured_request_timeout()
+ if configured is None:
+ return _DEFAULT_TIMEOUT
+ return httpx.Timeout(
+ timeout=configured, connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
+ )
+
+
_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS = 5.0
_STREAMING_ERROR_BODY_READ_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
max_workers=50,
@@ -1379,7 +1394,7 @@ def get_async_httpx_client(
_new_client = AsyncHTTPHandler(**handler_params)
else:
_new_client = AsyncHTTPHandler(
- timeout=_DEFAULT_TIMEOUT,
+ timeout=_default_cached_client_timeout(),
shared_session=shared_session,
)
@@ -1428,7 +1443,7 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler:
}
_new_client = HTTPHandler(**handler_params)
else:
- _new_client = HTTPHandler(timeout=_DEFAULT_TIMEOUT)
+ _new_client = HTTPHandler(timeout=_default_cached_client_timeout())
cache.set_cache(
key=_cache_key_name,
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 8ac5b47c6e7..948c90f9f99 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -1,5 +1,6 @@
import json
import ssl
+from functools import lru_cache
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
from typing import (
TYPE_CHECKING,
@@ -13,6 +14,7 @@ from typing import (
Tuple,
Union,
cast,
+ get_type_hints,
)
import httpx # type: ignore
@@ -26,6 +28,7 @@ from litellm._logging import _redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
+from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
@@ -101,6 +104,7 @@ from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
OpenAIFileObject,
ResponseInputParam,
+ ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
)
from litellm.types.rerank import RerankResponse
@@ -135,6 +139,7 @@ from litellm.utils import (
ImageResponse,
ModelResponse,
ProviderConfigManager,
+ async_pre_call_deployment_hook,
)
from .http_handler import get_shared_realtime_ssl_context
@@ -184,6 +189,47 @@ def _google_genai_streaming_hidden_params(
}
+@lru_cache(maxsize=None)
+def _responses_api_optional_request_param_names() -> frozenset[str]:
+ return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys())
+
+
+def _custom_logger_callbacks(logging_obj: Any) -> list[Any]:
+ from litellm.integrations.custom_logger import CustomLogger
+ from litellm.litellm_core_utils.litellm_logging import (
+ get_custom_logger_compatible_class,
+ )
+
+ dynamic_success_callbacks = getattr(logging_obj, "dynamic_success_callbacks", None)
+ callbacks = list(litellm.callbacks)
+ if isinstance(dynamic_success_callbacks, (list, tuple)):
+ callbacks.extend(dynamic_success_callbacks)
+
+ custom_loggers: list[Any] = []
+ for cb in callbacks:
+ if isinstance(cb, str):
+ resolved = get_custom_logger_compatible_class(cb) # type: ignore[arg-type]
+ if resolved is None:
+ continue
+ cb = resolved
+ if isinstance(cb, CustomLogger):
+ custom_loggers.append(cb)
+ return custom_loggers
+
+
+def _has_pre_call_deployment_hook(logging_obj: Any) -> bool:
+ from litellm.integrations.custom_logger import CustomLogger
+
+ base_func = CustomLogger.async_pre_call_deployment_hook
+ for cb in _custom_logger_callbacks(logging_obj):
+ cb_func = getattr(type(cb), "async_pre_call_deployment_hook", base_func)
+ if getattr(cb_func, "__func__", cb_func) is not getattr(
+ base_func, "__func__", base_func
+ ):
+ return True
+ return False
+
+
class BaseLLMHTTPHandler:
async def _make_common_async_call(
self,
@@ -1833,6 +1879,9 @@ class BaseLLMHTTPHandler:
data = provider_config.transform_search_request(
query=query,
optional_params=optional_params,
+ api_key=api_key,
+ api_base=api_base,
+ headers=headers or {},
)
# Get complete URL (pass data for providers that need request body for URL construction)
@@ -2224,12 +2273,92 @@ class BaseLLMHTTPHandler:
)
raise ValueError("anthropic_messages_handler is not implemented for sync calls")
+ def _run_sync_responses_pre_call_deployment_hook(
+ self,
+ *,
+ model: str,
+ input: Union[str, ResponseInputParam],
+ custom_llm_provider: str,
+ response_api_optional_request_params: dict[str, Any],
+ litellm_params: GenericLiteLLMParams,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> tuple[
+ str,
+ Union[str, ResponseInputParam],
+ str,
+ dict[str, Any],
+ GenericLiteLLMParams,
+ ]:
+ if not _has_pre_call_deployment_hook(logging_obj):
+ return (
+ model,
+ input,
+ custom_llm_provider,
+ response_api_optional_request_params,
+ litellm_params,
+ )
+
+ modified_kwargs = run_async_function(
+ async_pre_call_deployment_hook,
+ {
+ **dict(litellm_params),
+ **response_api_optional_request_params,
+ "model": model,
+ "input": input,
+ "custom_llm_provider": custom_llm_provider,
+ },
+ CallTypes.responses.value,
+ )
+ if modified_kwargs is None:
+ return (
+ model,
+ input,
+ custom_llm_provider,
+ response_api_optional_request_params,
+ litellm_params,
+ )
+
+ optional_param_names = _responses_api_optional_request_param_names()
+ updated_response_params = {
+ **response_api_optional_request_params,
+ **{
+ key: value
+ for key, value in modified_kwargs.items()
+ if key in optional_param_names
+ },
+ }
+ updated_litellm_params = GenericLiteLLMParams(
+ **{
+ **dict(litellm_params),
+ **{
+ key: value
+ for key, value in modified_kwargs.items()
+ if key not in optional_param_names
+ and key not in {"model", "input", "custom_llm_provider"}
+ },
+ }
+ )
+ return (
+ str(modified_kwargs["model"]) if "model" in modified_kwargs else model,
+ cast(
+ Union[str, ResponseInputParam],
+ modified_kwargs["input"] if "input" in modified_kwargs else input,
+ ),
+ (
+ str(modified_kwargs["custom_llm_provider"])
+ if "custom_llm_provider" in modified_kwargs
+ else custom_llm_provider
+ ),
+ updated_response_params,
+ updated_litellm_params,
+ )
+
def response_api_handler(
self,
model: str,
input: Union[str, ResponseInputParam],
responses_api_provider_config: BaseResponsesAPIConfig,
- response_api_optional_request_params: Dict,
+ response_api_optional_request_params: dict[str, Any],
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
@@ -2276,6 +2405,21 @@ class BaseLLMHTTPHandler:
shared_session=shared_session,
)
+ (
+ model,
+ input,
+ custom_llm_provider,
+ response_api_optional_request_params,
+ litellm_params,
+ ) = self._run_sync_responses_pre_call_deployment_hook(
+ model=model,
+ input=input,
+ custom_llm_provider=custom_llm_provider,
+ response_api_optional_request_params=response_api_optional_request_params,
+ litellm_params=litellm_params,
+ logging_obj=logging_obj,
+ )
+
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
@@ -2407,12 +2551,36 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
- return responses_api_provider_config.transform_response_api_response(
- model=model,
- raw_response=response,
- logging_obj=logging_obj,
+ initial_response = (
+ responses_api_provider_config.transform_response_api_response(
+ model=model,
+ raw_response=response,
+ logging_obj=logging_obj,
+ )
)
+ if self._has_agentic_completion_hook(logging_obj):
+ final_response = run_async_function(
+ self._call_agentic_completion_hooks,
+ response=initial_response,
+ model=model,
+ messages=(
+ input
+ if isinstance(input, list)
+ else [{"role": "user", "content": input}]
+ ),
+ anthropic_messages_provider_config=responses_api_provider_config,
+ anthropic_messages_optional_request_params=response_api_optional_request_params,
+ logging_obj=logging_obj,
+ stream=False,
+ custom_llm_provider=custom_llm_provider,
+ kwargs=dict(litellm_params),
+ api_surface="responses",
+ )
+ return final_response if final_response is not None else initial_response
+
+ return initial_response
+
async def async_response_api_handler(
self,
model: str,
@@ -2570,12 +2738,44 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
- return responses_api_provider_config.transform_response_api_response(
- model=model,
- raw_response=response,
- logging_obj=logging_obj,
+ initial_response = (
+ responses_api_provider_config.transform_response_api_response(
+ model=model,
+ raw_response=response,
+ logging_obj=logging_obj,
+ )
)
+ final_response = await self._call_agentic_completion_hooks(
+ response=initial_response,
+ model=model,
+ messages=(
+ input
+ if isinstance(input, list)
+ else [{"role": "user", "content": input}]
+ ),
+ anthropic_messages_provider_config=responses_api_provider_config,
+ anthropic_messages_optional_request_params=response_api_optional_request_params,
+ logging_obj=logging_obj,
+ stream=False,
+ custom_llm_provider=custom_llm_provider,
+ kwargs=dict(litellm_params),
+ api_surface="responses",
+ )
+
+ result = final_response if final_response is not None else initial_response
+ if litellm_params.get(
+ "_code_interpreter_interception_converted_stream"
+ ) and not litellm_params.get("_agentic_loop_depth"):
+ return self._wrap_responses_response_as_fake_stream(
+ result=result,
+ model=model,
+ responses_api_provider_config=responses_api_provider_config,
+ logging_obj=logging_obj,
+ custom_llm_provider=custom_llm_provider,
+ )
+ return result
+
async def async_delete_response_api_handler(
self,
response_id: str,
@@ -4734,22 +4934,9 @@ class BaseLLMHTTPHandler:
agentic callback is detected too.
"""
from litellm.integrations.custom_logger import CustomLogger
- from litellm.litellm_core_utils.litellm_logging import (
- get_custom_logger_compatible_class,
- )
base_func = CustomLogger.async_should_run_agentic_loop
- callbacks = litellm.callbacks + (
- getattr(logging_obj, "dynamic_success_callbacks", None) or []
- )
- for cb in callbacks:
- if isinstance(cb, str):
- resolved = get_custom_logger_compatible_class(cb) # type: ignore[arg-type]
- if resolved is None:
- continue
- cb = resolved
- if not isinstance(cb, CustomLogger):
- continue
+ for cb in _custom_logger_callbacks(logging_obj):
cb_func = getattr(type(cb), "async_should_run_agentic_loop", base_func)
if getattr(cb_func, "__func__", cb_func) is not getattr(
base_func, "__func__", base_func
@@ -4875,6 +5062,132 @@ class BaseLLMHTTPHandler:
return response
+ async def _execute_responses_agentic_plan(
+ self,
+ plan: AgenticLoopPlan,
+ model: str,
+ response_api_optional_request_params: dict,
+ logging_obj: "LiteLLMLoggingObj",
+ kwargs: dict,
+ depth: int,
+ max_loops: int,
+ fingerprints: list[str],
+ fingerprint: str,
+ callback: Any | None = None,
+ ) -> Any:
+ patch = plan.request_patch or AgenticLoopRequestPatch()
+ if patch.messages is None:
+ raise ValueError("Agentic loop plan missing patched responses input")
+
+ optional_params = dict(response_api_optional_request_params)
+ optional_params.update(patch.optional_params)
+ if patch.tools is not None:
+ optional_params["tools"] = patch.tools
+ optional_params = {
+ k: v
+ for k, v in optional_params.items()
+ if k != "stream" and k != "_code_interpreter_interception_converted_stream"
+ }
+
+ internal_keys = {"litellm_logging_obj"}
+ kwargs_for_followup = {
+ k: v
+ for k, v in kwargs.items()
+ if not k.startswith("_websearch_interception")
+ and not k.startswith("_compression_interception")
+ and k != "_code_interpreter_interception_converted_stream"
+ and k not in internal_keys
+ and k not in optional_params
+ }
+ kwargs_for_followup.update(patch.kwargs)
+ kwargs_for_followup["_agentic_loop_depth"] = depth + 1
+ kwargs_for_followup["max_agentic_loops"] = max_loops
+ kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
+
+ try:
+ response = await litellm.aresponses(
+ model=patch.model or model,
+ input=patch.messages,
+ **optional_params,
+ **kwargs_for_followup,
+ )
+
+ if callback is not None:
+ try:
+ response = await callback.async_post_agentic_loop_response_hook(
+ response=response, plan=plan, kwargs=kwargs
+ )
+ except Exception as e:
+ _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+ verbose_logger.exception(
+ "LiteLLM.AgenticHookError: Exception in "
+ "async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s",
+ _call_id,
+ model,
+ str(e),
+ )
+
+ return response
+ finally:
+ if callback is not None:
+ await self._run_agentic_loop_cleanup(
+ callback=callback,
+ plan=plan,
+ kwargs=kwargs,
+ logging_obj=logging_obj,
+ model=model,
+ )
+
+ @staticmethod
+ async def _run_agentic_loop_cleanup(
+ callback: Any,
+ plan: AgenticLoopPlan,
+ kwargs: dict,
+ logging_obj: "LiteLLMLoggingObj",
+ model: str,
+ ) -> None:
+ try:
+ await callback.async_agentic_loop_cleanup_hook(plan=plan, kwargs=kwargs)
+ except Exception as e:
+ _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+ verbose_logger.exception(
+ "LiteLLM.AgenticHookError: Exception in "
+ "async_agentic_loop_cleanup_hook [call_id=%s model=%s]: %s",
+ _call_id,
+ model,
+ str(e),
+ )
+
+ def _wrap_responses_response_as_fake_stream(
+ self,
+ result: Any,
+ model: str,
+ responses_api_provider_config: Any,
+ logging_obj: "LiteLLMLoggingObj",
+ custom_llm_provider: str,
+ ) -> Any:
+ """
+ Wrap a completed responses result as a synthetic stream.
+
+ Used when an interceptor forced stream=False to run the agentic loop on
+ the non-streaming path, but the caller originally asked for streaming.
+ """
+ import httpx
+
+ from litellm.responses.streaming_iterator import (
+ MockResponsesAPIStreamingIterator,
+ )
+
+ payload = result.model_dump() if hasattr(result, "model_dump") else result
+ raw_response = httpx.Response(status_code=200, json=payload)
+ return MockResponsesAPIStreamingIterator(
+ response=raw_response,
+ model=model,
+ responses_api_provider_config=responses_api_provider_config,
+ logging_obj=logging_obj,
+ custom_llm_provider=custom_llm_provider,
+ )
+
async def _execute_chat_completion_agentic_plan(
self,
plan: AgenticLoopPlan,
@@ -4940,6 +5253,7 @@ class BaseLLMHTTPHandler:
stream: bool,
custom_llm_provider: str,
kwargs: Dict,
+ api_surface: str = "anthropic_messages",
) -> Optional[Any]:
"""
Call agentic completion hooks for all custom loggers (Anthropic Messages API).
@@ -5046,6 +5360,20 @@ class BaseLLMHTTPHandler:
if not plan.run_agentic_loop:
continue
+ if api_surface == "responses":
+ return await self._execute_responses_agentic_plan(
+ plan=plan,
+ model=model,
+ response_api_optional_request_params=anthropic_messages_optional_request_params,
+ logging_obj=logging_obj,
+ kwargs=kwargs_with_provider,
+ depth=depth,
+ max_loops=max_loops,
+ fingerprints=fingerprints,
+ fingerprint=fingerprint,
+ callback=callback,
+ )
+
return await self._execute_anthropic_agentic_plan(
plan=plan,
model=model,
@@ -5083,7 +5411,7 @@ class BaseLLMHTTPHandler:
else False
)
- if websearch_converted_stream:
+ if api_surface == "anthropic_messages" and websearch_converted_stream:
from typing import cast
from litellm._logging import verbose_logger
@@ -5358,9 +5686,7 @@ class BaseLLMHTTPHandler:
import websockets
from websockets.asyncio.client import ClientConnection
- url = self._append_query_params(
- provider_config.get_complete_url(api_base, model, api_key), query_params
- )
+ url = provider_config.get_complete_url(api_base, model, api_key)
headers = provider_config.validate_environment(
headers=headers,
model=model,
diff --git a/litellm/llms/dataforseo/search/transformation.py b/litellm/llms/dataforseo/search/transformation.py
index 27c10d740b5..701db586b72 100644
--- a/litellm/llms/dataforseo/search/transformation.py
+++ b/litellm/llms/dataforseo/search/transformation.py
@@ -61,9 +61,18 @@ class DataForSEOSearchConfig(BaseSearchConfig):
password = get_secret_str("DATAFORSEO_PASSWORD")
# If api_key is provided in "login:password" format, use it
+ caller_supplied_credentials = bool(api_key and ":" in api_key)
if api_key and ":" in api_key:
login, password = api_key.split(":", 1)
+ if not caller_supplied_credentials and login and password:
+ self._assert_trusted_api_base_for_server_credential(
+ api_base,
+ self.DATAFORSEO_API_BASE,
+ "DATAFORSEO_API_BASE",
+ "DATAFORSEO_LOGIN",
+ )
+
if not login:
raise ValueError(
"DATAFORSEO_LOGIN is not set. Set `DATAFORSEO_LOGIN` environment variable or pass credentials in api_key parameter."
diff --git a/litellm/llms/e2b/__init__.py b/litellm/llms/e2b/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/llms/e2b/sandbox/__init__.py b/litellm/llms/e2b/sandbox/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/llms/e2b/sandbox/transformation.py b/litellm/llms/e2b/sandbox/transformation.py
new file mode 100644
index 00000000000..ecfc1642c97
--- /dev/null
+++ b/litellm/llms/e2b/sandbox/transformation.py
@@ -0,0 +1,212 @@
+"""
+e2b sandbox provider.
+
+Talks to e2b's REST API directly over httpx (no e2b SDK dependency):
+ - create: POST {api_base}/sandboxes
+ - run: POST https://{JUPYTER_PORT}-{sandboxID}.{domain}/execute (NDJSON stream)
+ - delete: DELETE {api_base}/sandboxes/{sandboxID}
+"""
+
+import json
+from typing import Union, cast
+
+import httpx
+
+from litellm.llms.base_llm.sandbox.transformation import (
+ BaseSandboxConfig,
+ CodeExecutionResult,
+ ContainerHandle,
+ SANDBOX_MAX_OUTPUT_BYTES,
+)
+from litellm.llms.custom_httpx.http_handler import (
+ AsyncHTTPHandler,
+ get_async_httpx_client,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.custom_http import httpxSpecialProvider
+
+E2B_API_BASE = "https://api.e2b.app"
+E2B_DEFAULT_TEMPLATE = "code-interpreter-v1"
+E2B_DEFAULT_DOMAIN = "e2b.app"
+JUPYTER_PORT = 49999
+DEFAULT_SANDBOX_TIMEOUT = 300
+MAX_OUTPUT_BYTES = SANDBOX_MAX_OUTPUT_BYTES
+
+
+class E2BSandboxConfig(BaseSandboxConfig):
+ def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler:
+ if client is not None:
+ return client
+ return get_async_httpx_client(llm_provider=httpxSpecialProvider.Sandbox)
+
+ def validate_environment(self, api_key: str | None = None, **kwargs) -> str:
+ key = api_key or get_secret_str("E2B_API_KEY")
+ if not key:
+ raise ValueError("E2B API key not set. Set E2B_API_KEY or pass api_key=...")
+ return key
+
+ async def acreate_sandbox(
+ self,
+ *,
+ template: str | None = None,
+ timeout: int | None = None,
+ allow_internet_access: bool | None = None,
+ api_key: str | None = None,
+ api_base: str | None = None,
+ metadata: dict | None = None,
+ client: AsyncHTTPHandler | None = None,
+ **kwargs,
+ ) -> ContainerHandle:
+ key = self.validate_environment(api_key=api_key)
+ base = api_base or E2B_API_BASE
+ body = {
+ "templateID": template or E2B_DEFAULT_TEMPLATE,
+ "timeout": timeout if timeout is not None else DEFAULT_SANDBOX_TIMEOUT,
+ "secure": True,
+ "allow_internet_access": (
+ True if allow_internet_access is None else allow_internet_access
+ ),
+ }
+ if metadata:
+ body["metadata"] = metadata
+
+ response = cast(
+ httpx.Response,
+ await self._http(client).post(
+ url=f"{base}/sandboxes",
+ headers={"X-API-Key": key, "Content-Type": "application/json"},
+ json=body,
+ ),
+ )
+ data = response.json()
+
+ handle = ContainerHandle(
+ id=data["sandboxID"],
+ provider="e2b",
+ domain=data.get("domain") or E2B_DEFAULT_DOMAIN,
+ )
+ handle._hidden_params = {
+ "envd_access_token": data.get("envdAccessToken"),
+ "traffic_access_token": data.get("trafficAccessToken"),
+ "api_key": key,
+ "api_base": base,
+ }
+ return handle
+
+ async def arun_code(
+ self,
+ *,
+ container: Union[ContainerHandle, str],
+ code: str,
+ api_key: str | None = None,
+ env_vars: dict | None = None,
+ client: AsyncHTTPHandler | None = None,
+ **kwargs,
+ ) -> CodeExecutionResult:
+ handle = self._as_handle(container)
+
+ token = handle._hidden_params.get("envd_access_token")
+ if not token:
+ raise ValueError(
+ "Cannot run code from a sandbox id alone. e2b secure sandboxes "
+ "require the access token returned by acreate_sandbox; pass the "
+ "ContainerHandle it returned instead of a bare sandbox id."
+ )
+
+ headers = {"Content-Type": "application/json", "X-Access-Token": token}
+ traffic_token = handle._hidden_params.get("traffic_access_token")
+ if traffic_token:
+ headers["E2B-Traffic-Access-Token"] = traffic_token
+
+ url = f"https://{JUPYTER_PORT}-{handle.id}.{handle.domain}/execute"
+ response = cast(
+ httpx.Response,
+ await self._http(client).post(
+ url=url,
+ headers=headers,
+ json={"code": code, "context_id": None, "env_vars": env_vars},
+ stream=True,
+ ),
+ )
+ lines = await self._read_capped_lines(response)
+ return self._parse_lines(lines)
+
+ async def adelete_sandbox(
+ self,
+ *,
+ container: Union[ContainerHandle, str],
+ api_key: str | None = None,
+ api_base: str | None = None,
+ client: AsyncHTTPHandler | None = None,
+ **kwargs,
+ ) -> bool:
+ handle = self._as_handle(container)
+ key = (
+ api_key
+ or handle._hidden_params.get("api_key")
+ or self.validate_environment()
+ )
+ base = api_base or handle._hidden_params.get("api_base") or E2B_API_BASE
+ try:
+ response = cast(
+ httpx.Response,
+ await self._http(client).delete(
+ url=f"{base}/sandboxes/{handle.id}",
+ headers={"X-API-Key": key},
+ ),
+ )
+ except httpx.HTTPStatusError as e:
+ if e.response.status_code == 404:
+ return False
+ raise
+ return 200 <= response.status_code < 300
+
+ @staticmethod
+ def _as_handle(container: Union[ContainerHandle, str]) -> ContainerHandle:
+ if isinstance(container, ContainerHandle):
+ return container
+ handle = ContainerHandle(
+ id=str(container), provider="e2b", domain=E2B_DEFAULT_DOMAIN
+ )
+ handle._hidden_params = {}
+ return handle
+
+ @staticmethod
+ def _parse_lines(lines: list[str]) -> CodeExecutionResult:
+ def _try_parse(stripped: str):
+ try:
+ return json.loads(stripped)
+ except json.JSONDecodeError:
+ return None
+
+ messages = tuple(
+ parsed
+ for line in lines
+ if (stripped := line.strip())
+ if (parsed := _try_parse(stripped)) is not None
+ )
+
+ def of_type(message_type: str):
+ return (m for m in messages if m.get("type") == message_type)
+
+ error = next(
+ (
+ {key: m.get(key) for key in ("name", "value", "traceback")}
+ for m in of_type("error")
+ ),
+ None,
+ )
+ execution_count = next(
+ (m.get("execution_count") for m in of_type("number_of_executions")),
+ None,
+ )
+
+ return CodeExecutionResult(
+ stdout="".join(m.get("text", "") for m in of_type("stdout")),
+ stderr="".join(m.get("text", "") for m in of_type("stderr")),
+ results=[
+ {k: v for k, v in m.items() if k != "type"} for m in of_type("result")
+ ],
+ error=error,
+ execution_count=execution_count,
+ )
diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py
index 7a34ededa6b..5cfd14aeaa9 100644
--- a/litellm/llms/exa_ai/search/transformation.py
+++ b/litellm/llms/exa_ai/search/transformation.py
@@ -65,7 +65,13 @@ class ExaAISearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("EXA_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("EXA_API_KEY",),
+ base_env_var="EXA_API_BASE",
+ default_api_base=self.EXA_AI_API_BASE,
+ )
if not api_key:
raise ValueError(
"EXA_API_KEY is not set. Set `EXA_API_KEY` environment variable."
diff --git a/litellm/llms/fastcrw/search/transformation.py b/litellm/llms/fastcrw/search/transformation.py
index ce702266e7b..b571a659cac 100644
--- a/litellm/llms/fastcrw/search/transformation.py
+++ b/litellm/llms/fastcrw/search/transformation.py
@@ -57,7 +57,13 @@ class FastCRWSearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("CRW_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("CRW_API_KEY",),
+ base_env_var="CRW_API_BASE",
+ default_api_base=self.FASTCRW_API_BASE,
+ )
if not api_key:
raise ValueError(
"CRW_API_KEY is not set. Set `CRW_API_KEY` environment variable."
diff --git a/litellm/llms/firecrawl/search/transformation.py b/litellm/llms/firecrawl/search/transformation.py
index 18cf1d28c4d..7e01ba58706 100644
--- a/litellm/llms/firecrawl/search/transformation.py
+++ b/litellm/llms/firecrawl/search/transformation.py
@@ -61,7 +61,13 @@ class FirecrawlSearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("FIRECRAWL_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("FIRECRAWL_API_KEY",),
+ base_env_var="FIRECRAWL_API_BASE",
+ default_api_base=self.FIRECRAWL_API_BASE,
+ )
if not api_key:
raise ValueError(
"FIRECRAWL_API_KEY is not set. Set `FIRECRAWL_API_KEY` environment variable."
diff --git a/litellm/llms/fireworks_ai/audio_transcription/transformation.py b/litellm/llms/fireworks_ai/audio_transcription/transformation.py
deleted file mode 100644
index 00bb5f26797..00000000000
--- a/litellm/llms/fireworks_ai/audio_transcription/transformation.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from typing import List
-
-from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
-
-from ...openai.transcriptions.whisper_transformation import (
- OpenAIWhisperAudioTranscriptionConfig,
-)
-from ..common_utils import FireworksAIMixin
-
-
-class FireworksAIAudioTranscriptionConfig(
- FireworksAIMixin, OpenAIWhisperAudioTranscriptionConfig
-):
- def get_supported_openai_params(
- self, model: str
- ) -> List[OpenAIAudioTranscriptionOptionalParams]:
- return ["language", "prompt", "response_format", "timestamp_granularities"]
diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py
index cca3b3da37a..7e4395959b9 100644
--- a/litellm/llms/fireworks_ai/chat/transformation.py
+++ b/litellm/llms/fireworks_ai/chat/transformation.py
@@ -1,5 +1,15 @@
import json
-from typing import Any, List, Literal, Optional, Tuple, Union, cast
+from typing import (
+ Any,
+ AsyncIterator,
+ Iterator,
+ List,
+ Literal,
+ Optional,
+ Tuple,
+ Union,
+ cast,
+)
import httpx
@@ -15,7 +25,6 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import (
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
- ChatCompletionImageObject,
ChatCompletionToolParam,
OpenAIChatCompletionToolParam,
)
@@ -25,6 +34,7 @@ from litellm.types.utils import (
Function,
Message,
ModelResponse,
+ ModelResponseStream,
ProviderSpecificModelInfo,
)
from litellm.utils import (
@@ -34,10 +44,34 @@ from litellm.utils import (
supports_tool_choice,
)
-from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+from ...openai.chat.gpt_transformation import (
+ OpenAIChatCompletionStreamingHandler,
+ OpenAIGPTConfig,
+)
from ..common_utils import FireworksAIException
+def _extract_fireworks_hidden_params(payload: dict) -> dict:
+ """
+ Collect Fireworks-specific response fields (perf_metrics, prompt_token_ids,
+ per-choice raw_output and token_ids) from a non-streaming completion payload
+ or a single streaming chunk, so the same data lands in ``_hidden_params`` on
+ both response paths.
+ """
+ choices = [c for c in (payload.get("choices") or []) if isinstance(c, dict)]
+ top_level = {
+ f"fireworks_{field}": payload[field]
+ for field in ("perf_metrics", "prompt_token_ids")
+ if field in payload
+ }
+ per_choice = {
+ f"fireworks_{dest}": [c[field] for c in choices if field in c]
+ for field, dest in (("raw_output", "raw_outputs"), ("token_ids", "token_ids"))
+ if any(field in c for c in choices)
+ }
+ return {**top_level, **per_choice}
+
+
class FireworksAIConfig(OpenAIGPTConfig):
"""
Reference: https://docs.fireworks.ai/api-reference/post-chatcompletions
@@ -60,8 +94,7 @@ class FireworksAIConfig(OpenAIGPTConfig):
logprobs: Optional[int] = None
reasoning_effort: Optional[str] = None
- # Non OpenAI parameters - Fireworks AI only params
- prompt_truncate_length: Optional[int] = None
+ prompt_truncate_len: Optional[int] = None
context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None
def __init__(
@@ -80,7 +113,7 @@ class FireworksAIConfig(OpenAIGPTConfig):
user: Optional[str] = None,
logprobs: Optional[int] = None,
reasoning_effort: Optional[str] = None,
- prompt_truncate_length: Optional[int] = None,
+ prompt_truncate_len: Optional[int] = None,
context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None,
) -> None:
locals_ = locals().copy()
@@ -108,8 +141,30 @@ class FireworksAIConfig(OpenAIGPTConfig):
"response_format",
"user",
"logprobs",
- "prompt_truncate_length",
+ "prompt_truncate_len",
"context_length_exceeded_behavior",
+ "seed",
+ "top_logprobs",
+ "min_p",
+ "typical_p",
+ "repetition_penalty",
+ "mirostat_target",
+ "mirostat_lr",
+ "logit_bias",
+ "echo",
+ "echo_last",
+ "ignore_eos",
+ "prompt_cache_key",
+ "prompt_cache_isolation_key",
+ "raw_output",
+ "perf_metrics_in_response",
+ "return_token_ids",
+ "safe_tokenization",
+ "service_tier",
+ "speculation",
+ "prediction",
+ "stream_options",
+ "sampling_mask",
]
# Only add tools for models that support function calling
@@ -133,9 +188,11 @@ class FireworksAIConfig(OpenAIGPTConfig):
if supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"):
supported_params.append("tool_choice")
- # Only add reasoning_effort for models that support it
+ # Only add reasoning params for models that support it
if supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
supported_params.append("reasoning_effort")
+ supported_params.append("reasoning_history")
+ supported_params.append("thinking")
return supported_params
@@ -151,6 +208,18 @@ class FireworksAIConfig(OpenAIGPTConfig):
param == "tools" and value is not None
for param, value in non_default_params.items()
)
+ if (
+ non_default_params.get("thinking") is not None
+ and non_default_params.get("reasoning_effort") is not None
+ ):
+ raise litellm.BadRequestError(
+ message=(
+ "Fireworks AI chat completions does not support specifying both "
+ "`thinking` and `reasoning_effort` in the same request."
+ ),
+ model=model,
+ llm_provider="fireworks_ai",
+ )
for param, value in non_default_params.items():
if param == "tool_choice":
@@ -174,40 +243,19 @@ class FireworksAIConfig(OpenAIGPTConfig):
optional_params["response_format"] = value
elif param == "max_completion_tokens":
optional_params["max_tokens"] = value
+ elif param == "reasoning_effort":
+ if value is True:
+ optional_params["reasoning_effort"] = "medium"
+ elif value is False:
+ optional_params["reasoning_effort"] = "none"
+ else:
+ optional_params["reasoning_effort"] = value
elif param in supported_openai_params:
if value is not None:
optional_params[param] = value
return optional_params
- def _add_transform_inline_image_block(
- self,
- content: ChatCompletionImageObject,
- model: str,
- disable_add_transform_inline_image_block: Optional[bool],
- ) -> ChatCompletionImageObject:
- """
- Add transform_inline to the image_url (allows non-vision models to parse documents/images/etc.)
- - ignore if model is a vision model
- - ignore if user has disabled this feature
- """
- if (
- "vision" in model or disable_add_transform_inline_image_block
- ): # allow user to toggle this feature.
- return content
- if isinstance(content["image_url"], str):
- # Skip base64 data URLs — appending #transform=inline corrupts the
- # base64 payload and causes an "Incorrect padding" decode error on
- # the Fireworks side. Data URLs are already inlined by definition.
- # Lower-case before checking: URI schemes are case-insensitive (RFC 3986).
- if not content["image_url"].lower().startswith("data:"):
- content["image_url"] = f"{content['image_url']}#transform=inline"
- elif isinstance(content["image_url"], dict):
- url = content["image_url"]["url"]
- if not url.lower().startswith("data:"):
- content["image_url"]["url"] = f"{url}#transform=inline"
- return content
-
def _transform_tools(
self, tools: List[OpenAIChatCompletionToolParam]
) -> List[OpenAIChatCompletionToolParam]:
@@ -225,36 +273,46 @@ class FireworksAIConfig(OpenAIGPTConfig):
self, messages: List[AllMessageValues], model: str, litellm_params: dict
) -> List[AllMessageValues]:
"""
- Add 'transform=inline' to the url of the image_url
+ Strip fields not permitted by FireworksAI from messages.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
- migrate_file_to_image_url,
)
- disable_add_transform_inline_image_block = cast(
- Optional[bool],
- litellm_params.get("disable_add_transform_inline_image_block")
- or litellm.disable_add_transform_inline_image_block,
+ supports_vision_value = self._get_model_cost_capability_exact(
+ model=model, capability="supports_vision"
)
- ## For any 'file' message type with pdf content, move to 'image_url' message type
- for message in messages:
- if message["role"] == "user":
- _message_content = message.get("content")
- if _message_content is not None and isinstance(_message_content, list):
- for idx, content in enumerate(_message_content):
- if content["type"] == "file":
- _message_content[idx] = migrate_file_to_image_url(content)
for message in messages:
if message["role"] == "user":
_message_content = message.get("content")
if _message_content is not None and isinstance(_message_content, list):
for content in _message_content:
- if content["type"] == "image_url":
- content = self._add_transform_inline_image_block(
- content=content,
+ if not isinstance(content, dict):
+ continue
+ if content.get("type") == "file":
+ raise litellm.BadRequestError(
+ message=(
+ "Fireworks AI chat completions does not support "
+ "file content blocks. For PDFs, convert pages to "
+ "images and send image_url blocks to a Fireworks "
+ "vision model, or extract text before calling a "
+ "text-only model."
+ ),
model=model,
- disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
+ llm_provider="fireworks_ai",
+ )
+ if (
+ content.get("type") == "image_url"
+ and supports_vision_value is False
+ ):
+ raise litellm.BadRequestError(
+ message=(
+ f"Fireworks AI model {model} does not support "
+ "image inputs. Use a Fireworks vision model or "
+ "remove image_url content blocks."
+ ),
+ model=model,
+ llm_provider="fireworks_ai",
)
filter_value_from_dict(cast(dict, message), "cache_control")
# Remove fields not permitted by FireworksAI (additionalProperties: false
@@ -317,43 +375,55 @@ class FireworksAIConfig(OpenAIGPTConfig):
return True
return ("-" + key_short + "-") in short_name
- def _get_model_cost_capability(self, model: str, capability: str) -> Optional[bool]:
+ @staticmethod
+ def _short_model_name(model: str) -> str:
short_name = model
if short_name.startswith("fireworks_ai/"):
short_name = short_name[len("fireworks_ai/") :]
if short_name.startswith("accounts/fireworks/models/"):
short_name = short_name[len("accounts/fireworks/models/") :]
+ return short_name
- candidate_keys = [
+ def _get_model_cost_capability_exact(
+ self, model: str, capability: str
+ ) -> Optional[bool]:
+ short_name = self._short_model_name(model)
+ candidate_keys = (
model,
f"fireworks_ai/{short_name}",
f"fireworks_ai/accounts/fireworks/models/{short_name}",
- ]
-
+ )
for candidate_key in candidate_keys:
model_info = litellm.model_cost.get(candidate_key)
if model_info is not None and model_info.get(capability) is not None:
return cast(Optional[bool], model_info.get(capability))
+ return None
- # Fallback: preserve historical substring matching for model name
- # variants (e.g. fine-tuned or regionally-suffixed versions of a
- # known model). Pick the *longest* matching entry so a more specific
- # known model (e.g. "qwen3-8b-instruct") wins over a less specific
- # one (e.g. "qwen3-8b") when the query model is more specific still.
- # Use hyphen-aligned matching to avoid false positives where a short
- # known model name is an unrelated substring of a longer one.
- best_match_short: Optional[str] = None
- best_match_value: Optional[bool] = None
- for key_short, model_info in self._get_fireworks_index():
- if model_info.get(capability) is None:
- continue
- if not self._matches_on_hyphen_boundary(short_name, key_short):
- continue
- if best_match_short is None or len(key_short) > len(best_match_short):
- best_match_short = key_short
- best_match_value = cast(Optional[bool], model_info.get(capability))
+ def _get_model_cost_capability(self, model: str, capability: str) -> Optional[bool]:
+ exact = self._get_model_cost_capability_exact(
+ model=model, capability=capability
+ )
+ if exact is not None:
+ return exact
- return best_match_value
+ # Fallback: substring matching for model name variants (e.g. fine-tuned
+ # or regionally-suffixed versions of a known model). Pick the *longest*
+ # matching entry so a more specific known model (e.g. "qwen3-8b-instruct")
+ # wins over a less specific one (e.g. "qwen3-8b"). Hyphen-aligned matching
+ # avoids false positives where a short known name is an unrelated
+ # substring of a longer one. This stays a soft signal: capability-gated
+ # hard rejections use the exact lookup so a fuzzy match never blocks a
+ # custom deployment.
+ short_name = self._short_model_name(model)
+ matches = [
+ (key_short, cast(Optional[bool], model_info.get(capability)))
+ for key_short, model_info in self._get_fireworks_index()
+ if model_info.get(capability) is not None
+ and self._matches_on_hyphen_boundary(short_name, key_short)
+ ]
+ if not matches:
+ return None
+ return max(matches, key=lambda match: len(match[0]))[1]
def get_provider_info(self, model: str) -> ProviderSpecificModelInfo:
supports_function_calling_value = self._get_model_cost_capability(
@@ -362,12 +432,16 @@ class FireworksAIConfig(OpenAIGPTConfig):
supports_reasoning_value = self._get_model_cost_capability(
model=model, capability="supports_reasoning"
)
+ supports_vision_value = self._get_model_cost_capability(
+ model=model, capability="supports_vision"
+ )
+ supports_pdf_input_value = self._get_model_cost_capability(
+ model=model, capability="supports_pdf_input"
+ )
provider_specific_model_info: ProviderSpecificModelInfo = {
"supports_function_calling": True,
"supports_prompt_caching": True, # https://docs.fireworks.ai/guides/prompt-caching
- "supports_pdf_input": True, # via document inlining
- "supports_vision": True, # via document inlining
}
if supports_function_calling_value is not None:
@@ -381,6 +455,14 @@ class FireworksAIConfig(OpenAIGPTConfig):
supports_reasoning_value
)
+ if supports_vision_value is not None:
+ provider_specific_model_info["supports_vision"] = supports_vision_value
+
+ if supports_pdf_input_value is not None:
+ provider_specific_model_info["supports_pdf_input"] = (
+ supports_pdf_input_value
+ )
+
return provider_specific_model_info
def transform_request(
@@ -392,13 +474,25 @@ class FireworksAIConfig(OpenAIGPTConfig):
headers: dict,
) -> dict:
if not model.startswith("accounts/") and "#" not in model:
- model = f"accounts/fireworks/models/{model}"
+ if model.endswith("-fast"):
+ model = f"accounts/fireworks/routers/{model}"
+ else:
+ model = f"accounts/fireworks/models/{model}"
messages = self._transform_messages_helper(
messages=messages, model=model, litellm_params=litellm_params
)
if "tools" in optional_params and optional_params["tools"] is not None:
tools = self._transform_tools(tools=optional_params["tools"])
optional_params["tools"] = tools
+ if optional_params.get("stream"):
+ stream_options = optional_params.get("stream_options")
+ if stream_options is None:
+ optional_params["stream_options"] = {"include_usage": True}
+ elif stream_options.get("include_usage") is not False:
+ optional_params["stream_options"] = {
+ **stream_options,
+ "include_usage": True,
+ }
return super().transform_request(
model=model,
messages=messages,
@@ -491,10 +585,25 @@ class FireworksAIConfig(OpenAIGPTConfig):
)
)
- response._hidden_params = {"additional_headers": additional_headers}
+ response._hidden_params = {
+ "additional_headers": additional_headers,
+ **_extract_fireworks_hidden_params(completion_response),
+ }
return response
+ def get_model_response_iterator(
+ self,
+ streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+ sync_stream: bool,
+ json_mode: Optional[bool] = False,
+ ) -> Any:
+ return FireworksAIChatCompletionStreamingHandler(
+ streaming_response=streaming_response,
+ sync_stream=sync_stream,
+ json_mode=json_mode,
+ )
+
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
@@ -551,3 +660,15 @@ class FireworksAIConfig(OpenAIGPTConfig):
or get_secret_str("FIREWORKSAI_API_KEY")
or get_secret_str("FIREWORKS_AI_TOKEN")
)
+
+
+class FireworksAIChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
+ def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+ parsed = super().chunk_parser(chunk)
+ fireworks_fields = _extract_fireworks_hidden_params(chunk)
+ if fireworks_fields:
+ parsed.provider_specific_fields = {
+ **(getattr(parsed, "provider_specific_fields", None) or {}),
+ **fireworks_fields,
+ }
+ return parsed
diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py
index 74f6cd4d831..e153d00e6ab 100644
--- a/litellm/llms/gemini/realtime/transformation.py
+++ b/litellm/llms/gemini/realtime/transformation.py
@@ -103,6 +103,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# bypassing spend and budget accounting.
self._pending_usage_metadata: Optional[dict] = None
+ def _include_function_response_id(self) -> bool:
+ """Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it."""
+ return True
+
@staticmethod
def _usage_detail_alias(details: Any, defaults: Dict[str, int]) -> Dict[str, Any]:
if not isinstance(details, dict):
@@ -604,10 +608,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
)
# Build Gemini toolResponse format
- function_response = {
- "id": call_id,
- "response": output_dict,
- }
+ function_response: dict[str, Any] = {"response": output_dict}
+ if self._include_function_response_id() and call_id:
+ function_response["id"] = call_id
if function_name:
function_response["name"] = function_name
diff --git a/litellm/llms/google_pse/search/transformation.py b/litellm/llms/google_pse/search/transformation.py
index a8aa109cbf0..5cd3f2085a8 100644
--- a/litellm/llms/google_pse/search/transformation.py
+++ b/litellm/llms/google_pse/search/transformation.py
@@ -85,7 +85,13 @@ class GooglePSESearchConfig(BaseSearchConfig):
Google PSE uses API key as a query parameter, not in headers.
This method is called but headers are not used for authentication.
"""
- api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("GOOGLE_PSE_API_KEY",),
+ base_env_var="GOOGLE_PSE_API_BASE",
+ default_api_base=self.GOOGLE_PSE_API_BASE,
+ )
if not api_key:
raise ValueError(
"GOOGLE_PSE_API_KEY is not set. Set `GOOGLE_PSE_API_KEY` environment variable."
@@ -137,6 +143,7 @@ class GooglePSESearchConfig(BaseSearchConfig):
query: Union[str, List[str]],
optional_params: dict,
api_key: Optional[str] = None,
+ api_base: str | None = None,
search_engine_id: Optional[str] = None,
**kwargs,
) -> Dict:
@@ -165,8 +172,16 @@ class GooglePSESearchConfig(BaseSearchConfig):
# Google PSE only supports single string queries
query = " ".join(query)
- # Get API credentials
- api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY")
+ # Get API credentials. The key is sent as a query param to api_base, so
+ # resolve it host-aware to avoid leaking a server-managed key to a
+ # caller-supplied host.
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("GOOGLE_PSE_API_KEY",),
+ base_env_var="GOOGLE_PSE_API_BASE",
+ default_api_base=self.GOOGLE_PSE_API_BASE,
+ )
search_engine_id = search_engine_id or get_secret_str("GOOGLE_PSE_ENGINE_ID")
if not api_key:
diff --git a/litellm/llms/linkup/search/transformation.py b/litellm/llms/linkup/search/transformation.py
index 2b17d5642ac..d27ae038f9e 100644
--- a/litellm/llms/linkup/search/transformation.py
+++ b/litellm/llms/linkup/search/transformation.py
@@ -61,7 +61,13 @@ class LinkupSearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("LINKUP_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("LINKUP_API_KEY",),
+ base_env_var="LINKUP_API_BASE",
+ default_api_base=self.LINKUP_API_BASE,
+ )
if not api_key:
raise ValueError(
"LINKUP_API_KEY is not set. Set `LINKUP_API_KEY` environment variable."
diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py
index f1ad3708236..8d0cf993814 100644
--- a/litellm/llms/mistral/chat/transformation.py
+++ b/litellm/llms/mistral/chat/transformation.py
@@ -247,6 +247,8 @@ class MistralConfig(OpenAIGPTConfig):
The above statement is not valid now. Need to plan to remove all the #1,2,3
Mistral API supports content as a list.
"""
+ messages = [self._strip_output_only_fields(m) for m in messages]
+
## 1. If 'image_url' or 'file' in content, then transform with base class and mistral-specific handling
for m in messages:
_content_block = m.get("content")
@@ -409,6 +411,25 @@ class MistralConfig(OpenAIGPTConfig):
return cleaned_tools
+ @classmethod
+ def _strip_output_only_fields(cls, message: AllMessageValues) -> AllMessageValues:
+ """
+ ``reasoning_content`` and ``thinking_blocks`` are output-only fields that
+ LiteLLM attaches to assistant responses. Mistral's input schema forbids
+ unknown fields, so replaying them verbatim in a follow-up turn triggers a
+ 422 ``extra_forbidden``. Drop them before the request is sent.
+ """
+ if message["role"] != "assistant":
+ return message
+ return cast(
+ AllMessageValues,
+ {
+ k: v
+ for k, v in message.items()
+ if k not in ("reasoning_content", "thinking_blocks")
+ },
+ )
+
@classmethod
def _handle_name_in_message(cls, message: AllMessageValues) -> AllMessageValues:
"""
diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py
index 5464b5bb7ee..b8b750b8c12 100644
--- a/litellm/llms/openai/chat/gpt_transformation.py
+++ b/litellm/llms/openai/chat/gpt_transformation.py
@@ -18,6 +18,9 @@ from typing import (
overload,
)
+import os
+from urllib.parse import urlparse
+
import httpx
import litellm
@@ -426,6 +429,32 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
)
return messages, tools
+ def _should_preserve_cache_control_for_endpoint(
+ self,
+ custom_llm_provider: str | None,
+ api_base: str | None,
+ ) -> bool:
+ """
+ The generic `openai` provider also reaches OpenAI-compatible endpoints
+ (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom
+ api_base. Those can understand cache_control, so it must survive there.
+ Real OpenAI cannot, so it is still stripped for an openai.com host.
+ """
+ if custom_llm_provider != "openai":
+ return False
+ resolved_api_base = (
+ api_base
+ or litellm.api_base
+ or os.getenv("OPENAI_BASE_URL")
+ or os.getenv("OPENAI_API_BASE")
+ )
+ if not resolved_api_base:
+ return False
+ hostname = urlparse(resolved_api_base).hostname
+ if hostname is None:
+ return False
+ return hostname != "openai.com" and not hostname.endswith(".openai.com")
+
def transform_request(
self,
model: str,
@@ -441,11 +470,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
dict: The transformed request. Sent as the body of the API call.
"""
messages = self._transform_messages(messages=messages, model=model)
- messages, tools = self.remove_cache_control_flag_from_messages_and_tools(
- model=model, messages=messages, tools=optional_params.get("tools", [])
- )
- if tools is not None and len(tools) > 0:
- optional_params["tools"] = tools
+ if not self._should_preserve_cache_control_for_endpoint(
+ litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
+ ):
+ messages, tools = self.remove_cache_control_flag_from_messages_and_tools(
+ model=model, messages=messages, tools=optional_params.get("tools", [])
+ )
+ if tools is not None and len(tools) > 0:
+ optional_params["tools"] = tools
optional_params.pop("max_retries", None)
@@ -466,16 +498,19 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
transformed_messages = await self._transform_messages(
messages=messages, model=model, is_async=True
)
- (
- transformed_messages,
- tools,
- ) = self.remove_cache_control_flag_from_messages_and_tools(
- model=model,
- messages=transformed_messages,
- tools=optional_params.get("tools", []),
- )
- if tools is not None and len(tools) > 0:
- optional_params["tools"] = tools
+ if not self._should_preserve_cache_control_for_endpoint(
+ litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
+ ):
+ (
+ transformed_messages,
+ tools,
+ ) = self.remove_cache_control_flag_from_messages_and_tools(
+ model=model,
+ messages=transformed_messages,
+ tools=optional_params.get("tools", []),
+ )
+ if tools is not None and len(tools) > 0:
+ optional_params["tools"] = tools
if self.__class__._is_base_class:
return {
"model": model,
diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json
index 0dda047d1ca..d87346fea70 100644
--- a/litellm/llms/openai_like/providers.json
+++ b/litellm/llms/openai_like/providers.json
@@ -115,6 +115,14 @@
"max_completion_tokens": "max_tokens"
}
},
+ "darkbloom": {
+ "base_url": "https://api.darkbloom.dev/v1",
+ "api_key_env": "DARKBLOOM_API_KEY",
+ "api_base_env": "DARKBLOOM_API_BASE",
+ "param_mappings": {
+ "max_completion_tokens": "max_tokens"
+ }
+ },
"neosantara": {
"base_url": "https://api.neosantara.xyz/v1",
"api_key_env": "NEOSANTARA_API_KEY",
@@ -159,5 +167,14 @@
"max_completion_tokens": "max_tokens"
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
+ },
+ "pinstripes": {
+ "base_url": "https://pinstripes.io/v1",
+ "api_key_env": "PINSTRIPES_API_KEY",
+ "api_base_env": "PINSTRIPES_API_BASE",
+ "param_mappings": {
+ "max_completion_tokens": "max_tokens"
+ },
+ "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
}
}
diff --git a/litellm/llms/opensandbox/__init__.py b/litellm/llms/opensandbox/__init__.py
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/litellm/llms/opensandbox/__init__.py
@@ -0,0 +1 @@
+
diff --git a/litellm/llms/opensandbox/sandbox/__init__.py b/litellm/llms/opensandbox/sandbox/__init__.py
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/litellm/llms/opensandbox/sandbox/__init__.py
@@ -0,0 +1 @@
+
diff --git a/litellm/llms/opensandbox/sandbox/transformation.py b/litellm/llms/opensandbox/sandbox/transformation.py
new file mode 100644
index 00000000000..dc9f8440d30
--- /dev/null
+++ b/litellm/llms/opensandbox/sandbox/transformation.py
@@ -0,0 +1,598 @@
+import asyncio
+import json
+import time
+from typing import Union, cast
+
+import httpx
+
+from litellm.constants import (
+ OPEN_SANDBOX_API_BASE_ENV_VAR,
+ OPEN_SANDBOX_API_KEY_ENV_VAR,
+ OPEN_SANDBOX_DEFAULT_CPU_LIMIT,
+ OPEN_SANDBOX_DEFAULT_ENTRYPOINT,
+ OPEN_SANDBOX_DEFAULT_LANGUAGE,
+ OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT,
+ OPEN_SANDBOX_DEFAULT_TEMPLATE,
+ OPEN_SANDBOX_DEFAULT_TIMEOUT,
+ OPEN_SANDBOX_EXECD_PORT,
+ OPEN_SANDBOX_POLL_INTERVAL,
+ OPEN_SANDBOX_READY_TIMEOUT,
+)
+from litellm.llms.base_llm.sandbox.transformation import (
+ BaseSandboxConfig,
+ CodeExecutionResult,
+ ContainerHandle,
+ SANDBOX_MAX_OUTPUT_BYTES,
+)
+from litellm.llms.custom_httpx.http_handler import (
+ AsyncHTTPHandler,
+ get_async_httpx_client,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.custom_http import httpxSpecialProvider
+
+DEFAULT_SANDBOX_TIMEOUT = OPEN_SANDBOX_DEFAULT_TIMEOUT
+DEFAULT_READY_TIMEOUT = OPEN_SANDBOX_READY_TIMEOUT
+DEFAULT_POLL_INTERVAL = OPEN_SANDBOX_POLL_INTERVAL
+MAX_OUTPUT_BYTES = SANDBOX_MAX_OUTPUT_BYTES
+
+
+class OpenSandboxSandboxConfig(BaseSandboxConfig):
+ def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler:
+ if client is not None:
+ return client
+ return get_async_httpx_client(llm_provider=httpxSpecialProvider.Sandbox)
+
+ def validate_environment(self, api_key: str | None = None, **kwargs) -> str:
+ if api_key is not None:
+ return api_key
+ return get_secret_str(OPEN_SANDBOX_API_KEY_ENV_VAR) or ""
+
+ async def acreate_sandbox(
+ self,
+ *,
+ template: str | None = None,
+ timeout: int | None = None,
+ allow_internet_access: bool | None = None,
+ api_key: str | None = None,
+ api_base: str | None = None,
+ metadata: dict[str, str] | None = None,
+ env_vars: dict[str, str] | None = None,
+ resource_limits: dict[str, str] | None = None,
+ resource_requests: dict[str, str] | None = None,
+ entrypoint: list[str] | tuple[str, ...] | None = None,
+ network_policy: dict[str, object] | None = None,
+ secure_access: bool = False,
+ use_server_proxy: bool = False,
+ ready_timeout: float | None = None,
+ poll_interval: float | None = None,
+ client: AsyncHTTPHandler | None = None,
+ **kwargs,
+ ) -> ContainerHandle:
+ key = self.validate_environment(api_key=api_key)
+ base = self._api_base(api_base)
+ ready_timeout_seconds = (
+ float(ready_timeout) if ready_timeout is not None else DEFAULT_READY_TIMEOUT
+ )
+ poll_interval_seconds = (
+ float(poll_interval) if poll_interval is not None else DEFAULT_POLL_INTERVAL
+ )
+ body = self._create_body(
+ template=template,
+ timeout=timeout,
+ allow_internet_access=allow_internet_access,
+ metadata=metadata,
+ env_vars=env_vars,
+ resource_limits=resource_limits,
+ resource_requests=resource_requests,
+ entrypoint=entrypoint,
+ network_policy=network_policy,
+ secure_access=secure_access,
+ )
+
+ response = cast(
+ httpx.Response,
+ await self._http(client).post(
+ url=f"{base}/sandboxes",
+ headers=self._lifecycle_headers(key),
+ json=body,
+ ),
+ )
+ data = response.json()
+ sandbox_id = str(data["id"])
+
+ if self._sandbox_state(data) != "Running":
+ await self._wait_until_running(
+ sandbox_id=sandbox_id,
+ api_base=base,
+ headers=self._lifecycle_headers(key),
+ client=client,
+ ready_timeout=ready_timeout_seconds,
+ poll_interval=poll_interval_seconds,
+ )
+
+ endpoint, endpoint_headers = await self._wait_for_execd_endpoint(
+ sandbox_id=sandbox_id,
+ api_base=base,
+ headers=self._lifecycle_headers(key),
+ use_server_proxy=use_server_proxy,
+ client=client,
+ ready_timeout=ready_timeout_seconds,
+ poll_interval=poll_interval_seconds,
+ )
+
+ handle = ContainerHandle(id=sandbox_id, provider="opensandbox", domain=base)
+ handle._hidden_params = {
+ "api_base": base,
+ "api_key": key,
+ "execd_endpoint": endpoint,
+ "execd_headers": endpoint_headers,
+ "use_server_proxy": use_server_proxy,
+ }
+ return handle
+
+ async def arun_code(
+ self,
+ *,
+ container: Union[ContainerHandle, str],
+ code: str,
+ api_key: str | None = None,
+ api_base: str | None = None,
+ language: str = OPEN_SANDBOX_DEFAULT_LANGUAGE,
+ use_server_proxy: bool = False,
+ ready_timeout: float | None = None,
+ poll_interval: float | None = None,
+ client: AsyncHTTPHandler | None = None,
+ **kwargs,
+ ) -> CodeExecutionResult:
+ handle = await self._ensure_handle(
+ container=container,
+ api_key=api_key,
+ api_base=api_base,
+ use_server_proxy=use_server_proxy,
+ ready_timeout=(
+ float(ready_timeout)
+ if ready_timeout is not None
+ else DEFAULT_READY_TIMEOUT
+ ),
+ poll_interval=(
+ float(poll_interval)
+ if poll_interval is not None
+ else DEFAULT_POLL_INTERVAL
+ ),
+ client=client,
+ )
+ endpoint = str(handle._hidden_params["execd_endpoint"])
+ endpoint_headers = self._as_str_dict(handle._hidden_params.get("execd_headers"))
+ base = str(
+ handle._hidden_params.get("api_base")
+ or handle.domain
+ or self._api_base(api_base)
+ )
+ lines = await self._post_code(
+ url=f"{self._endpoint_base_url(endpoint, base)}/code",
+ headers={
+ "Content-Type": "application/json",
+ "Accept": "text/event-stream",
+ "Cache-Control": "no-cache",
+ **endpoint_headers,
+ },
+ body={
+ "code": code,
+ "context": {"language": language},
+ },
+ client=client,
+ )
+ return self._parse_lines(lines)
+
+ async def adelete_sandbox(
+ self,
+ *,
+ container: Union[ContainerHandle, str],
+ api_key: str | None = None,
+ api_base: str | None = None,
+ client: AsyncHTTPHandler | None = None,
+ **kwargs,
+ ) -> bool:
+ handle = self._as_handle(container, api_base=api_base)
+ base = str(handle._hidden_params.get("api_base") or self._api_base(api_base))
+ key = self._api_key(api_key=api_key, handle=handle)
+ try:
+ response = cast(
+ httpx.Response,
+ await self._http(client).delete(
+ url=f"{base}/sandboxes/{handle.id}",
+ headers=self._lifecycle_headers(key),
+ ),
+ )
+ except httpx.HTTPStatusError as e:
+ if e.response.status_code == 404:
+ return False
+ raise
+ return 200 <= response.status_code < 300
+
+ async def _ensure_handle(
+ self,
+ *,
+ container: Union[ContainerHandle, str],
+ api_key: str | None,
+ api_base: str | None,
+ use_server_proxy: bool,
+ ready_timeout: float,
+ poll_interval: float,
+ client: AsyncHTTPHandler | None,
+ ) -> ContainerHandle:
+ handle = self._as_handle(container, api_base=api_base)
+ if handle._hidden_params.get("execd_endpoint"):
+ return handle
+
+ base = str(handle._hidden_params.get("api_base") or self._api_base(api_base))
+ key = self._api_key(api_key=api_key, handle=handle)
+ resolved_use_server_proxy = bool(
+ handle._hidden_params.get("use_server_proxy", use_server_proxy)
+ )
+ endpoint, endpoint_headers = await self._wait_for_execd_endpoint(
+ sandbox_id=handle.id,
+ api_base=base,
+ headers=self._lifecycle_headers(key),
+ use_server_proxy=resolved_use_server_proxy,
+ client=client,
+ ready_timeout=ready_timeout,
+ poll_interval=poll_interval,
+ )
+ handle.domain = base
+ handle._hidden_params = {
+ **handle._hidden_params,
+ "api_base": base,
+ "api_key": key,
+ "execd_endpoint": endpoint,
+ "execd_headers": endpoint_headers,
+ "use_server_proxy": resolved_use_server_proxy,
+ }
+ return handle
+
+ async def _wait_until_running(
+ self,
+ *,
+ sandbox_id: str,
+ api_base: str,
+ headers: dict[str, str],
+ client: AsyncHTTPHandler | None,
+ ready_timeout: float,
+ poll_interval: float,
+ ) -> None:
+ deadline = time.monotonic() + ready_timeout
+ while True:
+ response = cast(
+ httpx.Response,
+ await self._http(client).get(
+ url=f"{api_base}/sandboxes/{sandbox_id}",
+ headers=headers,
+ ),
+ )
+ data = response.json()
+ state = self._sandbox_state(data)
+ if state == "Running":
+ return
+ if state in {"Failed", "Stopping", "Terminated"}:
+ raise ValueError(f"OpenSandbox sandbox {sandbox_id} entered {state}")
+ if time.monotonic() >= deadline:
+ raise TimeoutError(
+ f"OpenSandbox sandbox {sandbox_id} was not Running within "
+ f"{ready_timeout} seconds"
+ )
+ await asyncio.sleep(poll_interval)
+
+ async def _wait_for_execd_endpoint(
+ self,
+ *,
+ sandbox_id: str,
+ api_base: str,
+ headers: dict[str, str],
+ use_server_proxy: bool,
+ client: AsyncHTTPHandler | None,
+ ready_timeout: float,
+ poll_interval: float,
+ ) -> tuple[str, dict[str, str]]:
+ deadline = time.monotonic() + ready_timeout
+ last_error: Exception | None = None
+ while True:
+ try:
+ return await self._get_execd_endpoint(
+ sandbox_id=sandbox_id,
+ api_base=api_base,
+ headers=headers,
+ use_server_proxy=use_server_proxy,
+ client=client,
+ )
+ except httpx.HTTPStatusError as e:
+ if e.response.status_code != 404:
+ raise
+ last_error = e
+ except ValueError as e:
+ last_error = e
+
+ if time.monotonic() >= deadline:
+ raise TimeoutError(
+ f"OpenSandbox execd endpoint for {sandbox_id} was not ready within "
+ f"{ready_timeout} seconds"
+ ) from last_error
+ await asyncio.sleep(poll_interval)
+
+ async def _get_execd_endpoint(
+ self,
+ *,
+ sandbox_id: str,
+ api_base: str,
+ headers: dict[str, str],
+ use_server_proxy: bool,
+ client: AsyncHTTPHandler | None,
+ ) -> tuple[str, dict[str, str]]:
+ response = cast(
+ httpx.Response,
+ await self._http(client).get(
+ url=f"{api_base}/sandboxes/{sandbox_id}/endpoints/{OPEN_SANDBOX_EXECD_PORT}",
+ headers=headers,
+ params={"use_server_proxy": use_server_proxy},
+ ),
+ )
+ data = response.json()
+ endpoint = data.get("endpoint")
+ if not endpoint:
+ raise ValueError(
+ f"OpenSandbox did not return an execd endpoint for {sandbox_id}"
+ )
+ return str(endpoint), self._as_str_dict(data.get("headers"))
+
+ async def _post_code(
+ self,
+ *,
+ url: str,
+ headers: dict[str, str],
+ body: dict[str, object],
+ client: AsyncHTTPHandler | None,
+ ) -> list[str]:
+ timeout = httpx.Timeout(connect=30.0, read=None, write=30.0, pool=None)
+ response = cast(
+ httpx.Response,
+ await self._http(client).post(
+ url=url,
+ headers=headers,
+ timeout=timeout,
+ json=body,
+ stream=True,
+ ),
+ )
+ return await self._read_capped_lines(response)
+
+ def _api_key(self, *, api_key: str | None, handle: ContainerHandle) -> str:
+ if api_key is not None:
+ return api_key
+ if "api_key" in handle._hidden_params:
+ return str(handle._hidden_params["api_key"])
+ return self.validate_environment()
+
+ @staticmethod
+ def _create_body(
+ *,
+ template: str | None,
+ timeout: int | None,
+ allow_internet_access: bool | None,
+ metadata: dict[str, str] | None,
+ env_vars: dict[str, str] | None,
+ resource_limits: dict[str, str] | None,
+ resource_requests: dict[str, str] | None,
+ entrypoint: list[str] | tuple[str, ...] | None,
+ network_policy: dict[str, object] | None,
+ secure_access: bool,
+ ) -> dict[str, object]:
+ body: dict[str, object] = {
+ "image": {"uri": template or OPEN_SANDBOX_DEFAULT_TEMPLATE},
+ "entrypoint": list(entrypoint or OPEN_SANDBOX_DEFAULT_ENTRYPOINT),
+ "timeout": timeout if timeout is not None else DEFAULT_SANDBOX_TIMEOUT,
+ "resourceLimits": resource_limits
+ or OpenSandboxSandboxConfig._default_resource_limits(),
+ }
+ if metadata:
+ body["metadata"] = metadata
+ if env_vars:
+ body["env"] = env_vars
+ if resource_requests:
+ body["resourceRequests"] = resource_requests
+ if network_policy is not None:
+ body["networkPolicy"] = network_policy
+ elif allow_internet_access is not True:
+ body["networkPolicy"] = {"defaultAction": "deny", "egress": []}
+ if secure_access:
+ body["secureAccess"] = True
+ return body
+
+ @staticmethod
+ def _default_resource_limits() -> dict[str, str]:
+ return {
+ "cpu": OPEN_SANDBOX_DEFAULT_CPU_LIMIT,
+ "memory": OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT,
+ }
+
+ @staticmethod
+ def _sandbox_state(data: object) -> str | None:
+ if not isinstance(data, dict):
+ return None
+ status = data.get("status")
+ if not isinstance(status, dict):
+ return None
+ state = status.get("state")
+ return str(state) if state is not None else None
+
+ @staticmethod
+ def _as_str_dict(value: object) -> dict[str, str]:
+ if not isinstance(value, dict):
+ return {}
+ return {str(k): str(v) for k, v in value.items()}
+
+ @staticmethod
+ def _api_base(api_base: str | None) -> str:
+ base = api_base or get_secret_str(OPEN_SANDBOX_API_BASE_ENV_VAR)
+ if not base:
+ raise ValueError(
+ "OpenSandbox api_base is required. Pass api_base or set "
+ f"{OPEN_SANDBOX_API_BASE_ENV_VAR}."
+ )
+ return str(base).rstrip("/")
+
+ @staticmethod
+ def _lifecycle_headers(api_key: str) -> dict[str, str]:
+ headers = {"Content-Type": "application/json"}
+ if api_key:
+ headers["OPEN-SANDBOX-API-KEY"] = api_key
+ return headers
+
+ @staticmethod
+ def _endpoint_base_url(endpoint: str, api_base: str) -> str:
+ normalized_endpoint = endpoint.rstrip("/")
+ if normalized_endpoint.startswith(("http://", "https://")):
+ return normalized_endpoint
+ protocol = api_base.split("://", 1)[0] if "://" in api_base else "http"
+ return f"{protocol}://{normalized_endpoint}"
+
+ @staticmethod
+ def _as_handle(
+ container: Union[ContainerHandle, str], *, api_base: str | None
+ ) -> ContainerHandle:
+ if isinstance(container, ContainerHandle):
+ return container
+ handle = ContainerHandle(
+ id=str(container),
+ provider="opensandbox",
+ domain=OpenSandboxSandboxConfig._api_base(api_base),
+ )
+ handle._hidden_params = {}
+ return handle
+
+ @staticmethod
+ def _parse_lines(lines: list[str]) -> CodeExecutionResult:
+ messages = tuple(
+ event
+ for line in lines
+ if (event := OpenSandboxSandboxConfig._parse_sse_line(line)) is not None
+ )
+
+ def of_type(message_type: str):
+ return (m for m in messages if m.get("type") == message_type)
+
+ error = next(
+ (OpenSandboxSandboxConfig._normalize_error(m) for m in of_type("error")),
+ None,
+ )
+ execution_count = next(
+ (
+ OpenSandboxSandboxConfig._as_int(m.get("execution_count"))
+ for m in of_type("execution_count")
+ if OpenSandboxSandboxConfig._as_int(m.get("execution_count"))
+ is not None
+ ),
+ None,
+ )
+
+ return CodeExecutionResult(
+ stdout="".join(str(m.get("text", "")) for m in of_type("stdout")),
+ stderr="".join(str(m.get("text", "")) for m in of_type("stderr")),
+ results=[
+ OpenSandboxSandboxConfig._normalize_result(m) for m in of_type("result")
+ ],
+ error=error,
+ execution_count=execution_count,
+ )
+
+ @staticmethod
+ def _parse_sse_line(line: str) -> dict[str, object] | None:
+ stripped = line.strip()
+ if not stripped or stripped.startswith(
+ (
+ ":",
+ "event:",
+ "id:",
+ "retry:",
+ )
+ ):
+ return None
+ data = stripped[5:].strip() if stripped.startswith("data:") else stripped
+ if not data:
+ return None
+ try:
+ parsed = json.loads(data)
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(parsed, dict):
+ return None
+ if "type" not in parsed and "code" in parsed and "message" in parsed:
+ return {
+ "type": "error",
+ "error": {
+ "ename": str(parsed["code"]),
+ "evalue": str(parsed["message"]),
+ "traceback": [],
+ },
+ }
+ return parsed
+
+ @staticmethod
+ def _normalize_result(message: dict[str, object]) -> dict[str, object]:
+ results = message.get("results")
+ if isinstance(results, dict):
+ return {str(k): v for k, v in results.items()}
+ return {
+ str(k): v
+ for k, v in message.items()
+ if k not in {"type", "timestamp", "execution_count"}
+ }
+
+ @staticmethod
+ def _normalize_error(message: dict[str, object]) -> dict[str, object]:
+ raw_error = message.get("error")
+ if isinstance(raw_error, dict):
+ name = OpenSandboxSandboxConfig._first_non_none_value(
+ raw_error, "ename", "name", default=""
+ )
+ value = OpenSandboxSandboxConfig._first_non_none_value(
+ raw_error, "evalue", "value", default=""
+ )
+ traceback = OpenSandboxSandboxConfig._first_non_none_value(
+ raw_error, "traceback", default=[]
+ )
+ return {
+ "name": name,
+ "value": value,
+ "traceback": traceback,
+ }
+ return {
+ "name": OpenSandboxSandboxConfig._first_non_none_value(
+ message, "name", default=""
+ ),
+ "value": OpenSandboxSandboxConfig._first_non_none_value(
+ message, "value", "text", default=""
+ ),
+ "traceback": OpenSandboxSandboxConfig._first_non_none_value(
+ message, "traceback", default=[]
+ ),
+ }
+
+ @staticmethod
+ def _as_int(value: object) -> int | None:
+ if isinstance(value, int):
+ return value
+ if isinstance(value, str):
+ try:
+ return int(value)
+ except ValueError:
+ return None
+ return None
+
+ @staticmethod
+ def _first_non_none_value(
+ values: dict[str, object], *keys: str, default: object
+ ) -> object:
+ return next(
+ (values[key] for key in keys if key in values and values[key] is not None),
+ default,
+ )
diff --git a/litellm/llms/parallel_ai/search/transformation.py b/litellm/llms/parallel_ai/search/transformation.py
index 85602bf1d86..35a0d84df40 100644
--- a/litellm/llms/parallel_ai/search/transformation.py
+++ b/litellm/llms/parallel_ai/search/transformation.py
@@ -67,10 +67,12 @@ class ParallelAISearchConfig(BaseSearchConfig):
api_base: Optional[str] = None,
**kwargs,
) -> Dict:
- api_key = (
- api_key
- or get_secret_str("PARALLEL_AI_API_KEY")
- or get_secret_str("PARALLEL_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"),
+ base_env_var="PARALLEL_AI_API_BASE",
+ default_api_base=self.PARALLEL_AI_API_BASE,
)
if not api_key:
raise ValueError(
diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py
index 0f9c3cad841..ec7ec397ea6 100644
--- a/litellm/llms/perplexity/cost_calculator.py
+++ b/litellm/llms/perplexity/cost_calculator.py
@@ -58,11 +58,8 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
## CALCULATE OUTPUT COST
output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token"))
- completion_cost: float = (usage.completion_tokens or 0) * output_cost_per_token
- ## ADD REASONING TOKENS COST (if present)
reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0
- # Also check completion_tokens_details if reasoning_tokens is not directly available
if (
reasoning_tokens == 0
and hasattr(usage, "completion_tokens_details")
@@ -73,9 +70,19 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
)
reasoning_cost_value = model_info.get("output_cost_per_reasoning_token")
+
+ # `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage
+ # convention (codified for the central path in PR #18607). When a reasoning rate is
+ # configured we subtract before the output-rate multiplication so the reasoning
+ # tokens are not billed twice.
if reasoning_tokens > 0 and reasoning_cost_value is not None:
- reasoning_cost_per_token = _safe_float_cast(reasoning_cost_value)
- completion_cost += reasoning_tokens * reasoning_cost_per_token
+ non_reasoning_completion_tokens = max(
+ 0, (usage.completion_tokens or 0) - reasoning_tokens
+ )
+ completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token
+ completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value)
+ else:
+ completion_cost = (usage.completion_tokens or 0) * output_cost_per_token
## ADD SEARCH QUERIES COST (if present)
num_search_queries = 0
@@ -91,10 +98,11 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
if num_search_queries > 0 and search_cost_value is not None:
# Handle both dict and float formats
if isinstance(search_cost_value, dict):
- # Use the "low" size as default - tests expect 0.005 / 1000
- search_cost_per_query = (
- _safe_float_cast(search_cost_value.get("search_context_size_low", 0))
- / 1000
+ # search_context_cost_per_query stores the per-request price in USD
+ # (e.g. sonar low = $0.005/request). Use it directly, matching the
+ # gemini cost calculator which reads the same field per request.
+ search_cost_per_query = _safe_float_cast(
+ search_cost_value.get("search_context_size_low", 0)
)
else:
search_cost_per_query = _safe_float_cast(search_cost_value)
diff --git a/litellm/llms/perplexity/search/transformation.py b/litellm/llms/perplexity/search/transformation.py
index ea96f87957c..55de52c5384 100644
--- a/litellm/llms/perplexity/search/transformation.py
+++ b/litellm/llms/perplexity/search/transformation.py
@@ -50,7 +50,13 @@ class PerplexitySearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("PERPLEXITYAI_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("PERPLEXITYAI_API_KEY",),
+ base_env_var="PERPLEXITY_API_BASE",
+ default_api_base=self.PERPLEXITY_API_BASE,
+ )
if not api_key:
raise ValueError(
"PERPLEXITYAI_API_KEY is not set. Set `PERPLEXITYAI_API_KEY` environment variable."
diff --git a/litellm/llms/searchapi/search/transformation.py b/litellm/llms/searchapi/search/transformation.py
index c04e1377f9c..ae8413684cc 100644
--- a/litellm/llms/searchapi/search/transformation.py
+++ b/litellm/llms/searchapi/search/transformation.py
@@ -74,7 +74,13 @@ class SearchAPIConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("SEARCHAPI_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("SEARCHAPI_API_KEY",),
+ base_env_var="SEARCHAPI_API_BASE",
+ default_api_base=self.SEARCHAPI_API_BASE,
+ )
if not api_key:
raise ValueError(
@@ -114,6 +120,7 @@ class SearchAPIConfig(BaseSearchConfig):
query: Union[str, List[str]],
optional_params: dict,
api_key: Optional[str] = None,
+ api_base: str | None = None,
search_engine_id: Optional[str] = None,
**kwargs,
) -> Dict:
@@ -137,8 +144,16 @@ class SearchAPIConfig(BaseSearchConfig):
if isinstance(query, list):
query = " ".join(query)
- # Get API key from parameter or environment
- api_key = api_key or get_secret_str("SEARCHAPI_API_KEY")
+ # Get API key from parameter or environment. The key is sent as a query
+ # param to api_base, so resolve it host-aware to avoid leaking a
+ # server-managed key to a caller-supplied host.
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("SEARCHAPI_API_KEY",),
+ base_env_var="SEARCHAPI_API_BASE",
+ default_api_base=self.SEARCHAPI_API_BASE,
+ )
if not api_key:
raise ValueError(
"SEARCHAPI_API_KEY is not set. Set `SEARCHAPI_API_KEY` environment variable."
diff --git a/litellm/llms/searxng/search/transformation.py b/litellm/llms/searxng/search/transformation.py
index ee6f3895721..ff68be5709e 100644
--- a/litellm/llms/searxng/search/transformation.py
+++ b/litellm/llms/searxng/search/transformation.py
@@ -61,7 +61,13 @@ class SearXNGSearchConfig(BaseSearchConfig):
Some instances may require authentication via headers.
"""
# SearXNG typically doesn't require API keys, but support optional auth
- api_key = api_key or get_secret_str("SEARXNG_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("SEARXNG_API_KEY",),
+ base_env_var="SEARXNG_API_BASE",
+ default_api_base=None,
+ )
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
headers["Content-Type"] = "application/json"
diff --git a/litellm/llms/serper/search/transformation.py b/litellm/llms/serper/search/transformation.py
index 0daccbe652b..dd43f2d2dc9 100644
--- a/litellm/llms/serper/search/transformation.py
+++ b/litellm/llms/serper/search/transformation.py
@@ -55,7 +55,13 @@ class SerperSearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("SERPER_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("SERPER_API_KEY",),
+ base_env_var="SERPER_API_BASE",
+ default_api_base=self.SERPER_API_BASE,
+ )
if not api_key:
raise ValueError(
"SERPER_API_KEY is not set. Set `SERPER_API_KEY` environment variable."
diff --git a/litellm/llms/tavily/search/transformation.py b/litellm/llms/tavily/search/transformation.py
index ec96db96f36..647cfb5fa84 100644
--- a/litellm/llms/tavily/search/transformation.py
+++ b/litellm/llms/tavily/search/transformation.py
@@ -64,7 +64,13 @@ class TavilySearchConfig(BaseSearchConfig):
"""
Validate environment and return headers.
"""
- api_key = api_key or get_secret_str("TAVILY_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("TAVILY_API_KEY",),
+ base_env_var="TAVILY_API_BASE",
+ default_api_base=self.TAVILY_API_BASE,
+ )
if not api_key:
raise ValueError(
"TAVILY_API_KEY is not set. Set `TAVILY_API_KEY` environment variable."
diff --git a/litellm/llms/tinyfish/search/__init__.py b/litellm/llms/tinyfish/search/__init__.py
new file mode 100644
index 00000000000..9777e735aac
--- /dev/null
+++ b/litellm/llms/tinyfish/search/__init__.py
@@ -0,0 +1,3 @@
+from litellm.llms.tinyfish.search.transformation import TinyfishSearchConfig
+
+__all__ = ["TinyfishSearchConfig"]
diff --git a/litellm/llms/tinyfish/search/transformation.py b/litellm/llms/tinyfish/search/transformation.py
new file mode 100644
index 00000000000..b92f7ca1aff
--- /dev/null
+++ b/litellm/llms/tinyfish/search/transformation.py
@@ -0,0 +1,170 @@
+"""
+TinyFish Search API.
+Endpoint: GET https://api.search.tinyfish.ai
+Docs: https://docs.tinyfish.ai/search-api
+"""
+
+from __future__ import annotations
+
+from typing import Literal, TypedDict
+from urllib.parse import urlencode
+
+import httpx
+from pydantic import BaseModel, TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.search.transformation import (
+ BaseSearchConfig,
+ SearchResponse,
+ SearchResult,
+)
+from litellm.secret_managers.main import get_secret_str
+
+
+class _TinyfishSearchRequestRequired(TypedDict):
+ query: str
+
+
+class TinyfishSearchRequest(_TinyfishSearchRequestRequired, total=False):
+ location: str
+ language: str
+ page: int
+ include_thumbnail: bool
+ max_results: int
+
+
+class _TinyfishResultItem(BaseModel, frozen=True):
+ title: str = ""
+ url: str = ""
+ snippet: str = ""
+
+
+class _TinyfishApiResponse(BaseModel, frozen=True):
+ results: tuple[_TinyfishResultItem, ...] = ()
+
+
+_UrlEncodableParams = TypeAdapter(dict[str, str | int | bool])
+_StrList = TypeAdapter(list[str])
+_StrFrozenSet = TypeAdapter(frozenset[str])
+
+_TINYFISH_PARAMS_KEY = "_tinyfish_params"
+
+
+class TinyfishSearchConfig(BaseSearchConfig):
+ TINYFISH_API_BASE = "https://api.search.tinyfish.ai"
+
+ @staticmethod
+ def ui_friendly_name() -> str:
+ return "TinyFish"
+
+ def get_http_method(self) -> Literal["GET", "POST"]:
+ return "GET"
+
+ def validate_environment(
+ self,
+ headers: dict[str, str],
+ api_key: str | None = None,
+ api_base: str | None = None,
+ **kwargs: object,
+ ) -> dict[str, str]:
+ resolved_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("TINYFISH_API_KEY",),
+ base_env_var="TINYFISH_API_BASE",
+ default_api_base=self.TINYFISH_API_BASE,
+ )
+ if not resolved_key:
+ raise ValueError(
+ "TINYFISH_API_KEY is not set. Set `TINYFISH_API_KEY` environment variable."
+ )
+ return {**headers, "X-API-Key": resolved_key, "Accept": "application/json"}
+
+ def get_complete_url(
+ self,
+ api_base: str | None,
+ optional_params: dict[str, object],
+ data: dict[str, object] | list[dict[str, object]] | None = None,
+ **kwargs: object,
+ ) -> str:
+ resolved_base = (
+ api_base or get_secret_str("TINYFISH_API_BASE") or self.TINYFISH_API_BASE
+ )
+ if isinstance(data, dict) and _TINYFISH_PARAMS_KEY in data:
+ validated_params = _UrlEncodableParams.validate_python(
+ data[_TINYFISH_PARAMS_KEY]
+ )
+ return f"{resolved_base}?{urlencode(validated_params, doseq=True)}"
+ return resolved_base
+
+ def transform_search_request(
+ self,
+ query: str | list[str],
+ optional_params: dict[str, object],
+ **kwargs: object,
+ ) -> dict[str, object]:
+ resolved_query = " ".join(query) if isinstance(query, list) else query
+
+ request_data: TinyfishSearchRequest = {"query": resolved_query}
+
+ country = optional_params.get("country")
+ if isinstance(country, str):
+ request_data["location"] = country
+
+ raw_max = optional_params.get("max_results")
+ if isinstance(raw_max, (int, float, str)):
+ request_data["max_results"] = max(1, min(int(raw_max), 20))
+
+ try:
+ domains = _StrList.validate_python(
+ optional_params.get("search_domain_filter")
+ )
+ except (ValidationError, TypeError):
+ domains = []
+ if domains:
+ request_data["query"] = _append_domain_filters(
+ request_data["query"], domains
+ )
+
+ result_data: dict[str, object] = dict(request_data)
+
+ raw_supported: object = (
+ self.get_supported_perplexity_optional_params() # any-ok: base class returns bare set
+ )
+ supported_perplexity = _StrFrozenSet.validate_python(raw_supported)
+ for param, value in optional_params.items():
+ if param not in supported_perplexity and param not in result_data:
+ result_data[param] = value
+
+ return {_TINYFISH_PARAMS_KEY: result_data}
+
+ def transform_search_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj | None,
+ **kwargs: object,
+ ) -> SearchResponse:
+ raw_json: object = raw_response.json() # any-ok: httpx Response.json() -> Any
+ parsed = _TinyfishApiResponse.model_validate(raw_json)
+
+ max_results_str: str = "20"
+ if raw_response.request:
+ raw_param: object = (
+ raw_response.request.url.params.get( # any-ok: httpx QueryParams.get() -> Any
+ "max_results", "20"
+ )
+ )
+ max_results_str = str(raw_param)
+ max_results: int = min(int(max_results_str), 20)
+
+ results = [
+ SearchResult(title=item.title, url=item.url, snippet=item.snippet)
+ for item in parsed.results[:max_results]
+ ]
+
+ return SearchResponse(results=results, object="search")
+
+
+def _append_domain_filters(query: str, domains: list[str]) -> str:
+ domain_clauses = " OR ".join(f"site:{d}" for d in domains)
+ return f"({query}) ({domain_clauses})"
diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py
index 85c23d8603c..5028c0cf5c8 100644
--- a/litellm/llms/vertex_ai/common_utils.py
+++ b/litellm/llms/vertex_ai/common_utils.py
@@ -271,7 +271,7 @@ def supports_response_json_schema(model: str) -> bool:
# Gemini 2.0+ and 2.5+ models support responseJsonSchema
# Pattern matches: gemini-2.0-*, gemini-2.5-*, gemini-3-*, etc.
- gemini_2_plus_pattern = re.compile(r"gemini-([2-9]|[1-9]\d+)\.")
+ gemini_2_plus_pattern = re.compile(r"gemini-(?:[2-9]|[1-9]\d+)(?:\.|\-)")
return bool(gemini_2_plus_pattern.search(model_lower))
diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py
index d6441db7856..1fe9f15c9f0 100644
--- a/litellm/llms/vertex_ai/realtime/transformation.py
+++ b/litellm/llms/vertex_ai/realtime/transformation.py
@@ -32,6 +32,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
self._project = project
self._location = location
+ def _include_function_response_id(self) -> bool:
+ return False
+
# ------------------------------------------------------------------
# URL
# ------------------------------------------------------------------
diff --git a/litellm/llms/watsonx/embed/transformation.py b/litellm/llms/watsonx/embed/transformation.py
index 930212e3ef3..ae873d63fe9 100644
--- a/litellm/llms/watsonx/embed/transformation.py
+++ b/litellm/llms/watsonx/embed/transformation.py
@@ -43,8 +43,19 @@ class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig):
api_params=watsonx_api_params,
)
+ if isinstance(input, str):
+ inputs: list[str] = [input]
+ elif isinstance(input, list):
+ if len(input) > 0 and isinstance(input[0], (list, int)):
+ raise ValueError(
+ "WatsonX embeddings require a string or list of strings"
+ )
+ inputs = input
+ else:
+ inputs = [input]
+
return {
- "inputs": input,
+ "inputs": inputs,
"parameters": optional_params,
**watsonx_auth_payload,
}
diff --git a/litellm/llms/you_com/search/transformation.py b/litellm/llms/you_com/search/transformation.py
index 3c94b991735..0c7916e4c05 100644
--- a/litellm/llms/you_com/search/transformation.py
+++ b/litellm/llms/you_com/search/transformation.py
@@ -64,7 +64,13 @@ class YouComSearchConfig(BaseSearchConfig):
endpoint with the `X-API-Key` header. Otherwise fall through to the
keyless free tier; no auth header is required.
"""
- api_key = api_key or get_secret_str("YOUCOM_API_KEY")
+ api_key = self.resolve_server_api_key(
+ caller_api_key=api_key,
+ caller_api_base=api_base,
+ key_env_vars=("YOUCOM_API_KEY",),
+ base_env_var="YOUCOM_API_BASE",
+ default_api_base=self.YOU_COM_API_BASE,
+ )
headers["Content-Type"] = "application/json"
# Pin Accept-Encoding to identity: the keyless `api.you.com/v1/agents/search`
# endpoint advertises gzip content-encoding but returns body bytes the
diff --git a/litellm/main.py b/litellm/main.py
index 63c5798e70a..c3d7ca28c49 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -81,11 +81,17 @@ from litellm.constants import (
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.asyncify import run_async_function
+from litellm.litellm_core_utils.chat_completion_agentic_loop import (
+ maybe_run_chat_completion_agentic_loop,
+)
from litellm.litellm_core_utils.audio_utils.utils import (
calculate_request_duration,
get_audio_file_for_health_check,
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
+from litellm.litellm_core_utils.request_timeout_resolver import (
+ get_configured_request_timeout,
+)
from litellm.litellm_core_utils.get_litellm_params import OPTIONAL_KWARGS_KEYS
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_provider_specific_headers import (
@@ -118,6 +124,10 @@ from litellm.llms.vertex_ai.common_utils import (
)
from litellm.realtime_api.main import _realtime_health_check
from litellm.secret_managers.main import get_secret_bool, get_secret_str
+from litellm.types.completion import (
+ _CompletionDispatchContext,
+ _CompletionDispatchResult,
+)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CustomPricingLiteLLMParams,
@@ -650,6 +660,39 @@ async def acompletion(
response_object=response,
model_response_object=litellm.ModelResponse(),
)
+ # Provider-agnostic dispatch point for the chat-completions agentic loop
+ # (code-interpreter interception, etc). Chat routing forks per provider
+ # before this (OpenAI goes through the OpenAI SDK in openai.py, others
+ # through the shared httpx handler), so a dispatch inside any single
+ # provider handler would miss the others. Here is where every fork
+ # reconverges, so the loop runs once for all providers. Responses needs
+ # no equivalent: every provider already funnels through one shared
+ # handler where the loop is dispatched.
+ if isinstance(response, litellm.ModelResponse):
+ looped = await maybe_run_chat_completion_agentic_loop(
+ response=response,
+ model=model,
+ messages=messages,
+ optional_params={
+ k: v
+ for k, v in completion_kwargs.items()
+ if v is not None
+ and k
+ not in (
+ "model",
+ "messages",
+ "stream",
+ "acompletion",
+ "deployment_id",
+ )
+ },
+ kwargs=kwargs,
+ logging_obj=kwargs.get("litellm_logging_obj"),
+ custom_llm_provider=custom_llm_provider,
+ stream=bool(stream),
+ )
+ if looped is not None:
+ response = looped
if isinstance(response, CustomStreamWrapper):
response.set_logging_event_loop(
loop=loop
@@ -1084,6 +1127,3825 @@ def _build_custom_pricing_entry(
return entry
+def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ _azure_detection_model = ctx._azure_detection_model
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ api_version = ctx.api_version
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ extra_headers = ctx.extra_headers
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ max_retries = ctx.max_retries
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ timeout = ctx.timeout
+
+ dynamic_params = False
+ if client is not None and (
+ isinstance(client, openai.AzureOpenAI)
+ or isinstance(client, openai.AsyncAzureOpenAI)
+ ):
+ dynamic_params = _check_dynamic_azure_params(
+ azure_client_params={"api_version": api_version},
+ azure_client=client,
+ )
+
+ api_type = get_secret("AZURE_API_TYPE") or "azure"
+
+ api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE")
+
+ api_version = (
+ api_version
+ or litellm.api_version
+ or get_secret_str("AZURE_API_VERSION")
+ or litellm.AZURE_DEFAULT_API_VERSION
+ )
+
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.azure_key
+ or get_secret_str("AZURE_OPENAI_API_KEY")
+ or get_secret_str("AZURE_API_KEY")
+ )
+
+ azure_ad_token = optional_params.get("extra_body", {}).pop(
+ "azure_ad_token", None
+ ) or get_secret_str("AZURE_AD_TOKEN")
+
+ azure_ad_token_provider = litellm_params.get("azure_ad_token_provider", None)
+
+ headers = headers or litellm.headers
+
+ if extra_headers is not None:
+ optional_params["extra_headers"] = extra_headers
+ if max_retries is not None:
+ optional_params["max_retries"] = max_retries
+
+ if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model):
+ ## LOAD CONFIG - if set
+ config = litellm.AzureOpenAIO1Config.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ response = azure_o1_chat_completions.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ api_key=api_key,
+ api_base=api_base,
+ api_version=api_version,
+ dynamic_params=dynamic_params,
+ azure_ad_token=azure_ad_token,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ logging_obj=logging,
+ acompletion=acompletion,
+ timeout=timeout, # type: ignore
+ client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
+ custom_llm_provider=custom_llm_provider,
+ )
+ else:
+ ## LOAD CONFIG - if set
+ config = litellm.AzureOpenAIConfig.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ ## COMPLETION CALL
+ response = azure_chat_completions.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ api_key=api_key,
+ api_base=api_base,
+ api_version=api_version,
+ api_type=api_type,
+ dynamic_params=dynamic_params,
+ azure_ad_token=azure_ad_token,
+ azure_ad_token_provider=azure_ad_token_provider,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ logging_obj=logging,
+ acompletion=acompletion,
+ timeout=timeout, # type: ignore
+ client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
+ )
+
+ if optional_params.get("stream", False):
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ additional_args={
+ "headers": headers,
+ "api_version": api_version,
+ "api_base": api_base,
+ },
+ )
+
+ return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ api_version = ctx.api_version
+ client = ctx.client
+ extra_headers = ctx.extra_headers
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ timeout = ctx.timeout
+
+ api_type = get_secret_str("AZURE_API_TYPE") or "azure"
+
+ api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
+
+ if api_base is None:
+ raise ValueError(
+ "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable."
+ )
+
+ api_version = (
+ api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
+ )
+
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.azure_key
+ or get_secret_str("AZURE_OPENAI_API_KEY")
+ or get_secret_str("AZURE_API_KEY")
+ )
+
+ azure_ad_token = optional_params.get("extra_body", {}).pop(
+ "azure_ad_token", None
+ ) or get_secret_str("AZURE_AD_TOKEN")
+
+ azure_ad_token_provider = litellm_params.get("azure_ad_token_provider", None)
+
+ headers = headers or litellm.headers
+
+ if extra_headers is not None:
+ optional_params["extra_headers"] = extra_headers
+
+ ## LOAD CONFIG - if set
+ config = litellm.AzureOpenAIConfig.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ ## COMPLETION CALL
+ response = azure_text_completions.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ api_key=api_key,
+ api_base=api_base,
+ api_version=cast(str, api_version),
+ api_type=api_type,
+ azure_ad_token=azure_ad_token,
+ azure_ad_token_provider=azure_ad_token_provider,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ logging_obj=logging,
+ acompletion=acompletion,
+ timeout=timeout,
+ client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
+ )
+
+ if optional_params.get("stream", False) or acompletion is True:
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ additional_args={
+ "headers": headers,
+ "api_version": api_version,
+ "api_base": api_base,
+ },
+ )
+
+ return response
+
+
+def _complete_deepseek(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ return response
+
+
+def _complete_azure_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ extra_headers = ctx.extra_headers
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
+ azure_ai_route = AzureFoundryModelInfo.get_azure_ai_route(model)
+
+ # Check if this is an agents route - model format: azure_ai/agents/
+ if azure_ai_route == "agents":
+ from litellm.llms.azure_ai.agents import AzureAIAgentsConfig
+
+ api_base = AzureFoundryModelInfo.get_api_base(api_base)
+ if api_base is None:
+ raise ValueError(
+ "Azure AI Agents requests require an api_base. "
+ "Set `api_base` or the AZURE_AI_API_BASE env var."
+ )
+ api_key = AzureFoundryModelInfo.get_api_key(api_key)
+
+ response = AzureAIAgentsConfig.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ api_key=api_key,
+ model_response=model_response,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout,
+ acompletion=acompletion,
+ stream=stream,
+ headers=headers or litellm.headers,
+ )
+
+ # Check if this is a Claude model - route to Azure Anthropic handler
+ elif "claude" in model.lower():
+ # Use Azure Anthropic handler for Claude models
+ api_base = AzureFoundryModelInfo.get_api_base(api_base)
+ if api_base is None:
+ raise ValueError(
+ "Azure Anthropic requests require an api_base. "
+ "Set `api_base` or the AZURE_AI_API_BASE env var."
+ )
+ api_key = AzureFoundryModelInfo.get_api_key(api_key)
+
+ # Ensure the URL ends with /v1/messages for Anthropic
+ if api_base:
+ api_base = api_base.rstrip("/")
+ if not api_base.endswith("/v1/messages"):
+ if "/anthropic" in api_base:
+ parts = api_base.split("/anthropic", 1)
+ api_base = parts[0] + "/anthropic"
+ else:
+ api_base = api_base + "/anthropic"
+ api_base = api_base + "/v1/messages"
+
+ response = azure_anthropic_chat_completions.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ acompletion=acompletion,
+ custom_prompt_dict=litellm.custom_prompt_dict,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ headers=headers,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ )
+ if optional_params.get("stream", False) or acompletion is True:
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ )
+ response = response
+ else:
+ # Non-Claude models use standard Azure AI flow
+ api_base = AzureFoundryModelInfo.get_api_base(api_base)
+ # set API KEY
+ api_key = AzureFoundryModelInfo.get_api_key(api_key)
+
+ headers = headers or litellm.headers
+
+ if extra_headers is not None:
+ optional_params["extra_headers"] = extra_headers
+
+ ## FOR COHERE
+ if "command-r" in model: # make sure tool call in messages are str
+ messages = stringify_json_tool_call_content(messages=messages)
+
+ ## COMPLETION CALL
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout, # type: ignore
+ client=client, # pass AsyncOpenAI, OpenAI client
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ if optional_params.get("stream", False):
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ additional_args={"headers": headers},
+ )
+
+ return response
+
+
+def _complete_text_completion_openai(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ text_completion = ctx.text_completion
+ timeout = ctx.timeout
+
+ openai.api_type = "openai"
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret("OPENAI_BASE_URL")
+ or get_secret("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+
+ openai.api_version = None
+ # set API KEY
+
+ api_key = (
+ api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
+ )
+
+ headers = headers or litellm.headers
+
+ ## LOAD CONFIG - if set
+ config = litellm.OpenAITextCompletionConfig.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+ if litellm.organization:
+ openai.organization = litellm.organization
+
+ ## COMPLETION CALL
+ _response = openai_text_completions.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ api_key=api_key,
+ custom_llm_provider=custom_llm_provider,
+ api_base=api_base,
+ acompletion=acompletion,
+ client=client, # pass AsyncOpenAI, OpenAI client
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ timeout=timeout, # type: ignore
+ )
+
+ if (
+ optional_params.get("stream", False) is False
+ and acompletion is False
+ and text_completion is False
+ ):
+ # convert to chat completion response
+ _response = (
+ litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
+ response_object=_response, model_response_object=model_response
+ )
+ )
+
+ if optional_params.get("stream", False) or acompletion is True:
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=_response,
+ additional_args={"headers": headers},
+ )
+ return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_fireworks_ai(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ return response
+
+
+def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+ except Exception as e:
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ return response
+
+
+def _complete_ragflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+ except Exception as e:
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ return response
+
+
+def _complete_xai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ return response
+
+
+def _complete_groq(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
+ or litellm.api_base
+ or get_secret("GROQ_API_BASE")
+ or "https://api.groq.com/openai/v1"
+ )
+
+ # set API KEY
+ api_key = (
+ api_key
+ or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
+ or litellm.groq_key
+ or get_secret("GROQ_API_KEY")
+ )
+
+ headers = headers or litellm.headers
+
+ ## LOAD CONFIG - if set
+ config = litellm.GroqChatConfig.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider=custom_llm_provider,
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+
+
+def _complete_bedrock_mantle(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE")
+ api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
+ headers = headers or litellm.headers
+ config = litellm.BedrockMantleChatConfig.get_config()
+ for k, v in config.items():
+ if k not in optional_params:
+ optional_params[k] = v
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider=custom_llm_provider,
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ client=client,
+ )
+
+
+def _complete_a2a(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ (
+ api_base,
+ api_key,
+ headers,
+ ) = litellm.A2AConfig.resolve_agent_config_from_registry(
+ model=model,
+ api_base=api_base,
+ api_key=api_key,
+ headers=headers,
+ optional_params=optional_params,
+ )
+
+ # Fall back to environment variables and defaults
+ api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE")
+
+ if api_base is None:
+ raise Exception(
+ "api_base is required for A2A provider. "
+ "Either provide api_base parameter, set A2A_API_BASE environment variable, "
+ "or register the agent in the proxy with model='a2a/'."
+ )
+
+ headers = headers or litellm.headers
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider=custom_llm_provider,
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ client=client,
+ provider_config=provider_config,
+ )
+
+
+def _complete_gigachat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.gigachat_key
+ or get_secret("GIGACHAT_API_KEY")
+ or get_secret("GIGACHAT_CREDENTIALS")
+ )
+
+ headers = headers or litellm.headers or {}
+
+ ## COMPLETION CALL
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ return response
+
+
+def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ headers = headers or litellm.headers
+ ## LOAD CONFIG - if set
+ config = litellm.GenAIHubOrchestrationConfig.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ return sap_gen_ai_hub_chat_completions.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ shared_session=shared_session,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ api_base=api_base,
+ stream=stream,
+ )
+
+
+def _complete_aiohttp_openai(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ extra_headers = ctx.extra_headers
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
+ or litellm.api_base
+ or get_secret("OPENAI_BASE_URL")
+ or get_secret("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+ # set API KEY
+ api_key = (
+ api_key
+ or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
+ or litellm.openai_key
+ or get_secret("OPENAI_API_KEY")
+ )
+
+ headers = headers or litellm.headers
+
+ if extra_headers is not None:
+ optional_params["extra_headers"] = extra_headers
+ return base_llm_aiohttp_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ )
+
+
+def _complete_cometapi(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.cometapi_key
+ or get_secret_str("COMETAPI_KEY")
+ or litellm.api_key
+ )
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("COMETAPI_API_BASE")
+ or "https://api.cometapi.com/v1"
+ )
+
+ ## COMPLETION CALL
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+
+ ## LOGGING
+ logging.post_call(input=messages, api_key=api_key, original_response=response)
+
+ return response
+
+
+def _complete_minimax(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("MINIMAX_API_BASE")
+ or "https://api.minimax.io/v1"
+ )
+
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ custom_llm_provider=custom_llm_provider,
+ model_response=model_response,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ optional_params=optional_params,
+ timeout=timeout,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ acompletion=acompletion,
+ stream=stream,
+ api_key=api_key,
+ headers=headers,
+ client=client,
+ provider_config=provider_config,
+ )
+ logging.post_call(input=messages, api_key=api_key, original_response=response)
+
+ return response
+
+
+def _complete_hosted_vllm(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE")
+
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ custom_llm_provider=custom_llm_provider,
+ model_response=model_response,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ optional_params=optional_params,
+ timeout=timeout,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ acompletion=acompletion,
+ stream=stream,
+ api_key=api_key,
+ headers=headers,
+ client=client,
+ provider_config=provider_config,
+ )
+ logging.post_call(input=messages, api_key=api_key, original_response=response)
+
+ return response
+
+
+def _complete_custom_openai(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ custom_prompt_dict = ctx.custom_prompt_dict
+ extra_headers = ctx.extra_headers
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ metadata = ctx.metadata
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ organization = ctx.organization
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
+ or litellm.api_base
+ or get_secret("OPENAI_BASE_URL")
+ or get_secret("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+ organization = (
+ organization
+ or litellm.organization
+ or get_secret("OPENAI_ORGANIZATION")
+ or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
+ )
+ openai.organization = organization
+ # set API KEY
+ api_key = (
+ api_key
+ or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
+ or litellm.openai_key
+ or get_secret("OPENAI_API_KEY")
+ )
+
+ headers = headers or litellm.headers
+
+ # Add GitHub Copilot headers (same as /responses endpoint does)
+ if custom_llm_provider == "github_copilot":
+ from litellm.llms.github_copilot.authenticator import Authenticator
+ from litellm.llms.github_copilot.common_utils import (
+ get_copilot_default_headers,
+ )
+
+ copilot_auth = Authenticator()
+ copilot_api_key = copilot_auth.get_api_key()
+ copilot_headers = get_copilot_default_headers(copilot_api_key)
+ if extra_headers:
+ copilot_headers.update(extra_headers)
+ extra_headers = copilot_headers
+
+ if extra_headers is not None:
+ optional_params["extra_headers"] = extra_headers
+
+ if (
+ litellm.enable_preview_features and metadata is not None
+ ): # [PREVIEW] allow metadata to be passed to OPENAI
+ openai_metadata = get_requester_metadata(metadata)
+ if openai_metadata is not None:
+ optional_params["metadata"] = openai_metadata
+
+ ## LOAD CONFIG - if set
+ config = litellm.OpenAIConfig.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ ## COMPLETION CALL
+ use_base_llm_http_handler = get_secret_bool(
+ "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER"
+ )
+
+ try:
+ if use_base_llm_http_handler:
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ custom_llm_provider=custom_llm_provider,
+ model_response=model_response,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ optional_params=optional_params,
+ timeout=timeout,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ acompletion=acompletion,
+ stream=stream,
+ api_key=api_key,
+ headers=headers,
+ client=client,
+ provider_config=provider_config,
+ )
+ else:
+ response = openai_chat_completions.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ timeout=timeout, # type: ignore
+ custom_prompt_dict=custom_prompt_dict,
+ client=client, # pass AsyncOpenAI, OpenAI client
+ organization=organization,
+ custom_llm_provider=custom_llm_provider,
+ shared_session=shared_session,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ if optional_params.get("stream", False):
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ additional_args={"headers": headers},
+ )
+
+ return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_mistral(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY")
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret("MISTRAL_API_BASE")
+ or "https://api.mistral.ai/v1"
+ )
+
+ return base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ custom_llm_provider=custom_llm_provider,
+ model_response=model_response,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ optional_params=optional_params,
+ timeout=timeout,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ acompletion=acompletion,
+ stream=stream,
+ api_key=api_key,
+ headers=headers,
+ client=client,
+ provider_config=provider_config,
+ )
+
+
+def _complete_replicate(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+
+ replicate_key = (
+ api_key
+ or litellm.replicate_key
+ or litellm.api_key
+ or get_secret("REPLICATE_API_KEY")
+ or get_secret("REPLICATE_API_TOKEN")
+ )
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret("REPLICATE_API_BASE")
+ or "https://api.replicate.com/v1"
+ )
+
+ custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
+
+ model_response = replicate_chat_completion( # type: ignore
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(), # for calculating input/output tokens
+ api_key=replicate_key,
+ logging_obj=logging,
+ custom_prompt_dict=custom_prompt_dict,
+ acompletion=acompletion,
+ headers=headers,
+ )
+
+ if optional_params.get("stream", False) is True:
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=replicate_key,
+ original_response=model_response,
+ )
+
+ return model_response
+
+
+def _complete_anthropic_text(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.anthropic_key
+ or litellm.api_key
+ or os.environ.get("ANTHROPIC_API_KEY")
+ )
+ custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
+ api_base = cast(
+ Optional[str],
+ api_base
+ or litellm.api_base
+ or get_secret("ANTHROPIC_API_BASE")
+ or get_secret("ANTHROPIC_BASE_URL")
+ or "https://api.anthropic.com/v1/complete",
+ )
+
+ # Check if we should disable automatic URL suffix appending
+ disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
+ if (
+ api_base is not None
+ and not disable_url_suffix
+ and not api_base.endswith("/v1/complete")
+ ):
+ api_base += "/v1/complete"
+ elif disable_url_suffix:
+ verbose_logger.debug(
+ "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix"
+ )
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="anthropic_text",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ )
+
+
+def _complete_anthropic(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.anthropic_key
+ or litellm.api_key
+ or os.environ.get("ANTHROPIC_API_KEY")
+ )
+ custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
+ # call /messages
+ # default route for all anthropic models
+ api_base = cast(
+ Optional[str],
+ api_base
+ or litellm.api_base
+ or get_secret("ANTHROPIC_API_BASE")
+ or get_secret("ANTHROPIC_BASE_URL")
+ or "https://api.anthropic.com/v1/messages",
+ )
+
+ # Check if we should disable automatic URL suffix appending
+ disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
+ if (
+ api_base is not None
+ and not disable_url_suffix
+ and not api_base.endswith("/v1/messages")
+ ):
+ api_base += "/v1/messages"
+ elif disable_url_suffix:
+ verbose_logger.debug(
+ "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix"
+ )
+
+ response = anthropic_chat_completions.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ acompletion=acompletion,
+ custom_prompt_dict=litellm.custom_prompt_dict,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(), # for calculating input/output tokens
+ api_key=api_key,
+ logging_obj=logging,
+ headers=headers,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ )
+ if optional_params.get("stream", False) or acompletion is True:
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ )
+ return response
+
+
+def _complete_nlp_cloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+
+ nlp_cloud_key = (
+ api_key
+ or litellm.nlp_cloud_key
+ or get_secret("NLP_CLOUD_API_KEY")
+ or litellm.api_key
+ )
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret("NLP_CLOUD_API_BASE")
+ or "https://api.nlpcloud.io/v1/gpu/"
+ )
+
+ response = nlp_cloud_chat_completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ api_key=nlp_cloud_key,
+ logging_obj=logging,
+ )
+
+ if "stream" in optional_params and optional_params["stream"] is True:
+ # don't try to access stream object,
+ response = CustomStreamWrapper(
+ response,
+ model,
+ custom_llm_provider="nlp_cloud",
+ logging_obj=logging,
+ )
+
+ if optional_params.get("stream", False) or acompletion is True:
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ )
+
+ return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_aleph_alpha(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+
+ aleph_alpha_key = (
+ api_key
+ or litellm.aleph_alpha_key
+ or get_secret("ALEPH_ALPHA_API_KEY")
+ or get_secret("ALEPHALPHA_API_KEY")
+ or litellm.api_key
+ )
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret("ALEPH_ALPHA_API_BASE")
+ or "https://api.aleph-alpha.com/complete"
+ )
+
+ model_response = aleph_alpha.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ default_max_tokens_to_sample=litellm.max_tokens,
+ api_key=aleph_alpha_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ )
+
+ if "stream" in optional_params and optional_params["stream"] is True:
+ # don't try to access stream object,
+ return CustomStreamWrapper(
+ model_response,
+ model,
+ custom_llm_provider="aleph_alpha",
+ logging_obj=logging,
+ )
+ return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_cohere_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ extra_headers = ctx.extra_headers
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ cohere_key = (
+ api_key
+ or litellm.cohere_key
+ or get_secret_str("COHERE_API_KEY")
+ or get_secret_str("CO_API_KEY")
+ or litellm.api_key
+ )
+
+ cohere_route = CohereModelInfo.get_cohere_route(model)
+ verbose_logger.debug(f"Cohere route: {cohere_route}")
+ # Set API base based on route
+ if cohere_route == "v2":
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("COHERE_API_BASE")
+ or "https://api.cohere.com/v2/chat"
+ )
+ # Remove v2/ prefix from model name for the actual API call
+ if "v2/" in model:
+ model = model.replace("v2/", "")
+ else:
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("COHERE_API_BASE")
+ or "https://api.cohere.ai/v1/chat"
+ )
+
+ headers = headers or litellm.headers or {}
+ if headers is None:
+ headers = {}
+
+ if extra_headers is not None:
+ headers.update(extra_headers)
+
+ verbose_logger.debug(f"Model: {model}, API Base: {api_base}")
+ verbose_logger.debug(f"Provider Config: {provider_config}")
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="cohere_chat",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=cohere_key,
+ provider_config=provider_config,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ )
+
+
+def _complete_maritalk(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_prompt_dict = ctx.custom_prompt_dict
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+
+ maritalk_key = (
+ api_key
+ or litellm.maritalk_key
+ or get_secret("MARITALK_API_KEY")
+ or litellm.api_key
+ )
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret("MARITALK_API_BASE")
+ or "https://chat.maritaca.ai/api"
+ )
+
+ return openai_like_chat_completion.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ api_key=maritalk_key,
+ logging_obj=logging,
+ custom_llm_provider="maritalk",
+ custom_prompt_dict=custom_prompt_dict,
+ )
+
+
+def _complete_amazon_nova(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_llm_provider = ctx.custom_llm_provider
+ custom_prompt_dict = ctx.custom_prompt_dict
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.amazon_nova_api_key
+ or get_secret_str("AMAZON_NOVA_API_KEY")
+ or litellm.api_key
+ )
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("AMAZON_NOVA_API_BASE")
+ or "https://api.nova.amazon.com/v1"
+ )
+ return openai_like_chat_completion.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ timeout=timeout,
+ custom_llm_provider=custom_llm_provider,
+ custom_prompt_dict=custom_prompt_dict,
+ )
+
+
+def _complete_huggingface(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ huggingface_key = (
+ api_key
+ or litellm.huggingface_key
+ or os.environ.get("HF_TOKEN")
+ or os.environ.get("HUGGINGFACE_API_KEY")
+ or litellm.api_key
+ )
+ hf_headers = headers or litellm.headers
+ return base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=hf_headers,
+ model_response=model_response,
+ api_key=huggingface_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ )
+
+
+def _complete_oci(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ return base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ )
+
+
+def _complete_compactifai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
+
+ api_base = api_base or "https://api.compactif.ai/v1"
+
+ ## COMPLETION CALL
+ return base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+
+
+def _complete_oobabooga(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ api_base = ctx.api_base
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+
+ model_response = oobabooga.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ api_base=api_base, # type: ignore
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ api_key=None,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ )
+ if "stream" in optional_params and optional_params["stream"] is True:
+ # don't try to access stream object,
+ return CustomStreamWrapper(
+ model_response,
+ model,
+ custom_llm_provider="oobabooga",
+ logging_obj=logging,
+ )
+ return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_databricks(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ api_base # for databricks we check in get_llm_provider and pass in the api base from there
+ or litellm.api_base
+ or os.getenv("DATABRICKS_API_BASE")
+ )
+
+ # set API KEY
+ api_key = (
+ api_key
+ or litellm.api_key # for databricks we check in get_llm_provider and pass in the api key from there
+ or litellm.databricks_key
+ or get_secret("DATABRICKS_API_KEY")
+ )
+
+ headers = headers or litellm.headers
+
+ ## COMPLETION CALL
+ try:
+ response = base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ custom_llm_provider="databricks",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ if optional_params.get("stream", False):
+ ## LOGGING
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=response,
+ additional_args={"headers": headers},
+ )
+
+ return response
+
+
+def _complete_datarobot(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ return base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=provider_config,
+ )
+
+
+def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("OPENROUTER_API_BASE")
+ or "https://openrouter.ai/api/v1"
+ )
+
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.openrouter_key
+ or get_secret_str("OPENROUTER_API_KEY")
+ or get_secret_str("OR_API_KEY")
+ )
+
+ openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai"
+ openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM"
+
+ openrouter_headers = {
+ "HTTP-Referer": openrouter_site_url,
+ "X-Title": openrouter_app_name,
+ }
+
+ _headers = headers or litellm.headers
+ if _headers:
+ openrouter_headers.update(_headers)
+
+ headers = openrouter_headers
+
+ ## Load Config
+ config = litellm.OpenrouterConfig.get_config()
+ for k, v in config.items():
+ if k == "extra_body":
+ # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models
+ if "extra_body" in optional_params:
+ optional_params[k].update(v)
+ else:
+ optional_params[k] = v
+ elif k not in optional_params:
+ optional_params[k] = v
+
+ ## COMPLETION CALL
+ response = base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="openrouter",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+ ## LOGGING
+ logging.post_call(
+ input=messages, api_key=openai.api_key, original_response=response
+ )
+
+ return response
+
+
+def _complete_vercel_ai_gateway(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
+ or "https://ai-gateway.vercel.sh/v1"
+ )
+
+ api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
+
+ vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
+ vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM"
+
+ vercel_headers = {
+ "http-referer": vercel_site_url,
+ "x-title": vercel_app_name,
+ }
+
+ _headers = headers or litellm.headers
+ if _headers:
+ vercel_headers.update(_headers)
+
+ headers = vercel_headers
+
+ ## Load Config
+ config = litellm.VercelAIGatewayConfig.get_config()
+ for k, v in config.items():
+ if k == "extra_body":
+ # we use openai 'extra_body' to pass vercel specific params - providerOptions
+ if "extra_body" in optional_params:
+ optional_params[k].update(v)
+ else:
+ optional_params[k] = v
+ elif k not in optional_params:
+ optional_params[k] = v
+
+ ## COMPLETION CALL
+ response = base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="vercel_ai_gateway",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+ ## LOGGING
+ logging.post_call(
+ input=messages, api_key=openai.api_key, original_response=response
+ )
+
+ return response
+
+
+def _complete_vertex_ai_beta(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ timeout = ctx.timeout
+
+ vertex_ai_project = (
+ optional_params.pop("vertex_project", None)
+ or optional_params.pop("vertex_ai_project", None)
+ or litellm.vertex_project
+ or get_secret("VERTEXAI_PROJECT")
+ )
+ vertex_ai_location = (
+ optional_params.pop("vertex_location", None)
+ or optional_params.pop("vertex_ai_location", None)
+ or litellm.vertex_location
+ or get_secret("VERTEXAI_LOCATION")
+ )
+ vertex_credentials = (
+ optional_params.pop("vertex_credentials", None)
+ or optional_params.pop("vertex_ai_credentials", None)
+ or get_secret("VERTEXAI_CREDENTIALS")
+ )
+
+ gemini_api_key = (
+ api_key
+ or get_api_key_from_env()
+ or get_secret("PALM_API_KEY") # older palm api key should also work
+ or litellm.api_key
+ )
+
+ api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE")
+ new_params = safe_deep_copy(optional_params or {})
+ return vertex_chat_completion.completion( # type: ignore
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=new_params,
+ litellm_params=litellm_params, # type: ignore
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ vertex_location=vertex_ai_location,
+ vertex_project=vertex_ai_project,
+ vertex_credentials=vertex_credentials,
+ gemini_api_key=gemini_api_key,
+ logging_obj=logging,
+ acompletion=acompletion,
+ timeout=timeout,
+ custom_llm_provider=custom_llm_provider, # type: ignore
+ client=client,
+ api_base=api_base,
+ extra_headers=headers,
+ )
+
+
+def _complete_vertex_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ vertex_ai_project = (
+ optional_params.pop("vertex_project", None)
+ or optional_params.pop("vertex_ai_project", None)
+ or litellm.vertex_project
+ or get_secret("VERTEXAI_PROJECT")
+ )
+ vertex_ai_location = (
+ optional_params.pop("vertex_location", None)
+ or optional_params.pop("vertex_ai_location", None)
+ or litellm.vertex_location
+ or get_secret("VERTEXAI_LOCATION")
+ )
+ vertex_credentials = (
+ optional_params.pop("vertex_credentials", None)
+ or optional_params.pop("vertex_ai_credentials", None)
+ or get_secret("VERTEXAI_CREDENTIALS")
+ )
+
+ api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE")
+
+ new_params = safe_deep_copy(optional_params or {})
+ model_route = get_vertex_ai_model_route(model=model, litellm_params=litellm_params)
+
+ if model_route == VertexAIModelRoute.PARTNER_MODELS:
+ model_response = vertex_partner_models_chat_completion.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=new_params,
+ litellm_params=litellm_params, # type: ignore
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ api_base=api_base,
+ vertex_location=vertex_ai_location,
+ vertex_project=vertex_ai_project,
+ vertex_credentials=vertex_credentials,
+ logging_obj=logging,
+ acompletion=acompletion,
+ headers=headers,
+ custom_prompt_dict=custom_prompt_dict,
+ timeout=timeout,
+ client=client,
+ )
+ elif model_route == VertexAIModelRoute.GEMINI:
+ model_response = vertex_chat_completion.completion( # type: ignore
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=new_params,
+ litellm_params=litellm_params, # type: ignore
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ vertex_location=vertex_ai_location,
+ vertex_project=vertex_ai_project,
+ vertex_credentials=vertex_credentials,
+ gemini_api_key=None,
+ logging_obj=logging,
+ acompletion=acompletion,
+ timeout=timeout,
+ custom_llm_provider=custom_llm_provider, # type: ignore
+ client=client,
+ api_base=api_base,
+ extra_headers=headers,
+ )
+ elif model_route == VertexAIModelRoute.GEMMA:
+ # Vertex Gemma Models with custom prediction endpoint
+ model_response = vertex_gemma_chat_completion.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=new_params,
+ litellm_params=litellm_params, # type: ignore
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ api_base=api_base,
+ vertex_location=vertex_ai_location,
+ vertex_project=vertex_ai_project,
+ vertex_credentials=vertex_credentials,
+ logging_obj=logging,
+ acompletion=acompletion,
+ headers=headers,
+ custom_prompt_dict=custom_prompt_dict,
+ timeout=timeout,
+ client=client,
+ )
+ elif model_route == VertexAIModelRoute.MODEL_GARDEN:
+ # Vertex Model Garden - OpenAI compatible models
+ model_response = vertex_model_garden_chat_completion.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=new_params,
+ litellm_params=litellm_params, # type: ignore
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ api_base=api_base,
+ vertex_location=vertex_ai_location,
+ vertex_project=vertex_ai_project,
+ vertex_credentials=vertex_credentials,
+ logging_obj=logging,
+ acompletion=acompletion,
+ headers=headers,
+ custom_prompt_dict=custom_prompt_dict,
+ timeout=timeout,
+ client=client,
+ )
+ elif model_route == VertexAIModelRoute.AGENT_ENGINE:
+ # Vertex AI Agent Engine (Reasoning Engines)
+ from litellm.llms.vertex_ai.agent_engine.transformation import (
+ VertexAgentEngineConfig,
+ )
+
+ vertex_agent_engine_config = VertexAgentEngineConfig()
+
+ # Update litellm_params with vertex credentials
+ litellm_params["vertex_project"] = vertex_ai_project
+ litellm_params["vertex_location"] = vertex_ai_location
+ litellm_params["vertex_credentials"] = vertex_credentials
+
+ model_response = base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ model_response=model_response,
+ optional_params=new_params,
+ litellm_params=litellm_params, # type: ignore
+ encoding=_get_encoding(),
+ api_key=None,
+ api_base=api_base,
+ logging_obj=logging,
+ acompletion=acompletion,
+ timeout=timeout,
+ client=client,
+ custom_llm_provider="vertex_ai",
+ provider_config=vertex_agent_engine_config,
+ headers=headers or {},
+ )
+ else: # VertexAIModelRoute.NON_GEMINI
+ model_response = vertex_ai_non_gemini.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=new_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ vertex_location=vertex_ai_location,
+ vertex_project=vertex_ai_project,
+ vertex_credentials=vertex_credentials,
+ logging_obj=logging,
+ acompletion=acompletion,
+ )
+
+ if (
+ "stream" in optional_params
+ and optional_params["stream"] is True
+ and acompletion is False
+ ):
+ return CustomStreamWrapper(
+ model_response,
+ model,
+ custom_llm_provider="vertex_ai",
+ logging_obj=logging,
+ )
+ return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_predibase(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_prompt_dict = ctx.custom_prompt_dict
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ timeout = ctx.timeout
+
+ tenant_id = (
+ optional_params.pop("tenant_id", None)
+ or optional_params.pop("predibase_tenant_id", None)
+ or litellm.predibase_tenant_id
+ or get_secret("PREDIBASE_TENANT_ID")
+ )
+
+ if tenant_id is None:
+ raise ValueError(
+ "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`."
+ )
+
+ api_base = (
+ api_base
+ or optional_params.pop("api_base", None)
+ or optional_params.pop("base_url", None)
+ or litellm.api_base
+ or get_secret("PREDIBASE_API_BASE")
+ )
+
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.predibase_key
+ or get_secret("PREDIBASE_API_KEY")
+ )
+
+ _model_response = predibase_chat_completions.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ acompletion=acompletion,
+ api_base=api_base,
+ custom_prompt_dict=custom_prompt_dict,
+ api_key=api_key,
+ tenant_id=tenant_id,
+ timeout=timeout,
+ )
+
+ if (
+ "stream" in optional_params
+ and optional_params["stream"] is True
+ and acompletion is False
+ ):
+ return _model_response
+ return _model_response
+
+
+def _complete_text_completion_codestral(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_prompt_dict = ctx.custom_prompt_dict
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ api_base
+ or optional_params.pop("api_base", None)
+ or optional_params.pop("base_url", None)
+ or litellm.api_base
+ or "https://codestral.mistral.ai/v1/fim/completions"
+ )
+
+ api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY")
+
+ text_completion_model_response = litellm.TextCompletionResponse(stream=stream)
+
+ _model_response = codestral_text_completions.completion( # type: ignore
+ model=model,
+ messages=messages,
+ model_response=text_completion_model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ acompletion=acompletion,
+ api_base=api_base,
+ custom_prompt_dict=custom_prompt_dict,
+ api_key=api_key,
+ timeout=timeout,
+ )
+
+ if (
+ "stream" in optional_params
+ and optional_params["stream"] is True
+ and acompletion is False
+ ):
+ return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+ return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_text_completion_inception(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ text_completion = ctx.text_completion
+ timeout = ctx.timeout
+
+ passed_api_base = (
+ api_base
+ or optional_params.pop("api_base", None)
+ or optional_params.pop("base_url", None)
+ )
+ api_base = (
+ passed_api_base
+ or get_secret_str("INCEPTION_API_BASE")
+ or "https://api.inceptionlabs.ai/v1"
+ )
+ # FIM is served at `/v1/fim/completions`; the OpenAI client appends
+ # `/completions`, so point it at the `/v1/fim` base.
+ api_base = api_base.rstrip("/")
+ if not api_base.endswith("/fim"):
+ api_base += "/fim"
+
+ # Don't forward the server-managed Inception key to a caller-supplied
+ # api_base; only resolve it for the default/server base, or when the
+ # caller passes their own key.
+ if passed_api_base is None or api_key:
+ api_key = (
+ api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY")
+ )
+
+ _response = openai_text_completions.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ api_key=api_key, # type: ignore[arg-type]
+ custom_llm_provider="text-completion-inception",
+ api_base=api_base,
+ acompletion=acompletion,
+ client=client,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ timeout=timeout, # type: ignore
+ )
+
+ if (
+ optional_params.get("stream", False) is False
+ and acompletion is False
+ and text_completion is False
+ ):
+ _response = (
+ litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
+ response_object=_response, model_response_object=model_response
+ )
+ )
+
+ if optional_params.get("stream", False) or acompletion is True:
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=_response,
+ additional_args={"headers": headers},
+ )
+ return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_sagemaker_chat(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ custom_llm_provider=custom_llm_provider,
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+
+
+def _complete_sagemaker(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ custom_prompt_dict = ctx.custom_prompt_dict
+ hf_model_name = ctx.hf_model_name
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+
+ return sagemaker_llm.completion(
+ model=model,
+ messages=messages,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ custom_prompt_dict=custom_prompt_dict,
+ hf_model_name=hf_model_name,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ acompletion=acompletion,
+ )
+
+
+def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ provider_config = ctx.provider_config
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
+
+ if "aws_bedrock_client" in optional_params:
+ verbose_logger.warning(
+ "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication."
+ )
+ # Extract credentials for legacy boto3 client and pass thru to httpx
+ aws_bedrock_client = optional_params.pop("aws_bedrock_client")
+ creds = aws_bedrock_client._get_credentials().get_frozen_credentials()
+
+ if creds.access_key:
+ optional_params["aws_access_key_id"] = creds.access_key
+ if creds.secret_key:
+ optional_params["aws_secret_access_key"] = creds.secret_key
+ if creds.token:
+ optional_params["aws_session_token"] = creds.token
+ if (
+ "aws_region_name" not in optional_params
+ or optional_params["aws_region_name"] is None
+ ):
+ optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
+
+ bedrock_route = BedrockModelInfo.get_bedrock_route(model)
+ if bedrock_route == "claude_platform":
+ provider_config = ProviderConfigManager.get_provider_chat_config(
+ model=model,
+ provider=LlmProviders.BEDROCK,
+ )
+ model = BedrockModelInfo.get_claude_platform_model(model)
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="bedrock",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ client=client,
+ provider_config=provider_config,
+ )
+ elif bedrock_route == "converse":
+ model = model.replace("converse/", "")
+ response = bedrock_converse_chat_completion.completion(
+ model=model,
+ messages=messages,
+ custom_prompt_dict=custom_prompt_dict,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params, # type: ignore
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ extra_headers=headers, # Use merged headers instead of original extra_headers
+ timeout=timeout,
+ acompletion=acompletion,
+ client=client,
+ api_base=api_base,
+ api_key=api_key,
+ )
+ elif bedrock_route == "converse_like":
+ model = model.replace("converse_like/", "")
+ response = base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ custom_llm_provider="bedrock",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+ else:
+ response = base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ custom_llm_provider="bedrock",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ client=client,
+ )
+
+ return response
+
+
+def _complete_watsonx(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ timeout = ctx.timeout
+
+ return watsonx_chat_completion.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ timeout=timeout, # type: ignore
+ custom_prompt_dict=custom_prompt_dict,
+ client=client, # pass AsyncOpenAI, OpenAI client
+ encoding=_get_encoding(),
+ custom_llm_provider="watsonx",
+ )
+
+
+def _complete_watsonx_text(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or optional_params.pop("apikey", None)
+ or get_secret_str("WATSONX_APIKEY")
+ or get_secret_str("WATSONX_API_KEY")
+ or get_secret_str("WX_API_KEY")
+ )
+
+ api_base = (
+ api_base
+ or optional_params.pop(
+ "url",
+ optional_params.pop("api_base", optional_params.pop("base_url", None)),
+ )
+ or get_secret_str("WATSONX_API_BASE")
+ or get_secret_str("WATSONX_URL")
+ or get_secret_str("WX_URL")
+ or get_secret_str("WML_URL")
+ )
+
+ wx_credentials = optional_params.pop(
+ "wx_credentials",
+ optional_params.pop(
+ "watsonx_credentials", None
+ ), # follow {provider}_credentials, same as vertex ai
+ )
+
+ token: Optional[str] = None
+ if wx_credentials is not None:
+ api_base = wx_credentials.get("url", api_base)
+ api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key))
+ token = wx_credentials.get(
+ "token",
+ wx_credentials.get(
+ "watsonx_token", None
+ ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..'
+ )
+
+ if token is not None:
+ optional_params["token"] = token
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="watsonx_text",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+
+
+def _complete_vllm(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ custom_prompt_dict = ctx.custom_prompt_dict
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+
+ custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
+ model_response = vllm_handler.completion(
+ model=model,
+ messages=messages,
+ custom_prompt_dict=custom_prompt_dict,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ )
+
+ if "stream" in optional_params and optional_params["stream"] is True: ## [BETA]
+ # don't try to access stream object,
+ return CustomStreamWrapper(
+ model_response,
+ model,
+ custom_llm_provider="vllm",
+ logging_obj=logging,
+ )
+
+ ## RESPONSE OBJECT
+ return model_response
+
+
+def _complete_ollama(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ litellm.api_base
+ or api_base
+ or get_secret("OLLAMA_API_BASE")
+ or "http://localhost:11434"
+ )
+ if api_key is not None and "Authorization" not in headers:
+ headers["Authorization"] = f"Bearer {api_key}"
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="ollama",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+
+
+def _complete_ollama_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = (
+ litellm.api_base
+ or api_base
+ or get_secret("OLLAMA_API_BASE")
+ or "http://localhost:11434"
+ )
+
+ api_key = (
+ api_key
+ or litellm.ollama_key
+ or os.environ.get("OLLAMA_API_KEY")
+ or litellm.api_key
+ )
+ if api_key is not None and "Authorization" not in headers:
+ headers["Authorization"] = f"Bearer {api_key}"
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="ollama_chat",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ client=client,
+ )
+
+
+def _complete_triton(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = litellm.api_base or api_base
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider=custom_llm_provider,
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ )
+
+
+def _complete_cloudflare(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.cloudflare_api_key
+ or litellm.api_key
+ or get_secret("CLOUDFLARE_API_KEY")
+ )
+ api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE")
+
+ custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="cloudflare",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
+ )
+
+
+def _complete_petals(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ api_base = ctx.api_base
+ client = ctx.client
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+
+ api_base = api_base or litellm.api_base
+
+ stream = optional_params.pop("stream", False)
+ model_response = petals_handler.completion(
+ model=model,
+ messages=messages,
+ api_base=api_base,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ encoding=_get_encoding(),
+ logging_obj=logging,
+ client=client,
+ )
+ if stream is True: ## [BETA]
+ # Fake streaming for petals
+ resp_string = model_response["choices"][0]["message"]["content"]
+ return CustomStreamWrapper(
+ resp_string,
+ model,
+ custom_llm_provider="petals",
+ logging_obj=logging,
+ )
+ return model_response
+
+
+def _complete_snowflake(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ try:
+ client = (
+ HTTPHandler(timeout=timeout) if stream is False else None
+ ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ )
+
+ except Exception as e:
+ ## LOGGING - log the original exception returned
+ logging.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=str(e),
+ additional_args={"headers": headers},
+ )
+ raise e
+
+ return response
+
+
+def _complete_gradient_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_base = litellm.api_base or api_base
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider="gradient_ai",
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ )
+
+
+def _complete_bytez(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.bytez_key
+ or get_secret_str("BYTEZ_API_KEY")
+ or litellm.api_key
+ )
+
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=bytez_transformation,
+ )
+
+ pass
+
+ return response
+
+
+def _complete_lemonade(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.lemonade_key
+ or get_secret_str("LEMONADE_API_KEY")
+ or litellm.api_key
+ )
+
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=lemonade_transformation,
+ )
+
+ pass
+
+ return response
+
+
+def _complete_ovhcloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ api_key = (
+ api_key
+ or litellm.ovhcloud_key
+ or get_secret_str("OVHCLOUD_API_KEY")
+ or litellm.api_key
+ )
+
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("OVHCLOUD_API_BASE")
+ or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1"
+ )
+
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=_get_encoding(),
+ stream=stream,
+ provider_config=ovhcloud_transformation,
+ )
+
+ pass
+
+ return response
+
+
+def _complete_custom(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ api_base = ctx.api_base
+ headers = ctx.headers
+ kwargs = ctx.kwargs
+ max_tokens = ctx.max_tokens
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ temperature = ctx.temperature
+ top_p = ctx.top_p
+
+ url = litellm.api_base or api_base or ""
+ if url is None or url == "":
+ raise ValueError(
+ "api_base not set. Set api_base or litellm.api_base for custom endpoints"
+ )
+
+ """
+ assume input to custom LLM api bases follow this format:
+ resp = litellm.module_level_client.post(
+ api_base,
+ json={
+ 'model': 'meta-llama/Llama-2-13b-hf', # model name
+ 'params': {
+ 'prompt': ["The capital of France is P"],
+ 'max_tokens': 32,
+ 'temperature': 0.7,
+ 'top_p': 1.0,
+ 'top_k': 40,
+ }
+ }
+ )
+
+ """
+ prompt = " ".join([message["content"] for message in messages]) # type: ignore
+ resp = litellm.module_level_client.post(
+ url,
+ headers=headers,
+ json={
+ "model": model,
+ "params": {
+ "prompt": [prompt],
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ "top_p": top_p,
+ "top_k": kwargs.get("top_k"),
+ },
+ **kwargs.get("extra_body", {}),
+ },
+ )
+ response_json = resp.json()
+ """
+ assume all responses from custom api_bases of this format:
+ {
+ 'data': [
+ {
+ 'prompt': 'The capital of France is P',
+ 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'],
+ 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}],
+ 'message': 'ok'
+ }
+ ]
+ }
+ """
+ string_response = response_json["data"][0]["output"][0]
+ ## RESPONSE OBJECT
+ model_response.choices[0].message.content = string_response # type: ignore
+ model_response.created = int(time.time())
+ model_response.model = model
+ return model_response
+
+
+def _complete_custom_providers(
+ ctx: _CompletionDispatchContext,
+) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ custom_prompt_dict = ctx.custom_prompt_dict
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logger_fn = ctx.logger_fn
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ custom_handler: Optional[CustomLLM] = None
+ for item in litellm.custom_provider_map:
+ if item["provider"] == custom_llm_provider:
+ custom_handler = item["custom_handler"]
+
+ if custom_handler is None:
+ raise LiteLLMUnknownProvider(
+ model=model, custom_llm_provider=custom_llm_provider
+ )
+
+ ## ROUTE LLM CALL ##
+ handler_fn = custom_chat_llm_router(
+ async_fn=acompletion, stream=stream, custom_llm=custom_handler
+ )
+
+ headers = headers or litellm.headers or {}
+
+ ## CALL FUNCTION
+ response = handler_fn(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ print_verbose=print_verbose,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ timeout=timeout, # type: ignore
+ custom_prompt_dict=custom_prompt_dict,
+ client=client, # pass AsyncOpenAI, OpenAI client
+ encoding=_get_encoding(),
+ )
+ if stream is True:
+ return CustomStreamWrapper(
+ completion_stream=response,
+ model=model,
+ custom_llm_provider=custom_llm_provider,
+ logging_obj=logging,
+ )
+
+ return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
+
+
+def _complete_langgraph(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ from litellm.llms.langgraph.chat.transformation import LangGraphConfig
+
+ (
+ api_base,
+ api_key,
+ ) = LangGraphConfig()._get_openai_compatible_provider_info(
+ api_base=api_base or litellm.api_base,
+ api_key=api_key or litellm.api_key,
+ )
+
+ headers = headers or litellm.headers
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider=custom_llm_provider,
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ client=client,
+ )
+
+
+def _complete_langflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+ acompletion = ctx.acompletion
+ api_base = ctx.api_base
+ api_key = ctx.api_key
+ client = ctx.client
+ custom_llm_provider = ctx.custom_llm_provider
+ headers = ctx.headers
+ litellm_params = ctx.litellm_params
+ logging = ctx.logging
+ messages = ctx.messages
+ model = ctx.model
+ model_response = ctx.model_response
+ optional_params = ctx.optional_params
+ shared_session = ctx.shared_session
+ stream = ctx.stream
+ timeout = ctx.timeout
+
+ from litellm.llms.langflow.chat.transformation import LangFlowConfig
+
+ (
+ api_base,
+ api_key,
+ ) = LangFlowConfig()._get_openai_compatible_provider_info(
+ api_base=api_base or litellm.api_base,
+ api_key=api_key or litellm.api_key,
+ )
+
+ headers = headers or litellm.headers
+
+ return base_llm_http_handler.completion(
+ model=model,
+ stream=stream,
+ messages=messages,
+ acompletion=acompletion,
+ api_base=api_base,
+ model_response=model_response,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ shared_session=shared_session,
+ custom_llm_provider=custom_llm_provider,
+ timeout=timeout,
+ headers=headers,
+ encoding=_get_encoding(),
+ api_key=api_key,
+ logging_obj=logging,
+ client=client,
+ )
+
+
@tracer.wrap()
@client
def completion( # type: ignore
@@ -1215,9 +5077,7 @@ def completion( # type: ignore
if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(
tools=tools_for_mcp
):
- # Return coroutine - acompletion will await it
- # completion() can return a coroutine when MCP tools are present, which acompletion() awaits
- return acompletion_with_mcp( # type: ignore[return-value]
+ return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it
model=model,
messages=messages,
functions=functions,
@@ -1389,12 +5249,16 @@ def completion( # type: ignore
logging: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, litellm_logging_obj)
fallbacks = fallbacks or litellm.model_fallbacks
if fallbacks is not None:
- return completion_with_fallbacks(**args)
+ return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolves to ModelResponse|CustomStreamWrapper at runtime
+ **args
+ )
if model_list is not None:
deployments = [
m["litellm_params"] for m in model_list if m["model_name"] == model
]
- return litellm.batch_completion_models(deployments=deployments, **args)
+ return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of responses, outside completion()'s single-response return type
+ deployments=deployments, **args
+ )
if litellm.model_alias_map and model in litellm.model_alias_map:
model = litellm.model_alias_map[
model
@@ -1454,7 +5318,7 @@ def completion( # type: ignore
timeout,
kwargs,
custom_llm_provider,
- global_timeout=getattr(litellm, "request_timeout", None),
+ global_timeout=get_configured_request_timeout(),
supports_httpx_timeout=supports_httpx_timeout,
)
@@ -1716,7 +5580,7 @@ def completion( # type: ignore
else:
optional_params["reasoning_effort"] = {"summary": rs_val}
- return responses_api_bridge.completion(
+ return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller
model=model,
messages=messages,
headers=headers,
@@ -1746,375 +5610,52 @@ def completion( # type: ignore
optional_params
)
+ _dispatch_ctx = _CompletionDispatchContext(
+ _azure_detection_model=_azure_detection_model,
+ acompletion=acompletion,
+ api_base=api_base,
+ api_key=api_key,
+ api_version=api_version,
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ custom_prompt_dict=custom_prompt_dict,
+ extra_headers=extra_headers,
+ headers=headers,
+ hf_model_name=hf_model_name,
+ kwargs=kwargs,
+ litellm_params=litellm_params,
+ logger_fn=logger_fn,
+ logging=logging,
+ max_retries=max_retries,
+ max_tokens=max_tokens,
+ messages=messages,
+ metadata=metadata,
+ model=model,
+ model_response=model_response,
+ optional_params=optional_params,
+ organization=organization,
+ provider_config=provider_config,
+ shared_session=shared_session,
+ stream=stream,
+ temperature=temperature,
+ text_completion=text_completion,
+ timeout=timeout,
+ top_p=top_p,
+ )
if custom_llm_provider == "azure":
# azure configs
## check dynamic params ##
- dynamic_params = False
- if client is not None and (
- isinstance(client, openai.AzureOpenAI)
- or isinstance(client, openai.AsyncAzureOpenAI)
- ):
- dynamic_params = _check_dynamic_azure_params(
- azure_client_params={"api_version": api_version},
- azure_client=client,
- )
-
- api_type = get_secret("AZURE_API_TYPE") or "azure"
-
- api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE")
-
- api_version = (
- api_version
- or litellm.api_version
- or get_secret_str("AZURE_API_VERSION")
- or litellm.AZURE_DEFAULT_API_VERSION
- )
-
- api_key = (
- api_key
- or litellm.api_key
- or litellm.azure_key
- or get_secret_str("AZURE_OPENAI_API_KEY")
- or get_secret_str("AZURE_API_KEY")
- )
-
- azure_ad_token = optional_params.get("extra_body", {}).pop(
- "azure_ad_token", None
- ) or get_secret_str("AZURE_AD_TOKEN")
-
- azure_ad_token_provider = litellm_params.get(
- "azure_ad_token_provider", None
- )
-
- headers = headers or litellm.headers
-
- if extra_headers is not None:
- optional_params["extra_headers"] = extra_headers
- if max_retries is not None:
- optional_params["max_retries"] = max_retries
-
- if litellm.AzureOpenAIO1Config().is_o_series_model(
- model=_azure_detection_model
- ):
- ## LOAD CONFIG - if set
- config = litellm.AzureOpenAIO1Config.get_config()
- for k, v in config.items():
- if (
- k not in optional_params
- ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
- optional_params[k] = v
-
- response = azure_o1_chat_completions.completion(
- model=model,
- messages=messages,
- headers=headers,
- api_key=api_key,
- api_base=api_base,
- api_version=api_version,
- dynamic_params=dynamic_params,
- azure_ad_token=azure_ad_token,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- logging_obj=logging,
- acompletion=acompletion,
- timeout=timeout, # type: ignore
- client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
- custom_llm_provider=custom_llm_provider,
- )
- else:
- ## LOAD CONFIG - if set
- config = litellm.AzureOpenAIConfig.get_config()
- for k, v in config.items():
- if (
- k not in optional_params
- ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
- optional_params[k] = v
-
- ## COMPLETION CALL
- response = azure_chat_completions.completion(
- model=model,
- messages=messages,
- headers=headers,
- api_key=api_key,
- api_base=api_base,
- api_version=api_version,
- api_type=api_type,
- dynamic_params=dynamic_params,
- azure_ad_token=azure_ad_token,
- azure_ad_token_provider=azure_ad_token_provider,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- logging_obj=logging,
- acompletion=acompletion,
- timeout=timeout, # type: ignore
- client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
- )
-
- if optional_params.get("stream", False):
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- additional_args={
- "headers": headers,
- "api_version": api_version,
- "api_base": api_base,
- },
- )
+ response = _complete_azure(_dispatch_ctx)
elif custom_llm_provider == "azure_text":
# azure configs
- api_type = get_secret_str("AZURE_API_TYPE") or "azure"
-
- api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
-
- if api_base is None:
- raise ValueError(
- "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable."
- )
-
- api_version = (
- api_version
- or litellm.api_version
- or get_secret_str("AZURE_API_VERSION")
- )
-
- api_key = (
- api_key
- or litellm.api_key
- or litellm.azure_key
- or get_secret_str("AZURE_OPENAI_API_KEY")
- or get_secret_str("AZURE_API_KEY")
- )
-
- azure_ad_token = optional_params.get("extra_body", {}).pop(
- "azure_ad_token", None
- ) or get_secret_str("AZURE_AD_TOKEN")
-
- azure_ad_token_provider = litellm_params.get(
- "azure_ad_token_provider", None
- )
-
- headers = headers or litellm.headers
-
- if extra_headers is not None:
- optional_params["extra_headers"] = extra_headers
-
- ## LOAD CONFIG - if set
- config = litellm.AzureOpenAIConfig.get_config()
- for k, v in config.items():
- if (
- k not in optional_params
- ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
- optional_params[k] = v
-
- ## COMPLETION CALL
- response = azure_text_completions.completion(
- model=model,
- messages=messages,
- headers=headers,
- api_key=api_key,
- api_base=api_base,
- api_version=cast(str, api_version),
- api_type=api_type,
- azure_ad_token=azure_ad_token,
- azure_ad_token_provider=azure_ad_token_provider,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- logging_obj=logging,
- acompletion=acompletion,
- timeout=timeout,
- client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
- )
-
- if optional_params.get("stream", False) or acompletion is True:
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- additional_args={
- "headers": headers,
- "api_version": api_version,
- "api_base": api_base,
- },
- )
+ response = _complete_azure_text(_dispatch_ctx)
elif custom_llm_provider == "deepseek":
## COMPLETION CALL
- try:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
+ response = _complete_deepseek(_dispatch_ctx)
elif custom_llm_provider == "azure_ai":
- from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
-
- azure_ai_route = AzureFoundryModelInfo.get_azure_ai_route(model)
-
- # Check if this is an agents route - model format: azure_ai/agents/
- if azure_ai_route == "agents":
- from litellm.llms.azure_ai.agents import AzureAIAgentsConfig
-
- api_base = AzureFoundryModelInfo.get_api_base(api_base)
- if api_base is None:
- raise ValueError(
- "Azure AI Agents requests require an api_base. "
- "Set `api_base` or the AZURE_AI_API_BASE env var."
- )
- api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
- response = AzureAIAgentsConfig.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- api_key=api_key,
- model_response=model_response,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout,
- acompletion=acompletion,
- stream=stream,
- headers=headers or litellm.headers,
- )
-
- # Check if this is a Claude model - route to Azure Anthropic handler
- elif "claude" in model.lower():
- # Use Azure Anthropic handler for Claude models
- api_base = AzureFoundryModelInfo.get_api_base(api_base)
- if api_base is None:
- raise ValueError(
- "Azure Anthropic requests require an api_base. "
- "Set `api_base` or the AZURE_AI_API_BASE env var."
- )
- api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
- # Ensure the URL ends with /v1/messages for Anthropic
- if api_base:
- api_base = api_base.rstrip("/")
- if not api_base.endswith("/v1/messages"):
- if "/anthropic" in api_base:
- parts = api_base.split("/anthropic", 1)
- api_base = parts[0] + "/anthropic"
- else:
- api_base = api_base + "/anthropic"
- api_base = api_base + "/v1/messages"
-
- response = azure_anthropic_chat_completions.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- acompletion=acompletion,
- custom_prompt_dict=litellm.custom_prompt_dict,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- headers=headers,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- )
- if optional_params.get("stream", False) or acompletion is True:
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- )
- response = response
- else:
- # Non-Claude models use standard Azure AI flow
- api_base = AzureFoundryModelInfo.get_api_base(api_base)
- # set API KEY
- api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
- headers = headers or litellm.headers
-
- if extra_headers is not None:
- optional_params["extra_headers"] = extra_headers
-
- ## FOR COHERE
- if "command-r" in model: # make sure tool call in messages are str
- messages = stringify_json_tool_call_content(messages=messages)
-
- ## COMPLETION CALL
- try:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout, # type: ignore
- client=client, # pass AsyncOpenAI, OpenAI client
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- )
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
-
- if optional_params.get("stream", False):
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- additional_args={"headers": headers},
- )
+ response = _complete_azure_ai(_dispatch_ctx)
elif (
custom_llm_provider == "text-completion-openai"
or "ft:babbage-002" in model
@@ -2123,535 +5664,42 @@ def completion( # type: ignore
in litellm.openai_text_completion_compatible_providers
and kwargs.get("text_completion") is True
):
- openai.api_type = "openai"
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("OPENAI_BASE_URL")
- or get_secret("OPENAI_API_BASE")
- or "https://api.openai.com/v1"
- )
-
- openai.api_version = None
- # set API KEY
-
- api_key = (
- api_key
- or litellm.api_key
- or litellm.openai_key
- or get_secret("OPENAI_API_KEY")
- )
-
- headers = headers or litellm.headers
-
- ## LOAD CONFIG - if set
- config = litellm.OpenAITextCompletionConfig.get_config()
- for k, v in config.items():
- if (
- k not in optional_params
- ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in
- optional_params[k] = v
- if litellm.organization:
- openai.organization = litellm.organization
-
- if (
- len(messages) > 0
- and "content" in messages[0]
- and isinstance(messages[0]["content"], list)
- ):
- # text-davinci-003 can accept a string or array, if it's an array, assume the array is set in messages[0]['content']
- # https://platform.openai.com/docs/api-reference/completions/create
- prompt = messages[0]["content"]
- else:
- prompt = " ".join([message["content"] for message in messages]) # type: ignore
-
- ## COMPLETION CALL
- _response = openai_text_completions.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- print_verbose=print_verbose,
- api_key=api_key,
- custom_llm_provider=custom_llm_provider,
- api_base=api_base,
- acompletion=acompletion,
- client=client, # pass AsyncOpenAI, OpenAI client
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- timeout=timeout, # type: ignore
- )
-
- if (
- optional_params.get("stream", False) is False
- and acompletion is False
- and text_completion is False
- ):
- # convert to chat completion response
- _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
- response_object=_response, model_response_object=model_response
- )
-
- if optional_params.get("stream", False) or acompletion is True:
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=_response,
- additional_args={"headers": headers},
- )
- response = _response
+ response = _complete_text_completion_openai(_dispatch_ctx)
elif custom_llm_provider == "fireworks_ai":
## COMPLETION CALL
- try:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
+ response = _complete_fireworks_ai(_dispatch_ctx)
elif custom_llm_provider == "heroku":
- try:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
- except Exception as e:
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
+ response = _complete_heroku(_dispatch_ctx)
elif custom_llm_provider == "ragflow":
## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths
- try:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
- except Exception as e:
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
+ response = _complete_ragflow(_dispatch_ctx)
elif custom_llm_provider == "xai":
## COMPLETION CALL
- try:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
+ response = _complete_xai(_dispatch_ctx)
elif custom_llm_provider == "groq":
- api_base = (
- api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
- or litellm.api_base
- or get_secret("GROQ_API_BASE")
- or "https://api.groq.com/openai/v1"
- )
-
- # set API KEY
- api_key = (
- api_key
- or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
- or litellm.groq_key
- or get_secret("GROQ_API_KEY")
- )
-
- headers = headers or litellm.headers
-
- ## LOAD CONFIG - if set
- config = litellm.GroqChatConfig.get_config()
- for k, v in config.items():
- if (
- k not in optional_params
- ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
- optional_params[k] = v
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider=custom_llm_provider,
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
+ response = _complete_groq(_dispatch_ctx)
elif custom_llm_provider == "bedrock_mantle":
- api_base = (
- api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE")
- )
- api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
- headers = headers or litellm.headers
- config = litellm.BedrockMantleChatConfig.get_config()
- for k, v in config.items():
- if k not in optional_params:
- optional_params[k] = v
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider=custom_llm_provider,
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- client=client,
- )
+ response = _complete_bedrock_mantle(_dispatch_ctx)
elif custom_llm_provider == "a2a":
# A2A (Agent-to-Agent) Protocol
# Resolve agent configuration from registry if model format is "a2a/"
- (
- api_base,
- api_key,
- headers,
- ) = litellm.A2AConfig.resolve_agent_config_from_registry(
- model=model,
- api_base=api_base,
- api_key=api_key,
- headers=headers,
- optional_params=optional_params,
- )
-
- # Fall back to environment variables and defaults
- api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE")
-
- if api_base is None:
- raise Exception(
- "api_base is required for A2A provider. "
- "Either provide api_base parameter, set A2A_API_BASE environment variable, "
- "or register the agent in the proxy with model='a2a/'."
- )
-
- headers = headers or litellm.headers
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider=custom_llm_provider,
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- client=client,
- provider_config=provider_config,
- )
+ response = _complete_a2a(_dispatch_ctx)
elif custom_llm_provider == "gigachat":
# GigaChat - Sber AI's LLM (Russia)
- api_key = (
- api_key
- or litellm.api_key
- or litellm.gigachat_key
- or get_secret("GIGACHAT_API_KEY")
- or get_secret("GIGACHAT_CREDENTIALS")
- )
-
- headers = headers or litellm.headers or {}
-
- ## COMPLETION CALL
- try:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
+ response = _complete_gigachat(_dispatch_ctx)
elif custom_llm_provider == "sap":
- headers = headers or litellm.headers
- ## LOAD CONFIG - if set
- config = litellm.GenAIHubOrchestrationConfig.get_config()
- for k, v in config.items():
- if (
- k not in optional_params
- ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
- optional_params[k] = v
-
- response = sap_gen_ai_hub_chat_completions.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout, # type: ignore
- shared_session=shared_session,
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- api_key=api_key,
- api_base=api_base,
- stream=stream,
- )
+ response = _complete_sap(_dispatch_ctx)
elif custom_llm_provider == "aiohttp_openai":
# NEW aiohttp provider for 10-100x higher RPS
- api_base = (
- api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
- or litellm.api_base
- or get_secret("OPENAI_BASE_URL")
- or get_secret("OPENAI_API_BASE")
- or "https://api.openai.com/v1"
- )
- # set API KEY
- api_key = (
- api_key
- or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
- or litellm.openai_key
- or get_secret("OPENAI_API_KEY")
- )
-
- headers = headers or litellm.headers
-
- if extra_headers is not None:
- optional_params["extra_headers"] = extra_headers
- response = base_llm_aiohttp_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- )
+ response = _complete_aiohttp_openai(_dispatch_ctx)
elif custom_llm_provider == "cometapi":
- api_key = (
- api_key
- or litellm.cometapi_key
- or get_secret_str("COMETAPI_KEY")
- or litellm.api_key
- )
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("COMETAPI_API_BASE")
- or "https://api.cometapi.com/v1"
- )
-
- ## COMPLETION CALL
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
-
- ## LOGGING
- logging.post_call(
- input=messages, api_key=api_key, original_response=response
- )
+ response = _complete_cometapi(_dispatch_ctx)
elif custom_llm_provider == "minimax":
- api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("MINIMAX_API_BASE")
- or "https://api.minimax.io/v1"
- )
-
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- custom_llm_provider=custom_llm_provider,
- model_response=model_response,
- encoding=_get_encoding(),
- logging_obj=logging,
- optional_params=optional_params,
- timeout=timeout,
- litellm_params=litellm_params,
- shared_session=shared_session,
- acompletion=acompletion,
- stream=stream,
- api_key=api_key,
- headers=headers,
- client=client,
- provider_config=provider_config,
- )
- logging.post_call(
- input=messages, api_key=api_key, original_response=response
- )
+ response = _complete_minimax(_dispatch_ctx)
elif custom_llm_provider == "hosted_vllm":
- api_base = (
- api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE")
- )
-
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- custom_llm_provider=custom_llm_provider,
- model_response=model_response,
- encoding=_get_encoding(),
- logging_obj=logging,
- optional_params=optional_params,
- timeout=timeout,
- litellm_params=litellm_params,
- shared_session=shared_session,
- acompletion=acompletion,
- stream=stream,
- api_key=api_key,
- headers=headers,
- client=client,
- provider_config=provider_config,
- )
- logging.post_call(
- input=messages, api_key=api_key, original_response=response
- )
+ response = _complete_hosted_vllm(_dispatch_ctx)
elif (
model in litellm.open_ai_chat_completion_models
or custom_llm_provider == "custom_openai"
@@ -2676,205 +5724,17 @@ def completion( # type: ignore
): # allow user to make an openai call with a custom base
# note: if a user sets a custom base - we should ensure this works
# allow for the setting of dynamic and stateful api-bases
- api_base = (
- api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
- or litellm.api_base
- or get_secret("OPENAI_BASE_URL")
- or get_secret("OPENAI_API_BASE")
- or "https://api.openai.com/v1"
- )
- organization = (
- organization
- or litellm.organization
- or get_secret("OPENAI_ORGANIZATION")
- or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
- )
- openai.organization = organization
- # set API KEY
- api_key = (
- api_key
- or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
- or litellm.openai_key
- or get_secret("OPENAI_API_KEY")
- )
-
- headers = headers or litellm.headers
-
- # Add GitHub Copilot headers (same as /responses endpoint does)
- if custom_llm_provider == "github_copilot":
- from litellm.llms.github_copilot.authenticator import Authenticator
- from litellm.llms.github_copilot.common_utils import (
- get_copilot_default_headers,
- )
-
- copilot_auth = Authenticator()
- copilot_api_key = copilot_auth.get_api_key()
- copilot_headers = get_copilot_default_headers(copilot_api_key)
- if extra_headers:
- copilot_headers.update(extra_headers)
- extra_headers = copilot_headers
-
- if extra_headers is not None:
- optional_params["extra_headers"] = extra_headers
-
- if (
- litellm.enable_preview_features and metadata is not None
- ): # [PREVIEW] allow metadata to be passed to OPENAI
- openai_metadata = get_requester_metadata(metadata)
- if openai_metadata is not None:
- optional_params["metadata"] = openai_metadata
-
- ## LOAD CONFIG - if set
- config = litellm.OpenAIConfig.get_config()
- for k, v in config.items():
- if (
- k not in optional_params
- ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
- optional_params[k] = v
-
- ## COMPLETION CALL
- use_base_llm_http_handler = get_secret_bool(
- "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER"
- )
-
- try:
- if use_base_llm_http_handler:
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- custom_llm_provider=custom_llm_provider,
- model_response=model_response,
- encoding=_get_encoding(),
- logging_obj=logging,
- optional_params=optional_params,
- timeout=timeout,
- litellm_params=litellm_params,
- shared_session=shared_session,
- acompletion=acompletion,
- stream=stream,
- api_key=api_key,
- headers=headers,
- client=client,
- provider_config=provider_config,
- )
- else:
- response = openai_chat_completions.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- print_verbose=print_verbose,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- timeout=timeout, # type: ignore
- custom_prompt_dict=custom_prompt_dict,
- client=client, # pass AsyncOpenAI, OpenAI client
- organization=organization,
- custom_llm_provider=custom_llm_provider,
- shared_session=shared_session,
- )
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
-
- if optional_params.get("stream", False):
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- additional_args={"headers": headers},
- )
+ response = _complete_custom_openai(_dispatch_ctx)
elif custom_llm_provider == "mistral":
- api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY")
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("MISTRAL_API_BASE")
- or "https://api.mistral.ai/v1"
- )
-
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- custom_llm_provider=custom_llm_provider,
- model_response=model_response,
- encoding=_get_encoding(),
- logging_obj=logging,
- optional_params=optional_params,
- timeout=timeout,
- litellm_params=litellm_params,
- shared_session=shared_session,
- acompletion=acompletion,
- stream=stream,
- api_key=api_key,
- headers=headers,
- client=client,
- provider_config=provider_config,
- )
+ response = _complete_mistral(_dispatch_ctx)
elif (
"replicate" in model
or custom_llm_provider == "replicate"
or model in litellm.replicate_models
):
# Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN")
- replicate_key = (
- api_key
- or litellm.replicate_key
- or litellm.api_key
- or get_secret("REPLICATE_API_KEY")
- or get_secret("REPLICATE_API_TOKEN")
- )
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("REPLICATE_API_BASE")
- or "https://api.replicate.com/v1"
- )
-
- custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
-
- model_response = replicate_chat_completion( # type: ignore
- model=model,
- messages=messages,
- api_base=api_base,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(), # for calculating input/output tokens
- api_key=replicate_key,
- logging_obj=logging,
- custom_prompt_dict=custom_prompt_dict,
- acompletion=acompletion,
- headers=headers,
- )
-
- if optional_params.get("stream", False) is True:
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=replicate_key,
- original_response=model_response,
- )
-
- response = model_response
+ response = _complete_replicate(_dispatch_ctx)
elif (
"clarifai" in model
or custom_llm_provider == "clarifai"
@@ -2882,614 +5742,36 @@ def completion( # type: ignore
):
pass # Deprecated - handled in the openai compatible provider section above
elif custom_llm_provider == "anthropic_text":
- api_key = (
- api_key
- or litellm.anthropic_key
- or litellm.api_key
- or os.environ.get("ANTHROPIC_API_KEY")
- )
- custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("ANTHROPIC_API_BASE")
- or get_secret("ANTHROPIC_BASE_URL")
- or "https://api.anthropic.com/v1/complete"
- )
-
- # Check if we should disable automatic URL suffix appending
- disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
- if (
- api_base is not None
- and not disable_url_suffix
- and not api_base.endswith("/v1/complete")
- ):
- api_base += "/v1/complete"
- elif disable_url_suffix:
- verbose_logger.debug(
- "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix"
- )
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="anthropic_text",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- )
+ response = _complete_anthropic_text(_dispatch_ctx)
elif custom_llm_provider == "anthropic":
- api_key = (
- api_key
- or litellm.anthropic_key
- or litellm.api_key
- or os.environ.get("ANTHROPIC_API_KEY")
- )
- custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
- # call /messages
- # default route for all anthropic models
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("ANTHROPIC_API_BASE")
- or get_secret("ANTHROPIC_BASE_URL")
- or "https://api.anthropic.com/v1/messages"
- )
-
- # Check if we should disable automatic URL suffix appending
- disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
- if (
- api_base is not None
- and not disable_url_suffix
- and not api_base.endswith("/v1/messages")
- ):
- api_base += "/v1/messages"
- elif disable_url_suffix:
- verbose_logger.debug(
- "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix"
- )
-
- response = anthropic_chat_completions.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- acompletion=acompletion,
- custom_prompt_dict=litellm.custom_prompt_dict,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(), # for calculating input/output tokens
- api_key=api_key,
- logging_obj=logging,
- headers=headers,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- )
- if optional_params.get("stream", False) or acompletion is True:
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- )
- response = response
+ response = _complete_anthropic(_dispatch_ctx)
elif custom_llm_provider == "nlp_cloud":
- nlp_cloud_key = (
- api_key
- or litellm.nlp_cloud_key
- or get_secret("NLP_CLOUD_API_KEY")
- or litellm.api_key
- )
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("NLP_CLOUD_API_BASE")
- or "https://api.nlpcloud.io/v1/gpu/"
- )
-
- response = nlp_cloud_chat_completion(
- model=model,
- messages=messages,
- api_base=api_base,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- api_key=nlp_cloud_key,
- logging_obj=logging,
- )
-
- if "stream" in optional_params and optional_params["stream"] is True:
- # don't try to access stream object,
- response = CustomStreamWrapper(
- response,
- model,
- custom_llm_provider="nlp_cloud",
- logging_obj=logging,
- )
-
- if optional_params.get("stream", False) or acompletion is True:
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- )
-
- response = response
+ response = _complete_nlp_cloud(_dispatch_ctx)
elif custom_llm_provider == "aleph_alpha":
- aleph_alpha_key = (
- api_key
- or litellm.aleph_alpha_key
- or get_secret("ALEPH_ALPHA_API_KEY")
- or get_secret("ALEPHALPHA_API_KEY")
- or litellm.api_key
- )
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("ALEPH_ALPHA_API_BASE")
- or "https://api.aleph-alpha.com/complete"
- )
-
- model_response = aleph_alpha.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- default_max_tokens_to_sample=litellm.max_tokens,
- api_key=aleph_alpha_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- )
-
- if "stream" in optional_params and optional_params["stream"] is True:
- # don't try to access stream object,
- response = CustomStreamWrapper(
- model_response,
- model,
- custom_llm_provider="aleph_alpha",
- logging_obj=logging,
- )
- return response
- response = model_response
+ response = _complete_aleph_alpha(_dispatch_ctx)
elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
- cohere_key = (
- api_key
- or litellm.cohere_key
- or get_secret_str("COHERE_API_KEY")
- or get_secret_str("CO_API_KEY")
- or litellm.api_key
- )
-
- cohere_route = CohereModelInfo.get_cohere_route(model)
- verbose_logger.debug(f"Cohere route: {cohere_route}")
- # Set API base based on route
- if cohere_route == "v2":
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("COHERE_API_BASE")
- or "https://api.cohere.com/v2/chat"
- )
- # Remove v2/ prefix from model name for the actual API call
- if "v2/" in model:
- model = model.replace("v2/", "")
- else:
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("COHERE_API_BASE")
- or "https://api.cohere.ai/v1/chat"
- )
-
- headers = headers or litellm.headers or {}
- if headers is None:
- headers = {}
-
- if extra_headers is not None:
- headers.update(extra_headers)
-
- verbose_logger.debug(f"Model: {model}, API Base: {api_base}")
- verbose_logger.debug(f"Provider Config: {provider_config}")
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="cohere_chat",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=cohere_key,
- provider_config=provider_config,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- )
+ response = _complete_cohere_chat(_dispatch_ctx)
elif custom_llm_provider == "maritalk":
- maritalk_key = (
- api_key
- or litellm.maritalk_key
- or get_secret("MARITALK_API_KEY")
- or litellm.api_key
- )
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("MARITALK_API_BASE")
- or "https://chat.maritaca.ai/api"
- )
-
- model_response = openai_like_chat_completion.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- api_key=maritalk_key,
- logging_obj=logging,
- custom_llm_provider="maritalk",
- custom_prompt_dict=custom_prompt_dict,
- )
-
- response = model_response
+ response = _complete_maritalk(_dispatch_ctx)
elif custom_llm_provider == "amazon_nova":
- api_key = (
- api_key
- or litellm.amazon_nova_api_key
- or get_secret_str("AMAZON_NOVA_API_KEY")
- or litellm.api_key
- )
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("AMAZON_NOVA_API_BASE")
- or "https://api.nova.amazon.com/v1"
- )
- response = openai_like_chat_completion.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- timeout=timeout,
- custom_llm_provider=custom_llm_provider,
- custom_prompt_dict=custom_prompt_dict,
- )
+ response = _complete_amazon_nova(_dispatch_ctx)
elif custom_llm_provider == "huggingface":
- huggingface_key = (
- api_key
- or litellm.huggingface_key
- or os.environ.get("HF_TOKEN")
- or os.environ.get("HUGGINGFACE_API_KEY")
- or litellm.api_key
- )
- hf_headers = headers or litellm.headers
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=hf_headers,
- model_response=model_response,
- api_key=huggingface_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- )
+ response = _complete_huggingface(_dispatch_ctx)
elif custom_llm_provider == "oci":
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- )
+ response = _complete_oci(_dispatch_ctx)
elif custom_llm_provider == "compactifai":
- api_key = (
- api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
- )
-
- api_base = api_base or "https://api.compactif.ai/v1"
-
- ## COMPLETION CALL
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout,
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
+ response = _complete_compactifai(_dispatch_ctx)
elif custom_llm_provider == "oobabooga":
- custom_llm_provider = "oobabooga"
- model_response = oobabooga.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- api_base=api_base, # type: ignore
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- api_key=None,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- logging_obj=logging,
- )
- if "stream" in optional_params and optional_params["stream"] is True:
- # don't try to access stream object,
- response = CustomStreamWrapper(
- model_response,
- model,
- custom_llm_provider="oobabooga",
- logging_obj=logging,
- )
- return response
- response = model_response
+ response = _complete_oobabooga(_dispatch_ctx)
elif custom_llm_provider == "databricks":
- api_base = (
- api_base # for databricks we check in get_llm_provider and pass in the api base from there
- or litellm.api_base
- or os.getenv("DATABRICKS_API_BASE")
- )
-
- # set API KEY
- api_key = (
- api_key
- or litellm.api_key # for databricks we check in get_llm_provider and pass in the api key from there
- or litellm.databricks_key
- or get_secret("DATABRICKS_API_KEY")
- )
-
- headers = headers or litellm.headers
-
- ## COMPLETION CALL
- try:
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- custom_llm_provider="databricks",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
-
- if optional_params.get("stream", False):
- ## LOGGING
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=response,
- additional_args={"headers": headers},
- )
+ response = _complete_databricks(_dispatch_ctx)
elif custom_llm_provider == "datarobot":
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=provider_config,
- )
+ response = _complete_datarobot(_dispatch_ctx)
elif custom_llm_provider == "openrouter":
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("OPENROUTER_API_BASE")
- or "https://openrouter.ai/api/v1"
- )
-
- api_key = (
- api_key
- or litellm.api_key
- or litellm.openrouter_key
- or get_secret_str("OPENROUTER_API_KEY")
- or get_secret_str("OR_API_KEY")
- )
-
- openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai"
- openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM"
-
- openrouter_headers = {
- "HTTP-Referer": openrouter_site_url,
- "X-Title": openrouter_app_name,
- }
-
- _headers = headers or litellm.headers
- if _headers:
- openrouter_headers.update(_headers)
-
- headers = openrouter_headers
-
- ## Load Config
- config = litellm.OpenrouterConfig.get_config()
- for k, v in config.items():
- if k == "extra_body":
- # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models
- if "extra_body" in optional_params:
- optional_params[k].update(v)
- else:
- optional_params[k] = v
- elif k not in optional_params:
- optional_params[k] = v
-
- data = {"model": model, "messages": messages, **optional_params}
-
- ## COMPLETION CALL
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="openrouter",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
- ## LOGGING
- logging.post_call(
- input=messages, api_key=openai.api_key, original_response=response
- )
+ response = _complete_openrouter(_dispatch_ctx)
elif custom_llm_provider == "vercel_ai_gateway":
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
- or "https://ai-gateway.vercel.sh/v1"
- )
-
- api_key = (
- api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
- )
-
- vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
- vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM"
-
- vercel_headers = {
- "http-referer": vercel_site_url,
- "x-title": vercel_app_name,
- }
-
- _headers = headers or litellm.headers
- if _headers:
- vercel_headers.update(_headers)
-
- headers = vercel_headers
-
- ## Load Config
- config = litellm.VercelAIGatewayConfig.get_config()
- for k, v in config.items():
- if k == "extra_body":
- # we use openai 'extra_body' to pass vercel specific params - providerOptions
- if "extra_body" in optional_params:
- optional_params[k].update(v)
- else:
- optional_params[k] = v
- elif k not in optional_params:
- optional_params[k] = v
-
- data = {"model": model, "messages": messages, **optional_params}
-
- ## COMPLETION CALL
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="vercel_ai_gateway",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
- ## LOGGING
- logging.post_call(
- input=messages, api_key=openai.api_key, original_response=response
- )
+ response = _complete_vercel_ai_gateway(_dispatch_ctx)
elif (
custom_llm_provider == "together_ai"
or ("togethercomputer" in model)
@@ -3504,1114 +5786,75 @@ def completion( # type: ignore
"Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en"
)
elif custom_llm_provider == "vertex_ai_beta" or custom_llm_provider == "gemini":
- vertex_ai_project = (
- optional_params.pop("vertex_project", None)
- or optional_params.pop("vertex_ai_project", None)
- or litellm.vertex_project
- or get_secret("VERTEXAI_PROJECT")
- )
- vertex_ai_location = (
- optional_params.pop("vertex_location", None)
- or optional_params.pop("vertex_ai_location", None)
- or litellm.vertex_location
- or get_secret("VERTEXAI_LOCATION")
- )
- vertex_credentials = (
- optional_params.pop("vertex_credentials", None)
- or optional_params.pop("vertex_ai_credentials", None)
- or get_secret("VERTEXAI_CREDENTIALS")
- )
-
- gemini_api_key = (
- api_key
- or get_api_key_from_env()
- or get_secret("PALM_API_KEY") # older palm api key should also work
- or litellm.api_key
- )
-
- api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE")
- new_params = safe_deep_copy(optional_params or {})
- response = vertex_chat_completion.completion( # type: ignore
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=new_params,
- litellm_params=litellm_params, # type: ignore
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- vertex_location=vertex_ai_location,
- vertex_project=vertex_ai_project,
- vertex_credentials=vertex_credentials,
- gemini_api_key=gemini_api_key,
- logging_obj=logging,
- acompletion=acompletion,
- timeout=timeout,
- custom_llm_provider=custom_llm_provider, # type: ignore
- client=client,
- api_base=api_base,
- extra_headers=headers,
- )
+ response = _complete_vertex_ai_beta(_dispatch_ctx)
elif custom_llm_provider == "vertex_ai":
- vertex_ai_project = (
- optional_params.pop("vertex_project", None)
- or optional_params.pop("vertex_ai_project", None)
- or litellm.vertex_project
- or get_secret("VERTEXAI_PROJECT")
- )
- vertex_ai_location = (
- optional_params.pop("vertex_location", None)
- or optional_params.pop("vertex_ai_location", None)
- or litellm.vertex_location
- or get_secret("VERTEXAI_LOCATION")
- )
- vertex_credentials = (
- optional_params.pop("vertex_credentials", None)
- or optional_params.pop("vertex_ai_credentials", None)
- or get_secret("VERTEXAI_CREDENTIALS")
- )
-
- api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE")
-
- new_params = safe_deep_copy(optional_params or {})
- model_route = get_vertex_ai_model_route(
- model=model, litellm_params=litellm_params
- )
-
- if model_route == VertexAIModelRoute.PARTNER_MODELS:
- model_response = vertex_partner_models_chat_completion.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=new_params,
- litellm_params=litellm_params, # type: ignore
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- api_base=api_base,
- vertex_location=vertex_ai_location,
- vertex_project=vertex_ai_project,
- vertex_credentials=vertex_credentials,
- logging_obj=logging,
- acompletion=acompletion,
- headers=headers,
- custom_prompt_dict=custom_prompt_dict,
- timeout=timeout,
- client=client,
- )
- elif model_route == VertexAIModelRoute.GEMINI:
- model_response = vertex_chat_completion.completion( # type: ignore
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=new_params,
- litellm_params=litellm_params, # type: ignore
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- vertex_location=vertex_ai_location,
- vertex_project=vertex_ai_project,
- vertex_credentials=vertex_credentials,
- gemini_api_key=None,
- logging_obj=logging,
- acompletion=acompletion,
- timeout=timeout,
- custom_llm_provider=custom_llm_provider, # type: ignore
- client=client,
- api_base=api_base,
- extra_headers=headers,
- )
- elif model_route == VertexAIModelRoute.GEMMA:
- # Vertex Gemma Models with custom prediction endpoint
- model_response = vertex_gemma_chat_completion.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=new_params,
- litellm_params=litellm_params, # type: ignore
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- api_base=api_base,
- vertex_location=vertex_ai_location,
- vertex_project=vertex_ai_project,
- vertex_credentials=vertex_credentials,
- logging_obj=logging,
- acompletion=acompletion,
- headers=headers,
- custom_prompt_dict=custom_prompt_dict,
- timeout=timeout,
- client=client,
- )
- elif model_route == VertexAIModelRoute.MODEL_GARDEN:
- # Vertex Model Garden - OpenAI compatible models
- model_response = vertex_model_garden_chat_completion.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=new_params,
- litellm_params=litellm_params, # type: ignore
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- api_base=api_base,
- vertex_location=vertex_ai_location,
- vertex_project=vertex_ai_project,
- vertex_credentials=vertex_credentials,
- logging_obj=logging,
- acompletion=acompletion,
- headers=headers,
- custom_prompt_dict=custom_prompt_dict,
- timeout=timeout,
- client=client,
- )
- elif model_route == VertexAIModelRoute.AGENT_ENGINE:
- # Vertex AI Agent Engine (Reasoning Engines)
- from litellm.llms.vertex_ai.agent_engine.transformation import (
- VertexAgentEngineConfig,
- )
-
- vertex_agent_engine_config = VertexAgentEngineConfig()
-
- # Update litellm_params with vertex credentials
- litellm_params["vertex_project"] = vertex_ai_project
- litellm_params["vertex_location"] = vertex_ai_location
- litellm_params["vertex_credentials"] = vertex_credentials
-
- model_response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- model_response=model_response,
- optional_params=new_params,
- litellm_params=litellm_params, # type: ignore
- encoding=_get_encoding(),
- api_key=None,
- api_base=api_base,
- logging_obj=logging,
- acompletion=acompletion,
- timeout=timeout,
- client=client,
- custom_llm_provider="vertex_ai",
- provider_config=vertex_agent_engine_config,
- headers=headers or {},
- )
- else: # VertexAIModelRoute.NON_GEMINI
- model_response = vertex_ai_non_gemini.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=new_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- vertex_location=vertex_ai_location,
- vertex_project=vertex_ai_project,
- vertex_credentials=vertex_credentials,
- logging_obj=logging,
- acompletion=acompletion,
- )
-
- if (
- "stream" in optional_params
- and optional_params["stream"] is True
- and acompletion is False
- ):
- response = CustomStreamWrapper(
- model_response,
- model,
- custom_llm_provider="vertex_ai",
- logging_obj=logging,
- )
- return response
- response = model_response
+ response = _complete_vertex_ai(_dispatch_ctx)
elif custom_llm_provider == "predibase":
- tenant_id = (
- optional_params.pop("tenant_id", None)
- or optional_params.pop("predibase_tenant_id", None)
- or litellm.predibase_tenant_id
- or get_secret("PREDIBASE_TENANT_ID")
- )
-
- if tenant_id is None:
- raise ValueError(
- "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`."
- )
-
- api_base = (
- api_base
- or optional_params.pop("api_base", None)
- or optional_params.pop("base_url", None)
- or litellm.api_base
- or get_secret("PREDIBASE_API_BASE")
- )
-
- api_key = (
- api_key
- or litellm.api_key
- or litellm.predibase_key
- or get_secret("PREDIBASE_API_KEY")
- )
-
- _model_response = predibase_chat_completions.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- logging_obj=logging,
- acompletion=acompletion,
- api_base=api_base,
- custom_prompt_dict=custom_prompt_dict,
- api_key=api_key,
- tenant_id=tenant_id,
- timeout=timeout,
- )
-
- if (
- "stream" in optional_params
- and optional_params["stream"] is True
- and acompletion is False
- ):
- return _model_response
- response = _model_response
+ response = _complete_predibase(_dispatch_ctx)
elif custom_llm_provider == "text-completion-codestral":
- api_base = (
- api_base
- or optional_params.pop("api_base", None)
- or optional_params.pop("base_url", None)
- or litellm.api_base
- or "https://codestral.mistral.ai/v1/fim/completions"
- )
-
- api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY")
-
- text_completion_model_response = litellm.TextCompletionResponse(
- stream=stream
- )
-
- _model_response = codestral_text_completions.completion( # type: ignore
- model=model,
- messages=messages,
- model_response=text_completion_model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- logging_obj=logging,
- acompletion=acompletion,
- api_base=api_base,
- custom_prompt_dict=custom_prompt_dict,
- api_key=api_key,
- timeout=timeout,
- )
-
- if (
- "stream" in optional_params
- and optional_params["stream"] is True
- and acompletion is False
- ):
- return _model_response
- response = _model_response
+ response = _complete_text_completion_codestral(_dispatch_ctx)
elif custom_llm_provider == "text-completion-inception":
- passed_api_base = (
- api_base
- or optional_params.pop("api_base", None)
- or optional_params.pop("base_url", None)
- )
- api_base = (
- passed_api_base
- or get_secret_str("INCEPTION_API_BASE")
- or "https://api.inceptionlabs.ai/v1"
- )
- # FIM is served at `/v1/fim/completions`; the OpenAI client appends
- # `/completions`, so point it at the `/v1/fim` base.
- api_base = api_base.rstrip("/")
- if not api_base.endswith("/fim"):
- api_base += "/fim"
-
- # Don't forward the server-managed Inception key to a caller-supplied
- # api_base; only resolve it for the default/server base, or when the
- # caller passes their own key.
- if passed_api_base is None or api_key:
- api_key = (
- api_key
- or litellm.inception_key
- or get_secret_str("INCEPTION_API_KEY")
- )
-
- _response = openai_text_completions.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- api_key=api_key, # type: ignore[arg-type]
- custom_llm_provider="text-completion-inception",
- api_base=api_base,
- acompletion=acompletion,
- client=client,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- timeout=timeout, # type: ignore
- )
-
- if (
- optional_params.get("stream", False) is False
- and acompletion is False
- and text_completion is False
- ):
- _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
- response_object=_response, model_response_object=model_response
- )
-
- if optional_params.get("stream", False) or acompletion is True:
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=_response,
- additional_args={"headers": headers},
- )
- response = _response
+ response = _complete_text_completion_inception(_dispatch_ctx)
elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"):
# boto3 reads keys from .env
# sagemaker_chat: HF Messages API endpoints
# sagemaker_nova: Nova models on SageMaker (OpenAI-compatible)
- model_response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- custom_llm_provider=custom_llm_provider,
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
-
- ## RESPONSE OBJECT
- response = model_response
+ response = _complete_sagemaker_chat(_dispatch_ctx)
elif custom_llm_provider == "sagemaker":
# boto3 reads keys from .env
- model_response = sagemaker_llm.completion(
- model=model,
- messages=messages,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- custom_prompt_dict=custom_prompt_dict,
- hf_model_name=hf_model_name,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- logging_obj=logging,
- acompletion=acompletion,
- )
-
- ## RESPONSE OBJECT
- response = model_response
+ response = _complete_sagemaker(_dispatch_ctx)
elif custom_llm_provider == "bedrock":
# boto3 reads keys from .env
- custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
-
- if "aws_bedrock_client" in optional_params:
- verbose_logger.warning(
- "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication."
- )
- # Extract credentials for legacy boto3 client and pass thru to httpx
- aws_bedrock_client = optional_params.pop("aws_bedrock_client")
- creds = aws_bedrock_client._get_credentials().get_frozen_credentials()
-
- if creds.access_key:
- optional_params["aws_access_key_id"] = creds.access_key
- if creds.secret_key:
- optional_params["aws_secret_access_key"] = creds.secret_key
- if creds.token:
- optional_params["aws_session_token"] = creds.token
- if (
- "aws_region_name" not in optional_params
- or optional_params["aws_region_name"] is None
- ):
- optional_params["aws_region_name"] = (
- aws_bedrock_client.meta.region_name
- )
-
- bedrock_route = BedrockModelInfo.get_bedrock_route(model)
- if bedrock_route == "claude_platform":
- provider_config = ProviderConfigManager.get_provider_chat_config(
- model=model,
- provider=LlmProviders.BEDROCK,
- )
- model = BedrockModelInfo.get_claude_platform_model(model)
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="bedrock",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- client=client,
- provider_config=provider_config,
- )
- return response
- elif bedrock_route == "converse":
- model = model.replace("converse/", "")
- response = bedrock_converse_chat_completion.completion(
- model=model,
- messages=messages,
- custom_prompt_dict=custom_prompt_dict,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params, # type: ignore
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- logging_obj=logging,
- extra_headers=headers, # Use merged headers instead of original extra_headers
- timeout=timeout,
- acompletion=acompletion,
- client=client,
- api_base=api_base,
- api_key=api_key,
- )
- elif bedrock_route == "converse_like":
- model = model.replace("converse_like/", "")
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- custom_llm_provider="bedrock",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
- else:
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- custom_llm_provider="bedrock",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- client=client,
- )
+ response = _complete_bedrock(_dispatch_ctx)
elif custom_llm_provider == "watsonx":
- response = watsonx_chat_completion.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- print_verbose=print_verbose,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- timeout=timeout, # type: ignore
- custom_prompt_dict=custom_prompt_dict,
- client=client, # pass AsyncOpenAI, OpenAI client
- encoding=_get_encoding(),
- custom_llm_provider="watsonx",
- )
+ response = _complete_watsonx(_dispatch_ctx)
elif custom_llm_provider == "watsonx_text":
- api_key = (
- api_key
- or optional_params.pop("apikey", None)
- or get_secret_str("WATSONX_APIKEY")
- or get_secret_str("WATSONX_API_KEY")
- or get_secret_str("WX_API_KEY")
- )
-
- api_base = (
- api_base
- or optional_params.pop(
- "url",
- optional_params.pop(
- "api_base", optional_params.pop("base_url", None)
- ),
- )
- or get_secret_str("WATSONX_API_BASE")
- or get_secret_str("WATSONX_URL")
- or get_secret_str("WX_URL")
- or get_secret_str("WML_URL")
- )
-
- wx_credentials = optional_params.pop(
- "wx_credentials",
- optional_params.pop(
- "watsonx_credentials", None
- ), # follow {provider}_credentials, same as vertex ai
- )
-
- token: Optional[str] = None
- if wx_credentials is not None:
- api_base = wx_credentials.get("url", api_base)
- api_key = wx_credentials.get(
- "apikey", wx_credentials.get("api_key", api_key)
- )
- token = wx_credentials.get(
- "token",
- wx_credentials.get(
- "watsonx_token", None
- ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..'
- )
-
- if token is not None:
- optional_params["token"] = token
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="watsonx_text",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
+ response = _complete_watsonx_text(_dispatch_ctx)
elif custom_llm_provider == "vllm":
- custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
- model_response = vllm_handler.completion(
- model=model,
- messages=messages,
- custom_prompt_dict=custom_prompt_dict,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- logging_obj=logging,
- )
-
- if (
- "stream" in optional_params and optional_params["stream"] is True
- ): ## [BETA]
- # don't try to access stream object,
- response = CustomStreamWrapper(
- model_response,
- model,
- custom_llm_provider="vllm",
- logging_obj=logging,
- )
- return response
-
- ## RESPONSE OBJECT
- response = model_response
+ response = _complete_vllm(_dispatch_ctx)
elif custom_llm_provider == "ollama":
- api_base = (
- litellm.api_base
- or api_base
- or get_secret("OLLAMA_API_BASE")
- or "http://localhost:11434"
- )
- if api_key is not None and "Authorization" not in headers:
- headers["Authorization"] = f"Bearer {api_key}"
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="ollama",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
+ response = _complete_ollama(_dispatch_ctx)
elif custom_llm_provider == "ollama_chat":
- api_base = (
- litellm.api_base
- or api_base
- or get_secret("OLLAMA_API_BASE")
- or "http://localhost:11434"
- )
-
- api_key = (
- api_key
- or litellm.ollama_key
- or os.environ.get("OLLAMA_API_KEY")
- or litellm.api_key
- )
- if api_key is not None and "Authorization" not in headers:
- headers["Authorization"] = f"Bearer {api_key}"
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="ollama_chat",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- client=client,
- )
+ response = _complete_ollama_chat(_dispatch_ctx)
elif custom_llm_provider == "triton":
- api_base = litellm.api_base or api_base
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider=custom_llm_provider,
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- )
+ response = _complete_triton(_dispatch_ctx)
elif custom_llm_provider == "cloudflare":
- api_key = (
- api_key
- or litellm.cloudflare_api_key
- or litellm.api_key
- or get_secret("CLOUDFLARE_API_KEY")
- )
- account_id = get_secret("CLOUDFLARE_ACCOUNT_ID")
- api_base = (
- api_base
- or litellm.api_base
- or get_secret("CLOUDFLARE_API_BASE")
- or f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/"
- )
-
- custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="cloudflare",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
- )
+ response = _complete_cloudflare(_dispatch_ctx)
elif custom_llm_provider == "petals" or model in litellm.petals_models:
- api_base = api_base or litellm.api_base
-
- custom_llm_provider = "petals"
- stream = optional_params.pop("stream", False)
- model_response = petals_handler.completion(
- model=model,
- messages=messages,
- api_base=api_base,
- model_response=model_response,
- print_verbose=print_verbose,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- encoding=_get_encoding(),
- logging_obj=logging,
- client=client,
- )
- if stream is True: ## [BETA]
- # Fake streaming for petals
- resp_string = model_response["choices"][0]["message"]["content"]
- response = CustomStreamWrapper(
- resp_string,
- model,
- custom_llm_provider="petals",
- logging_obj=logging,
- )
- return response
- response = model_response
+ response = _complete_petals(_dispatch_ctx)
elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models:
- try:
- client = (
- HTTPHandler(timeout=timeout) if stream is False else None
- ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- )
-
- except Exception as e:
- ## LOGGING - log the original exception returned
- logging.post_call(
- input=messages,
- api_key=api_key,
- original_response=str(e),
- additional_args={"headers": headers},
- )
- raise e
+ response = _complete_snowflake(_dispatch_ctx)
elif custom_llm_provider == "gradient_ai":
- api_base = litellm.api_base or api_base
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider="gradient_ai",
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- )
+ response = _complete_gradient_ai(_dispatch_ctx)
elif custom_llm_provider == "bytez":
- api_key = (
- api_key
- or litellm.bytez_key
- or get_secret_str("BYTEZ_API_KEY")
- or litellm.api_key
- )
-
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=bytez_transformation,
- )
-
- pass
+ response = _complete_bytez(_dispatch_ctx)
elif custom_llm_provider == "lemonade":
- api_key = (
- api_key
- or litellm.lemonade_key
- or get_secret_str("LEMONADE_API_KEY")
- or litellm.api_key
- )
-
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=lemonade_transformation,
- )
-
- pass
+ response = _complete_lemonade(_dispatch_ctx)
elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models:
- api_key = (
- api_key
- or litellm.ovhcloud_key
- or get_secret_str("OVHCLOUD_API_KEY")
- or litellm.api_key
- )
-
- api_base = (
- api_base
- or litellm.api_base
- or get_secret_str("OVHCLOUD_API_BASE")
- or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1"
- )
-
- response = base_llm_http_handler.completion(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- timeout=timeout, # type: ignore
- client=client,
- custom_llm_provider=custom_llm_provider,
- encoding=_get_encoding(),
- stream=stream,
- provider_config=ovhcloud_transformation,
- )
-
- pass
+ response = _complete_ovhcloud(_dispatch_ctx)
elif custom_llm_provider == "custom":
- url = litellm.api_base or api_base or ""
- if url is None or url == "":
- raise ValueError(
- "api_base not set. Set api_base or litellm.api_base for custom endpoints"
- )
-
- """
- assume input to custom LLM api bases follow this format:
- resp = litellm.module_level_client.post(
- api_base,
- json={
- 'model': 'meta-llama/Llama-2-13b-hf', # model name
- 'params': {
- 'prompt': ["The capital of France is P"],
- 'max_tokens': 32,
- 'temperature': 0.7,
- 'top_p': 1.0,
- 'top_k': 40,
- }
- }
- )
-
- """
- prompt = " ".join([message["content"] for message in messages]) # type: ignore
- resp = litellm.module_level_client.post(
- url,
- headers=headers,
- json={
- "model": model,
- "params": {
- "prompt": [prompt],
- "max_tokens": max_tokens,
- "temperature": temperature,
- "top_p": top_p,
- "top_k": kwargs.get("top_k"),
- },
- **kwargs.get("extra_body", {}),
- },
- )
- response_json = resp.json()
- """
- assume all responses from custom api_bases of this format:
- {
- 'data': [
- {
- 'prompt': 'The capital of France is P',
- 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'],
- 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}],
- 'message': 'ok'
- }
- ]
- }
- """
- string_response = response_json["data"][0]["output"][0]
- ## RESPONSE OBJECT
- model_response.choices[0].message.content = string_response # type: ignore
- model_response.created = int(time.time())
- model_response.model = model
- response = model_response
+ response = _complete_custom(_dispatch_ctx)
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider
# Get the Custom Handler
- custom_handler: Optional[CustomLLM] = None
- for item in litellm.custom_provider_map:
- if item["provider"] == custom_llm_provider:
- custom_handler = item["custom_handler"]
-
- if custom_handler is None:
- raise LiteLLMUnknownProvider(
- model=model, custom_llm_provider=custom_llm_provider
- )
-
- ## ROUTE LLM CALL ##
- handler_fn = custom_chat_llm_router(
- async_fn=acompletion, stream=stream, custom_llm=custom_handler
- )
-
- headers = headers or litellm.headers or {}
-
- ## CALL FUNCTION
- response = handler_fn(
- model=model,
- messages=messages,
- headers=headers,
- model_response=model_response,
- print_verbose=print_verbose,
- api_key=api_key,
- api_base=api_base,
- acompletion=acompletion,
- logging_obj=logging,
- optional_params=optional_params,
- litellm_params=litellm_params,
- logger_fn=logger_fn,
- timeout=timeout, # type: ignore
- custom_prompt_dict=custom_prompt_dict,
- client=client, # pass AsyncOpenAI, OpenAI client
- encoding=_get_encoding(),
- )
- if stream is True:
- return CustomStreamWrapper(
- completion_stream=response,
- model=model,
- custom_llm_provider=custom_llm_provider,
- logging_obj=logging,
- )
+ response = _complete_custom_providers(_dispatch_ctx)
elif custom_llm_provider == "langgraph":
# LangGraph - Agent Runtime Provider
- from litellm.llms.langgraph.chat.transformation import LangGraphConfig
-
- (
- api_base,
- api_key,
- ) = LangGraphConfig()._get_openai_compatible_provider_info(
- api_base=api_base or litellm.api_base,
- api_key=api_key or litellm.api_key,
- )
-
- headers = headers or litellm.headers
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider=custom_llm_provider,
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- client=client,
- )
+ response = _complete_langgraph(_dispatch_ctx)
elif custom_llm_provider == "langflow":
# LangFlow - Visual AI Agent Platform
- from litellm.llms.langflow.chat.transformation import LangFlowConfig
-
- (
- api_base,
- api_key,
- ) = LangFlowConfig()._get_openai_compatible_provider_info(
- api_base=api_base or litellm.api_base,
- api_key=api_key or litellm.api_key,
- )
-
- headers = headers or litellm.headers
-
- response = base_llm_http_handler.completion(
- model=model,
- stream=stream,
- messages=messages,
- acompletion=acompletion,
- api_base=api_base,
- model_response=model_response,
- optional_params=optional_params,
- litellm_params=litellm_params,
- shared_session=shared_session,
- custom_llm_provider=custom_llm_provider,
- timeout=timeout,
- headers=headers,
- encoding=_get_encoding(),
- api_key=api_key,
- logging_obj=logging,
- client=client,
- )
+ response = _complete_langflow(_dispatch_ctx)
else:
raise LiteLLMUnknownProvider(
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 0ee4a33c4ca..1dc984d5fda 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -571,7 +571,7 @@
"output_vector_size": 1536
},
"amazon.titan-embed-text-v2:0": {
- "input_cost_per_token": 2e-07,
+ "input_cost_per_token": 2e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 8192,
"max_tokens": 8192,
@@ -10684,6 +10684,268 @@
"mode": "chat",
"output_cost_per_token": 1.923e-06
},
+ "cloudflare/@cf/openai/gpt-oss-120b": {
+ "input_cost_per_token": 3.5e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 128000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 7.5e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/google/gemma-2b-it-lora": {
+ "input_cost_per_token": 0.0,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
+ "mode": "chat",
+ "output_cost_per_token": 0.0
+ },
+ "cloudflare/@cf/meta/llama-3.2-3b-instruct": {
+ "input_cost_per_token": 5.09e-08,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 80000,
+ "max_output_tokens": 80000,
+ "max_tokens": 80000,
+ "mode": "chat",
+ "output_cost_per_token": 3.35e-07
+ },
+ "cloudflare/@cf/meta/llama-guard-3-8b": {
+ "input_cost_per_token": 4.84e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 3e-08
+ },
+ "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": {
+ "input_cost_per_token": 0.0,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 15000,
+ "max_output_tokens": 15000,
+ "max_tokens": 15000,
+ "mode": "chat",
+ "output_cost_per_token": 0.0
+ },
+ "cloudflare/@cf/moonshotai/kimi-k2.7-code": {
+ "cache_read_input_token_cost": 1.9e-07,
+ "input_cost_per_token": 9.5e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": {
+ "input_cost_per_token": 4.97e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 80000,
+ "max_output_tokens": 80000,
+ "max_tokens": 80000,
+ "mode": "chat",
+ "output_cost_per_token": 4.881e-06,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": {
+ "input_cost_per_token": 1.52e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 32000,
+ "max_output_tokens": 32000,
+ "max_tokens": 32000,
+ "mode": "chat",
+ "output_cost_per_token": 2.87e-07
+ },
+ "cloudflare/@cf/meta/llama-3.2-1b-instruct": {
+ "input_cost_per_token": 2.7e-08,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 60000,
+ "max_output_tokens": 60000,
+ "max_tokens": 60000,
+ "mode": "chat",
+ "output_cost_per_token": 2.01e-07
+ },
+ "cloudflare/@cf/moonshotai/kimi-k2.6": {
+ "cache_read_input_token_cost": 1.6e-07,
+ "input_cost_per_token": 9.5e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/zai-org/glm-4.7-flash": {
+ "input_cost_per_token": 6.05e-08,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/meta-llama/llama-2-7b-chat-hf-lora": {
+ "input_cost_per_token": 0.0,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
+ "mode": "chat",
+ "output_cost_per_token": 0.0
+ },
+ "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": {
+ "input_cost_per_token": 2.93e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 24000,
+ "max_output_tokens": 24000,
+ "max_tokens": 24000,
+ "mode": "chat",
+ "output_cost_per_token": 2.253e-06,
+ "supports_function_calling": true
+ },
+ "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": {
+ "input_cost_per_token": 1.7e-08,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 131000,
+ "max_output_tokens": 131000,
+ "max_tokens": 131000,
+ "mode": "chat",
+ "output_cost_per_token": 1.12e-07,
+ "supports_function_calling": true
+ },
+ "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": {
+ "input_cost_per_token": 6.6e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 32768,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 1e-06
+ },
+ "cloudflare/@cf/zai-org/glm-5.2": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/nvidia/nemotron-3-120b-a12b": {
+ "input_cost_per_token": 5e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 256000,
+ "max_output_tokens": 256000,
+ "max_tokens": 256000,
+ "mode": "chat",
+ "output_cost_per_token": 1.5e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/aisingapore/gemma-sea-lion-v4-27b-it": {
+ "input_cost_per_token": 3.51e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 128000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5.55e-07
+ },
+ "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": {
+ "input_cost_per_token": 5.09e-08,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 32768,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 3.35e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/google/gemma-7b-it-lora": {
+ "input_cost_per_token": 0.0,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 3500,
+ "max_output_tokens": 3500,
+ "max_tokens": 3500,
+ "mode": "chat",
+ "output_cost_per_token": 0.0
+ },
+ "cloudflare/@cf/google/gemma-4-26b-a4b-it": {
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 256000,
+ "max_output_tokens": 256000,
+ "max_tokens": 256000,
+ "mode": "chat",
+ "output_cost_per_token": 3e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/mistralai/mistral-small-3.1-24b-instruct": {
+ "input_cost_per_token": 3.51e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 128000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5.55e-07,
+ "supports_function_calling": true
+ },
+ "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": {
+ "input_cost_per_token": 4.85e-08,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 128000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 6.76e-07,
+ "supports_vision": true
+ },
+ "cloudflare/@cf/openai/gpt-oss-20b": {
+ "input_cost_per_token": 2e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 128000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 3e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true
+ },
+ "cloudflare/@cf/meta/llama-4-scout-17b-16e-instruct": {
+ "input_cost_per_token": 2.7e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 131000,
+ "max_output_tokens": 131000,
+ "max_tokens": 131000,
+ "mode": "chat",
+ "output_cost_per_token": 8.5e-07,
+ "supports_function_calling": true
+ },
+ "cloudflare/@cf/qwen/qwq-32b": {
+ "input_cost_per_token": 6.6e-07,
+ "litellm_provider": "cloudflare",
+ "max_input_tokens": 24000,
+ "max_output_tokens": 24000,
+ "max_tokens": 24000,
+ "mode": "chat",
+ "output_cost_per_token": 1e-06,
+ "supports_reasoning": true
+ },
"codestral/codestral-2405": {
"input_cost_per_token": 0.0,
"litellm_provider": "codestral",
@@ -10912,13 +11174,13 @@
"supports_tool_choice": true
},
"command-r7b-12-2024": {
- "input_cost_per_token": 1.5e-07,
+ "input_cost_per_token": 3.75e-08,
"litellm_provider": "cohere_chat",
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
- "output_cost_per_token": 3.75e-08,
+ "output_cost_per_token": 1.5e-07,
"source": "https://docs.cohere.com/v2/docs/command-r7b",
"supports_function_calling": true,
"supports_tool_choice": true
@@ -14612,6 +14874,38 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
+ "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 384000,
+ "max_tokens": 384000,
+ "mode": "chat",
+ "output_cost_per_token": 2.8e-07,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": {
+ "cache_read_input_token_cost": 1.45e-07,
+ "input_cost_per_token": 1.74e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 384000,
+ "max_tokens": 384000,
+ "mode": "chat",
+ "output_cost_per_token": 3.48e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
"fireworks_ai/accounts/fireworks/models/firefunction-v2": {
"input_cost_per_token": 9e-07,
"litellm_provider": "fireworks_ai",
@@ -14687,43 +14981,64 @@
"input_cost_per_token": 1.4e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 202800,
- "max_output_tokens": 202800,
- "max_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
- "source": "https://fireworks.ai/models/fireworks/glm-5p1",
+ "source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
- "supports_tool_choice": true
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/accounts/fireworks/models/glm-5p2": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/gpt-oss-120b": {
+ "cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6e-07,
- "source": "https://fireworks.ai/pricing",
+ "source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
- "supports_tool_choice": true
+ "supports_tool_choice": true,
+ "supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/gpt-oss-20b": {
- "input_cost_per_token": 5e-08,
+ "cache_read_input_token_cost": 3.5e-08,
+ "input_cost_per_token": 7e-08,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
"mode": "chat",
- "output_cost_per_token": 2e-07,
- "source": "https://fireworks.ai/pricing",
+ "output_cost_per_token": 3e-07,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
- "supports_tool_choice": true
+ "supports_tool_choice": true,
+ "supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": {
"input_cost_per_token": 6e-07,
@@ -14779,6 +15094,38 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
+ "fireworks_ai/accounts/fireworks/models/kimi-k2p6": {
+ "cache_read_input_token_cost": 1.6e-07,
+ "input_cost_per_token": 9.5e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "fireworks_ai/accounts/fireworks/models/kimi-k2p7-code": {
+ "cache_read_input_token_cost": 1.9e-07,
+ "input_cost_per_token": 9.5e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": {
"input_cost_per_token": 3e-06,
"litellm_provider": "fireworks_ai",
@@ -14896,6 +15243,38 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
+ "fireworks_ai/accounts/fireworks/models/minimax-m2p7": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 196608,
+ "max_output_tokens": 196608,
+ "max_tokens": 196608,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/accounts/fireworks/models/minimax-m3": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 512000,
+ "max_output_tokens": 512000,
+ "max_tokens": 512000,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
"litellm_provider": "fireworks_ai",
@@ -14948,6 +15327,38 @@
"supports_response_schema": true,
"supports_tool_choice": false
},
+ "fireworks_ai/deepseek-v4-flash": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 384000,
+ "max_tokens": 384000,
+ "mode": "chat",
+ "output_cost_per_token": 2.8e-07,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/deepseek-v4-pro": {
+ "cache_read_input_token_cost": 1.45e-07,
+ "input_cost_per_token": 1.74e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 384000,
+ "max_tokens": 384000,
+ "mode": "chat",
+ "output_cost_per_token": 3.48e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
"fireworks_ai/glm-4p7": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 6e-07,
@@ -14968,15 +15379,80 @@
"input_cost_per_token": 1.4e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 202800,
- "max_output_tokens": 202800,
- "max_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
- "source": "https://fireworks.ai/models/fireworks/glm-5p1",
+ "source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
- "supports_tool_choice": true
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/glm-5p1-fast": {
+ "cache_read_input_token_cost": 5.2e-07,
+ "input_cost_per_token": 2.8e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 8.8e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/glm-5p2": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/gpt-oss-120b": {
+ "cache_read_input_token_cost": 1.5e-08,
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 6e-07,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/gpt-oss-20b": {
+ "cache_read_input_token_cost": 3.5e-08,
+ "input_cost_per_token": 7e-08,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 3e-07,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
},
"fireworks_ai/kimi-k2p5": {
"cache_read_input_token_cost": 1e-07,
@@ -14992,6 +15468,70 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
+ "fireworks_ai/kimi-k2p6": {
+ "cache_read_input_token_cost": 1.6e-07,
+ "input_cost_per_token": 9.5e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "fireworks_ai/kimi-k2p6-fast": {
+ "cache_read_input_token_cost": 3e-07,
+ "input_cost_per_token": 2e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 8e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "fireworks_ai/kimi-k2p7-code": {
+ "cache_read_input_token_cost": 1.9e-07,
+ "input_cost_per_token": 9.5e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "fireworks_ai/kimi-k2p7-code-fast": {
+ "cache_read_input_token_cost": 3.8e-07,
+ "input_cost_per_token": 1.9e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 8e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"fireworks_ai/minimax-m2p1": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
@@ -15006,6 +15546,54 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
+ "fireworks_ai/minimax-m2p7": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 196608,
+ "max_output_tokens": 196608,
+ "max_tokens": 196608,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/minimax-m3": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 512000,
+ "max_output_tokens": 512000,
+ "max_tokens": 512000,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "fireworks_ai/qwen3p7-plus": {
+ "cache_read_input_token_cost": 8e-08,
+ "input_cost_per_token": 4e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.6e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"fireworks_ai/nomic-ai/nomic-embed-text-v1": {
"input_cost_per_token": 8e-09,
"litellm_provider": "fireworks_ai-embedding-models",
@@ -39467,6 +40055,22 @@
"litellm_provider": "fireworks_ai",
"mode": "chat"
},
+ "fireworks_ai/accounts/fireworks/models/qwen3p7-plus": {
+ "cache_read_input_token_cost": 8e-08,
+ "input_cost_per_token": 4e-07,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.6e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"fireworks_ai/accounts/fireworks/models/qwq-32b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@@ -39566,24 +40170,6 @@
"litellm_provider": "fireworks_ai",
"mode": "chat"
},
- "fireworks_ai/accounts/fireworks/models/whisper-v3": {
- "max_tokens": 4096,
- "max_input_tokens": 4096,
- "max_output_tokens": 4096,
- "input_cost_per_token": 0.0,
- "output_cost_per_token": 0.0,
- "litellm_provider": "fireworks_ai",
- "mode": "audio_transcription"
- },
- "fireworks_ai/accounts/fireworks/models/whisper-v3-turbo": {
- "max_tokens": 4096,
- "max_input_tokens": 4096,
- "max_output_tokens": 4096,
- "input_cost_per_token": 0.0,
- "output_cost_per_token": 0.0,
- "litellm_provider": "fireworks_ai",
- "mode": "audio_transcription"
- },
"fireworks_ai/accounts/fireworks/models/yi-34b": {
"max_tokens": 4096,
"max_input_tokens": 4096,
@@ -39629,6 +40215,54 @@
"litellm_provider": "fireworks_ai",
"mode": "chat"
},
+ "fireworks_ai/accounts/fireworks/routers/glm-5p1-fast": {
+ "cache_read_input_token_cost": 5.2e-07,
+ "input_cost_per_token": 2.8e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 8.8e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast": {
+ "cache_read_input_token_cost": 3e-07,
+ "input_cost_per_token": 2e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 8e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast": {
+ "cache_read_input_token_cost": 3.8e-07,
+ "input_cost_per_token": 1.9e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 8e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"novita/deepseek/deepseek-v3.2": {
"litellm_provider": "novita",
"mode": "chat",
@@ -41993,6 +42627,7 @@
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
+ "supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_reasoning": true,
@@ -42007,6 +42642,7 @@
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
+ "supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_reasoning": true,
@@ -42021,6 +42657,7 @@
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
+ "supported_endpoints": ["/v1/chat/completions"],
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@@ -42034,6 +42671,7 @@
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
+ "supported_endpoints": ["/v1/chat/completions"],
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@@ -42087,6 +42725,8 @@
"max_output_tokens": 256000,
"max_tokens": 256000,
"mode": "chat",
+ "use_openai_responses_path": true,
+ "supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_reasoning": true,
@@ -42101,6 +42741,8 @@
"max_output_tokens": 256000,
"max_tokens": 256000,
"mode": "chat",
+ "use_openai_responses_path": true,
+ "supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_reasoning": true,
@@ -42115,6 +42757,8 @@
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
+ "use_openai_responses_path": true,
+ "supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_reasoning": true,
@@ -42413,6 +43057,19 @@
],
"supports_audio_input": true
},
+ "soniox/stt-async-v5": {
+ "litellm_provider": "soniox",
+ "max_output_tokens": 8000,
+ "max_tokens": 8000,
+ "input_cost_per_second": 0.0,
+ "output_cost_per_second": 0.0000277778,
+ "mode": "audio_transcription",
+ "source": "https://soniox.com/pricing",
+ "supported_endpoints": [
+ "/v1/audio/transcriptions"
+ ],
+ "supports_audio_input": true
+ },
"tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": {
"litellm_provider": "tensormesh",
"mode": "chat",
@@ -42648,6 +43305,40 @@
"supports_tool_choice": true,
"supports_vision": false
},
+ "darkbloom/gemma-4-26b": {
+ "input_cost_per_token": 3e-08,
+ "litellm_provider": "darkbloom",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 1.65e-07,
+ "source": "https://www.darkbloom.dev/",
+ "supported_endpoints": [
+ "/v1/chat/completions"
+ ],
+ "supports_function_calling": true,
+ "supports_native_streaming": true,
+ "supports_system_messages": true,
+ "supports_tool_choice": true
+ },
+ "darkbloom/gpt-oss-20b": {
+ "input_cost_per_token": 1.45e-08,
+ "litellm_provider": "darkbloom",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 7e-08,
+ "source": "https://www.darkbloom.dev/",
+ "supported_endpoints": [
+ "/v1/chat/completions"
+ ],
+ "supports_function_calling": true,
+ "supports_native_streaming": true,
+ "supports_system_messages": true,
+ "supports_tool_choice": true
+ },
"deepseek/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 3.625e-09,
diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py
index b27082c361a..3a9ef8db804 100644
--- a/litellm/ocr/main.py
+++ b/litellm/ocr/main.py
@@ -10,7 +10,7 @@ import os
import re
from functools import partial
from io import IOBase
-from typing import Any, Coroutine, Dict, Optional, Union
+from typing import Any, Callable, Coroutine, Dict, Optional, Union, cast
import httpx
@@ -20,6 +20,7 @@ from litellm.constants import request_timeout
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.ocr.rust_bridge import RustOcr, load_rust_ocr, rust_ocr_enabled
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ProviderConfigManager, client
@@ -28,6 +29,82 @@ base_llm_http_handler = BaseLLMHTTPHandler()
#################################################
+def _timeout_to_seconds(
+ timeout: Optional[Union[float, httpx.Timeout]],
+) -> Optional[float]:
+ """Convert the Python OCR timeout to a single seconds value for the Rust bridge.
+
+ The Rust HTTP client takes one duration; ``httpx.Timeout`` carries separate
+ connect/read/write/pool values, so pick the read deadline as the closest
+ analog to a total-request timeout.
+ """
+ if timeout is None:
+ return None
+ if isinstance(timeout, httpx.Timeout):
+ return timeout.read
+ return float(timeout)
+
+
+def _run_rust_ocr(
+ rust_ocr: RustOcr,
+ logging_obj: LiteLLMLoggingObj,
+ provider_config: BaseOCRConfig,
+ resolve_api_key: Callable[[str], Optional[str]],
+ model: str,
+ document: dict[str, object],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ optional_params: dict[str, object],
+ litellm_params: dict[str, object],
+ timeout_seconds: Optional[float],
+) -> OCRResponse:
+ """Run the Mistral OCR call through the Rust bridge and wrap the result.
+
+ Resolves the key the same way the Python path does so secret-manager backends
+ (AWS/Azure/GCP/Vault) work; the Rust bridge's own fallback only reads the
+ process environment. The request that Rust actually sends (resolved URL and
+ headers) is mirrored into pre_call so logs match the wire. Dependencies are
+ injected so this stays unit-testable without patching module globals.
+ """
+ resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY")
+ resolved_headers = provider_config.validate_environment(
+ headers={},
+ model=model,
+ api_key=resolved_api_key,
+ api_base=api_base,
+ litellm_params=litellm_params,
+ )
+ resolved_complete_url = provider_config.get_complete_url(
+ api_base=api_base,
+ model=model,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+ logging_obj.pre_call(
+ input="OCR document processing",
+ api_key=resolved_api_key,
+ additional_args={
+ "complete_input_dict": {
+ "model": model,
+ "document": document,
+ **optional_params,
+ },
+ "api_base": resolved_complete_url,
+ "headers": resolved_headers,
+ },
+ )
+ return OCRResponse.model_validate(
+ rust_ocr(
+ model=model,
+ document=document,
+ api_key=resolved_api_key,
+ api_base=api_base,
+ optional_params=optional_params,
+ timeout_seconds=timeout_seconds,
+ )
+ )
+
+
@client
async def aocr(
model: str,
@@ -220,7 +297,7 @@ def ocr(
"""
local_vars = locals()
try:
- litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
+ litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj"))
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
_is_async = kwargs.pop("aocr", False) is True
@@ -261,7 +338,6 @@ def ocr(
if dynamic_api_base:
api_base = dynamic_api_base
- # Get provider config
ocr_provider_config: Optional[BaseOCRConfig] = (
ProviderConfigManager.get_provider_ocr_config(
model=model,
@@ -278,17 +354,14 @@ def ocr(
f"OCR call - model: {model}, provider: {custom_llm_provider}"
)
- # Get litellm params using GenericLiteLLMParams (same as responses API)
litellm_params = GenericLiteLLMParams(**kwargs)
- # Extract OCR-specific parameters from kwargs
supported_params = ocr_provider_config.get_supported_ocr_params(model=model)
non_default_params = {}
for param in supported_params:
if param in kwargs:
non_default_params[param] = kwargs.pop(param)
- # Map parameters to provider-specific format
optional_params = ocr_provider_config.map_ocr_params(
non_default_params=non_default_params,
optional_params={},
@@ -297,7 +370,8 @@ def ocr(
verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}")
- # Pre Call logging
+ effective_timeout = timeout or request_timeout
+
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model,
@@ -309,12 +383,35 @@ def ocr(
custom_llm_provider=custom_llm_provider,
)
- # Call the handler - pass document dict directly
+ # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge.
+ if custom_llm_provider == "mistral" and rust_ocr_enabled():
+ rust_ocr = load_rust_ocr()
+ if rust_ocr is None:
+ verbose_logger.debug(
+ "Rust OCR bridge unavailable; falling back to Python path"
+ )
+ else:
+ from litellm.secret_managers.main import get_secret_str
+
+ return _run_rust_ocr(
+ rust_ocr=rust_ocr,
+ logging_obj=litellm_logging_obj,
+ provider_config=ocr_provider_config,
+ resolve_api_key=get_secret_str,
+ model=model,
+ document=document,
+ api_key=api_key,
+ api_base=api_base,
+ optional_params=optional_params,
+ litellm_params=dict(litellm_params),
+ timeout_seconds=_timeout_to_seconds(effective_timeout),
+ )
+
response = base_llm_http_handler.ocr(
model=model,
- document=document, # Pass the entire document dict
+ document=document,
optional_params=optional_params,
- timeout=timeout or request_timeout,
+ timeout=effective_timeout,
logging_obj=litellm_logging_obj,
api_key=api_key,
api_base=api_base,
diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py
new file mode 100644
index 00000000000..61f9e8ca69a
--- /dev/null
+++ b/litellm/ocr/rust_bridge.py
@@ -0,0 +1,74 @@
+"""
+Optional Rust-backed OCR path.
+
+Enable with ``litellm.use_litellm_rust()``; the sync ``litellm.ocr()`` entrypoint
+then routes supported Mistral calls through the compiled ``litellm_python_bridge``
+extension, which performs the whole OCR call (URL, headers, HTTP, parse) in Rust.
+
+No module-level ``litellm`` imports keep this a leaf so ``litellm/ocr/main.py``
+can import it statically without forming an import cycle.
+"""
+
+from __future__ import annotations
+
+from typing import Final, Protocol, cast
+
+
+class RustOcr(Protocol):
+ """Signature of the compiled ``litellm_python_bridge.ocr`` entrypoint."""
+
+ def __call__(
+ self,
+ model: str,
+ document: dict[str, object],
+ api_key: str | None,
+ api_base: str | None,
+ optional_params: dict[str, object],
+ timeout_seconds: float | None,
+ ) -> dict[str, object]: ...
+
+
+class _Unset:
+ """Sentinel type so ``ocr=None`` can clear a prior injection while omission preserves it."""
+
+
+_UNSET: Final[_Unset] = _Unset()
+
+_rust_ocr_enabled = False
+_rust_ocr_impl: RustOcr | None = None
+
+
+def use_litellm_rust(
+ enabled: bool = True, *, ocr: RustOcr | None | _Unset = _UNSET
+) -> None:
+ """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension.
+
+ ``ocr`` injects the bridge callable; when omitted the compiled extension is
+ loaded on demand and any previously injected bridge is preserved. Pass
+ ``ocr=None`` explicitly to clear a prior injection.
+ """
+ global _rust_ocr_enabled, _rust_ocr_impl
+ _rust_ocr_enabled = enabled
+ if not isinstance(ocr, _Unset):
+ _rust_ocr_impl = ocr
+
+
+def rust_ocr_enabled() -> bool:
+ """Whether the Rust OCR path has been turned on via ``use_litellm_rust()``."""
+ return _rust_ocr_enabled
+
+
+def load_rust_ocr() -> RustOcr | None:
+ """Return the Rust OCR callable, or ``None`` when no bridge is available.
+
+ Prefers an injected implementation, otherwise loads the compiled
+ ``litellm_python_bridge`` extension; a missing extension yields ``None`` so
+ the caller can fall back to the Python path instead of hard-failing.
+ """
+ if _rust_ocr_impl is not None:
+ return _rust_ocr_impl
+ try:
+ import litellm_python_bridge
+ except ImportError:
+ return None
+ return cast(RustOcr, litellm_python_bridge.ocr)
diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json
index db6183edaa0..dd7712aabca 100644
--- a/litellm/provider_endpoints_support_backup.json
+++ b/litellm/provider_endpoints_support_backup.json
@@ -1835,6 +1835,23 @@
"interactions": true
}
},
+ "darkbloom": {
+ "display_name": "Darkbloom (`darkbloom`)",
+ "url": "https://docs.litellm.ai/docs/providers/darkbloom",
+ "endpoints": {
+ "chat_completions": true,
+ "messages": false,
+ "responses": false,
+ "embeddings": false,
+ "image_generations": false,
+ "audio_transcriptions": false,
+ "audio_speech": false,
+ "moderations": false,
+ "batches": false,
+ "rerank": false,
+ "a2a": false
+ }
+ },
"predibase": {
"display_name": "Predibase (`predibase`)",
"url": "https://docs.litellm.ai/docs/providers/predibase",
diff --git a/litellm/proxy/_experimental/mcp_server/AGENTS.md b/litellm/proxy/_experimental/mcp_server/AGENTS.md
new file mode 100644
index 00000000000..8eebc3ea3b3
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/AGENTS.md
@@ -0,0 +1,95 @@
+# Experimental MCP Server Change Guidelines
+
+Read @../../../../CLAUDE.md and @CLAUDE.md before changing this package.
+
+This directory owns the proxy-hosted MCP server implementation. Keep changes
+inside the module that owns the behavior, and only reach outside this package
+when the public type contract, database schema, dashboard, or cross-proxy route
+wiring must change with it.
+
+## File Structure
+
+Respect the current package boundaries:
+
+```text
+litellm/proxy/_experimental/mcp_server/
+ AGENTS.md
+ CLAUDE.md
+ server.py # ASGI/MCP route handling, sessions, tool calls [PR7: 7-arm only — move BYOK/OAuth pre-fetch into resolver]
+ mcp_server_manager.py # upstream server registry, clients, tool routing [PR7: _create_mcp_client swaps resolve_mcp_auth -> resolve_credentials]
+ auth/
+ user_api_key_auth_mcp.py # LiteLLM admission auth and MCP request headers
+ token_exchange.py # OAuth token exchange handling [unchanged; V1TokenExchangeAdapter delegates here]
+ litellm_auth_handler.py # authenticated-user adapter for MCP sessions
+ outbound_credentials/ # NEW — typed upstream-credential resolution (resolve_credentials + arms)
+ __init__.py # public surface: resolve_credentials, the configs, CredError
+ result.py # Ok | Error union (pure stdlib)
+ types.py # AuthConfig union, CredError, Subject, ServerSpec
+ httpx_auth.py # NoOpAuth, StaticHeaderAuth (every mode -> one httpx.Auth)
+ resolver.py # resolve_credentials(): exhaustive per-mode match + assert_never
+ seams.py # injected Protocols (one per cache-touching mode)
+ v1_adapters.py # v1-backed seam bodies; delegate to auth/oauth2/db owners
+ adapter.py # to_subject / to_server_spec / raise_public (v1 <-> v2 boundary)
+ discoverable_endpoints.py # MCP OAuth metadata, authorize, token, callback
+ byok_oauth_endpoints.py # BYOK OAuth UI/API flow
+ oauth_utils.py # redirect URI and proxy base URL validation
+ oauth2_token_cache.py # OAuth2 and per-user token resolution/cache [PR7: resolve_mcp_auth removed; cache class stays, V1OAuth2CacheAdapter delegates to async_get_token]
+ db.py # MCP server, credential, env var, submission DB access [unchanged; V1ByokStore delegates to _get_byok_credential / get_user_credential]
+ toolset_db.py # MCP toolset DB access
+ rest_endpoints.py # proxy REST facade for listing/calling MCP tools [PR7: 7-arm only — pass identity + inbound token down instead of mcp_auth_header]
+ openapi_to_mcp_generator.py# OpenAPI spec to MCP tool generation
+ sampling_handler.py # MCP sampling to LiteLLM completion flow
+ elicitation_handler.py # MCP elicitation relay flow
+ semantic_tool_filter.py # semantic filtering of available MCP tools
+ guardrail_translation/
+ handler.py # MCP guardrail result translation
+ sse_transport.py # SSE transport implementation
+ mcp_context.py # contextvars for MCP request/session metadata
+ mcp_debug.py # debug helpers
+ tool_registry.py # in-memory MCP tool registry helpers
+ cost_calculator.py # MCP tool cost calculation
+ ui_session_utils.py # dashboard session auth context helpers
+ utils.py # shared primitives used by several modules
+```
+
+Do not add broad catch-all modules. Prefer the existing owner above, and add a
+new file only for a distinct capability that would otherwise make an existing
+module materially harder to understand.
+
+## Implementation Rules
+
+- Preserve the boundary between LiteLLM admission auth and upstream MCP auth.
+ Admission belongs in `auth/user_api_key_auth_mcp.py`; upstream token exchange,
+ delegated auth, per-user OAuth, BYOK, and raw header forwarding belong in the
+ dedicated OAuth/header modules.
+- Treat `none`, bearer/API key, OAuth, OAuth token exchange, delegated upstream
+ auth, SSE, streamable HTTP, and stdio as separate flows. Do not collapse them
+ behind a single generic branch unless tests prove every mode still behaves
+ correctly.
+- Be especially careful with `available_on_public_internet: false` combined with
+ `delegate_auth_to_upstream: true`. The local `CLAUDE.md` explains the anonymous
+ upstream PKCE path that must remain intentional.
+- Keep database-backed fields in sync across migrations, typed models under
+ `litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this
+ package, and dashboard state when the field is user-visible.
+- Use the official MCP SDK types and established LiteLLM Pydantic models where
+ they exist. Avoid untyped protocol dictionaries at package boundaries.
+- Keep security-sensitive logic easy to audit. Header forwarding, IP filtering,
+ public internet checks, token storage, env var interpolation, and credential
+ encryption need focused tests for both allowed and rejected paths.
+- Avoid adding comments to new code unless they explain non-obvious security or
+ protocol behavior. Prefer clear names and small functions.
+
+## Tests
+
+Mirror this package under `tests/test_litellm/proxy/_experimental/mcp_server/`.
+For regressions, extend the existing mapped test file instead of creating a new
+one. Use subdirectories that match the implementation path, such as
+`auth/test_token_exchange.py` for `auth/token_exchange.py` and
+`guardrail_translation/test_mcp_guardrail_handler.py` for
+`guardrail_translation/handler.py`.
+
+Use `tests/mcp_tests/` only when extending an existing broader MCP integration
+scenario that already lives there. Route, auth, tool listing, tool execution,
+OAuth, sampling, elicitation, DB, and dashboard-session changes should have
+focused coverage in the mirrored `tests/test_litellm/...` path first.
diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
index e47fc84b533..90108de25c3 100644
--- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
+++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
@@ -12,6 +12,7 @@ from litellm.proxy._types import (
LiteLLM_TeamTable,
ProxyException,
SpecialHeaders,
+ SpecialMCPServerNames,
UserAPIKeyAuth,
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
@@ -642,6 +643,15 @@ class MCPRequestHandler:
user_api_key_auth
)
)
+
+ # The key explicitly opted out of every MCP server. This overrides
+ # team inheritance and additive grants (mirrors no-default-models).
+ if (
+ SpecialMCPServerNames.no_mcp_servers.value
+ in allowed_mcp_servers_for_key
+ ):
+ return []
+
allowed_mcp_servers_for_team = (
await MCPRequestHandler._get_allowed_mcp_servers_for_team(
user_api_key_auth
@@ -1058,6 +1068,13 @@ class MCPRequestHandler:
if key_object_permission is None:
return []
+ # Sentinel opt-out: surface it unexpanded so the caller can short-circuit
+ # to zero servers instead of inheriting the team.
+ if SpecialMCPServerNames.no_mcp_servers.value in (
+ key_object_permission.mcp_servers or []
+ ):
+ return [SpecialMCPServerNames.no_mcp_servers.value]
+
# Permission entries may be server_ids OR names/aliases — expand to ids.
direct_mcp_servers = global_mcp_server_manager.expand_permission_list(
key_object_permission.mcp_servers or []
diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
index afec884cd96..5e704b889ae 100644
--- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
@@ -80,6 +80,7 @@ from litellm.proxy._types import (
MCPEnvVar,
MCPTransport,
MCPTransportType,
+ SpecialMCPServerNames,
UserAPIKeyAuth,
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
@@ -1349,6 +1350,17 @@ class MCPServerManager:
allow_all_server_ids = self.get_allow_all_keys_server_ids()
try:
+ # The key explicitly opted out of every MCP server. Return zero before
+ # layering on allow_all_keys servers so the opt-out is absolute.
+ key_object_permission = (
+ user_api_key_auth.object_permission if user_api_key_auth else None
+ )
+ if key_object_permission is not None and (
+ SpecialMCPServerNames.no_mcp_servers.value
+ in (key_object_permission.mcp_servers or [])
+ ):
+ return []
+
# Check if object_permission.mcp_servers is explicitly set
has_explicit_object_permission = False
if user_api_key_auth and user_api_key_auth.object_permission:
diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py
new file mode 100644
index 00000000000..73166a45d6e
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py
@@ -0,0 +1,73 @@
+"""Typed upstream-credential resolution for MCP servers.
+
+This subpackage houses the typed credential vocabulary and the ``resolve_credentials``
+dispatch. A server declares one per-mode config from the ``AuthConfig`` discriminated union;
+``UpstreamCredentialProvider.resolve_credentials`` selects one arm and returns an ``httpx.Auth``
+or a typed ``CredError``. Failures are modeled as values via :mod:`.result` (``Result[T,
+CredError]``) rather than raised, so every seam is total. Nothing here is wired onto a live
+request path yet.
+"""
+
+from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import (
+ NoOpAuth,
+ StaticHeaderAuth,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import (
+ UpstreamCredentialProvider,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
+ Error,
+ Ok,
+ Result,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
+ Ambient,
+ ApiKeyConfig,
+ ApiKeySource,
+ AssumeRole,
+ AuthConfig,
+ AuthorizationCodeConfig,
+ AuthSpecKind,
+ AwsCredentialSource,
+ AwsSigV4Config,
+ Byok,
+ ClientCredentialsConfig,
+ CredError,
+ NoneConfig,
+ PassthroughConfig,
+ ServerSpec,
+ SharedKey,
+ StaticKeys,
+ Subject,
+ TokenExchangeConfig,
+ parse_auth_spec_kind,
+)
+
+__all__ = [
+ "Ok",
+ "Error",
+ "Result",
+ "NoOpAuth",
+ "StaticHeaderAuth",
+ "UpstreamCredentialProvider",
+ "AuthSpecKind",
+ "CredError",
+ "Subject",
+ "ServerSpec",
+ "AuthConfig",
+ "parse_auth_spec_kind",
+ "AuthorizationCodeConfig",
+ "ClientCredentialsConfig",
+ "TokenExchangeConfig",
+ "ApiKeyConfig",
+ "ApiKeySource",
+ "SharedKey",
+ "Byok",
+ "PassthroughConfig",
+ "NoneConfig",
+ "AwsSigV4Config",
+ "AwsCredentialSource",
+ "StaticKeys",
+ "AssumeRole",
+ "Ambient",
+]
diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py
new file mode 100644
index 00000000000..2345fa98123
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py
@@ -0,0 +1,45 @@
+"""Concrete `httpx.Auth` objects the resolver returns for the self-contained modes.
+
+These are the egress credential as the SDK consumes it: an `httpx.Auth` attached to the
+upstream `AsyncClient`. The OAuth-flow modes (`authorization_code`, `client_credentials`,
+`token_exchange`) return SDK-provided auth objects instead and land later.
+
+`auth_flow` mutating the outbound request is the `httpx.Auth` contract, not a house-style
+violation: the request is httpx's object, and these carry no state of their own.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+
+import httpx
+from pydantic import SecretStr
+
+
+class NoOpAuth(httpx.Auth):
+ """Attaches nothing — the `none` mode (and the seam-level default)."""
+
+ def auth_flow(
+ self, request: httpx.Request
+ ) -> Generator[httpx.Request, httpx.Response, None]:
+ yield request
+
+
+class StaticHeaderAuth(httpx.Auth):
+ """Sets one fixed header on every request — the `api_key` family and `passthrough`.
+
+ The header value is a live credential (a bearer token, an API key, a forwarded user
+ token), so it is held as a `SecretStr` and unwrapped only when written onto the request.
+ That keeps it masked in reprs, `vars()`, tracebacks, and structured logs, matching the
+ `SecretStr` discipline the config models use.
+ """
+
+ def __init__(self, header_value: str, header_name: str = "Authorization") -> None:
+ self.header_name = header_name
+ self._header_value = SecretStr(header_value)
+
+ def auth_flow(
+ self, request: httpx.Request
+ ) -> Generator[httpx.Request, httpx.Response, None]:
+ request.headers[self.header_name] = self._header_value.get_secret_value()
+ yield request
diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py
new file mode 100644
index 00000000000..7bcdb3e6529
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py
@@ -0,0 +1,70 @@
+"""The one credential resolver: dispatch on the declared mode, fail closed.
+
+`resolve_credentials` selects exactly one arm off the server's typed `config` and either
+produces an `httpx.Auth` or returns a typed `CredError`. The `match` is over the `AuthConfig`
+variant, so each arm receives its own fully-typed config with no field-presence inference and
+no precedence cascade. It is wildcard-free with an `assert_never` tail, so adding a mode without
+an arm fails the type gate (basedpyright `reportMatchNotExhaustive`); a bypassed gate fails loudly
+at runtime instead of returning `None`.
+
+This skeleton ships every arm as a `not_implemented` stub. Each mode's real body, with its
+injected seam, lands in its own follow-up PR; until then the arm returns a typed error rather
+than silently producing no credential. Pure v2: no imports from v1.
+"""
+
+from __future__ import annotations
+
+import httpx
+from typing_extensions import assert_never
+
+from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
+ Error,
+ Result,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
+ ApiKeyConfig,
+ AuthorizationCodeConfig,
+ AuthSpecKind,
+ AwsSigV4Config,
+ ClientCredentialsConfig,
+ CredError,
+ NoneConfig,
+ PassthroughConfig,
+ ServerSpec,
+ Subject,
+ TokenExchangeConfig,
+)
+
+
+class UpstreamCredentialProvider:
+ """Produces the one `httpx.Auth` for a `(subject, upstream)` pair, per declared mode.
+
+ Collaborators (the per-mode credential stores and token fetchers) are injected as each arm
+ is built; the skeleton needs none, since every arm is a stub.
+ """
+
+ async def resolve_credentials(
+ self, subject: Subject, server: ServerSpec
+ ) -> Result[httpx.Auth, CredError]:
+ match server.config:
+ case NoneConfig():
+ return _not_implemented(AuthSpecKind.none)
+ case ApiKeyConfig():
+ return _not_implemented(AuthSpecKind.api_key)
+ case PassthroughConfig():
+ return _not_implemented(AuthSpecKind.passthrough)
+ case ClientCredentialsConfig():
+ return _not_implemented(AuthSpecKind.client_credentials)
+ case TokenExchangeConfig():
+ return _not_implemented(AuthSpecKind.token_exchange)
+ case AuthorizationCodeConfig():
+ return _not_implemented(AuthSpecKind.authorization_code)
+ case AwsSigV4Config():
+ return _not_implemented(AuthSpecKind.aws_sigv4)
+ assert_never(server.config)
+
+
+def _not_implemented(kind: AuthSpecKind) -> Result[httpx.Auth, CredError]:
+ return Error(
+ CredError.of_not_implemented(f"{kind.value}: resolver arm not implemented yet")
+ )
diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py
new file mode 100644
index 00000000000..a612e8510f5
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py
@@ -0,0 +1,54 @@
+"""A tagged-union ``Result`` the type checker can actually narrow.
+
+``Ok`` and ``Error`` are separate frozen classes joined by a ``Union`` alias, so
+reaching for ``result.ok`` before eliminating the ``Error`` arm (via ``isinstance``
+or a ``match`` pattern) is a type error rather than a runtime ``AttributeError``. A
+single class carrying both payload fields would make that unguarded access invisible
+to the type checker.
+
+Both variants are covariant and frozen; the absent side defaults to ``Never`` so a
+bare ``Ok(value)`` or ``Error(err)`` infers fully and is assignable to any ``Result``
+whose matching side fits.
+
+``is_ok`` / ``is_error`` are runtime predicates that also narrow via their ``Literal``
+returns; inside strictly typed code, discriminate with ``match`` or ``isinstance``.
+
+This is the shared ``Result`` shape for the ``outbound_credentials`` resolver: every
+seam returns ``Result[T, CredError]`` instead of raising, so each failure is a value
+the caller must handle rather than an exception that can slip past the type checker.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Generic, Literal, TypeAlias
+
+from typing_extensions import Never, TypeVar
+
+_TOk_co = TypeVar("_TOk_co", covariant=True, default=Never)
+_TError_co = TypeVar("_TError_co", covariant=True, default=Never)
+
+
+@dataclass(frozen=True)
+class Ok(Generic[_TOk_co, _TError_co]):
+ ok: _TOk_co
+
+ def is_ok(self) -> Literal[True]:
+ return True
+
+ def is_error(self) -> Literal[False]:
+ return False
+
+
+@dataclass(frozen=True)
+class Error(Generic[_TOk_co, _TError_co]):
+ error: _TError_co
+
+ def is_ok(self) -> Literal[False]:
+ return False
+
+ def is_error(self) -> Literal[True]:
+ return True
+
+
+Result: TypeAlias = Ok[_TOk_co, _TError_co] | Error[_TOk_co, _TError_co]
diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py
new file mode 100644
index 00000000000..2088dc77252
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py
@@ -0,0 +1,334 @@
+"""The upstream-credential vocabulary — the typed seam the resolver dispatches on.
+
+This module ships the data types only; the resolver lands in a later PR. It is the contract
+the credential build implements and the spec tests assert against.
+
+Design invariants encoded here:
+
+- **Mode is the single source of truth.** A server declares exactly one per-mode `config`
+ (the `AuthConfig` discriminated union); `auth_spec_kind` is *derived* from it, never a
+ second field that can drift. The resolver dispatches on the config variant, one arm per
+ mode. No field-presence inference, no precedence cascade.
+- **Illegal states unrepresentable.** Each mode's config is its own frozen model holding
+ only that mode's fields — an `aws_sigv4` server cannot hold OAuth fields, and a config
+ missing a required field is rejected at construction, not at call time.
+- **Fail-closed at the boundary.** A raw mode string can only enter through
+ `parse_auth_spec_kind()`, which returns a typed `CredError`.
+- **Errors as values.** Every seam returns `Result[_, CredError]`; only edge adapters raise.
+- **No v1 imports.** This vocabulary stays free of `MCPServer` and the rest of v1; the
+ v1 -> v2 adapter maps onto these types in a later PR.
+
+Sum types are Expression `@tagged_union`s discriminated on a `Literal` `tag`, matched via
+`self.tag` with an `assert_never` tail; `Result` is this package's vendored `Ok | Error`
+union (see `result.py`), not `expression.Result`.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Annotated, Literal
+
+from expression import case, tag, tagged_union
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+from typing_extensions import assert_never
+
+from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
+ Error,
+ Ok,
+ Result,
+)
+
+
+class AuthSpecKind(str, Enum):
+ """The server's statically-declared upstream-auth mode — derived from its `config`.
+
+ Covers v1's full `MCPAuth` surface, not only OAuth grants: the three grant modes, the
+ collapsed static-header family, client passthrough, no-auth, and AWS request signing.
+ BYOK is *not* a member: it is the `api_key` mode seeded per-user, a source selector
+ inside that arm. The static-header schemes v1 splits into separate `MCPAuth` values
+ (`bearer_token`/`api_key`/`basic`/`token`/`authorization`) collapse into `api_key`; the
+ scheme is a parameter the arm carries, not its own mode.
+ """
+
+ authorization_code = "authorization_code" # per-user 3LO; gateway-stored token
+ client_credentials = "client_credentials" # gateway service account (M2M)
+ token_exchange = "token_exchange" # RFC 8693: token endpoint + subject_token (OBO)
+ api_key = "api_key" # static header, any scheme (BYOK = per-user-seeded source)
+ passthrough = "passthrough" # client forwards an upstream-audience token
+ none = "none" # no upstream credential; resolve yields a no-op auth, never an error
+ aws_sigv4 = "aws_sigv4" # AWS SigV4 per-request signing (e.g. Bedrock AgentCore)
+
+
+@tagged_union(frozen=True)
+class CredError:
+ """Why a credential could not be produced. Fail-closed: an arm yields this or an `httpx.Auth`.
+
+ Discriminated on the `Literal` `tag`; consumers `match self.tag` (see `summary`) so the
+ type checker can prove exhaustiveness. Construct via the `of_*` factories.
+ """
+
+ tag: Literal[
+ "unauthorized",
+ "misconfigured",
+ "upstream_unavailable",
+ "unsupported_mode",
+ "precondition_required",
+ "not_implemented",
+ ] = tag()
+
+ unauthorized: str = (
+ case()
+ ) # no usable credential for this (subject, server) -> 401 challenge
+ misconfigured: str = (
+ case()
+ ) # the declared mode is missing required config -> 5xx (operator)
+ upstream_unavailable: str = (
+ case()
+ ) # the IdP / token endpoint could not be reached -> 503
+ unsupported_mode: str = (
+ case()
+ ) # a raw mode string did not parse into AuthSpecKind (boundary)
+ precondition_required: str = (
+ case()
+ ) # a required per-user value (e.g. an env var) has not been provided -> 412
+ not_implemented: str = (
+ case()
+ ) # the declared mode's resolver arm is not built yet -> 501 (not operator error)
+
+ @staticmethod
+ def of_unauthorized(detail: str) -> CredError:
+ return CredError(unauthorized=detail)
+
+ @staticmethod
+ def of_misconfigured(detail: str) -> CredError:
+ return CredError(misconfigured=detail)
+
+ @staticmethod
+ def of_upstream_unavailable(detail: str) -> CredError:
+ return CredError(upstream_unavailable=detail)
+
+ @staticmethod
+ def of_unsupported_mode(detail: str) -> CredError:
+ return CredError(unsupported_mode=detail)
+
+ @staticmethod
+ def of_precondition_required(detail: str) -> CredError:
+ return CredError(precondition_required=detail)
+
+ @staticmethod
+ def of_not_implemented(detail: str) -> CredError:
+ return CredError(not_implemented=detail)
+
+ @property
+ def summary(self) -> str:
+ # Exhaustiveness: every Literal tag has an arm; the trailing assert_never typechecks
+ # only while that stays true (a `case _` would defeat reportMatchNotExhaustive).
+ match self.tag:
+ case "unauthorized":
+ return f"unauthorized: {self.unauthorized}"
+ case "misconfigured":
+ return f"misconfigured: {self.misconfigured}"
+ case "upstream_unavailable":
+ return f"upstream unavailable: {self.upstream_unavailable}"
+ case "unsupported_mode":
+ return self.unsupported_mode
+ case "precondition_required":
+ return f"precondition required: {self.precondition_required}"
+ case "not_implemented":
+ return f"not implemented: {self.not_implemented}"
+ assert_never(self.tag)
+
+
+class AuthorizationCodeConfig(BaseModel):
+ """Per-user 3LO; the gateway is the OAuth client and stores the user's token.
+
+ Endpoints are discovered (RFC 9728 -> RFC 8414) and the client is registered via DCR
+ (RFC 7591), so the common case carries none of the fields below; they are optional manual
+ overrides for IdPs without discovery / DCR. The per-user token is read from the token store
+ at resolve time, not held here.
+ """
+
+ model_config = ConfigDict(frozen=True)
+ kind: Literal[AuthSpecKind.authorization_code] = AuthSpecKind.authorization_code
+ scopes: tuple[str, ...] = ()
+ client_id: str | None = None
+ client_secret: SecretStr | None = None
+ authorization_url: str | None = None
+ token_url: str | None = None
+
+
+class ClientCredentialsConfig(BaseModel):
+ """M2M service account; one upstream identity for every user.
+
+ Fields are optional so the config can be built incomplete: a value may be supplied at
+ runtime (`token_url` via RFC 8414 discovery, `client_id`/`secret` via DCR), and the
+ resolver arm raises `CredError.misconfigured` when a needed field is still absent.
+ """
+
+ model_config = ConfigDict(frozen=True)
+ kind: Literal[AuthSpecKind.client_credentials] = AuthSpecKind.client_credentials
+ client_id: str | None = None
+ client_secret: SecretStr | None = None
+ token_url: str | None = None
+ scopes: tuple[str, ...] = ()
+
+
+class TokenExchangeConfig(BaseModel):
+ """RFC 8693 OBO; swap the caller's live subject_token for a token bound to the upstream's
+ audience (`server.resource`, RFC 8707). The gateway authenticates to the exchange endpoint
+ as an OAuth client (`client_id`/`client_secret`); the inbound token is sent only to that
+ endpoint, never to the upstream.
+ """
+
+ model_config = ConfigDict(frozen=True)
+ kind: Literal[AuthSpecKind.token_exchange] = AuthSpecKind.token_exchange
+ subject_token_type: str = "urn:ietf:params:oauth:token-type:access_token"
+ token_exchange_endpoint: str | None = None
+ client_id: str | None = None
+ client_secret: SecretStr | None = None
+ scopes: tuple[str, ...] = ()
+
+
+class SharedKey(BaseModel):
+ """A fixed key configured on the server, identical for every caller."""
+
+ model_config = ConfigDict(frozen=True)
+ source: Literal["shared"] = "shared"
+ value: SecretStr
+
+
+class Byok(BaseModel):
+ """A key the user brings via the entry flow, stored per-user and pulled from the credential
+ store at resolve time. Missing means the user must provide it, a 401 + WWW-Authenticate
+ challenge."""
+
+ model_config = ConfigDict(frozen=True)
+ source: Literal["byok"] = "byok"
+
+
+ApiKeySource = Annotated[SharedKey | Byok, Field(discriminator="source")]
+
+
+class ApiKeyConfig(BaseModel):
+ """A fixed credential injected as a header. The value is shared (in config) or seeded
+ per-user (pulled from the store); `header_name` and `value_prefix` say where and how it is
+ written, modeled like OpenAPI's apiKey scheme so any upstream convention is expressible
+ (Authorization + Bearer, a raw value on X-API-Key, Ocp-Apim-Subscription-Key, etc.).
+ """
+
+ model_config = ConfigDict(frozen=True)
+ kind: Literal[AuthSpecKind.api_key] = AuthSpecKind.api_key
+ header_name: str = "Authorization"
+ value_prefix: str = "Bearer"
+ key_source: ApiKeySource
+
+ def header(self, value: str) -> tuple[str, str]:
+ formatted = f"{self.value_prefix} {value}" if self.value_prefix else value
+ return self.header_name, formatted
+
+
+class PassthroughConfig(BaseModel):
+ """Client-driven upstream OAuth; the gateway forwards the client's upstream token."""
+
+ model_config = ConfigDict(frozen=True)
+ kind: Literal[AuthSpecKind.passthrough] = AuthSpecKind.passthrough
+
+
+class NoneConfig(BaseModel):
+ """No upstream credential; the request is sent unauthenticated."""
+
+ model_config = ConfigDict(frozen=True)
+ kind: Literal[AuthSpecKind.none] = AuthSpecKind.none
+
+
+class StaticKeys(BaseModel):
+ """Long-lived AWS access keys configured on the server."""
+
+ model_config = ConfigDict(frozen=True)
+ source: Literal["static_keys"] = "static_keys"
+ access_key_id: str
+ secret_access_key: SecretStr
+ session_token: SecretStr | None = None
+
+
+class AssumeRole(BaseModel):
+ """An IAM role the gateway assumes via STS for short-lived, auto-refreshed credentials."""
+
+ model_config = ConfigDict(frozen=True)
+ source: Literal["assume_role"] = "assume_role"
+ role_arn: str
+ session_name: str | None = None
+ external_id: str | None = None
+
+
+class Ambient(BaseModel):
+ """The environment's default AWS credential chain (instance profile, IRSA, env vars)."""
+
+ model_config = ConfigDict(frozen=True)
+ source: Literal["ambient"] = "ambient"
+
+
+AwsCredentialSource = Annotated[
+ StaticKeys | AssumeRole | Ambient, Field(discriminator="source")
+]
+
+
+class AwsSigV4Config(BaseModel):
+ """AWS SigV4 per-request signing for an AWS-hosted upstream (e.g. Bedrock AgentCore). The
+ gateway signs with its own AWS identity, never the caller's; `credentials` selects how that
+ identity is obtained, defaulting to the ambient credential chain."""
+
+ model_config = ConfigDict(frozen=True)
+ kind: Literal[AuthSpecKind.aws_sigv4] = AuthSpecKind.aws_sigv4
+ region: str
+ service: str = "bedrock-agentcore"
+ credentials: AwsCredentialSource = Ambient()
+
+
+AuthConfig = Annotated[
+ AuthorizationCodeConfig
+ | ClientCredentialsConfig
+ | TokenExchangeConfig
+ | ApiKeyConfig
+ | PassthroughConfig
+ | NoneConfig
+ | AwsSigV4Config,
+ Field(discriminator="kind"),
+]
+
+
+class Subject(BaseModel):
+ """The validated inbound principal. NOT the v1 request object and NOT the LiteLLM key."""
+
+ model_config = ConfigDict(frozen=True)
+
+ tenant_id: str
+ subject_id: str
+ # Opaque, already-validated inbound identity. Only `token_exchange` / `passthrough` read it.
+ inbound_token: SecretStr | None = None
+
+
+class ServerSpec(BaseModel):
+ """The declared upstream. A v2-native type; the v1 -> v2 adapter maps onto this."""
+
+ model_config = ConfigDict(frozen=True)
+
+ server_id: str
+ resource: str # RFC 8707 audience URI this upstream's tokens are bound to
+ config: AuthConfig
+
+ @property
+ def auth_spec_kind(self) -> AuthSpecKind:
+ return self.config.kind
+
+
+def parse_auth_spec_kind(raw: str) -> Result[AuthSpecKind, CredError]:
+ """Boundary parser — the *only* place an unknown mode is handled, and it fails closed.
+
+ Inside the core the mode is always a valid `AuthSpecKind`, so the resolver never needs a
+ wildcard arm and basedpyright can prove its `match` exhaustive.
+ """
+ try:
+ return Ok(AuthSpecKind(raw))
+ except ValueError:
+ return Error(CredError.of_unsupported_mode(f"unknown auth_spec_kind: {raw!r}"))
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 08e42e918e9..e891425274f 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -63,7 +63,11 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import (
+ ProxyException,
+ SpecialMCPServerNames,
+ UserAPIKeyAuth,
+)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
@@ -229,6 +233,28 @@ def _jsonrpc_text_has_top_level_method(text: str) -> bool:
return False
+def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException:
+ """Map a ``ProxyException`` to an ``HTTPException`` that preserves its real
+ status code and headers.
+
+ ``user_api_key_auth`` raises ``ProxyException`` (not ``HTTPException``) on
+ auth failures. The MCP ASGI handlers re-raise ``HTTPException`` to keep the
+ status and any ``WWW-Authenticate`` challenge, but a ``ProxyException`` would
+ otherwise fall through to their generic handler and be flattened to a 500 —
+ dropping the 401 + challenge an OAuth client needs to re-authenticate, so the
+ tool call surfaces as a cancelled/terminated session instead.
+ """
+ try:
+ status_code = int(exc.code)
+ except (TypeError, ValueError):
+ status_code = 500
+ return HTTPException(
+ status_code=status_code,
+ detail=exc.message,
+ headers=exc.headers or None,
+ )
+
+
if MCP_AVAILABLE:
from mcp.server import Server
from mcp.server.lowlevel.server import NotificationOptions
@@ -3352,6 +3378,19 @@ if MCP_AVAILABLE:
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
+ # A key scoped to no MCP servers opts out of every MCP path. Enforce it
+ # here too, since toolset scoping replaces mcp_servers and would otherwise
+ # drop the sentinel. Checked before the admin branch, mirroring
+ # get_allowed_mcp_servers.
+ original_op = user_api_key_auth.object_permission
+ if original_op is not None and SpecialMCPServerNames.no_mcp_servers.value in (
+ original_op.mcp_servers or []
+ ):
+ raise HTTPException(
+ status_code=403,
+ detail="API key is scoped to no MCP servers; toolset access is denied.",
+ )
+
# Access control: non-admin keys must have this toolset in their grant list.
# Use _user_has_admin_view so that PROXY_ADMIN_VIEW_ONLY is also treated as admin.
is_admin = _user_has_admin_view(user_api_key_auth)
@@ -4006,6 +4045,12 @@ if MCP_AVAILABLE:
except HTTPException:
# Re-raise HTTP exceptions to preserve status codes and details
raise
+ except ProxyException as e:
+ # Auth failures from user_api_key_auth arrive as ProxyException, not
+ # HTTPException. Preserve the real status (e.g. 401 + WWW-Authenticate)
+ # so OAuth clients can re-authenticate instead of receiving a generic
+ # 500 that surfaces as a cancelled tool call.
+ raise _proxy_exception_to_http_exception(e)
except Exception as e:
verbose_logger.exception(f"Error handling MCP request: {e}")
# Try to send a graceful error response for non-HTTP exceptions
@@ -4123,6 +4168,12 @@ if MCP_AVAILABLE:
# Re-raise HTTP exceptions to preserve status codes and details
# (e.g. 401 + WWW-Authenticate challenges from OAuth pass-through).
raise
+ except ProxyException as e:
+ # Auth failures from user_api_key_auth arrive as ProxyException, not
+ # HTTPException. Preserve the real status (e.g. 401 + WWW-Authenticate)
+ # so OAuth clients can re-authenticate instead of receiving a generic
+ # 500 that surfaces as a cancelled tool call.
+ raise _proxy_exception_to_http_exception(e)
except Exception as e:
verbose_logger.exception(f"Error handling MCP request: {e}")
# Try to send a graceful error response for non-HTTP exceptions
diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py
index 97cfa74ea45..b0141d3207c 100644
--- a/litellm/proxy/_experimental/mcp_server/utils.py
+++ b/litellm/proxy/_experimental/mcp_server/utils.py
@@ -23,9 +23,20 @@ import os
from urllib.parse import quote
# Constants
-LITELLM_MCP_SERVER_NAME = "litellm-mcp-server"
+#
+# NOTE: The environment-backed values below are read once, when this module is
+# first imported, and cached for the lifetime of the process. Changing the
+# corresponding environment variables after import has no effect unless the
+# module is reloaded (e.g. ``importlib.reload``). Tests that override these
+# variables must reload this module — see
+# ``tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py``.
+LITELLM_MCP_SERVER_NAME = os.environ.get(
+ "LITELLM_MCP_SERVER_NAME", "litellm-mcp-server"
+)
LITELLM_MCP_SERVER_VERSION = "1.0.0"
-LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM"
+LITELLM_MCP_SERVER_DESCRIPTION = os.environ.get(
+ "LITELLM_MCP_SERVER_DESCRIPTION", "MCP Server for LiteLLM"
+)
MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-")
MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}"
diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html
index 45de348c4d5..6d246c81652 100644
--- a/litellm/proxy/_experimental/out/404.html
+++ b/litellm/proxy/_experimental/out/404.html
@@ -1 +1 @@
-404: This page could not be found.LiteLLM Dashboard
404
This page could not be found.
\ No newline at end of file
+404: This page could not be found.LiteLLM Dashboard
404
This page could not be found.
\ No newline at end of file
diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html
index 45de348c4d5..6d246c81652 100644
--- a/litellm/proxy/_experimental/out/404/index.html
+++ b/litellm/proxy/_experimental/out/404/index.html
@@ -1 +1 @@
-404: This page could not be found.LiteLLM Dashboard
404
This page could not be found.
\ No newline at end of file
+404: This page could not be found.LiteLLM Dashboard